From b94ef5e5ac602b0829e1420dc0a3d225e44fe66c Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 2 Nov 2017 09:25:19 +0100 Subject: [PATCH 1/4] added secret password to /rowers/record-progress --- rowers/longtask.py | 5 +++-- rowers/views.py | 24 +++++++++++++++++------- rowsandall_app/settings.py | 4 ++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/rowers/longtask.py b/rowers/longtask.py index f1fb8823..fda8bf90 100644 --- a/rowers/longtask.py +++ b/rowers/longtask.py @@ -61,7 +61,7 @@ def longtask(aantal,jobid=None,debug=False, return 1 -def longtask2(aantal,jobid=None,debug=False): +def longtask2(aantal,jobid=None,debug=False,secret=''): counter = 0 channel = 'tasks' @@ -80,7 +80,8 @@ def longtask2(aantal,jobid=None,debug=False): url = SITE_URL url += "/rowers/record-progress/" url += str(progress)+"/"+jobid - s = requests.get(url) + post_data = {"secret":secret} + s = requests.post(url, data=post_data) if debug: print url print s diff --git a/rowers/views.py b/rowers/views.py index 7b9114fa..17f9ba5f 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -14,6 +14,7 @@ from django.views.generic.base import TemplateView from django.db.models import Q from django import template from django.db import IntegrityError, transaction +from django.views.decorators.csrf import csrf_exempt from django.shortcuts import render from django.http import ( @@ -349,7 +350,8 @@ def test_job_view(request,aantal=100): def test_job_view2(request,aantal=100): - job = myqueue(queuehigh,long_test_task2,int(aantal)) + job = myqueue(queuehigh,long_test_task2,int(aantal), + secret=settings.PROGRESS_CACHE_SECRET) try: @@ -361,14 +363,22 @@ def test_job_view2(request,aantal=100): return HttpResponseRedirect(url) +@csrf_exempt def post_progress(request,id=None,value=0): - if id: - cache.set(id,value,3600) + if request.method == 'POST': + secret = request.POST['secret'] + if secret == settings.PROGRESS_CACHE_SECRET: + if id: + cache.set(id,value,3600) + # test + result = cache.get(id) - # test - result = cache.get(id) + return HttpResponse('progress cached '+str(result),status=200) + else: + return HttpResponse('access denied',status=400) - return HttpResponse('progress cached '+str(result),status=200) + else: + return HttpResponse('hi',status=200) def get_all_queued_jobs(userid=0): r = StrictRedis() @@ -9508,7 +9518,7 @@ def strokedataform(request,id=0): # Process the POSTed stroke data according to the API definition # Return the GET stroke data according to the API definition from rest_framework_swagger.renderers import OpenAPIRenderer, SwaggerUIRenderer -from django.views.decorators.csrf import csrf_exempt + @csrf_exempt @login_required() @api_view(['GET','POST']) diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index 7f8a63b2..ac9b9a8d 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -221,6 +221,10 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/' LOGIN_URL = '/login/' LOGOUT_URL = '/logout/' +# Update Cache with task progress password + +PROGRESS_CACHE_SECRET = CFG['progress_cache_secret'] + # Concept 2 C2_CLIENT_ID = CFG['c2_client_id'] C2_CLIENT_SECRET = CFG['c2_client_secret'] From c871d69097f8ca2f3a2ce68fc596d5b7e28fd9a1 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 2 Nov 2017 10:27:05 +0100 Subject: [PATCH 2/4] emailprocessing stops in a controlled way --- rowers/dataprep.py | 42 +++++++++++++++++++++++++++++++--------- rowers/mailprocessing.py | 2 ++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index ac9f8d7e..d583c73d 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -847,52 +847,65 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', def handle_nonpainsled(f2, fileformat, summary=''): oarlength = 2.89 inboard = 0.88 + hasrecognized = False # handle RowPro: if (fileformat == 'rp'): row = RowProParser(f2) + hasrecognized = True # handle TCX if (fileformat == 'tcx'): row = TCXParser(f2) + hasrecognized = True # handle Mystery if (fileformat == 'mystery'): row = MysteryParser(f2) + hasrecognized = True # handle RowPerfect if (fileformat == 'rowperfect3'): row = RowPerfectParser(f2) + hasrecognized = True # handle ErgData if (fileformat == 'ergdata'): row = ErgDataParser(f2) + hasrecognized = True # handle CoxMate if (fileformat == 'coxmate'): row = CoxMateParser(f2) + hasrecognized = True # handle Mike if (fileformat == 'bcmike'): row = BoatCoachAdvancedParser(f2) + hasrecognized = True # handle BoatCoach if (fileformat == 'boatcoach'): row = BoatCoachParser(f2) + hasrecognized = True # handle BoatCoach OTW if (fileformat == 'boatcoachotw'): row = BoatCoachOTWParser(f2) + hasrecognized = True # handle painsled desktop if (fileformat == 'painsleddesktop'): row = painsledDesktopParser(f2) + hasrecognized = True # handle speed coach GPS if (fileformat == 'speedcoach'): row = speedcoachParser(f2) + hasrecognized = True # handle speed coach GPS 2 if (fileformat == 'speedcoach2'): row = SpeedCoach2Parser(f2) + hasrecognized = True try: oarlength, inboard = get_empower_rigging(f2) summary = row.allstats() @@ -902,10 +915,12 @@ def handle_nonpainsled(f2, fileformat, summary=''): # handle ErgStick if (fileformat == 'ergstick'): row = ErgStickParser(f2) + hasrecognized = True # handle FIT if (fileformat == 'fit'): row = FITParser(f2) + hasrecognized = True try: s = fitsummarydata(f2) s.setsummary() @@ -913,6 +928,13 @@ def handle_nonpainsled(f2, fileformat, summary=''): except: pass + # Handle c2log + if (fileformat == 'c2log' or fileformat == 'rowprolog'): + return (0,0,0,0) + + if not hasrecognized: + return (0,0,0,0) + f_to_be_deleted = f2 # should delete file f2 = f2[:-4] + 'o.csv' @@ -971,7 +993,7 @@ def new_workout_from_file(r, f2, # Some people try to upload Concept2 logbook summaries if fileformat == 'c2log': os.remove(f2) - message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook." + message = "This summary does not contain stroke data. Use the files containing stroke by stroke data." return (0, message, f2) if fileformat == 'nostrokes': @@ -1021,14 +1043,16 @@ def new_workout_from_file(r, f2, return (0, message, '') dosummary = (fileformat != 'fit') - id, message = save_workout_database(f2, r, - workouttype=workouttype, - makeprivate=makeprivate, - dosummary=dosummary, - workoutsource=fileformat, - summary=summary, - inboard=inboard, oarlength=oarlength, - title=title) + id, message = save_workout_database( + f2, r, + workouttype=workouttype, + makeprivate=makeprivate, + dosummary=dosummary, + workoutsource=fileformat, + summary=summary, + inboard=inboard, oarlength=oarlength, + title=title + ) return (id, message, f2) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index b6489e61..bc478527 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -115,6 +115,8 @@ def make_new_workout_from_email(rower, datafile, name, cntr=0): if fileformat != 'csv': filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled( 'media/' + datafilename, fileformat, summary) + if not filename_mediadir: + return 0 else: filename_mediadir = 'media/' + datafilename inboard = 0.88 From c974927192fc9dece807eba4bcf835a1daf127fc Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 2 Nov 2017 15:53:54 +0100 Subject: [PATCH 3/4] processes files > 1 MB offline --- rowers/dataprep.py | 19 +++---- rowers/forms.py | 9 +++- rowers/templates/document_form.html | 29 ++++++++++- rowers/templates/team_document_form.html | 27 ++++++++++ rowers/views.py | 66 +++++++++++++++++++----- templates/basebase.html | 8 +-- 6 files changed, 129 insertions(+), 29 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index d583c73d..b7efd793 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -53,7 +53,7 @@ import sys import utils import datautils -from utils import lbstoN +from utils import lbstoN,myqueue from timezonefinder import TimezoneFinder @@ -975,17 +975,12 @@ def new_workout_from_file(r, f2, if len(fileformat) == 3 and fileformat[0] == 'zip': f_to_be_deleted = f2 title = os.path.basename(f2) - if settings.DEBUG: - res = handle_zip_file.delay( - r.user.email, title, f2 - ) - - else: - res = queuelow.enqueue( - handle_zip_file, - r.user.email, - title, - f2 + res = myqueue( + queuelow, + handle_zip_file, + r.user.email, + title, + f2 ) return -1, message, f2 diff --git a/rowers/forms.py b/rowers/forms.py index 4923a517..f2f40d1f 100644 --- a/rowers/forms.py +++ b/rowers/forms.py @@ -57,8 +57,15 @@ class DocumentsForm(forms.Form): notes = forms.CharField(required=False, widget=forms.Textarea) + offline = forms.BooleanField(initial=False,required=False, + label='Process in Background') class Meta: - fields = ['title','file','workouttype','fileformat'] + fields = ['title','file','workouttype','fileformat','offline'] + + def __init__(self, *args, **kwargs): + from django.forms.widgets import HiddenInput + super(DocumentsForm, self).__init__(*args, **kwargs) + self.fields['offline'].widget = HiddenInput() from utils import ( workflowleftpanel,workflowmiddlepanel, diff --git a/rowers/templates/document_form.html b/rowers/templates/document_form.html index 3f065765..53c88b5d 100644 --- a/rowers/templates/document_form.html +++ b/rowers/templates/document_form.html @@ -4,8 +4,35 @@ {% block title %}File loading{% endblock %} +{% block meta %} + + + +{% endblock %} + {% block content %} -
+

Upload Workout File

{% if user.is_authenticated and user|is_manager %} diff --git a/rowers/templates/team_document_form.html b/rowers/templates/team_document_form.html index aadc4f9e..37fc6a0a 100644 --- a/rowers/templates/team_document_form.html +++ b/rowers/templates/team_document_form.html @@ -3,6 +3,33 @@ {% block title %}File loading{% endblock %} +{% block meta %} + + + +{% endblock %} + {% block content %}
diff --git a/rowers/views.py b/rowers/views.py index 17f9ba5f..af5609da 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -101,7 +101,8 @@ from rowers.tasks import handle_makeplot,handle_otwsetpower,handle_sendemailtcx, from rowers.tasks import ( handle_sendemail_unrecognized,handle_sendemailnewcomment, handle_sendemailnewresponse, handle_updatedps, - handle_updatecp,long_test_task,long_test_task2 + handle_updatecp,long_test_task,long_test_task2, + handle_zip_file ) from scipy.signal import savgol_filter @@ -8244,6 +8245,7 @@ def workout_upload_view(request, } notes = form.cleaned_data['notes'] + offline = form.cleaned_data['offline'] if optionsform.is_valid(): make_plot = optionsform.cleaned_data['make_plot'] @@ -8276,12 +8278,29 @@ def workout_upload_view(request, f1 = res[0] # file name f2 = res[1] # file name incl media directory - - id,message,f2 = dataprep.new_workout_from_file(r,f2, - workouttype=workouttype, - makeprivate=makeprivate, - title = t, - notes='') + if not offline: + id,message,f2 = dataprep.new_workout_from_file( + r,f2, + workouttype=workouttype, + makeprivate=makeprivate, + title = t, + notes='' + ) + else: + job = myqueue( + queuehigh, + handle_zip_file, + r.user.email, + t, + f2) + + messages.info( + request, + "The file was too large to process in real time. It will be processed in a background process. You will receive an email when it is ready") + url = reverse(workout_upload_view) + response = HttpResponseRedirect(url) + return response + if not id: messages.error(request,message) url = reverse(workout_upload_view) @@ -8450,6 +8469,7 @@ def team_workout_upload_view(request,message="", f = request.FILES['file'] res = handle_uploaded_file(f) t = form.cleaned_data['title'] + offline = form.cleaned_data['offline'] if rowerform.is_valid(): u = rowerform.cleaned_data['user'] if u: @@ -8490,11 +8510,33 @@ def team_workout_upload_view(request,message="", f2 = res[1] # file name incl media directory - id,message,f2 = dataprep.new_workout_from_file(r,f2, - workouttype=workouttype, - makeprivate=False, - title = t, - notes='') + if not offline: + id,message,f2 = dataprep.new_workout_from_file( + r,f2, + workouttype=workouttype, + makeprivate=False, + title = t, + notes='' + ) + else: + job = myqueue( + queuehigh, + handle_zip_file, + r.user.email, + t, + f2) + + messages.info( + request, + "The file was too large to process in real time. It will be processed in a background process. The user will receive an email when it is ready" + ) + + + url = reverse(team_workout_upload_view) + response = HttpResponseRedirect(url) + return response + + if not id: messages.error(request,message) url = reverse(team_workout_upload_view) diff --git a/templates/basebase.html b/templates/basebase.html index 6fe7b455..f3ca6ebb 100644 --- a/templates/basebase.html +++ b/templates/basebase.html @@ -226,7 +226,7 @@ var current = null;
-
+
{% if messages %} {% for message in messages %} {% if message.level == DEFAULT_MESSAGE_LEVELS.ERROR %} @@ -241,14 +241,16 @@ var current = null; {% block message %} {% if message %}

- {{ message }} + {{ message }}

{% endif %} {% if successmessage %}

- {{ successmessage }} + {{ successmessage }}

{% endif %} +
+
{% endblock %}
From bee89206cdb8471bf918792a5fc42dff539a7443 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 2 Nov 2017 18:20:13 +0100 Subject: [PATCH 4/4] new welcome email --- rowers/templates/rower_form.html | 2 ++ rowers/views.py | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/rowers/templates/rower_form.html b/rowers/templates/rower_form.html index 4f06fc9e..fc9a574b 100644 --- a/rowers/templates/rower_form.html +++ b/rowers/templates/rower_form.html @@ -4,6 +4,8 @@ {% block content %}
+

User Settings

+

Need help? Click to read the tutorial

Heart Rate Zones

diff --git a/rowers/views.py b/rowers/views.py index af5609da..752b02bd 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -712,13 +712,20 @@ def rower_register_view(request): # Create and send email fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "Thank you for registering on rowsandall.com" - message = "Thank you for registering on rowsandall.com. You can now login using the credentials you provided.\n" - message += "The first thing you might want to do is check and edit the heart rate band values. After logging in, click the button with your first name.\n" - message += "You can also check our videos page at http://rowsandall.com/rowers/videos for some helpful instruction videos.\n\n" - message += "User name:"+username+"\n" - message += "For all your questions, just reply to this email.\n\n" - message += "Happy rowing!\n\n\n" - message += "Oh, one more thing. The site is currently in beta and is developing fast. Bear with us. Don't hesitate to contact me if anything is broken or doesn't seem to work as advertised." + message = """ +Thank you for registering on rowsandall.com. You can now login using the credentials you provided. The first thing you should do is go to the Settings page and make yourself familiar with the various options available to personalize your experience on the website. To get there, click the button with your first name on it in the upper right corner of the opening page. + +As a minimum to get you started, enter your data in the Heart Rate Zones and Power Zones sections and your Functional Threshold Power. Then check your Account Information to make sure it is accurate and reflects your preferences. + +For additional details on these settings and the buttons at the bottom of the page, read the settings tutorial at http://analytics.rowsandall.com/2017/11/02/rowsandall-settings-page-tutorial/. + +Also check out our instructional videos at http://rowsandall.com/rowers/videos. + +This website is a labor of love "by rowers, for rowers". If you find it to be useful, please help us cover our hosting costs and gain access to additional functionality by signing on as a Pro member: https://rowsandall.com/rowers/promembership + +Oh, one more thing. The site is currently in beta and is developing fast. Bear with us. Don't hesitate to contact me at info@rowsandall.com if anything is broken or doesn't seem to work as advertised. +""" + send_mail(subject, message, 'Sander Roosendaal ', [fullemail])