From 9101a5aca53c71a30d0fb5390c75f2e62804913b Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Oct 2017 20:28:03 +0200 Subject: [PATCH 01/15] added links to workflow page --- rowers/interactiveplots.py | 11 +++++++- rowers/templates/advancededit.html | 7 ++++- rowers/templates/advancedotw.html | 7 ++++- rowers/templates/flexchart3otw.html | 7 ++++- rowers/templates/map_view.html | 8 +++++- rowers/templates/otwsetpower.html | 10 +++++-- rowers/templates/panel_map.html | 9 ++++++ rowers/templates/panel_mapview.html | 5 ++++ rowers/templates/streamedit.html | 5 +++- rowers/templates/summary_edit.html | 2 +- rowers/templates/windedit.html | 5 +++- rowers/templates/workout_form.html | 7 ++++- rowers/templates/workoutstats.html | 2 +- rowers/utils.py | 4 ++- rowers/views.py | 44 +++++++++++++++++++++++++++-- 15 files changed, 118 insertions(+), 15 deletions(-) create mode 100644 rowers/templates/panel_map.html create mode 100644 rowers/templates/panel_mapview.html diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 242e6a53..12c0a1ec 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -2510,7 +2510,16 @@ def thumbnails_set(r,id,favorites): columns += [f.yparam2 for f in favorites] columns += ['time'] - rowdata = dataprep.getsmallrowdata_db(columns,ids=[id],doclean=True) + try: + rowdata = dataprep.getsmallrowdata_db(columns,ids=[id],doclean=True) + except: + return [ + {'script':"", + 'div':"", + 'notes':"" + }] + + rowdata.dropna(axis=1,how='all',inplace=True) if rowdata.empty: diff --git a/rowers/templates/advancededit.html b/rowers/templates/advancededit.html index a79ebad3..0df1fff3 100644 --- a/rowers/templates/advancededit.html +++ b/rowers/templates/advancededit.html @@ -24,7 +24,12 @@ Edit Workout

-
+
+

+ Workflow View +

+
+

Export

diff --git a/rowers/templates/advancedotw.html b/rowers/templates/advancedotw.html index 7bd5340b..3c4999fc 100644 --- a/rowers/templates/advancedotw.html +++ b/rowers/templates/advancedotw.html @@ -23,7 +23,12 @@ Edit Workout

-
+
+

+ Workflow View +

+
+

Export

diff --git a/rowers/templates/flexchart3otw.html b/rowers/templates/flexchart3otw.html index 5da6e48c..02e698d3 100644 --- a/rowers/templates/flexchart3otw.html +++ b/rowers/templates/flexchart3otw.html @@ -32,7 +32,12 @@ Edit Workout

-
+
+

+ Workflow View +

+
+

Advanced Edit

diff --git a/rowers/templates/map_view.html b/rowers/templates/map_view.html index e48ec411..cde62d7f 100644 --- a/rowers/templates/map_view.html +++ b/rowers/templates/map_view.html @@ -43,7 +43,13 @@ Edit Workout

-
+
+

+ Workflow View +

+ +
+

Advanced Edit

diff --git a/rowers/templates/otwsetpower.html b/rowers/templates/otwsetpower.html index 30ad309d..51218c6a 100644 --- a/rowers/templates/otwsetpower.html +++ b/rowers/templates/otwsetpower.html @@ -14,7 +14,13 @@ Edit Workout

-
+
+

+ Workflow View +

+ +
+

Advanced Edit

@@ -62,4 +68,4 @@ -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/rowers/templates/panel_map.html b/rowers/templates/panel_map.html new file mode 100644 index 00000000..9fd40d8a --- /dev/null +++ b/rowers/templates/panel_map.html @@ -0,0 +1,9 @@ +
+ {{ mapdiv|safe }} + + + {{ mapscript|safe }} +
+ + + diff --git a/rowers/templates/panel_mapview.html b/rowers/templates/panel_mapview.html new file mode 100644 index 00000000..0d819ffa --- /dev/null +++ b/rowers/templates/panel_mapview.html @@ -0,0 +1,5 @@ +
+

+ Map View +

+
diff --git a/rowers/templates/streamedit.html b/rowers/templates/streamedit.html index 1e4ba1a0..b12def3c 100644 --- a/rowers/templates/streamedit.html +++ b/rowers/templates/streamedit.html @@ -29,7 +29,10 @@
-
+
+

Workflow View

+
+

Wind Edit

diff --git a/rowers/templates/summary_edit.html b/rowers/templates/summary_edit.html index 8c3cf177..2d49567a 100644 --- a/rowers/templates/summary_edit.html +++ b/rowers/templates/summary_edit.html @@ -23,7 +23,7 @@

- Export + Workflow View

diff --git a/rowers/templates/windedit.html b/rowers/templates/windedit.html index d49dd272..e0eb56bb 100644 --- a/rowers/templates/windedit.html +++ b/rowers/templates/windedit.html @@ -28,7 +28,10 @@
-
+
+

Workflow View

+
+

Stream Edit

diff --git a/rowers/templates/workout_form.html b/rowers/templates/workout_form.html index 37cd8956..26edc221 100644 --- a/rowers/templates/workout_form.html +++ b/rowers/templates/workout_form.html @@ -44,7 +44,12 @@
-
+
+

+ Workflow View +

+
+

Map View

diff --git a/rowers/templates/workoutstats.html b/rowers/templates/workoutstats.html index 75dc0f8c..3a02f214 100644 --- a/rowers/templates/workoutstats.html +++ b/rowers/templates/workoutstats.html @@ -18,7 +18,7 @@

- Export + Workflow View

diff --git a/rowers/utils.py b/rowers/utils.py index 205c74e9..89f181d2 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -9,6 +9,7 @@ workflowmiddlepanel = ( ('panel_statcharts.html','Static Charts'), ('flexthumbnails.html','Flex Charts'), ('panel_summary.html','Summary'), + ('panel_map.html','Map'), ) defaultmiddle = ['panel_statcharts.html', @@ -25,7 +26,8 @@ workflowleftpanel = ( ('panel_geekyheader.html','Geeky Header'), ('panel_editwind.html','Edit Wind Data'), ('panel_editstream.html','Edit Stream Data'), - ('panel_otwpower.html','Run OTW Power Calculations') + ('panel_otwpower.html','Run OTW Power Calculations'), + ('panel_mapview.html','Map'), ) defaultleft = [ diff --git a/rowers/views.py b/rowers/views.py index 0faed633..7cbeaaf6 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -157,6 +157,27 @@ from interactiveplots import * # Define the API documentation schema_view = get_swagger_view(title='Rowsandall API') +# Test if row data include candidates +def rowhascoordinates(row): + # create interactive plot + f1 = row.csvfilename + u = row.user.user + r = getrower(u) + rowdata = rdata(f1) + hascoordinates = 1 + if rowdata != 0: + try: + latitude = rowdata.df[' latitude'] + if not latitude.std(): + hascoordinates = 0 + except KeyError,AttributeError: + hascoordinates = 0 + + else: + hascoordinates = 0 + + return hascoordinates + # Custom error pages with Rowsandall headers def error500_view(request): response = render_to_response('500.html', {}, @@ -5231,7 +5252,11 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): kwargs = { 'id':int(id)} - url = reverse(workout_advanced_view,kwargs=kwargs) + try: + url = request.session['referer'] + except KeyError: + url = reverse(workout_advanced_view,kwargs=kwargs) + response = HttpResponseRedirect(url) return response @@ -6006,8 +6031,21 @@ def workout_workflow_view(request,id): charts = [] - if favorites: + if favorites and 'flexthumbnails.html' in r.workflowmiddlepanel: charts = thumbnails_set(r,id,favorites) + if charts[0]['script'] == '': + charts = [] + + if 'panel_map.html' in r.workflowmiddlepanel and rowhascoordinates(row): + rowdata = rdata(row.csvfilename) + mapscript,mapdiv = leaflet_chart2(rowdata.df[' latitude'], + rowdata.df[' longitude'], + row.name) + else: + mapscript = '' + mapdiv = '' + + statcharts = GraphImage.objects.filter(workout=row) @@ -6023,6 +6061,8 @@ def workout_workflow_view(request,id): 'leftTemplates':leftTemplates, 'charts':charts, 'workout':row, + 'mapscript':mapscript, + 'mapdiv':mapdiv, 'statcharts':statcharts, }) From a0a3ef9f60bc9b1b84bc82d72a7e06b4f8ea42cc Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Oct 2017 20:38:29 +0200 Subject: [PATCH 02/15] added basic info part to workflow middle panel --- rowers/templates/panel_comments.html | 31 ++++++++++++++++++++++++++++ rowers/utils.py | 1 + 2 files changed, 32 insertions(+) create mode 100644 rowers/templates/panel_comments.html diff --git a/rowers/templates/panel_comments.html b/rowers/templates/panel_comments.html new file mode 100644 index 00000000..99de3bbc --- /dev/null +++ b/rowers/templates/panel_comments.html @@ -0,0 +1,31 @@ +{% load rowerfilters %} +{% load tz %} +
+ + +{% localtime on %} + +{% endlocaltime %} + + + + + + + + + + + + + + +
Date/Time:{{ workout.startdatetime|localtime}}
Distance:{{ workout.distance }}m
Duration:{{ workout.duration |durationprint:"%H:%M:%S.%f" }}
Public link to this workout + https://rowsandall.com/rowers/workout/{{ workout.id }} +
Comments + Comment ({{ aantalcomments }}) +
Public link to interactive chart + https://rowsandall.com/rowers/workout/{{ workout.id }}/interactiveplot + +
+
diff --git a/rowers/utils.py b/rowers/utils.py index 89f181d2..8f2e62a8 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -10,6 +10,7 @@ workflowmiddlepanel = ( ('flexthumbnails.html','Flex Charts'), ('panel_summary.html','Summary'), ('panel_map.html','Map'), + ('panel_comments.html','Basic Info and Links'), ) defaultmiddle = ['panel_statcharts.html', From 157c0373fed1e2aa9c068279be723cca24fc7846 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Oct 2017 21:32:37 +0200 Subject: [PATCH 03/15] implemented most of Mike's suggestions --- rowers/templates/export.html | 7 +++++- rowers/templates/panel_delete.html | 5 +++++ rowers/templates/panel_export.html | 5 +++++ rowers/templates/panel_middlesocial.html | 14 ++++++++++++ rowers/templates/panel_social.html | 14 ++++++++++++ rowers/templates/workflow.html | 1 - rowers/templatetags/do_include_maybe.py | 28 ++++++++++++++++++++++++ rowers/templatetags/include_maybe.py | 28 ++++++++++++++++++++++++ rowers/templatetags/rowerfilters.py | 1 - rowers/utils.py | 4 ++++ rowers/views.py | 20 ++++++++++++++--- 11 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 rowers/templates/panel_delete.html create mode 100644 rowers/templates/panel_export.html create mode 100644 rowers/templates/panel_middlesocial.html create mode 100644 rowers/templates/panel_social.html create mode 100644 rowers/templatetags/do_include_maybe.py create mode 100644 rowers/templatetags/include_maybe.py diff --git a/rowers/templates/export.html b/rowers/templates/export.html index 6ccd0bb6..b01ee514 100644 --- a/rowers/templates/export.html +++ b/rowers/templates/export.html @@ -15,7 +15,12 @@ Edit Workout

-
+
+

+ Workflow View +

+
+

Advanced Edit

diff --git a/rowers/templates/panel_delete.html b/rowers/templates/panel_delete.html new file mode 100644 index 00000000..1668a635 --- /dev/null +++ b/rowers/templates/panel_delete.html @@ -0,0 +1,5 @@ +
+

+ Delete +

+
diff --git a/rowers/templates/panel_export.html b/rowers/templates/panel_export.html new file mode 100644 index 00000000..543df100 --- /dev/null +++ b/rowers/templates/panel_export.html @@ -0,0 +1,5 @@ +
+

+ Export Workout +

+
diff --git a/rowers/templates/panel_middlesocial.html b/rowers/templates/panel_middlesocial.html new file mode 100644 index 00000000..b1c917d2 --- /dev/null +++ b/rowers/templates/panel_middlesocial.html @@ -0,0 +1,14 @@ +
+
+
+ Share +
+
+
+ +
+ + diff --git a/rowers/templates/panel_social.html b/rowers/templates/panel_social.html new file mode 100644 index 00000000..a1c993de --- /dev/null +++ b/rowers/templates/panel_social.html @@ -0,0 +1,14 @@ +
+
+ Share +
+
+
+

 

+ +
+ + diff --git a/rowers/templates/workflow.html b/rowers/templates/workflow.html index 01a0d55c..1a632bae 100644 --- a/rowers/templates/workflow.html +++ b/rowers/templates/workflow.html @@ -3,7 +3,6 @@ {% load rowerfilters %} {% load tz %} - {% get_current_timezone as TIME_ZONE %} {% block title %}{{ workout.name }}{% endblock %} diff --git a/rowers/templatetags/do_include_maybe.py b/rowers/templatetags/do_include_maybe.py new file mode 100644 index 00000000..d6c0a39e --- /dev/null +++ b/rowers/templatetags/do_include_maybe.py @@ -0,0 +1,28 @@ +from django import template +from django.template.loader_tags import do_include +from django.template.defaulttags import CommentNode +register = template.Library() + +@register.tag('do_include_maybe') +def do_include_maybe(parser, token): + "Source: http://stackoverflow.com/a/18951166/15690" + bits = token.split_contents() + if len(bits) < 2: + raise template.TemplateSyntaxError( + "%r tag takes at least one argument: " + "the name of the template to be included." % bits[0]) + + try: + silent_node = do_include(parser, token) + except template.TemplateDoesNotExist: + # Django < 1.7 + return CommentNode() + + _orig_render = silent_node.render + def wrapped_render(*args, **kwargs): + try: + return _orig_render(*args, **kwargs) + except template.TemplateDoesNotExist: + return CommentNode() + silent_node.render = wrapped_render + return silent_node diff --git a/rowers/templatetags/include_maybe.py b/rowers/templatetags/include_maybe.py new file mode 100644 index 00000000..3e3d2e9b --- /dev/null +++ b/rowers/templatetags/include_maybe.py @@ -0,0 +1,28 @@ +from django import template +from django.template.loader_tags import do_include +from django.template.defaulttags import CommentNode +register = template.Library() + +@register.tag('include_maybe') +def do_include_maybe(parser, token): + "Source: http://stackoverflow.com/a/18951166/15690" + bits = token.split_contents() + if len(bits) < 2: + raise template.TemplateSyntaxError( + "%r tag takes at least one argument: " + "the name of the template to be included." % bits[0]) + + try: + silent_node = do_include(parser, token) + except template.TemplateDoesNotExist: + # Django < 1.7 + return CommentNode() + + _orig_render = silent_node.render + def wrapped_render(*args, **kwargs): + try: + return _orig_render(*args, **kwargs) + except template.TemplateDoesNotExist: + return CommentNode() + silent_node.render = wrapped_render + return silent_node diff --git a/rowers/templatetags/rowerfilters.py b/rowers/templatetags/rowerfilters.py index 12cd75b1..123faa9e 100644 --- a/rowers/templatetags/rowerfilters.py +++ b/rowers/templatetags/rowerfilters.py @@ -118,7 +118,6 @@ def user_teams(user): return teams - @register.filter def has_teams(user): try: diff --git a/rowers/utils.py b/rowers/utils.py index 8f2e62a8..46ce0af2 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -11,6 +11,7 @@ workflowmiddlepanel = ( ('panel_summary.html','Summary'), ('panel_map.html','Map'), ('panel_comments.html','Basic Info and Links'), + ('panel_middlesocial.html','Social Media Share Buttons'), ) defaultmiddle = ['panel_statcharts.html', @@ -20,6 +21,9 @@ defaultmiddle = ['panel_statcharts.html', workflowleftpanel = ( ('panel_navigationheader.html','Navigation Header'), ('panel_editbuttons.html','Edit Workout Button'), + ('panel_delete.html','Delete Workout Button'), + ('panel_export.html','Export Workout Button'), + ('panel_social.html','Social Media Share Buttons'), ('panel_advancededit.html','Advanced Workout Edit Button'), ('panel_editintervals.html','Edit Intervals Button'), ('panel_stats.html','Workout Statistics Button'), diff --git a/rowers/views.py b/rowers/views.py index 7cbeaaf6..09238cbc 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -12,6 +12,7 @@ from PIL import Image from numbers import Number from django.views.generic.base import TemplateView from django.db.models import Q +from django import template from django.db import IntegrityError, transaction from django.shortcuts import render from django.http import ( @@ -6049,11 +6050,24 @@ def workout_workflow_view(request,id): statcharts = GraphImage.objects.filter(workout=row) - # This will be user configurable in the future - middleTemplates = r.workflowmiddlepanel - leftTemplates = r.workflowleftpanel + middleTemplates = [] + for t in r.workflowmiddlepanel: + try: + template.loader.get_template(t) + middleTemplates.append(t) + except template.TemplateDoesNotExist: + pass + leftTemplates = [] + for t in r.workflowleftpanel: + try: + template.loader.get_template(t) + leftTemplates.append(t) + except template.TemplateDoesNotExist: + pass + + return render(request, 'workflow.html', { From aa01ed6221c4e1c12a1a57f0d78ee1619d480a02 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Oct 2017 21:38:22 +0200 Subject: [PATCH 04/15] added some explanation --- rowers/templates/flexthumbnails.html | 1 + rowers/templates/workflow.html | 3 --- rowers/utils.py | 6 ++++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rowers/templates/flexthumbnails.html b/rowers/templates/flexthumbnails.html index 24d88ffc..e036ad44 100644 --- a/rowers/templates/flexthumbnails.html +++ b/rowers/templates/flexthumbnails.html @@ -1,5 +1,6 @@ {% if charts %}

Flex Charts

+

Click on the thumbnails to view the full chart.

{% for chart in charts %}
{{ forloop.counter }} diff --git a/rowers/templates/workflow.html b/rowers/templates/workflow.html index 1a632bae..87bae1e9 100644 --- a/rowers/templates/workflow.html +++ b/rowers/templates/workflow.html @@ -39,9 +39,6 @@ {% include templateName %} {% endfor %} {% endblock %} -
-

Click on the thumbnails to view the full chart

-
{% block middle_panel %} diff --git a/rowers/utils.py b/rowers/utils.py index 46ce0af2..23a04080 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -14,9 +14,11 @@ workflowmiddlepanel = ( ('panel_middlesocial.html','Social Media Share Buttons'), ) -defaultmiddle = ['panel_statcharts.html', +defaultmiddle = ['panel_middlesocial.html', + 'panel_statcharts.html', 'flexthumbnails.html', - 'panel_summary.html'] + 'panel_summary.html', + 'panel_map.html'] workflowleftpanel = ( ('panel_navigationheader.html','Navigation Header'), From c1626d8252772eaa1619d18362ddcdea620762bf Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Oct 2017 21:39:08 +0200 Subject: [PATCH 05/15] removed 2 unused files --- rowers/templatetags/do_include_maybe.py | 28 ------------------------- rowers/templatetags/include_maybe.py | 28 ------------------------- 2 files changed, 56 deletions(-) delete mode 100644 rowers/templatetags/do_include_maybe.py delete mode 100644 rowers/templatetags/include_maybe.py diff --git a/rowers/templatetags/do_include_maybe.py b/rowers/templatetags/do_include_maybe.py deleted file mode 100644 index d6c0a39e..00000000 --- a/rowers/templatetags/do_include_maybe.py +++ /dev/null @@ -1,28 +0,0 @@ -from django import template -from django.template.loader_tags import do_include -from django.template.defaulttags import CommentNode -register = template.Library() - -@register.tag('do_include_maybe') -def do_include_maybe(parser, token): - "Source: http://stackoverflow.com/a/18951166/15690" - bits = token.split_contents() - if len(bits) < 2: - raise template.TemplateSyntaxError( - "%r tag takes at least one argument: " - "the name of the template to be included." % bits[0]) - - try: - silent_node = do_include(parser, token) - except template.TemplateDoesNotExist: - # Django < 1.7 - return CommentNode() - - _orig_render = silent_node.render - def wrapped_render(*args, **kwargs): - try: - return _orig_render(*args, **kwargs) - except template.TemplateDoesNotExist: - return CommentNode() - silent_node.render = wrapped_render - return silent_node diff --git a/rowers/templatetags/include_maybe.py b/rowers/templatetags/include_maybe.py deleted file mode 100644 index 3e3d2e9b..00000000 --- a/rowers/templatetags/include_maybe.py +++ /dev/null @@ -1,28 +0,0 @@ -from django import template -from django.template.loader_tags import do_include -from django.template.defaulttags import CommentNode -register = template.Library() - -@register.tag('include_maybe') -def do_include_maybe(parser, token): - "Source: http://stackoverflow.com/a/18951166/15690" - bits = token.split_contents() - if len(bits) < 2: - raise template.TemplateSyntaxError( - "%r tag takes at least one argument: " - "the name of the template to be included." % bits[0]) - - try: - silent_node = do_include(parser, token) - except template.TemplateDoesNotExist: - # Django < 1.7 - return CommentNode() - - _orig_render = silent_node.render - def wrapped_render(*args, **kwargs): - try: - return _orig_render(*args, **kwargs) - except template.TemplateDoesNotExist: - return CommentNode() - silent_node.render = wrapped_render - return silent_node From f0bfb810a45b00a9aac6dac7faed2fd580fd09d3 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 11:11:30 +0200 Subject: [PATCH 06/15] made a landing page option --- rowers/forms.py | 4 ++++ rowers/models.py | 9 ++++++--- rowers/templates/list_workouts.html | 4 ++-- rowers/urls.py | 6 ++++-- rowers/utils.py | 6 ++++++ rowers/views.py | 15 ++++++++++++--- 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/rowers/forms.py b/rowers/forms.py index 7b7f2cc4..83de2878 100644 --- a/rowers/forms.py +++ b/rowers/forms.py @@ -12,6 +12,7 @@ import dataprep import types import datetime from django.forms import formset_factory +from utils import landingpages # login form class LoginForm(forms.Form): @@ -180,6 +181,9 @@ class UploadOptionsForm(forms.Form): makeprivate = forms.BooleanField(initial=False,required=False, label='Make Workout Private') + landingpage = forms.ChoiceField(choices=landingpages, + initial='workout_edit_view') + class Meta: fields = ['make_plot','plottype','upload_toc2','makeprivate'] diff --git a/rowers/models.py b/rowers/models.py index 5b7a89ac..fc40651a 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -198,7 +198,7 @@ class TeamRequest(models.Model): from utils import ( workflowleftpanel,workflowmiddlepanel, - defaultleft,defaultmiddle + defaultleft,defaultmiddle,landingpages ) # Extension of User with rowing specific data @@ -281,8 +281,10 @@ class Rower(models.Model): # Site Settings workflowleftpanel = TemplateListField(default=defaultleft) - workflowmiddlepanel = TemplateListField(default=defaultmiddle) + defaultlandingpage = models.CharField(default='workout_edit_view', + max_length=200, + choices=landingpages) # Access tokens c2token = models.CharField(default='',max_length=200,blank=True,null=True) @@ -889,7 +891,8 @@ class AccountRowerForm(ModelForm): class Meta: model = Rower fields = ['weightcategory','getemailnotifications', - 'defaulttimezone','showfavoritechartnotes'] + 'defaulttimezone','showfavoritechartnotes', + 'defaultlandingpage'] class UserForm(ModelForm): class Meta: diff --git a/rowers/templates/list_workouts.html b/rowers/templates/list_workouts.html index 9b61d44c..c04b7727 100644 --- a/rowers/templates/list_workouts.html +++ b/rowers/templates/list_workouts.html @@ -84,9 +84,9 @@ [RANKING PIECE] {% endif %} {% if workout.name != '' %} - {{ workout.name }} + {{ workout.name }} {% else %} - No Name + No Name {% endif %} {% else %} {% if workout.name != '' %} diff --git a/rowers/urls.py b/rowers/urls.py index 1691e9ae..a218c9b3 100644 --- a/rowers/urls.py +++ b/rowers/urls.py @@ -191,7 +191,8 @@ urlpatterns = [ url(r'^workout/compare/(?P\d+)/$',views.workout_comparison_list), url(r'^workout/compare2/(?P\d+)/(?P\d+)/(?P\w+.*)/(?P\w+.*)/$',views.workout_comparison_view), url(r'^workout/compare/(?P\d+)/(?P\d+-\d+-\d+)/(?P\w+.*)$',views.workout_comparison_list), - url(r'^workout/(?P\d+)/edit$',views.workout_edit_view), + url(r'^workout/(?P\d+)/edit$',views.workout_edit_view, + name='workout_edit_view'), url(r'^workout/(?P\d+)/navionics$',views.workout_edit_view_navionics), url(r'^workout/(?P\d+)/map$',views.workout_map_view), url(r'^workout/(?P\d+)/setprivate$',views.workout_setprivate_view), @@ -329,7 +330,8 @@ urlpatterns = [ url(r'^legal', TemplateView.as_view(template_name='legal.html'),name='legal'), url(r'^register$',views.rower_register_view), url(r'^register/thankyou/$', TemplateView.as_view(template_name='registerthankyou.html'), name='registerthankyou'), - url(r'^workout/(?P\d+)/workflow$',views.workout_workflow_view), + url(r'^workout/(?P\d+)/workflow$',views.workout_workflow_view, + name='workout_workflow_view'), url(r'^workout/(?P\d+)/flexchart/(?P\w+.*)/(?P\w+.*)/(?P\w+.*)/(?P\w+)/$',views.workout_flexchart3_view), url(r'^workout/(?P\d+)/flexchart/(?P\w+.*)/(?P\w+.*)/(?P\w+.*)/(?P\w+.*)$',views.workout_flexchart3_view), url(r'^workout/(?P\d+)/flexchart/(?P\w+.*)/(?P\w+.*)/(?P\w+.*)$',views.workout_flexchart3_view), diff --git a/rowers/utils.py b/rowers/utils.py index 23a04080..27133bf5 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -5,6 +5,12 @@ import colorsys lbstoN = 4.44822 +landingpages = ( + ('workout_edit_view','Edit View'), + ('workout_workflow_view','Workflow View'), +) + + workflowmiddlepanel = ( ('panel_statcharts.html','Static Charts'), ('flexthumbnails.html','Flex Charts'), diff --git a/rowers/views.py b/rowers/views.py index 09238cbc..ec138f07 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -4454,6 +4454,7 @@ def workouts_view(request,message='',successmessage='', return render(request, 'list_workouts.html', {'workouts': workouts, + 'rower':r, 'dateform':dateform, 'startdate':startdate, 'enddate':enddate, @@ -7587,13 +7588,17 @@ def workout_upload_view(request, 'make_plot':False, 'upload_to_C2':False, 'plottype':'timeplot', + 'landingpage':'workout_edit_view', }, docformoptions={ 'workouttype':'rower', }): + r = getrower(request.user) + if 'uploadoptions' in request.session: uploadoptions = request.session['uploadoptions'] + uploadoptions['landingpage'] = r.defaultlandingpage else: request.session['uploadoptions'] = uploadoptions @@ -7651,7 +7656,6 @@ def workout_upload_view(request, except KeyError: upload_totp = False - r = getrower(request.user) if request.method == 'POST': form = DocumentsForm(request.POST,request.FILES) optionsform = UploadOptionsForm(request.POST) @@ -7676,6 +7680,7 @@ def workout_upload_view(request, upload_to_ua = optionsform.cleaned_data['upload_to_MapMyFitness'] upload_to_tp = optionsform.cleaned_data['upload_to_TrainingPeaks'] makeprivate = optionsform.cleaned_data['makeprivate'] + landingpage = optionsform.cleaned_data['landingpage'] uploadoptions = { 'makeprivate':makeprivate, @@ -7687,6 +7692,7 @@ def workout_upload_view(request, 'upload_to_RunKeeper':upload_to_rk, 'upload_to_MapMyFitness':upload_to_ua, 'upload_to_TrainingPeaks':upload_to_tp, + 'landingpage':r.defaultlandingpage, } @@ -7809,8 +7815,8 @@ def workout_upload_view(request, messages.info(request,message) else: messages.error(request,message) - - url = reverse(workout_edit_view, + + url = reverse(landingpage, kwargs = { 'id':w.id, }) @@ -7826,6 +7832,7 @@ def workout_upload_view(request, return response else: form = DocumentsForm(initial=docformoptions) + print uploadoptions optionsform = UploadOptionsForm(initial=uploadoptions) return render(request, 'document_form.html', {'form':form, @@ -8778,6 +8785,7 @@ def rower_edit_view(request,message=""): first_name = ucd['first_name'] last_name = ucd['last_name'] email = ucd['email'] + defaultlandingpage = cd['defaultlandingpage'] weightcategory = cd['weightcategory'] getemailnotifications = cd['getemailnotifications'] defaulttimezone=cd['defaulttimezone'] @@ -8794,6 +8802,7 @@ def rower_edit_view(request,message=""): r.defaulttimezone=defaulttimezone r.weightcategory = weightcategory r.getemailnotifications = getemailnotifications + r.defaultlandingpage = defaultlandingpage r.save() form = RowerForm(instance=r) powerform = RowerPowerForm(instance=r) From dd05cf2d1ba6ab0a77eece753c25bf708150ca50 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 11:33:18 +0200 Subject: [PATCH 07/15] cleaning up --- rowers/dataprep.py | 1315 +++++++++++++-------------- rowers/forms.py | 3 +- rowers/templates/document_form.html | 17 +- rowers/views.py | 30 +- 4 files changed, 678 insertions(+), 687 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 92c5c37b..72d7cb91 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1,6 +1,6 @@ # All the data preparation, data cleaning and data mangling should # be defined here -from rowers.models import Workout, User, Rower,StrokeData +from rowers.models import Workout, User, Rower, StrokeData from rowingdata import rowingdata as rrdata from rowers.tasks import handle_sendemail_unrecognized @@ -9,25 +9,25 @@ import pytz from rowingdata import rower as rrower from rowingdata import main as rmain -from rowingdata import get_file_type,get_empower_rigging +from rowingdata import get_file_type, get_empower_rigging -from pandas import DataFrame,Series -from pytz import timezone as tz,utc +from pandas import DataFrame, Series +from pytz import timezone as tz, utc from django.utils import timezone -from time import strftime,strptime,mktime,time,daylight +from time import strftime, strptime, mktime, time, daylight import arrow from django.utils.timezone import get_current_timezone thetimezone = get_current_timezone() from rowingdata import ( - TCXParser,RowProParser,ErgDataParser, + TCXParser, RowProParser, ErgDataParser, CoxMateParser, - BoatCoachParser,RowPerfectParser,BoatCoachAdvancedParser, - MysteryParser,BoatCoachOTWParser, - painsledDesktopParser,speedcoachParser,ErgStickParser, - SpeedCoach2Parser,FITParser,fitsummarydata, + BoatCoachParser, RowPerfectParser, BoatCoachAdvancedParser, + MysteryParser, BoatCoachOTWParser, + painsledDesktopParser, speedcoachParser, ErgStickParser, + SpeedCoach2Parser, FITParser, fitsummarydata, make_cumvalues, - summarydata,get_file_type, - ) + summarydata, get_file_type, +) from rowers.models import Team from rowers.metrics import axes @@ -39,9 +39,9 @@ import numpy as np import itertools import math from tasks import ( - handle_sendemail_unrecognized,handle_sendemail_breakthrough, + handle_sendemail_unrecognized, handle_sendemail_breakthrough, handle_sendemail_hard - ) +) from django.conf import settings from sqlalchemy import create_engine @@ -61,7 +61,6 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') - user = settings.DATABASES['default']['USER'] password = settings.DATABASES['default']['PASSWORD'] database_name = settings.DATABASES['default']['NAME'] @@ -74,59 +73,61 @@ database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format( database_name=database_name, host=host, port=port, - ) +) # Use SQLite local database when we're in debug mode -if settings.DEBUG or user=='': +if settings.DEBUG or user == '': # database_url = 'sqlite:///db.sqlite3' - database_url = 'sqlite:///'+database_name + database_url = 'sqlite:///' + database_name # mapping the DB column names to the CSV file column names columndict = { - 'time':'TimeStamp (sec)', - 'hr':' HRCur (bpm)', - 'pace':' Stroke500mPace (sec/500m)', - 'spm':' Cadence (stokes/min)', - 'power':' Power (watts)', - 'averageforce':' AverageDriveForce (lbs)', - 'drivelength':' DriveLength (meters)', - 'peakforce':' PeakDriveForce (lbs)', - 'distance':' Horizontal (meters)', - 'catch':'catch', - 'finish':'finish', - 'peakforceangle':'peakforceangle', - 'wash':'wash', - 'slip':'slip', - 'workoutstate':' WorkoutState', - 'cumdist':'cum_dist', - } - + 'time': 'TimeStamp (sec)', + 'hr': ' HRCur (bpm)', + 'pace': ' Stroke500mPace (sec/500m)', + 'spm': ' Cadence (stokes/min)', + 'power': ' Power (watts)', + 'averageforce': ' AverageDriveForce (lbs)', + 'drivelength': ' DriveLength (meters)', + 'peakforce': ' PeakDriveForce (lbs)', + 'distance': ' Horizontal (meters)', + 'catch': 'catch', + 'finish': 'finish', + 'peakforceangle': 'peakforceangle', + 'wash': 'wash', + 'slip': 'slip', + 'workoutstate': ' WorkoutState', + 'cumdist': 'cum_dist', +} + from scipy.signal import savgol_filter import datetime + def get_latlon(id): try: w = Workout.objects.get(id=id) except Workout.DoesNotExist: return False - + rowdata = rdata(w.csvfilename) try: try: - latitude = rowdata.df.ix[:,' latitude'] - longitude = rowdata.df.ix[:,' longitude'] + latitude = rowdata.df.ix[:, ' latitude'] + longitude = rowdata.df.ix[:, ' longitude'] except KeyError: - latitude = 0*rowdata.df.ix[:,'TimeStamp (sec)'] - longitude = 0*rowdata.df.ix[:,'TimeStamp (sec)'] - return [latitude,longitude] + latitude = 0 * rowdata.df.ix[:, 'TimeStamp (sec)'] + longitude = 0 * rowdata.df.ix[:, 'TimeStamp (sec)'] + return [latitude, longitude] except AttributeError: - return [pd.Series([]),pd.Series([])] - - return [pd.Series([]),pd.Series([])] + return [pd.Series([]), pd.Series([])] -def get_workouts(ids,userid): + return [pd.Series([]), pd.Series([])] + + +def get_workouts(ids, userid): goodids = [] for id in ids: w = Workout.objects.get(id=id) @@ -135,21 +136,20 @@ def get_workouts(ids,userid): return [Workout.objects.get(id=id) for id in goodids] -def filter_df(datadf,fieldname,value,largerthan=True): + +def filter_df(datadf, fieldname, value, largerthan=True): try: x = datadf[fieldname] except KeyError: return datadf - if largerthan: mask = datadf[fieldname] < value else: mask = datadf[fieldname] >= value - datadf.loc[mask,fieldname] = np.nan - + datadf.loc[mask, fieldname] = np.nan return datadf @@ -160,9 +160,9 @@ def df_resample(datadf): datadf['timestamps'] = timestamps newdf = datadf.groupby(['timestamps']).mean() return newdf - -def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True, + +def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, ignoreadvanced=False): # clean data remove zeros and negative values @@ -173,18 +173,17 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True, pass try: - datadf['peakforceangle'] = datadf['peakforceangle']+1000 + datadf['peakforceangle'] = datadf['peakforceangle'] + 1000 except KeyError: pass - + try: - datadf['hr'] = datadf['hr']+10 + datadf['hr'] = datadf['hr'] + 10 except KeyError: pass - datadf=datadf.clip(lower=0) - datadf.replace(to_replace=0,value=np.nan,inplace=True) - + datadf = datadf.clip(lower=0) + datadf.replace(to_replace=0, value=np.nan, inplace=True) # return from positive domain to negative try: @@ -193,147 +192,131 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True, pass try: - datadf['peakforceangle'] = datadf['peakforceangle']-1000 + datadf['peakforceangle'] = datadf['peakforceangle'] - 1000 except KeyError: pass try: - datadf['hr'] = datadf['hr']-10 + datadf['hr'] = datadf['hr'] - 10 except KeyError: pass # clean data for useful ranges per column if not ignorehr: try: - mask = datadf['hr'] < 30 - datadf.loc[mask,'hr'] = np.nan + mask = datadf['hr'] < 30 + datadf.loc[mask, 'hr'] = np.nan except KeyError: pass try: mask = datadf['spm'] < 10 - datadf.loc[mask,'spm'] = np.nan + datadf.loc[mask, 'spm'] = np.nan except KeyError: pass - try: - mask = datadf['pace']/1000. > 300. - datadf.loc[mask,'pace'] = np.nan + mask = datadf['pace'] / 1000. > 300. + datadf.loc[mask, 'pace'] = np.nan except KeyError: pass try: mask = datadf['efficiency'] < 0. - datadf.loc[mask,'efficiency'] = np.nan - except KeyError: - pass - - try: - mask = datadf['pace']/1000. < 60. - datadf.loc[mask,'pace'] = np.nan - except KeyError: - pass - - try: - mask = datadf['spm'] > 60 - datadf.loc[mask,'spm'] = np.nan + datadf.loc[mask, 'efficiency'] = np.nan except KeyError: pass + try: + mask = datadf['pace'] / 1000. < 60. + datadf.loc[mask, 'pace'] = np.nan + except KeyError: + pass + + try: + mask = datadf['spm'] > 60 + datadf.loc[mask, 'spm'] = np.nan + except KeyError: + pass try: mask = datadf['wash'] < 1 - datadf.loc[mask,'wash'] = np.nan + datadf.loc[mask, 'wash'] = np.nan except KeyError: pass - if not ignoreadvanced: try: mask = datadf['rhythm'] < 5 - datadf.loc[mask,'rhythm'] = np.nan + datadf.loc[mask, 'rhythm'] = np.nan except KeyError: pass - try: mask = datadf['rhythm'] > 70 - datadf.loc[mask,'rhythm'] = np.nan + datadf.loc[mask, 'rhythm'] = np.nan except KeyError: pass - try: mask = datadf['power'] < 20 - datadf.loc[mask,'power'] = np.nan + datadf.loc[mask, 'power'] = np.nan except KeyError: pass - try: mask = datadf['drivelength'] < 0.5 - datadf.loc[mask,'drivelength'] = np.nan + datadf.loc[mask, 'drivelength'] = np.nan except KeyError: pass - try: mask = datadf['forceratio'] < 0.2 - datadf.loc[mask,'forceratio'] = np.nan + datadf.loc[mask, 'forceratio'] = np.nan except KeyError: pass - try: mask = datadf['forceratio'] > 1.0 - datadf.loc[mask,'forceratio'] = np.nan + datadf.loc[mask, 'forceratio'] = np.nan except KeyError: pass - - try: mask = datadf['drivespeed'] < 0.5 - datadf.loc[mask,'drivespeed'] = np.nan + datadf.loc[mask, 'drivespeed'] = np.nan except KeyError: pass - try: mask = datadf['drivespeed'] > 4 - datadf.loc[mask,'drivespeed'] = np.nan + datadf.loc[mask, 'drivespeed'] = np.nan except KeyError: pass - try: mask = datadf['driveenergy'] > 2000 - datadf.loc[mask,'driveenergy'] = np.nan + datadf.loc[mask, 'driveenergy'] = np.nan except KeyError: pass - try: mask = datadf['driveenergy'] < 100 - datadf.loc[mask,'driveenergy'] = np.nan + datadf.loc[mask, 'driveenergy'] = np.nan except KeyError: pass - - try: mask = datadf['catch'] > -30. - datadf.loc[mask,'catch'] = np.nan + datadf.loc[mask, 'catch'] = np.nan except KeyError: pass - - workoutstateswork = [1,4,5,8,9,6,7] + workoutstateswork = [1, 4, 5, 8, 9, 6, 7] workoutstatesrest = [3] - workoutstatetransition = [0,2,10,11,12,13] + workoutstatetransition = [0, 2, 10, 11, 12, 13] - if workstrokesonly=='True' or workstrokesonly==True: + if workstrokesonly == 'True' or workstrokesonly == True: try: datadf = datadf[~datadf['workoutstate'].isin(workoutstatesrest)] except: @@ -341,13 +324,14 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True, return datadf + def getstatsfields(): # Get field names and remove those that are not useful in stats fields = StrokeData._meta.get_fields() - fielddict = {field.name:field.verbose_name for field in fields} - - #fielddict.pop('workoutid') + fielddict = {field.name: field.verbose_name for field in fields} + + # fielddict.pop('workoutid') fielddict.pop('ergpace') fielddict.pop('hr_an') fielddict.pop('hr_tr') @@ -370,57 +354,62 @@ def getstatsfields(): fielddict.pop('hr_bottom') fielddict.pop('cumdist') - fieldlist = [field for field,value in fielddict.iteritems()] + fieldlist = [field for field, value in fielddict.iteritems()] - return fieldlist,fielddict + return fieldlist, fielddict # A string representation for time deltas def niceformat(values): out = [] for v in values: - formattedv = strfdelta(v) - out.append(formattedv) - + formattedv = strfdelta(v) + out.append(formattedv) + return out # A nice printable format for time delta values + + def strfdelta(tdelta): try: - minutes,seconds = divmod(tdelta.seconds,60) - tenths = int(tdelta.microseconds/1e5) + minutes, seconds = divmod(tdelta.seconds, 60) + tenths = int(tdelta.microseconds / 1e5) except AttributeError: - minutes,seconds = divmod(tdelta.view(np.int64),60e9) - seconds,rest = divmod(seconds,1e9) - tenths = int(rest/1e8) + minutes, seconds = divmod(tdelta.view(np.int64), 60e9) + seconds, rest = divmod(seconds, 1e9) + tenths = int(rest / 1e8) res = "{minutes:0>2}:{seconds:0>2}.{tenths:0>1}".format( - minutes=minutes, - seconds=seconds, - tenths=tenths, - ) - + minutes=minutes, + seconds=seconds, + tenths=tenths, + ) + return res # A nice printable format for pace values + + def nicepaceformat(values): out = [] for v in values: - formattedv = strfdelta(v) - out.append(formattedv) - + formattedv = strfdelta(v) + out.append(formattedv) return out # Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace + + def timedeltaconv(x): - if np.isfinite(x) and x != 0 and x>0 and x<175000: + if np.isfinite(x) and x != 0 and x > 0 and x < 175000: dt = datetime.timedelta(seconds=x) else: dt = datetime.timedelta(seconds=350.) - - + return dt + def paceformatsecs(values): out = [] for v in values: @@ -431,27 +420,29 @@ def paceformatsecs(values): return out # Processes painsled CSV file to database -def save_workout_database(f2,r,dosmooth=True,workouttype='rower', - dosummary=True,title='Workout', + + +def save_workout_database(f2, r, dosmooth=True, workouttype='rower', + dosummary=True, title='Workout', workoutsource='unknown', - notes='',totaldist=0,totaltime=0, + notes='', totaldist=0, totaltime=0, summary='', makeprivate=False, - oarlength=2.89,inboard=0.88, + oarlength=2.89, inboard=0.88, forceunit='lbs', consistencychecks=False): message = None - powerperc = 100*np.array([r.pw_ut2, - r.pw_ut1, - r.pw_at, - r.pw_tr,r.pw_an])/r.ftp + powerperc = 100 * np.array([r.pw_ut2, + r.pw_ut1, + r.pw_at, + r.pw_tr, r.pw_an]) / r.ftp # make workout and put in database - rr = rrower(hrmax=r.max,hrut2=r.ut2, - hrut1=r.ut1,hrat=r.at, - hrtr=r.tr,hran=r.an,ftp=r.ftp, - powerperc=powerperc,powerzones=r.powerzones) - row = rdata(f2,rower=rr) + rr = rrower(hrmax=r.max, hrut2=r.ut2, + hrut1=r.ut1, hrat=r.at, + hrtr=r.tr, hran=r.an, ftp=r.ftp, + powerperc=powerperc, powerzones=r.powerzones) + row = rdata(f2, rower=rr) dtavg = row.df['TimeStamp (sec)'].diff().mean() @@ -461,16 +452,17 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', os.remove(f2) except: pass - return new_workout_from_df(r,newdf, + return new_workout_from_df(r, newdf, title=title) try: checks = row.check_consistency() allchecks = 1 - for key,value in checks.iteritems(): + for key, value in checks.iteritems(): if not value: allchecks = 0 if consistencychecks: - a_messages.error(r.user,'Failed consistency check: '+key+', autocorrected') + a_messages.error( + r.user, 'Failed consistency check: ' + key + ', autocorrected') else: pass # a_messages.error(r.user,'Failed consistency check: '+key+', not corrected') @@ -480,92 +472,92 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', if not allchecks and consistencychecks: # row.repair() pass - - + if row == 0: - return (0,'Error: CSV data file not found') + return (0, 'Error: CSV data file not found') if dosmooth: # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values - velo = 500./pace - + velo = 500. / pace + f = row.df['TimeStamp (sec)'].diff().mean() - if f !=0 and not np.isnan(f): - windowsize = 2*(int(10./(f)))+1 + if f != 0 and not np.isnan(f): + windowsize = 2 * (int(10. / (f))) + 1 else: windowsize = 1 if not 'originalvelo' in row.df: - row.df['originalvelo'] = velo + row.df['originalvelo'] = velo - if windowsize > 3 and windowsize 3 and windowsize < len(velo): + velo2 = savgol_filter(velo, windowsize, 3) else: - velo2 = velo + velo2 = velo velo3 = pd.Series(velo2) - velo3 = velo3.replace([-np.inf,np.inf],np.nan) + velo3 = velo3.replace([-np.inf, np.inf], np.nan) velo3 = velo3.fillna(method='ffill') - - pace2 = 500./abs(velo3) - + + pace2 = 500. / abs(velo3) + row.df[' Stroke500mPace (sec/500m)'] = pace2 row.df = row.df.fillna(0) - row.write_csv(f2,gzip=True) + row.write_csv(f2, gzip=True) try: os.remove(f2) except: pass - + # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': - try: - row.erg_recalculatepower() - row.write_csv(f2,gzip=True) - except: - pass - + try: + row.erg_recalculatepower() + row.write_csv(f2, gzip=True) + except: + pass + averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() - + if totaldist == 0: totaldist = row.df['cum_dist'].max() if totaltime == 0: - totaltime = row.df['TimeStamp (sec)'].max()-row.df['TimeStamp (sec)'].min() + totaltime = row.df['TimeStamp (sec)'].max( + ) - row.df['TimeStamp (sec)'].min() try: - totaltime = totaltime+row.df.ix[0,' ElapsedTime (sec)'] + totaltime = totaltime + row.df.ix[0, ' ElapsedTime (sec)'] except KeyError: pass if np.isnan(totaltime): totaltime = 0 - hours = int(totaltime/3600.) - if hours>23: + hours = int(totaltime / 3600.) + if hours > 23: message = 'Warning: The workout duration was longer than 23 hours. ' hours = 23 - minutes = int((totaltime - 3600.*hours)/60.) - if minutes>59: + minutes = int((totaltime - 3600. * hours) / 60.) + if minutes > 59: minutes = 59 if not message: message = 'Warning: there is something wrong with the workout duration' - - seconds = int(totaltime - 3600.*hours - 60.*minutes) + + seconds = int(totaltime - 3600. * hours - 60. * minutes) if seconds > 59: seconds = 59 if not message: message = 'Warning: there is something wrong with the workout duration' - tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds)) + tenths = int(10 * (totaltime - 3600. * hours - 60. * minutes - seconds)) if tenths > 9: tenths = 9 if not message: message = 'Warning: there is something wrong with the workout duration' - - duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) + + duration = "%s:%s:%s.%s" % (hours, minutes, seconds, tenths) if dosummary: summary = row.allstats() @@ -573,7 +565,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', #summary += '\n' #summary += row.intervalstats() - #workoutstartdatetime = row.rowdatetime timezone_str = 'UTC' try: @@ -581,15 +572,13 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', except ValueError: workoutstartdatetime = row.rowdatetime - - try: latavg = row.df[' latitude'].mean() lonavg = row.df[' longitude'].mean() - + tf = TimezoneFinder() try: - timezone_str = tf.timezone_at(lng=lonavg,lat=latavg) + timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) except ValueError: timezone_str = 'UTC' if timezone_str == None: @@ -604,12 +593,10 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', except ValueError: workoutstartdatetime = workoutstartdatetime.astimezone( pytz.timezone(timezone_str) - ) + ) except KeyError: timezone_str = r.defaulttimezone - - workoutdate = workoutstartdatetime.astimezone( pytz.timezone(timezone_str) ).strftime('%Y-%m-%d') @@ -623,7 +610,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', privacy = 'visible' # checking for inf values - + totaldist = np.nan_to_num(totaldist) maxhr = np.nan_to_num(maxhr) averagehr = np.nan_to_num(averagehr) @@ -631,59 +618,57 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', # check for duplicate start times and duration ws = Workout.objects.filter(startdatetime=workoutstartdatetime, distance=totaldist, - user=r) + user=r) if (len(ws) != 0): - message = "Warning: This workout probably already exists in the database" + message = "Warning: This workout probably already exists in the database" privacy = 'hidden' - - - - w = Workout(user=r,name=title,date=workoutdate, - workouttype=workouttype, - duration=duration,distance=totaldist, - weightcategory=r.weightcategory, - starttime=workoutstarttime, + w = Workout(user=r, name=title, date=workoutdate, + workouttype=workouttype, + duration=duration, distance=totaldist, + weightcategory=r.weightcategory, + starttime=workoutstarttime, workoutsource=workoutsource, forceunit=forceunit, - csvfilename=f2,notes=notes,summary=summary, - maxhr=maxhr,averagehr=averagehr, - startdatetime=workoutstartdatetime, - inboard=inboard,oarlength=oarlength, + csvfilename=f2, notes=notes, summary=summary, + maxhr=maxhr, averagehr=averagehr, + startdatetime=workoutstartdatetime, + inboard=inboard, oarlength=oarlength, timezone=timezone_str, privacy=privacy) - w.save() isbreakthrough = False ishard = False if workouttype == 'water': - df = getsmallrowdata_db(['power','workoutid','time'],ids=[w.id]) + df = getsmallrowdata_db(['power', 'workoutid', 'time'], ids=[w.id]) # delta,cpvalues,avgpower = datautils.getsinglecp(row.df) thesecs = totaltime - maxt = 1.05*thesecs + maxt = 1.05 * thesecs if maxt > 0: logarr = datautils.getlogarr(maxt) dfgrouped = df.groupby(['workoutid']) - delta,cpvalues,avgpower = datautils.getcp(dfgrouped,logarr) - - res,btvalues,res2 = utils.isbreakthrough(delta,cpvalues,r.p0,r.p1,r.p2,r.p3,r.cpratio) + delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) + + res, btvalues, res2 = utils.isbreakthrough( + delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) else: res = 0 res2 = 0 if res: isbreakthrough = True - res = datautils.updatecp(delta,cpvalues,r) + res = datautils.updatecp(delta, cpvalues, r) if res2 and not isbreakthrough: ishard = True # submit email task to send email about breakthrough workout if isbreakthrough: - a_messages.info(r.user,'It looks like you have a new breakthrough workout') + a_messages.info( + r.user, 'It looks like you have a new breakthrough workout') if settings.DEBUG and r.getemailnotifications: - res = handle_sendemail_breakthrough.delay(w.id,r.user.email, - r.user.first_name, + res = handle_sendemail_breakthrough.delay(w.id, r.user.email, + r.user.first_name, r.user.last_name, btvalues=btvalues.to_json()) elif r.getemailnotifications: @@ -700,12 +685,12 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', pass # submit email task to send email about breakthrough workout if ishard: - a_messages.info(r.user,'That was a pretty hard workout') + a_messages.info(r.user, 'That was a pretty hard workout') if settings.DEBUG and r.getemailnotifications: - res = handle_sendemail_hard.delay(w.id,r.user.email, - r.user.first_name, - r.user.last_name, - btvalues=btvalues.to_json()) + res = handle_sendemail_hard.delay(w.id, r.user.email, + r.user.first_name, + r.user.last_name, + btvalues=btvalues.to_json()) elif r.getemailnotifications: try: res = queuehigh.enqueue( @@ -725,100 +710,100 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', w.team.add(t) # put stroke data in database - res = dataprep(row.df,id=w.id,bands=True, - barchart=True,otwpower=True,empower=True,inboard=inboard) + res = dataprep(row.df, id=w.id, bands=True, + barchart=True, otwpower=True, empower=True, inboard=inboard) - return (w.id,message) + return (w.id, message) -def handle_nonpainsled(f2,fileformat,summary=''): + +def handle_nonpainsled(f2, fileformat, summary=''): oarlength = 2.89 inboard = 0.88 # handle RowPro: if (fileformat == 'rp'): - row = RowProParser(f2) - # handle TCX - if (fileformat == 'tcx'): - row = TCXParser(f2) - + row = RowProParser(f2) + # handle TCX + if (fileformat == 'tcx'): + row = TCXParser(f2) + # handle Mystery if (fileformat == 'mystery'): - row = MysteryParser(f2) + row = MysteryParser(f2) # handle RowPerfect if (fileformat == 'rowperfect3'): row = RowPerfectParser(f2) - + # handle ErgData if (fileformat == 'ergdata'): - row = ErgDataParser(f2) + row = ErgDataParser(f2) - # handle CoxMate if (fileformat == 'coxmate'): - row = CoxMateParser(f2) + row = CoxMateParser(f2) # handle Mike if (fileformat == 'bcmike'): row = BoatCoachAdvancedParser(f2) - + # handle BoatCoach if (fileformat == 'boatcoach'): - row = BoatCoachParser(f2) + row = BoatCoachParser(f2) # handle BoatCoach OTW if (fileformat == 'boatcoachotw'): row = BoatCoachOTWParser(f2) - + # handle painsled desktop if (fileformat == 'painsleddesktop'): - row = painsledDesktopParser(f2) + row = painsledDesktopParser(f2) # handle speed coach GPS if (fileformat == 'speedcoach'): - row = speedcoachParser(f2) + row = speedcoachParser(f2) - # handle speed coach GPS 2 + # handle speed coach GPS 2 if (fileformat == 'speedcoach2'): - row = SpeedCoach2Parser(f2) + row = SpeedCoach2Parser(f2) try: - oarlength,inboard = get_empower_rigging(f2) + oarlength, inboard = get_empower_rigging(f2) summary = row.allstats() except: pass - # handle ErgStick if (fileformat == 'ergstick'): - row = ErgStickParser(f2) - + row = ErgStickParser(f2) + # handle FIT if (fileformat == 'fit'): - row = FITParser(f2) + row = FITParser(f2) try: - s = fitsummarydata(f2) + s = fitsummarydata(f2) s.setsummary() - summary = s.summarytext + summary = s.summarytext except: pass - f_to_be_deleted = f2 # should delete file - f2 = f2[:-4]+'o.csv' - row.write_csv(f2,gzip=True) - - #os.remove(f2) - try: - os.remove(f_to_be_deleted) - except: - os.remove(f_to_be_deleted+'.gz') + f2 = f2[:-4] + 'o.csv' + row.write_csv(f2, gzip=True) - return (f2,summary,oarlength,inboard) + # os.remove(f2) + try: + os.remove(f_to_be_deleted) + except: + os.remove(f_to_be_deleted + '.gz') + + return (f2, summary, oarlength, inboard) # Create new workout from file and store it in the database # This routine should be used everywhere in views.py and mailprocessing.py # Currently there is code duplication -def new_workout_from_file(r,f2, + + +def new_workout_from_file(r, f2, workouttype='rower', title='Workout', makeprivate=False, @@ -829,132 +814,128 @@ def new_workout_from_file(r,f2, except IOError: os.remove(f2) message = "Rowsandall could not process this file. The extension is supported but the file seems corrupt. Contact info@rowsandall.com if you think this is incorrect." - return (0,message,f2) - + return (0, message, f2) + summary = '' oarlength = 2.89 inboard = 0.88 - if len(fileformat)==3 and fileformat[0]=='zip': + if len(fileformat) == 3 and fileformat[0] == 'zip': f_to_be_deleted = f2 title = os.path.basename(f2) - if settings.DEBUG: - res = handle_zip_file.delay( - r.user.email,title,f2 + if settings.DEBUG: + res = handle_zip_file.delay( + r.user.email, title, f2 ) - else: - res = queuelow.enqueue( + else: + res = queuelow.enqueue( handle_zip_file, r.user.email, title, f2 ) - return -1,message,f2 + return -1, message, f2 # Some people try to upload Concept2 logbook summaries if fileformat == 'c2log': os.remove(f2) message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook." - return (0,message,f2) + return (0, message, f2) if fileformat == 'nostrokes': os.remove(f2) message = "It looks like this file doesn't contain stroke data." - return (0,message,f2) + return (0, message, f2) # Some people upload corrupted zip files if fileformat == 'notgzip': os.remove(f2) message = "Rowsandall could not process this file. The extension is supported but the file seems corrupt. Contact info@rowsandall.com if you think this is incorrect." - return (0,message,f2) + return (0, message, f2) - # Some people try to upload RowPro summary logs if fileformat == 'rowprolog': os.remove(f2) message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log." - return (0,message,f2) + return (0, message, f2) # Sometimes people try an unsupported file type. # Send an email to info@rowsandall.com with the file attached # for me to check if it is a bug, or a new file type # worth supporting if fileformat == 'unknown': - message = "We couldn't recognize the file type" - if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, - r.user.email) + message = "We couldn't recognize the file type" + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(f2, + r.user.email) + + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + f2, r.user.email) + return (0, message, f2) - else: - res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2,r.user.email) - return (0,message,f2) - # handle non-Painsled by converting it to painsled compatible CSV if (fileformat != 'csv'): try: - f2,summary,oarlength,inboard = handle_nonpainsled(f2, - fileformat, - summary=summary) + f2, summary, oarlength, inboard = handle_nonpainsled(f2, + fileformat, + summary=summary) except: errorstring = str(sys.exc_info()[0]) - message = 'Something went wrong: '+errorstring - return (0,message,'') - - + message = 'Something went wrong: ' + errorstring + return (0, message, '') dosummary = (fileformat != 'fit') - id,message = save_workout_database(f2,r, - workouttype=workouttype, - makeprivate=makeprivate, - dosummary=dosummary, - workoutsource=fileformat, - summary=summary, - inboard=inboard,oarlength=oarlength, - title=title) + id, message = save_workout_database(f2, r, + workouttype=workouttype, + makeprivate=makeprivate, + dosummary=dosummary, + workoutsource=fileformat, + summary=summary, + inboard=inboard, oarlength=oarlength, + title=title) - return (id,message,f2) + return (id, message, f2) -def split_workout(r,parent,splitsecond,splitmode): - data,row = getrowdata_db(id=parent.id) - latitude,longitude = get_latlon(parent.id) + +def split_workout(r, parent, splitsecond, splitmode): + data, row = getrowdata_db(id=parent.id) + latitude, longitude = get_latlon(parent.id) if not latitude.empty and not longitude.empty: data[' latitude'] = latitude data[' longitude'] = longitude - data['time'] = data['time']/1000. - - data1 = data[data['time']<=splitsecond].copy() - data2 = data[data['time']>splitsecond].copy() + data['time'] = data['time'] / 1000. + + data1 = data[data['time'] <= splitsecond].copy() + data2 = data[data['time'] > splitsecond].copy() data1 = data1.sort_values(['time']) - data1 = data1.interpolate(method='linear',axis=0,limit_direction='both', - limit=10) - data1.fillna(method='bfill',inplace=True) + data1 = data1.interpolate(method='linear', axis=0, limit_direction='both', + limit=10) + data1.fillna(method='bfill', inplace=True) - # Some new stuff to try out - data1 = data1.groupby('time',axis=0).mean() + # Some new stuff to try out + data1 = data1.groupby('time', axis=0).mean() data1['time'] = data1.index - data1.reset_index(drop=True,inplace=True) - + data1.reset_index(drop=True, inplace=True) data2 = data2.sort_values(['time']) - data2 = data2.interpolate(method='linear',axis=0,limit_direction='both', - limit=10) - data2.fillna(method='bfill',inplace=True) + data2 = data2.interpolate(method='linear', axis=0, limit_direction='both', + limit=10) + data2.fillna(method='bfill', inplace=True) - # Some new stuff to try out - data2 = data2.groupby('time',axis=0).mean() + # Some new stuff to try out + data2 = data2.groupby('time', axis=0).mean() data2['time'] = data2.index - data2.reset_index(drop=True,inplace=True) - - data1['pace'] = data1['pace']/1000. - data2['pace'] = data2['pace']/1000. - - - data1.drop_duplicates(subset='time',inplace=True) - data2.drop_duplicates(subset='time',inplace=True) + data2.reset_index(drop=True, inplace=True) + + data1['pace'] = data1['pace'] / 1000. + data2['pace'] = data2['pace'] / 1000. + + data1.drop_duplicates(subset='time', inplace=True) + data2.drop_duplicates(subset='time', inplace=True) messages = [] ids = [] @@ -965,17 +946,17 @@ def split_workout(r,parent,splitsecond,splitmode): else: setprivate = False - id,message = new_workout_from_df(r,data1, - title=parent.name+' (1)', - parent=parent, - setprivate=setprivate, - forceunit='N') + id, message = new_workout_from_df(r, data1, + title=parent.name + ' (1)', + parent=parent, + setprivate=setprivate, + forceunit='N') messages.append(message) ids.append(id) if 'keep second' in splitmode: - data2['cumdist'] = data2['cumdist'] - data2.ix[0,'cumdist'] - data2['distance'] = data2['distance'] - data2.ix[0,'distance'] - data2['time'] = data2['time'] - data2.ix[0,'time'] + data2['cumdist'] = data2['cumdist'] - data2.ix[0, 'cumdist'] + data2['distance'] = data2['distance'] - data2.ix[0, 'distance'] + data2['time'] = data2['time'] - data2.ix[0, 'time'] if 'secondprivate' in splitmode: setprivate = True else: @@ -983,19 +964,18 @@ def split_workout(r,parent,splitsecond,splitmode): dt = datetime.timedelta(seconds=splitsecond) - id,message = new_workout_from_df(r,data2, - title=parent.name+' (2)', - parent=parent, - setprivate=setprivate, - dt=dt,forceunit='N') + id, message = new_workout_from_df(r, data2, + title=parent.name + ' (2)', + parent=parent, + setprivate=setprivate, + dt=dt, forceunit='N') messages.append(message) ids.append(id) - if not 'keep original' in splitmode: if 'keep second' in splitmode or 'keep first' in splitmode: parent.delete() - messages.append('Deleted Workout: '+parent.name) + messages.append('Deleted Workout: ' + parent.name) else: messages.append('That would delete your workout') ids.append(parent.id) @@ -1003,12 +983,14 @@ def split_workout(r,parent,splitsecond,splitmode): parent.privacy = 'hidden' parent.save() - return ids,messages - + return ids, messages + # Create new workout from data frame and store it in the database # This routine should be used everywhere in views.py and mailprocessing.py # Currently there is code duplication -def new_workout_from_df(r,df, + + +def new_workout_from_df(r, df, title='New Workout', parent=None, setprivate=False, @@ -1022,62 +1004,59 @@ def new_workout_from_df(r,df, oarlength = parent.oarlength inboard = parent.inboard workouttype = parent.workouttype - notes=parent.notes - summary=parent.summary + notes = parent.notes + summary = parent.summary if parent.privacy == 'hidden': - makeprivate=True + makeprivate = True else: - makeprivate=False - - startdatetime=parent.startdatetime+dt - else: + makeprivate = False + + startdatetime = parent.startdatetime + dt + else: oarlength = 2.89 inboard = 0.88 workouttype = 'rower' - notes='' - summary='' - makeprivate=False + notes = '' + summary = '' + makeprivate = False startdatetime = timezone.now() if setprivate: - makeprivate=True + makeprivate = True - timestr = strftime("%Y%m%d-%H%M%S") - csvfilename ='media/df_'+timestr+'.csv' + csvfilename = 'media/df_' + timestr + '.csv' if forceunit == 'N': # change to lbs for now df['peakforce'] /= lbstoN df['averageforce'] /= lbstoN - df.rename(columns = columndict,inplace=True) + df.rename(columns=columndict, inplace=True) #starttimeunix = mktime(startdatetime.utctimetuple()) starttimeunix = arrow.get(startdatetime).timestamp df[' ElapsedTime (sec)'] = df['TimeStamp (sec)'] - df['TimeStamp (sec)'] = df['TimeStamp (sec)']+starttimeunix + df['TimeStamp (sec)'] = df['TimeStamp (sec)'] + starttimeunix row = rrdata(df=df) - row.write_csv(csvfilename,gzip=True) - - #res = df.to_csv(csvfilename+'.gz',index_label='index', + row.write_csv(csvfilename, gzip=True) + + # res = df.to_csv(csvfilename+'.gz',index_label='index', # compression='gzip') - id,message = save_workout_database(csvfilename,r, - workouttype=workouttype, - title=title, - notes=notes, - oarlength=oarlength, - inboard=inboard, - makeprivate=makeprivate, - dosmooth=False, - consistencychecks=False) - - - return (id,message) + id, message = save_workout_database(csvfilename, r, + workouttype=workouttype, + title=title, + notes=notes, + oarlength=oarlength, + inboard=inboard, + makeprivate=makeprivate, + dosmooth=False, + consistencychecks=False) + return (id, message) # Compare the data from the CSV file and the database @@ -1092,7 +1071,7 @@ def compare_data(id): except AttributeError: rowdata = 0 l1 = 0 - + engine = create_engine(database_url, echo=False) query = sa.text('SELECT COUNT(*) FROM strokedata WHERE workoutid={id};'.format( id=id, @@ -1107,30 +1086,32 @@ def compare_data(id): engine.dispose() lfile = l1 ldb = l2 - return l1==l2 and l1 != 0,ldb,lfile + return l1 == l2 and l1 != 0, ldb, lfile # Repair data for workouts where the CSV file is lost (or the DB entries # don't exist) + + def repair_data(verbose=False): ws = Workout.objects.all() for w in ws: if verbose: sys.stdout.write(".") - test,ldb,lfile = compare_data(w.id) + test, ldb, lfile = compare_data(w.id) if not test: if verbose: - print w.id,lfile,ldb + print w.id, lfile, ldb try: rowdata = rdata(w.csvfilename) if rowdata and len(rowdata.df): - update_strokedata(w.id,rowdata.df) + update_strokedata(w.id, rowdata.df) except IOError, AttributeError: pass - if lfile==0: + if lfile == 0: # if not ldb - delete workout - + try: data = read_df_sql(w.id) try: @@ -1139,34 +1120,38 @@ def repair_data(verbose=False): datalength = 0 if datalength != 0: - data.rename(columns = columndict,inplace=True) - res = data.to_csv(w.csvfilename+'.gz', + data.rename(columns=columndict, inplace=True) + res = data.to_csv(w.csvfilename + '.gz', index_label='index', compression='gzip') print 'adding csv file' else: - print w.id,' No stroke records anywhere' + print w.id, ' No stroke records anywhere' w.delete() except: print 'failed' print str(sys.exc_info()[0]) pass - + # A wrapper around the rowingdata class, with some error catching -def rdata(file,rower=rrower()): + + +def rdata(file, rower=rrower()): try: - res = rrdata(csvfile=file,rower=rower) - except IOError,IndexError: + res = rrdata(csvfile=file, rower=rower) + except IOError, IndexError: try: - res = rrdata(csvfile=file+'.gz',rower=rower) - except IOError,IndexError: - res = 0 + res = rrdata(csvfile=file + '.gz', rower=rower) + except IOError, IndexError: + res = 0 except: res = 0 return res # Remove all stroke data for workout ID from database + + def delete_strokedata(id): engine = create_engine(database_url, echo=False) query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( @@ -1181,80 +1166,86 @@ def delete_strokedata(id): engine.dispose() # Replace stroke data in DB with data from CSV file -def update_strokedata(id,df): + + +def update_strokedata(id, df): delete_strokedata(id) - rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True) + rowdata = dataprep(df, id=id, bands=True, barchart=True, otwpower=True) # Test that all data are of a numerical time -def testdata(time,distance,pace,spm): - t1 = np.issubdtype(time,np.number) - t2 = np.issubdtype(distance,np.number) - t3 = np.issubdtype(pace,np.number) - t4 = np.issubdtype(spm,np.number) + + +def testdata(time, distance, pace, spm): + t1 = np.issubdtype(time, np.number) + t2 = np.issubdtype(distance, np.number) + t3 = np.issubdtype(pace, np.number) + t4 = np.issubdtype(spm, np.number) return t1 and t2 and t3 and t4 # Get data from DB for one workout (fetches all data). If data # is not in DB, read from CSV file (and create DB entry) -def getrowdata_db(id=0,doclean=False,convertnewtons=True): + + +def getrowdata_db(id=0, doclean=False, convertnewtons=True): data = read_df_sql(id) - data['x_right'] = data['x_right']/1.0e6 + data['x_right'] = data['x_right'] / 1.0e6 if data.empty: - rowdata,row = getrowdata(id=id) + rowdata, row = getrowdata(id=id) if rowdata: - data = dataprep(rowdata.df,id=id,bands=True,barchart=True,otwpower=True) + data = dataprep(rowdata.df, id=id, bands=True, + barchart=True, otwpower=True) else: - data = pd.DataFrame() # returning empty dataframe + data = pd.DataFrame() # returning empty dataframe else: row = Workout.objects.get(id=id) if data['efficiency'].mean() == 0 and data['power'].mean() != 0: data = add_efficiency(id=id) - + if doclean: - data = clean_df_stats(data,ignorehr=True) + data = clean_df_stats(data, ignorehr=True) - - return data,row + return data, row # Fetch a subset of the data from the DB -def getsmallrowdata_db(columns,ids=[],doclean=True,workstrokesonly=True): + + +def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True): prepmultipledata(ids) - data = read_cols_df_sql(ids,columns) + data = read_cols_df_sql(ids, columns) # convert newtons - - if doclean: - data = clean_df_stats(data,ignorehr=True, + data = clean_df_stats(data, ignorehr=True, workstrokesonly=workstrokesonly) - - return data # Fetch both the workout and the workout stroke data (from CSV file) + + def getrowdata(id=0): # check if valid ID exists (workout exists) row = Workout.objects.get(id=id) f1 = row.csvfilename - + # get user r = row.user u = r.user - rr = rrower(hrmax=r.max,hrut2=r.ut2, - hrut1=r.ut1,hrat=r.at, - hrtr=r.tr,hran=r.an,ftp=r.ftp) + rr = rrower(hrmax=r.max, hrut2=r.ut2, + hrut1=r.ut1, hrat=r.at, + hrtr=r.tr, hran=r.an, ftp=r.ftp) - rowdata = rdata(f1,rower=rr) + rowdata = rdata(f1, rower=rr) - return rowdata,row + return rowdata, row # Checks if all rows for a list of workout IDs have entries in the # stroke_data table. If this is not the case, it creates the stroke @@ -1262,7 +1253,9 @@ def getrowdata(id=0): # In theory, this should never yield any work, but it's a good # safety net for programming errors elsewhere in the app # Also used heavily when I moved from CSV file only to CSV+Stroke data -def prepmultipledata(ids,verbose=False): + + +def prepmultipledata(ids, verbose=False): query = sa.text('SELECT DISTINCT workoutid FROM strokedata') engine = create_engine(database_url, echo=False) @@ -1276,74 +1269,81 @@ def prepmultipledata(ids,verbose=False): ids2 = [int(id) for id in ids] except ValueError: ids2 = ids - - res = list(set(ids2)-set(res)) + + res = list(set(ids2) - set(res)) for id in res: - rowdata,row = getrowdata(id=id) + rowdata, row = getrowdata(id=id) if verbose: print id if rowdata and len(rowdata.df): - data = dataprep(rowdata.df,id=id,bands=True,barchart=True,otwpower=True) + data = dataprep(rowdata.df, id=id, bands=True, + barchart=True, otwpower=True) return res # Read a set of columns for a set of workout ids, returns data as a # pandas dataframe -def read_cols_df_sql(ids,columns,convertnewtons=True): + + +def read_cols_df_sql(ids, columns, convertnewtons=True): # drop columns that are not in offical list -# axx = [ax[0] for ax in axes] + # axx = [ax[0] for ax in axes] axx = [f.name for f in StrokeData._meta.get_fields()] - + for c in columns: if not c in axx: columns.remove(c) - - columns = list(columns)+['distance','spm','workoutid'] + + columns = list(columns) + ['distance', 'spm', 'workoutid'] columns = [x for x in columns if x != 'None'] columns = list(set(columns)) cls = '' engine = create_engine(database_url, echo=False) for column in columns: - cls += column+', ' + cls += column + ', ' cls = cls[:-2] if len(ids) == 0: query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid=0'.format( - columns = cls, - )) + columns=cls, + )) elif len(ids) == 1: query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid={id}'.format( - id = ids[0], - columns = cls, - )) + id=ids[0], + columns=cls, + )) else: query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid IN {ids}'.format( - columns = cls, - ids = tuple(ids), + columns=cls, + ids=tuple(ids), )) - connection = engine.raw_connection() - df = pd.read_sql_query(query,engine) + df = pd.read_sql_query(query, engine) df = df.fillna(value=0) if 'peakforce' in columns: - funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) - for id,u in funits: - if u=='lbs': - mask = df['workoutid']==id - df.loc[mask,'peakforce'] = df.loc[mask,'peakforce']*lbstoN + funits = ((w.id, w.forceunit) + for w in Workout.objects.filter(id__in=ids)) + for id, u in funits: + if u == 'lbs': + mask = df['workoutid'] == id + df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN if 'averageforce' in columns: - funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) - for id,u in funits: - if u=='lbs': - mask = df['workoutid']==id - df.loc[mask,'averageforce'] = df.loc[mask,'averageforce']*lbstoN + funits = ((w.id, w.forceunit) + for w in Workout.objects.filter(id__in=ids)) + for id, u in funits: + if u == 'lbs': + mask = df['workoutid'] == id + df.loc[mask, 'averageforce'] = df.loc[mask, + 'averageforce'] * lbstoN engine.dispose() return df - + # Read stroke data from the DB for a Workout ID. Returns a pandas dataframe + + def read_df_sql(id): engine = create_engine(database_url, echo=False) @@ -1354,15 +1354,15 @@ def read_df_sql(id): df = df.fillna(value=0) funit = Workout.objects.get(id=id).forceunit - - if funit=='lbs': + + if funit == 'lbs': try: - df['peakforce'] = df['peakforce']*lbstoN + df['peakforce'] = df['peakforce'] * lbstoN except KeyError: pass try: - df['averageforce'] = df['averageforce']*lbstoN + df['averageforce'] = df['averageforce'] * lbstoN except KeyError: pass @@ -1370,7 +1370,9 @@ def read_df_sql(id): # Get the necessary data from the strokedata table in the DB. # For the flex plot -def smalldataprep(therows,xparam,yparam1,yparam2): + + +def smalldataprep(therows, xparam, yparam1, yparam2): df = pd.DataFrame() if yparam2 == 'None': yparam2 = 'power' @@ -1384,14 +1386,14 @@ def smalldataprep(therows,xparam,yparam1,yparam2): try: rowdata = dataprep(rrdata(f1).df) - + rowdata = pd.DataFrame({xparam: rowdata[xparam], yparam1: rowdata[yparam1], yparam2: rowdata[yparam2], 'distance': rowdata['distance'], 'spm': rowdata['spm'], - } - ) + } + ) if workout.forceunit == 'lbs': try: rowdata['peakforce'] *= lbstoN @@ -1402,18 +1404,18 @@ def smalldataprep(therows,xparam,yparam1,yparam2): rowdata['averageforce'] *= lbstoN except KeyError: pass - - df = pd.concat([df,rowdata],ignore_index=True) + + df = pd.concat([df, rowdata], ignore_index=True) except IOError: try: - rowdata = dataprep(rrdata(f1+'.gz').df) + rowdata = dataprep(rrdata(f1 + '.gz').df) rowdata = pd.DataFrame({xparam: rowdata[xparam], yparam1: rowdata[yparam1], yparam2: rowdata[yparam2], 'distance': rowdata['distance'], 'spm': rowdata['spm'], - } - ) + } + ) if workout.forceunit == 'lbs': try: rowdata['peakforce'] *= lbstoN @@ -1424,99 +1426,99 @@ def smalldataprep(therows,xparam,yparam1,yparam2): rowdata['averageforce'] *= lbstoN except KeyError: pass - df = pd.concat([df,rowdata],ignore_index=True) + df = pd.concat([df, rowdata], ignore_index=True) except IOError: pass - return df # data fusion -def datafusion(id1,id2,columns,offset): + + +def datafusion(id1, id2, columns, offset): workout1 = Workout.objects.get(id=id1) workout2 = Workout.objects.get(id=id2) - - df1,w1 = getrowdata_db(id=id1) - df1 = df1.drop([#'cumdist', - 'hr_ut2', - 'hr_ut1', - 'hr_at', - 'hr_tr', - 'hr_an', - 'hr_max', - 'ftime', - 'fpace', - 'workoutid', - 'id'], - 1,errors='ignore') - + + df1, w1 = getrowdata_db(id=id1) + df1 = df1.drop([ # 'cumdist', + 'hr_ut2', + 'hr_ut1', + 'hr_at', + 'hr_tr', + 'hr_an', + 'hr_max', + 'ftime', + 'fpace', + 'workoutid', + 'id'], + 1, errors='ignore') + # Add coordinates to DataFrame - latitude,longitude = get_latlon(id1) - + latitude, longitude = get_latlon(id1) + df1[' latitude'] = latitude df1[' longitude'] = longitude - - df2 = getsmallrowdata_db(['time']+columns,ids=[id2],doclean=False) + df2 = getsmallrowdata_db(['time'] + columns, ids=[id2], doclean=False) forceunit = 'N' - - offsetmillisecs = offset.seconds*1000+offset.microseconds/1000. - offsetmillisecs += offset.days*(3600*24*1000) - df2['time'] = df2['time']+offsetmillisecs - - keep1 = {c:c for c in set(df1.columns)} + offsetmillisecs = offset.seconds * 1000 + offset.microseconds / 1000. + offsetmillisecs += offset.days * (3600 * 24 * 1000) + df2['time'] = df2['time'] + offsetmillisecs + + keep1 = {c: c for c in set(df1.columns)} for c in columns: keep1.pop(c) - - + for c in df1.columns: if not c in keep1: - df1 = df1.drop(c,1,errors='ignore') + df1 = df1.drop(c, 1, errors='ignore') - df = pd.concat([df1,df2],ignore_index=True) + df = pd.concat([df1, df2], ignore_index=True) df = df.sort_values(['time']) - df = df.interpolate(method='linear',axis=0,limit_direction='both', + df = df.interpolate(method='linear', axis=0, limit_direction='both', limit=10) - df.fillna(method='bfill',inplace=True) + df.fillna(method='bfill', inplace=True) - # Some new stuff to try out - df = df.groupby('time',axis=0).mean() + # Some new stuff to try out + df = df.groupby('time', axis=0).mean() df['time'] = df.index - df.reset_index(drop=True,inplace=True) - - df['time'] = df['time']/1000. - df['pace'] = df['pace']/1000. - df['cum_dist'] = df['cumdist'] - - return df,forceunit + df.reset_index(drop=True, inplace=True) -def fix_newtons(id=0,limit=3000): + df['time'] = df['time'] / 1000. + df['pace'] = df['pace'] / 1000. + df['cum_dist'] = df['cumdist'] + + return df, forceunit + + +def fix_newtons(id=0, limit=3000): # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) - rowdata = getsmallrowdata_db(['peakforce'],ids=[id],doclean=False) + rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False) try: #avgforce = rowdata['averageforce'] peakforce = rowdata['peakforce'] if peakforce.mean() > limit: w = Workout.objects.get(id=id) - print "fixing ",id + print "fixing ", id rowdata = rdata(w.csvfilename) if rowdata and len(rowdata.df): - update_strokedata(w.id,rowdata.df) + update_strokedata(w.id, rowdata.df) except KeyError: pass -def add_efficiency(id=0): - rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) - power = rowdata['power'] - pace = rowdata['pace']/1.0e3 - velo = 500./pace - ergpw = 2.8*velo**3 - efficiency = 100.*ergpw/power - efficiency = efficiency.replace([-np.inf,np.inf],np.nan) +def add_efficiency(id=0): + rowdata, row = getrowdata_db(id=id, doclean=False, convertnewtons=False) + power = rowdata['power'] + pace = rowdata['pace'] / 1.0e3 + velo = 500. / pace + ergpw = 2.8 * velo**3 + efficiency = 100. * ergpw / power + + efficiency = efficiency.replace([-np.inf, np.inf], np.nan) efficiency.fillna(method='ffill') rowdata['efficiency'] = efficiency delete_strokedata(id) @@ -1524,79 +1526,80 @@ def add_efficiency(id=0): rowdata['workoutid'] = id engine = create_engine(database_url, echo=False) with engine.connect() as conn, conn.begin(): - rowdata.to_sql('strokedata',engine,if_exists='append',index=False) + rowdata.to_sql('strokedata', engine, + if_exists='append', index=False) conn.close() - engine.dispose() + engine.dispose() return rowdata # This is the main routine. # it reindexes, sorts, filters, and smooths the data, then # saves it to the stroke_data table in the database # Takes a rowingdata object's DataFrame as input -def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, - empower=True,inboard=0.88,forceunit='lbs'): + + +def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, + empower=True, inboard=0.88, forceunit='lbs'): if rowdatadf.empty: return 0 - rowdatadf.set_index([range(len(rowdatadf))],inplace=True) - t = rowdatadf.ix[:,'TimeStamp (sec)'] - t = pd.Series(t-rowdatadf.ix[0,'TimeStamp (sec)']) + rowdatadf.set_index([range(len(rowdatadf))], inplace=True) + t = rowdatadf.ix[:, 'TimeStamp (sec)'] + t = pd.Series(t - rowdatadf.ix[0, 'TimeStamp (sec)']) - row_index = rowdatadf.ix[:,' Stroke500mPace (sec/500m)'] > 3000 - rowdatadf.loc[row_index,' Stroke500mPace (sec/500m)'] = 3000. + row_index = rowdatadf.ix[:, ' Stroke500mPace (sec/500m)'] > 3000 + rowdatadf.loc[row_index, ' Stroke500mPace (sec/500m)'] = 3000. - p = rowdatadf.ix[:,' Stroke500mPace (sec/500m)'] - hr = rowdatadf.ix[:,' HRCur (bpm)'] - spm = rowdatadf.ix[:,' Cadence (stokes/min)'] - cumdist = rowdatadf.ix[:,'cum_dist'] - power = rowdatadf.ix[:,' Power (watts)'] - averageforce = rowdatadf.ix[:,' AverageDriveForce (lbs)'] - drivelength = rowdatadf.ix[:,' DriveLength (meters)'] + p = rowdatadf.ix[:, ' Stroke500mPace (sec/500m)'] + hr = rowdatadf.ix[:, ' HRCur (bpm)'] + spm = rowdatadf.ix[:, ' Cadence (stokes/min)'] + cumdist = rowdatadf.ix[:, 'cum_dist'] + power = rowdatadf.ix[:, ' Power (watts)'] + averageforce = rowdatadf.ix[:, ' AverageDriveForce (lbs)'] + drivelength = rowdatadf.ix[:, ' DriveLength (meters)'] try: - workoutstate = rowdatadf.ix[:,' WorkoutState'] + workoutstate = rowdatadf.ix[:, ' WorkoutState'] except KeyError: - workoutstate = 0*hr - - peakforce = rowdatadf.ix[:,' PeakDriveForce (lbs)'] + workoutstate = 0 * hr - forceratio = averageforce/peakforce + peakforce = rowdatadf.ix[:, ' PeakDriveForce (lbs)'] + + forceratio = averageforce / peakforce forceratio = forceratio.fillna(value=0) try: - drivetime = rowdatadf.ix[:,' DriveTime (ms)'] - recoverytime = rowdatadf.ix[:,' StrokeRecoveryTime (ms)'] - rhythm = 100.*drivetime/(recoverytime+drivetime) + drivetime = rowdatadf.ix[:, ' DriveTime (ms)'] + recoverytime = rowdatadf.ix[:, ' StrokeRecoveryTime (ms)'] + rhythm = 100. * drivetime / (recoverytime + drivetime) rhythm = rhythm.fillna(value=0) except: - rhythm = 0.0*forceratio - + rhythm = 0.0 * forceratio + f = rowdatadf['TimeStamp (sec)'].diff().mean() if f != 0: - windowsize = 2*(int(10./(f)))+1 + windowsize = 2 * (int(10. / (f))) + 1 else: windowsize = 1 if windowsize <= 3: - windowsize = 5 + windowsize = 5 - if windowsize > 3 and windowsize 3 and windowsize < len(hr): + spm = savgol_filter(spm, windowsize, 3) + hr = savgol_filter(hr, windowsize, 3) + drivelength = savgol_filter(drivelength, windowsize, 3) + forceratio = savgol_filter(forceratio, windowsize, 3) try: t2 = t.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) except TypeError: - t2 = 0*t + t2 = 0 * t - p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) try: - drivespeed = drivelength/rowdatadf[' DriveTime (ms)']*1.0e3 + drivespeed = drivelength / rowdatadf[' DriveTime (ms)'] * 1.0e3 except TypeError: - drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] - + drivespeed = 0.0 * rowdatadf['TimeStamp (sec)'] drivespeed = drivespeed.fillna(value=0) @@ -1604,159 +1607,155 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, driveenergy = rowdatadf['driveenergy'] except KeyError: if forceunit == 'lbs': - driveenergy = drivelength*averageforce*lbstoN + driveenergy = drivelength * averageforce * lbstoN else: - drivenergy = drivelength*averageforce - - distance = rowdatadf.ix[:,'cum_dist'] - velo = 500./p + drivenergy = drivelength * averageforce - distanceperstroke = 60.*velo/spm + distance = rowdatadf.ix[:, 'cum_dist'] + velo = 500. / p + + distanceperstroke = 60. * velo / spm - data = DataFrame( - dict( - time = t*1e3, - hr = hr, - pace = p*1e3, - spm = spm, - cumdist = cumdist, - ftime = niceformat(t2), - fpace = nicepaceformat(p2), - driveenergy=driveenergy, - power=power, + dict( + time=t * 1e3, + hr=hr, + pace=p * 1e3, + spm=spm, + cumdist=cumdist, + ftime=niceformat(t2), + fpace=nicepaceformat(p2), + driveenergy=driveenergy, + power=power, workoutstate=workoutstate, - averageforce=averageforce, - drivelength=drivelength, - peakforce=peakforce, + averageforce=averageforce, + drivelength=drivelength, + peakforce=peakforce, forceratio=forceratio, - distance=distance, - drivespeed=drivespeed, + distance=distance, + drivespeed=drivespeed, rhythm=rhythm, distanceperstroke=distanceperstroke, - ) - ) + ) + ) if bands: - # HR bands - data['hr_ut2'] = rowdatadf.ix[:,'hr_ut2'] - data['hr_ut1'] = rowdatadf.ix[:,'hr_ut1'] - data['hr_at'] = rowdatadf.ix[:,'hr_at'] - data['hr_tr'] = rowdatadf.ix[:,'hr_tr'] - data['hr_an'] = rowdatadf.ix[:,'hr_an'] - data['hr_max'] = rowdatadf.ix[:,'hr_max'] - data['hr_bottom'] = 0.0*data['hr'] + # HR bands + data['hr_ut2'] = rowdatadf.ix[:, 'hr_ut2'] + data['hr_ut1'] = rowdatadf.ix[:, 'hr_ut1'] + data['hr_at'] = rowdatadf.ix[:, 'hr_at'] + data['hr_tr'] = rowdatadf.ix[:, 'hr_tr'] + data['hr_an'] = rowdatadf.ix[:, 'hr_an'] + data['hr_max'] = rowdatadf.ix[:, 'hr_max'] + data['hr_bottom'] = 0.0 * data['hr'] try: - tel = rowdatadf.ix[:,' ElapsedTime (sec)'] + tel = rowdatadf.ix[:, ' ElapsedTime (sec)'] except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] if barchart: - # time increments for bar chart - time_increments = rowdatadf.ix[:,' ElapsedTime (sec)'].diff() - time_increments[0] = time_increments[1] - time_increments = 0.5*time_increments+0.5*np.abs(time_increments) - x_right = (t2+time_increments.apply(lambda x:timedeltaconv(x))) + # time increments for bar chart + time_increments = rowdatadf.ix[:, ' ElapsedTime (sec)'].diff() + time_increments[0] = time_increments[1] + time_increments = 0.5 * time_increments + 0.5 * np.abs(time_increments) + x_right = (t2 + time_increments.apply(lambda x: timedeltaconv(x))) - data['x_right'] = x_right + data['x_right'] = x_right if empower: try: - wash = rowdatadf.ix[:,'wash'] + wash = rowdatadf.ix[:, 'wash'] except KeyError: - wash = 0*power + wash = 0 * power try: - catch = rowdatadf.ix[:,'catch'] + catch = rowdatadf.ix[:, 'catch'] except KeyError: - catch = 0*power + catch = 0 * power try: - finish = rowdatadf.ix[:,'finish'] + finish = rowdatadf.ix[:, 'finish'] except KeyError: - finish = 0*power + finish = 0 * power try: - peakforceangle = rowdatadf.ix[:,'peakforceangle'] + peakforceangle = rowdatadf.ix[:, 'peakforceangle'] except KeyError: - peakforceangle = 0*power - + peakforceangle = 0 * power if data['driveenergy'].mean() == 0: try: - driveenergy = rowdatadf.ix[:,'driveenergy'] + driveenergy = rowdatadf.ix[:, 'driveenergy'] except KeyError: - driveenergy = power*60/spm + driveenergy = power * 60 / spm else: driveenergy = data['driveenergy'] - - arclength = (inboard-0.05)*(np.radians(finish)-np.radians(catch)) - if arclength.mean()>0: + arclength = (inboard - 0.05) * (np.radians(finish) - np.radians(catch)) + if arclength.mean() > 0: drivelength = arclength elif drivelength.mean() == 0: - drivelength = driveenergy/(averageforce*4.44822) + drivelength = driveenergy / (averageforce * 4.44822) try: - slip = rowdatadf.ix[:,'slip'] + slip = rowdatadf.ix[:, 'slip'] except KeyError: - slip = 0*power - + slip = 0 * power + try: - totalangle = finish-catch - effectiveangle = finish-wash-catch-slip + totalangle = finish - catch + effectiveangle = finish - wash - catch - slip except ValueError: - totalangle = 0*power - effectiveangle = 0*power + totalangle = 0 * power + effectiveangle = 0 * power - if windowsize > 3 and windowsize 3 and windowsize < len(slip): try: - wash = savgol_filter(wash,windowsize,3) + wash = savgol_filter(wash, windowsize, 3) except TypeError: pass try: - slip = savgol_filter(slip,windowsize,3) + slip = savgol_filter(slip, windowsize, 3) except TypeError: pass try: - catch = savgol_filter(catch,windowsize,3) + catch = savgol_filter(catch, windowsize, 3) except TypeError: pass try: - finish = savgol_filter(finish,windowsize,3) + finish = savgol_filter(finish, windowsize, 3) except TypeError: pass try: - peakforceangle = savgol_filter(peakforceangle,windowsize,3) + peakforceangle = savgol_filter(peakforceangle, windowsize, 3) except TypeError: pass try: - driveenergy = savgol_filter(driveenergy,windowsize,3) + driveenergy = savgol_filter(driveenergy, windowsize, 3) except TypeError: pass try: - drivelength = savgol_filter(drivelength,windowsize,3) + drivelength = savgol_filter(drivelength, windowsize, 3) except TypeError: pass try: - totalangle = savgol_filter(totalangle,windowsize,3) + totalangle = savgol_filter(totalangle, windowsize, 3) except TypeError: pass try: - effectiveangle = savgol_filter(effectiveangle,windowsize,3) + effectiveangle = savgol_filter(effectiveangle, windowsize, 3) except TypeError: pass - velo = 500./p + velo = 500. / p + ergpw = 2.8 * velo**3 + efficiency = 100. * ergpw / power - ergpw = 2.8*velo**3 - efficiency = 100.*ergpw/power - - efficiency = efficiency.replace([-np.inf,np.inf],np.nan) + efficiency = efficiency.replace([-np.inf, np.inf], np.nan) efficiency.fillna(method='ffill') - + try: data['wash'] = wash data['catch'] = catch @@ -1770,39 +1769,39 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, data['efficiency'] = efficiency except ValueError: pass - + if otwpower: - try: - nowindpace = rowdatadf.ix[:,'nowindpace'] - except KeyError: - nowindpace = p - try: - equivergpower = rowdatadf.ix[:,'equivergpower'] - except KeyError: - equivergpower = 0*p+50. - - nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x)) - ergvelo = (equivergpower/2.8)**(1./3.) + try: + nowindpace = rowdatadf.ix[:, 'nowindpace'] + except KeyError: + nowindpace = p + try: + equivergpower = rowdatadf.ix[:, 'equivergpower'] + except KeyError: + equivergpower = 0 * p + 50. - ergpace = 500./ergvelo - ergpace[ergpace == np.inf] = 240. - ergpace2 = ergpace.apply(lambda x: timedeltaconv(x)) + nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x)) + ergvelo = (equivergpower / 2.8)**(1. / 3.) - data['ergpace'] = ergpace*1e3 - data['nowindpace'] = nowindpace*1e3 - data['equivergpower'] = equivergpower - data['fergpace'] = nicepaceformat(ergpace2) - data['fnowindpace'] = nicepaceformat(nowindpace2) + ergpace = 500. / ergvelo + ergpace[ergpace == np.inf] = 240. + ergpace2 = ergpace.apply(lambda x: timedeltaconv(x)) - data = data.replace([-np.inf,np.inf],np.nan) + data['ergpace'] = ergpace * 1e3 + data['nowindpace'] = nowindpace * 1e3 + data['equivergpower'] = equivergpower + data['fergpace'] = nicepaceformat(ergpace2) + data['fnowindpace'] = nicepaceformat(nowindpace2) + + data = data.replace([-np.inf, np.inf], np.nan) data = data.fillna(method='ffill') - + # write data if id given if id != 0: data['workoutid'] = id engine = create_engine(database_url, echo=False) with engine.connect() as conn, conn.begin(): - data.to_sql('strokedata',engine,if_exists='append',index=False) + data.to_sql('strokedata', engine, if_exists='append', index=False) conn.close() - engine.dispose() + engine.dispose() return data diff --git a/rowers/forms.py b/rowers/forms.py index 83de2878..34b14e4d 100644 --- a/rowers/forms.py +++ b/rowers/forms.py @@ -182,7 +182,8 @@ class UploadOptionsForm(forms.Form): label='Make Workout Private') landingpage = forms.ChoiceField(choices=landingpages, - initial='workout_edit_view') + initial='workout_edit_view', + label='Landing Page') class Meta: fields = ['make_plot','plottype','upload_toc2','makeprivate'] diff --git a/rowers/templates/document_form.html b/rowers/templates/document_form.html index 53a2c06d..3f065765 100644 --- a/rowers/templates/document_form.html +++ b/rowers/templates/document_form.html @@ -38,23 +38,10 @@ You can select one static plot to be generated immediately for this workout. You can select to export to major fitness platforms automatically. - If you check "make private", this workout will not be visible to your followers and will not show up in your teams' workouts list. + If you check "make private", this workout will not be visible to your followers and will not show up in your teams' workouts list. With the Landing Page option, you can select to which (workout related) page you will be + taken after a successfull upload.

-

- Valid file types are: -

    -
  • Painsled iOS Stroke Export (CSV)
    • -
  • Painsled desktop version Stroke Export (CSV)
    • -
  • A TCX file with location data (lat,long) - with or without Heart Rate value, for example from RiM or CrewNerd
    • -
  • RowPro CSV export
    • -
  • SpeedCoach GPS and SpeedCoach GPS 2 CSV export
    • -
  • ErgData CSV export
    • -
  • ErgStick CSV export
    • -
  • BoatCoach CSV export
    • -
  • A FIT file with location data (experimental)
    • -
-

diff --git a/rowers/views.py b/rowers/views.py index ec138f07..8b2c5b75 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -7627,34 +7627,39 @@ def workout_upload_view(request, plottype = 'timeplot' try: - upload_toc2 = uploadoptions['upload_to_C2'] + landingpage = uploadoptions['landingpage'] except KeyError: - upload_toc2 = False + landingpage = r.defaultlandingpage try: - upload_tostrava = uploadoptions['upload_to_Strava'] + upload_to_c2 = uploadoptions['upload_to_C2'] except KeyError: - upload_tostrava = False + upload_to_c2 = False try: - upload_tost = uploadoptions['upload_to_SportTracks'] + upload_to_strava = uploadoptions['upload_to_Strava'] except KeyError: - upload_tost = False + upload_to_strava = False try: - upload_tork = uploadoptions['upload_to_RunKeeper'] + upload_to_st = uploadoptions['upload_to_SportTracks'] except KeyError: - upload_tork = False + upload_to_st = False try: - upload_toua = uploadoptions['upload_to_MapMyFitness'] + upload_to_rk = uploadoptions['upload_to_RunKeeper'] except KeyError: - upload_toua = False + upload_to_rk = False try: - upload_totp = uploadoptions['upload_to_TrainingPeaks'] + upload_to_ua = uploadoptions['upload_to_MapMyFitness'] except KeyError: - upload_totp = False + upload_to_ua = False + + try: + upload_to_tp = uploadoptions['upload_to_TrainingPeaks'] + except KeyError: + upload_to_tp = False if request.method == 'POST': form = DocumentsForm(request.POST,request.FILES) @@ -7832,7 +7837,6 @@ def workout_upload_view(request, return response else: form = DocumentsForm(initial=docformoptions) - print uploadoptions optionsform = UploadOptionsForm(initial=uploadoptions) return render(request, 'document_form.html', {'form':form, From a866b253a19e3da1e4e39201052fdf556c9e44c1 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 11:49:47 +0200 Subject: [PATCH 08/15] cleaned up processemail --- rowers/mailprocessing.py | 250 +++++++++++---------- rowers/management/commands/processemail.py | 157 +++++++------ 2 files changed, 202 insertions(+), 205 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index eefd0031..df6760c7 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -2,21 +2,21 @@ import time from django.conf import settings from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm +from django_mailbox.models import Mailbox, Message, MessageAttachment +from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm from django.core.files.base import ContentFile -from django.core.mail import send_mail, BadHeaderError,EmailMessage +from django.core.mail import send_mail, BadHeaderError, EmailMessage from rowsandall_app.settings import BASE_DIR from rowingdata import rower as rrower from rowingdata import main as rmain from rowingdata import rowingdata as rrdata -from rowingdata import TCXParser,RowProParser,ErgDataParser -from rowingdata import MysteryParser,BoatCoachParser -from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser -from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata +from rowingdata import TCXParser, RowProParser, ErgDataParser +from rowingdata import MysteryParser, BoatCoachParser +from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser +from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type +from rowingdata import summarydata, get_file_type from scipy.signal import savgol_filter @@ -30,19 +30,20 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u,name,link,options): + + +def send_confirm(u, name, link, options): fullemail = u.email - subject = 'Workout added: '+name - message = 'Dear '+u.first_name+',\n\n' + subject = 'Workout added: ' + name + message = 'Dear ' + u.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" - message += "Link to workout: "+link+"\n\n" + message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" if options: - message += "\n\n"+str(options) + message += "\n\n" + str(options) - - email = EmailMessage(subject,message, + email = EmailMessage(subject, message, 'Rowsandall ', [fullemail]) @@ -51,203 +52,204 @@ def send_confirm(u,name,link,options): return 1 # Reads a "rowingdata" object, plus some error protections -def rdata(file,rower=rrower()): + + +def rdata(file, rower=rrower()): try: - res = rrdata(file,rower=rower) + res = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file+'.gz',rower=rower) + res = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + res = 0 return res # Some error protection around process attachments + + def safeprocessattachments(): try: - return processattachments() + return processattachments() except: - return [0] + return [0] # This is duplicated in management/commands/processemail # Need to double check the code there, update here, and only # use the code here. + + def processattachments(): # in res, we store the ids of the new workouts res = [] attachments = MessageAttachment.objects.all() for a in attachments: - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - # move attachment and make workout - try: - wid = [make_new_workout_from_email(rr,a.document,name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if wid != 1: - dd = send_confirm(u,name,link) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 - except Rower.DoesNotExist: - pass - - - # remove attachment - if donotdelete == 0: - a.delete() + donotdelete = 0 + m = Message.objects.get(id=a.message_id) + from_address = m.from_address[0] + name = m.subject - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + # get a list of users + theusers = User.objects.filter(email=from_address) + for u in theusers: + try: + rr = Rower.objects.get(user=u.id) + # move attachment and make workout + try: + wid = [make_new_workout_from_email(rr, a.document, name)] + res += wid + link = 'https://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' + if wid != 1: + dd = send_confirm(u, name, link) + except: + # replace with code to process error + res += ['fail: ' + name] + donotdelete = 1 + except Rower.DoesNotExist: + pass + + # remove attachment + if donotdelete == 0: + a.delete() + + if m.attachments.exists() == False: + # no attachments, so can be deleted + m.delete() # Delete remaining messages (which should not have attachments) mm = Message.objects.all() for m in mm: - if m.attachments.exists()==False: - m.delete() + if m.attachments.exists() == False: + m.delete() return res # As above, but with some print commands for debugging purposes + + def processattachments_debug(): res = [] attachments = MessageAttachment.objects.all() for a in attachments: - donotdelete = 1 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) + donotdelete = 1 + m = Message.objects.get(id=a.message_id) + from_address = m.from_address[0] + name = m.subject + + # get a list of users + theusers = User.objects.filter(email=from_address) print theusers - for u in theusers: + for u in theusers: try: - rr = Rower.objects.get(user=u.id) + rr = Rower.objects.get(user=u.id) doorgaan = 1 except: doorgaan = 0 if doorgaan: - # move attachment and make workout + # move attachment and make workout print a.document print name - wid = [make_new_workout_from_email(rr,a.document,name)] + wid = [make_new_workout_from_email(rr, a.document, name)] res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'https://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if wid != 1: - dd = send_confirm(u,name,link) - - - - # remove attachment - if donotdelete == 0: - a.delete() + dd = send_confirm(u, name, link) - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + # remove attachment + if donotdelete == 0: + a.delete() + + if m.attachments.exists() == False: + # no attachments, so can be deleted + m.delete() mm = Message.objects.all() for m in mm: - if m.attachments.exists()==False: - m.delete() + if m.attachments.exists() == False: + m.delete() return res # Process the attachment file, create new workout # The code here is duplication of the code in views.py (workout_upload_view) # Need to move the code to a subroutine used both in views.py and here -def make_new_workout_from_email(rr,f2,name,cntr=0): + + +def make_new_workout_from_email(rr, f2, name, cntr=0): workouttype = 'rower' try: f2 = f2.name - fileformat = get_file_type('media/'+f2) + fileformat = get_file_type('media/' + f2) except IOError: - f2 = f2.name+'.gz' - fileformat = get_file_type('media/'+f2) + f2 = f2.name + '.gz' + fileformat = get_file_type('media/' + f2) except AttributeError: - fileformat = get_file_type('media/'+f2) + fileformat = get_file_type('media/' + f2) - if len(fileformat)==3 and fileformat[0]=='zip': + if len(fileformat) == 3 and fileformat[0] == 'zip': f_to_be_deleted = f2 - with zipfile.ZipFile('media/'+f2) as z: - f2 = z.extract(z.namelist()[0],path='media/')[6:] + with zipfile.ZipFile('media/' + f2) as z: + f2 = z.extract(z.namelist()[0], path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, - "roosendaalsander@gmail.com") + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(f2, + "roosendaalsander@gmail.com") - else: - res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2,"roosendaalsander@gmail.com") + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + f2, "roosendaalsander@gmail.com") + + return 1 - return 1 - summary = '' # handle non-Painsled if fileformat != 'csv': - f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary) + f3, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + f2, fileformat, summary) else: - f3 = 'media/'+f2 + f3 = 'media/' + f2 inboard = 0.88 oarlength = 2.89 - - - # make workout and put in database - #r = rrower(hrmax=rr.max,hrut2=rr.ut2, - # hrut1=rr.ut1,hrat=rr.at, - # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) - row = rdata(f3) #,rower=r) + # r = rrower(hrmax=rr.max,hrut2=rr.ut2, + # hrut1=rr.ut1,hrat=rr.at, + # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) + row = rdata(f3) # ,rower=r) if row == 0: - return 0 - + return 0 # change filename if f2[:5] != 'media': - timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/'+timestr+str(cntr)+'o.csv' + timestr = time.strftime("%Y%m%d-%H%M%S") + f2 = 'media/' + timestr + str(cntr) + 'o.csv' try: - avglat = row.df[' latitude'].mean() - avglon = row.df[' longitude'].mean() + avglat = row.df[' latitude'].mean() + avglon = row.df[' longitude'].mean() if avglat != 0 or avglon != 0: workouttype = 'water' except KeyError: pass - row.write_csv(f2,gzip=True) + row.write_csv(f2, gzip=True) dosummary = (fileformat != 'fit') if name == '': - name = 'imported through email' - - id,message = dataprep.save_workout_database(f2,rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + name = 'imported through email' + id, message = dataprep.save_workout_database(f2, rr, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email') return id - - - - diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 6d06150a..0a7cf1ca 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -1,139 +1,137 @@ #!/srv/venv/bin/python +""" Process emails """ import sys import os + +import zipfile + +from django.core.management.base import BaseCommand + +import time +from time import strftime +from django.conf import settings + +from django_mailbox.models import Message, MessageAttachment +from rowers.models import Workout, Rower + + +from rowingdata import rower as rrower + +from rowingdata import rowingdata as rrdata + +from rowers.mailprocessing import make_new_workout_from_email, send_confirm +import rowers.uploads as uploads + # If you find a solution that does not need the two paths, please comment! sys.path.append('$path_to_root_of_project$') sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' -import zipfile - -from django.core.management.base import BaseCommand, CommandError -from django.conf import settings -#from rowers.mailprocessing import processattachments -import time -from time import strftime -from django.conf import settings -from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm -from django.core.files.base import ContentFile - -from rowsandall_app.settings import BASE_DIR - -from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata - -from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type - -from scipy.signal import savgol_filter -from rowers.mailprocessing import make_new_workout_from_email,send_confirm -import rowers.uploads as uploads - -def rdata(file,rower=rrower()): +def rdata(file, rower=rrower()): + """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file,rower=rower) + res = rrdata(file, rower=rower) except IOError: - res = 0 + res = 0 return res - class Command(BaseCommand): + """Run the Email processing command """ def handle(self, *args, **options): - res = [] - attachments = MessageAttachment.objects.all() - cntr = 0 - for a in attachments: + res = [] + attachments = MessageAttachment.objects.all() + cntr = 0 + for a in attachments: extension = a.document.name[-3:].lower() - donotdelete = 0 - m = Message.objects.get(id=a.message_id) + donotdelete = 0 + m = Message.objects.get(id=a.message_id) body = "\n".join(m.text.splitlines()) uploadoptions = uploads.upload_options(body) - from_address = m.from_address[0].lower() - name = m.subject - cntr += 1 - # get a list of users - # theusers = User.objects.filter(email=from_address) + from_address = m.from_address[0].lower() + name = m.subject + cntr += 1 + # get a list of users + # theusers = User.objects.filter(email=from_address) ther = [ r for r in Rower.objects.all() if r.user.email.lower() == from_address ] - for rr in ther: + for rr in ther: if extension == 'zip': z = zipfile.ZipFile(a.document) for f in z.namelist(): - f2 = z.extract(f,path='media/') + f2 = z.extract(f, path='media/') title = os.path.basename(f2) wid = [ - make_new_workout_from_email(rr,f2[6:],title) + make_new_workout_from_email(rr, f2[6:], title) ] res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if uploadoptions and not 'error' in uploadoptions: w = Workout.objects.get(id=wid[0]) r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + f1 = w.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, + imagename = f1 + timestr + '.png' + resu = uploads.make_plot(r, w, f1, w.csvfilename, - plottype,name, + plottype, name, imagename=imagename) try: if wid != 1: - dd = send_confirm(rr.user,title,link, + dd = send_confirm(rr.user, title, link, uploadoptions) time.sleep(10) except: try: time.sleep(10) if wid != 1: - dd = send_confirm(rr.user,title,link, + dd = send_confirm(rr.user, title, link, uploadoptions) except: pass else: - # move attachment and make workout - try: - wid = [ + # move attachment and make workout + try: + wid = [ make_new_workout_from_email(rr, a.document, name) ] res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if uploadoptions: w = Workout.objects.get(id=wid[0]) r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + f1 = w.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, + imagename = f1 + timestr + '.png' + resu = uploads.make_plot(r, w, f1, w.csvfilename, - plottype,name, + plottype, name, imagename=imagename) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 + except: + # replace with code to process error + res += ['fail: ' + name] + donotdelete = 1 wid = 1 try: if wid != 1: - dd = send_confirm(rr.user,name,link, + dd = send_confirm(rr.user, name, link, uploadoptions) time.sleep(10) except: @@ -144,19 +142,16 @@ class Command(BaseCommand): except IOError: pass # remove attachment - #if donotdelete == 0: + # if donotdelete == 0: - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + if m.attachments.exists() is False: + # no attachments, so can be deleted + m.delete() - mm = Message.objects.all() - for m in mm: - if m.attachments.exists()==False: - m.delete() + mm = Message.objects.all() + for m in mm: + if m.attachments.exists() is False: + m.delete() - self.stdout.write(self.style.SUCCESS('Successfully processed email attachments')) - - - - + self.stdout.write(self.style.SUCCESS( + 'Successfully processed email attachments')) From f7ab21560d941565b5331cefd227faaac7e66b13 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 13:01:16 +0200 Subject: [PATCH 09/15] refactored, not tested --- rowers/mailprocessing.py | 151 +++------------------ rowers/management/commands/processemail.py | 130 +++++++----------- 2 files changed, 69 insertions(+), 212 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index df6760c7..db3bccd0 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -1,24 +1,19 @@ # Processes emails sent to workouts@rowsandall.com +""" Processes emails sent to workouts@rowsandall.com """ +import shutil import time from django.conf import settings from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox, Message, MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm -from django.core.files.base import ContentFile -from django.core.mail import send_mail, BadHeaderError, EmailMessage -from rowsandall_app.settings import BASE_DIR +from django_mailbox.models import Message, MessageAttachment +from rowers.models import User, Rower, RowerForm + +from django.core.mail import EmailMessage from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata -from rowingdata import TCXParser, RowProParser, ErgDataParser -from rowingdata import MysteryParser, BoatCoachParser -from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser -from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata -from rowingdata import make_cumvalues -from rowingdata import summarydata, get_file_type -from scipy.signal import savgol_filter +from rowingdata import rowingdata as rrdata + +from rowingdata import get_file_type import zipfile import os @@ -33,6 +28,7 @@ queuehigh = django_rq.get_queue('default') def send_confirm(u, name, link, options): + """ Send confirmation email to user when email has been processed """ fullemail = u.email subject = 'Workout added: ' + name message = 'Dear ' + u.first_name + ',\n\n' @@ -55,6 +51,7 @@ def send_confirm(u, name, link, options): def rdata(file, rower=rrower()): + """ Reads rowingdata data or returns 0 on Error """ try: res = rrdata(file, rower=rower) except IOError: @@ -65,119 +62,10 @@ def rdata(file, rower=rrower()): return res -# Some error protection around process attachments - - -def safeprocessattachments(): - try: - return processattachments() - except: - return [0] - -# This is duplicated in management/commands/processemail -# Need to double check the code there, update here, and only -# use the code here. - - -def processattachments(): - # in res, we store the ids of the new workouts - res = [] - attachments = MessageAttachment.objects.all() - for a in attachments: - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - # move attachment and make workout - try: - wid = [make_new_workout_from_email(rr, a.document, name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if wid != 1: - dd = send_confirm(u, name, link) - except: - # replace with code to process error - res += ['fail: ' + name] - donotdelete = 1 - except Rower.DoesNotExist: - pass - - # remove attachment - if donotdelete == 0: - a.delete() - - if m.attachments.exists() == False: - # no attachments, so can be deleted - m.delete() - - # Delete remaining messages (which should not have attachments) - mm = Message.objects.all() - for m in mm: - if m.attachments.exists() == False: - m.delete() - - return res - -# As above, but with some print commands for debugging purposes - - -def processattachments_debug(): - res = [] - attachments = MessageAttachment.objects.all() - for a in attachments: - donotdelete = 1 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - print theusers - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - doorgaan = 1 - except: - doorgaan = 0 - if doorgaan: - # move attachment and make workout - print a.document - print name - wid = [make_new_workout_from_email(rr, a.document, name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if wid != 1: - dd = send_confirm(u, name, link) - - # remove attachment - if donotdelete == 0: - a.delete() - - if m.attachments.exists() == False: - # no attachments, so can be deleted - m.delete() - - mm = Message.objects.all() - for m in mm: - if m.attachments.exists() == False: - m.delete() - - return res - -# Process the attachment file, create new workout -# The code here is duplication of the code in views.py (workout_upload_view) -# Need to move the code to a subroutine used both in views.py and here def make_new_workout_from_email(rr, f2, name, cntr=0): + """ This one is used in processemail """ workouttype = 'rower' try: @@ -196,15 +84,18 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): fileformat = fileformat[2] if fileformat == 'unknown': + fcopy = "copy_of_"+f2 + with open(f2, 'r') as f_in, open(fcopy, 'w') as f_out: + shutil.copyfileobj(f_in,f_out) if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, + res = handle_sendemail_unrecognized.delay(fcopy, "roosendaalsander@gmail.com") else: res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2, "roosendaalsander@gmail.com") + fcopy, "roosendaalsander@gmail.com") - return 1 + return 0 summary = '' # handle non-Painsled @@ -216,11 +107,7 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): inboard = 0.88 oarlength = 2.89 - # make workout and put in database - # r = rrower(hrmax=rr.max,hrut2=rr.ut2, - # hrut1=rr.ut1,hrat=rr.at, - # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) - row = rdata(f3) # ,rower=r) + row = rdata(f3) if row == 0: return 0 diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 0a7cf1ca..db17b354 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -5,22 +5,19 @@ import os import zipfile -from django.core.management.base import BaseCommand - import time from time import strftime -from django.conf import settings +from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment from rowers.models import Workout, Rower from rowingdata import rower as rrower - from rowingdata import rowingdata as rrdata -from rowers.mailprocessing import make_new_workout_from_email, send_confirm import rowers.uploads as uploads +from rowers.mailprocessing import make_new_workout_from_email, send_confirm # If you find a solution that does not need the two paths, please comment! sys.path.append('$path_to_root_of_project$') @@ -28,15 +25,58 @@ sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' -def rdata(file, rower=rrower()): +def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file, rower=rower) + res = rrdata(file_obj, rower=rower) except IOError: res = 0 return res +def processattachment(rr, f2, title, uploadoptions): + wid = [ + make_new_workout_from_email(rr, f2[6:], title) + ] + if wid: + res += wid + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' + if uploadoptions and not 'error' in uploadoptions: + w = Workout.objects.get(id=wid[0]) + r = w.user + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) + if 'make_plot' in uploadoptions: + plottype = uploadoptions['plottype'] + f1 = w.csvfilename[6:-4] + timestr = strftime("%Y%m%d-%H%M%S") + imagename = f1 + timestr + '.png' + resu = uploads.make_plot( + r, w, f1, + w.csvfilename, + plottype, name, + imagename=imagename + ) + try: + if wid: + dd = send_confirm( + rr.user, title, link, + uploadoptions + ) + time.sleep(10) + except: + try: + time.sleep(10) + if wid: + dd = send_confirm( + rr.user, title, link, + uploadoptions + ) + except: + pass + + return wid class Command(BaseCommand): """Run the Email processing command """ @@ -46,7 +86,6 @@ class Command(BaseCommand): cntr = 0 for a in attachments: extension = a.document.name[-3:].lower() - donotdelete = 0 m = Message.objects.get(id=a.message_id) body = "\n".join(m.text.splitlines()) uploadoptions = uploads.upload_options(body) @@ -64,85 +103,16 @@ class Command(BaseCommand): for f in z.namelist(): f2 = z.extract(f, path='media/') title = os.path.basename(f2) - wid = [ - make_new_workout_from_email(rr, f2[6:], title) - ] - res += wid - link = 'http://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if uploadoptions and not 'error' in uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w, uploadoptions) - uploads.make_private(w, uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1 + timestr + '.png' - resu = uploads.make_plot(r, w, f1, - w.csvfilename, - plottype, name, - imagename=imagename) - try: - if wid != 1: - dd = send_confirm(rr.user, title, link, - uploadoptions) - time.sleep(10) - except: - try: - time.sleep(10) - if wid != 1: - dd = send_confirm(rr.user, title, link, - uploadoptions) - except: - pass - + wid = processattachment(rr, f2, title, uploadoptions) else: # move attachment and make workout - try: - wid = [ - make_new_workout_from_email(rr, - a.document, - name) - ] - res += wid - link = 'http://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w, uploadoptions) - uploads.make_private(w, uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1 + timestr + '.png' - resu = uploads.make_plot(r, w, f1, - w.csvfilename, - plottype, name, - imagename=imagename) - - except: - # replace with code to process error - res += ['fail: ' + name] - donotdelete = 1 - wid = 1 - try: - if wid != 1: - dd = send_confirm(rr.user, name, link, - uploadoptions) - time.sleep(10) - except: - pass + wid = processattachment(rr, a.document, name, uploadoptions) + # We're done with the attachment. It can be deleted try: a.delete() except IOError: pass - # remove attachment - # if donotdelete == 0: if m.attachments.exists() is False: # no attachments, so can be deleted From f36911d73bc4221da1a8949a3cb4db1e5b713ff8 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 14:38:37 +0200 Subject: [PATCH 10/15] clarify variable naming I have made variable names longer to make it easier to follow the logic of the code. Also removed a few unused ones. Not tested for functionality, yet. --- rowers/management/commands/processemail.py | 108 +++++++++++---------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index db17b354..bc2ff090 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -10,8 +10,9 @@ from time import strftime from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment -from rowers.models import Workout, Rower +from django.core.urlresolvers import reverse +from rowers.models import Workout, Rower from rowingdata import rower as rrower from rowingdata import rowingdata as rrdata @@ -28,100 +29,105 @@ os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file_obj, rower=rower) + result = rrdata(file_obj, rower=rower) except IOError: - res = 0 + result = 0 - return res + return result -def processattachment(rr, f2, title, uploadoptions): - wid = [ - make_new_workout_from_email(rr, f2[6:], title) +def processattachment(rower, filename, title, uploadoptions): + workoutid = [ + make_new_workout_from_email(rower, filename[6:], title) ] - if wid: - res += wid - link = 'http://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' + if workoutid: + link = 'https://rowsandall.com'+reverse( + rower.defaultlandingpage, + kwargs = { + 'id':workoutid, + } + ) + if uploadoptions and not 'error' in uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w, uploadoptions) - uploads.make_private(w, uploadoptions) + workout = Workout.objects.get(id=workoutid[0]) + uploads.do_sync(workout, uploadoptions) + uploads.make_private(workout, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + workoutcsvfilename = workout.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1 + timestr + '.png' - resu = uploads.make_plot( - r, w, f1, - w.csvfilename, - plottype, name, + imagename = workoutcsvfilename + timestr + '.png' + result = uploads.make_plot( + workout.user, workout, workoutcsvfilename, + workout.csvfilename, + plottype, title, imagename=imagename ) try: - if wid: - dd = send_confirm( - rr.user, title, link, + if workoutid: + email_sent = send_confirm( + rower.user, title, link, uploadoptions ) time.sleep(10) except: try: time.sleep(10) - if wid: - dd = send_confirm( - rr.user, title, link, + if workoutid: + email_sent = send_confirm( + rower.user, title, link, uploadoptions ) except: pass - return wid + return workoutid class Command(BaseCommand): """Run the Email processing command """ def handle(self, *args, **options): - res = [] attachments = MessageAttachment.objects.all() cntr = 0 - for a in attachments: - extension = a.document.name[-3:].lower() - m = Message.objects.get(id=a.message_id) - body = "\n".join(m.text.splitlines()) + for attachment in attachments: + extension = attachment.document.name[-3:].lower() + message = Message.objects.get(id=attachment.message_id) + body = "\n".join(message.text.splitlines()) uploadoptions = uploads.upload_options(body) - from_address = m.from_address[0].lower() - name = m.subject - cntr += 1 + from_address = message.from_address[0].lower() + name = message.subject # get a list of users # theusers = User.objects.filter(email=from_address) - ther = [ + rowers = [ r for r in Rower.objects.all() if r.user.email.lower() == from_address ] - for rr in ther: + for rower in rowers: if extension == 'zip': - z = zipfile.ZipFile(a.document) - for f in z.namelist(): - f2 = z.extract(f, path='media/') - title = os.path.basename(f2) - wid = processattachment(rr, f2, title, uploadoptions) + zip_file = zipfile.ZipFile(attachment.document) + for filename in zip_file.namelist(): + datafile = zip_file.extract(filename, path='media/') + title = os.path.basename(datafile) + workoutid = processattachment( + rower, datafile, title, uploadoptions + ) else: # move attachment and make workout - wid = processattachment(rr, a.document, name, uploadoptions) + workoutid = processattachment( + rower, attachment.document, name, uploadoptions + ) # We're done with the attachment. It can be deleted try: - a.delete() + attachment.delete() except IOError: pass - if m.attachments.exists() is False: + if message.attachments.exists() is False: # no attachments, so can be deleted - m.delete() + message.delete() - mm = Message.objects.all() - for m in mm: - if m.attachments.exists() is False: - m.delete() + messages = Message.objects.all() + for message in messages: + if message.attachments.exists() is False: + message.delete() self.stdout.write(self.style.SUCCESS( 'Successfully processed email attachments')) From 973bbb9f0bd38cb9ac3b4942d36f3e4f09620f27 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 16:04:03 +0200 Subject: [PATCH 11/15] Complete and tested email processing Processed successfully individual files, zip files, unrecognized files, zips containing unrecognized files --- rowers/mailprocessing.py | 79 ++++++++++++---------- rowers/management/commands/processemail.py | 45 ++++++++---- rowsandall_app/settings.py | 3 + rowsandall_app/settings_dev.py | 2 + 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index db3bccd0..282fedbe 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -27,11 +27,11 @@ queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u, name, link, options): +def send_confirm(user, name, link, options): """ Send confirmation email to user when email has been processed """ - fullemail = u.email + fullemail = user.email subject = 'Workout added: ' + name - message = 'Dear ' + u.first_name + ',\n\n' + message = 'Dear ' + user.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" @@ -53,68 +53,73 @@ def send_confirm(u, name, link, options): def rdata(file, rower=rrower()): """ Reads rowingdata data or returns 0 on Error """ try: - res = rrdata(file, rower=rower) + result = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file + '.gz', rower=rower) + result = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + result = 0 - return res + return result -def make_new_workout_from_email(rr, f2, name, cntr=0): +def make_new_workout_from_email(rower, datafile, name, cntr=0): """ This one is used in processemail """ workouttype = 'rower' try: - f2 = f2.name - fileformat = get_file_type('media/' + f2) + datafilename = datafile.name + fileformat = get_file_type('media/' + datafilename) except IOError: - f2 = f2.name + '.gz' - fileformat = get_file_type('media/' + f2) + datafilename = datafile.name + '.gz' + fileformat = get_file_type('media/' + datafilename) except AttributeError: - fileformat = get_file_type('media/' + f2) + datafilename = datafile + fileformat = get_file_type('media/' + datafile) if len(fileformat) == 3 and fileformat[0] == 'zip': - f_to_be_deleted = f2 - with zipfile.ZipFile('media/' + f2) as z: - f2 = z.extract(z.namelist()[0], path='media/')[6:] + with zipfile.ZipFile('media/' + datafilename) as zip_file: + datafilename = zip_file.extract( + zip_file.namelist()[0], + path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - fcopy = "copy_of_"+f2 - with open(f2, 'r') as f_in, open(fcopy, 'w') as f_out: - shutil.copyfileobj(f_in,f_out) +# extension = datafilename[-4:].lower() +# fcopy = "media/"+datafilename[:-4]+"_copy"+extension +# with open('media/'+datafilename, 'r') as f_in, open(fcopy, 'w') as f_out: +# shutil.copyfileobj(f_in,f_out) + fcopy = "media/"+datafilename if settings.DEBUG: res = handle_sendemail_unrecognized.delay(fcopy, - "roosendaalsander@gmail.com") + rower.user.email) else: res = queuehigh.enqueue(handle_sendemail_unrecognized, - fcopy, "roosendaalsander@gmail.com") + fcopy, + rower.user.email) return 0 summary = '' # handle non-Painsled if fileformat != 'csv': - f3, summary, oarlength, inboard = dataprep.handle_nonpainsled( - 'media/' + f2, fileformat, summary) + filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + datafilename, fileformat, summary) else: - f3 = 'media/' + f2 + filename_mediadir = 'media/' + datafilename inboard = 0.88 oarlength = 2.89 - row = rdata(f3) + row = rdata(filename_mediadir) if row == 0: return 0 # change filename - if f2[:5] != 'media': + if datafilename[:5] != 'media': timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/' + timestr + str(cntr) + 'o.csv' + datafilename = 'media/' + timestr + str(cntr) + 'o.csv' try: avglat = row.df[' latitude'].mean() @@ -124,19 +129,21 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): except KeyError: pass - row.write_csv(f2, gzip=True) + row.write_csv(datafilename, gzip=True) dosummary = (fileformat != 'fit') if name == '': name = 'imported through email' - id, message = dataprep.save_workout_database(f2, rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + id, message = dataprep.save_workout_database( + datafilename, rower, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email' + ) return id diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index bc2ff090..5bd2d6f1 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -11,6 +11,7 @@ from time import strftime from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment from django.core.urlresolvers import reverse +from django.conf import settings from rowers.models import Workout, Rower @@ -26,6 +27,9 @@ sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' +if not getattr(__builtins__, "WindowsError", None): + class WindowsError(OSError): pass + def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: @@ -35,15 +39,20 @@ def rdata(file_obj, rower=rrower()): return result -def processattachment(rower, filename, title, uploadoptions): +def processattachment(rower, fileobj, title, uploadoptions): + try: + filename = fileobj.name + except AttributeError: + filename = fileobj[6:] + workoutid = [ - make_new_workout_from_email(rower, filename[6:], title) + make_new_workout_from_email(rower, filename, title) ] - if workoutid: - link = 'https://rowsandall.com'+reverse( + if workoutid[0]: + link = settings.SITE_URL+reverse( rower.defaultlandingpage, kwargs = { - 'id':workoutid, + 'id':workoutid[0], } ) @@ -62,23 +71,23 @@ def processattachment(rower, filename, title, uploadoptions): plottype, title, imagename=imagename ) + try: + if workoutid: + email_sent = send_confirm( + rower.user, title, link, + uploadoptions + ) + time.sleep(10) + except: try: + time.sleep(10) if workoutid: email_sent = send_confirm( rower.user, title, link, uploadoptions ) - time.sleep(10) except: - try: - time.sleep(10) - if workoutid: - email_sent = send_confirm( - rower.user, title, link, - uploadoptions - ) - except: - pass + pass return workoutid @@ -119,6 +128,12 @@ class Command(BaseCommand): attachment.delete() except IOError: pass + except WindowsError: + time.sleep(2) + try: + attachment.delete() + except WindowsError: + pass if message.attachments.exists() is False: # no attachments, so can be deleted diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index a7f25d88..beadcf86 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -261,6 +261,9 @@ TP_CLIENT_SECRET = CFG["tp_client_secret"] TP_REDIRECT_URI = CFG["tp_redirect_uri"] TP_CLIENT_KEY = TP_CLIENT_ID +# Full Site URL +SITE_URL = "https://rowsandall.com" + # RQ stuff RQ_QUEUES = { diff --git a/rowsandall_app/settings_dev.py b/rowsandall_app/settings_dev.py index f851d022..b37aaed8 100644 --- a/rowsandall_app/settings_dev.py +++ b/rowsandall_app/settings_dev.py @@ -76,6 +76,8 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/' SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" +SITE_URL = "http://localhost:8000" + #EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' #EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend' From 74e924a3c1d09ce9f4b2363d9a44c50964f20b37 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 23 Oct 2017 08:56:10 +0200 Subject: [PATCH 12/15] send email to sender of unrecognized file The email processing sends a notification message to the person who sent a unrecognized file to the workouts@rowsandall.com email address. This way, people will know that their email has worked and perhaps stop sending 99 or more emails with the same unsupported file. --- rowers/mailprocessing.py | 16 +- rowers/tasks.py | 523 +++++++++++++++++++++------------------ 2 files changed, 296 insertions(+), 243 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index 282fedbe..b6489e61 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -92,14 +92,22 @@ def make_new_workout_from_email(rower, datafile, name, cntr=0): # shutil.copyfileobj(f_in,f_out) fcopy = "media/"+datafilename if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(fcopy, - rower.user.email) - + res = handle_sendemail_unrecognized.delay( + fcopy, + rower.user.email + ) + res = handle_sendemail_unrecognizedowner.delay( + rower.user.email, + rower.user.first_name + ) else: res = queuehigh.enqueue(handle_sendemail_unrecognized, fcopy, rower.user.email) - + res = queuehigh.enqueue(handle_sendemail_unrecognizedowner, + rower.user.email, + rower.user.first_name + ) return 0 summary = '' diff --git a/rowers/tasks.py b/rowers/tasks.py index 743e6af8..bea8eddf 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -1,4 +1,4 @@ -from celery import Celery,app +""" Background tasks done by Celery (develop) or QR (production) """ import os import time import gc @@ -7,67 +7,73 @@ import shutil import numpy as np import rowingdata -from rowingdata import main as rmain + from rowingdata import rowingdata as rdata -import rowingdata -from async_messages import message_user,messages + +from celery import app from matplotlib.backends.backend_agg import FigureCanvas #from matplotlib.backends.backend_cairo import FigureCanvasCairo as FigureCanvas import matplotlib.pyplot as plt -from matplotlib import figure -import stravalib + + import pandas as pd from django_rq import job -from utils import serialize_list,deserialize_list +from utils import deserialize_list from rowers.dataprepnodjango import ( update_strokedata, new_workout_from_file, - getsmallrowdata_db,read_df_sql,columndict, -) + getsmallrowdata_db, +) -from django.core.mail import send_mail, BadHeaderError,EmailMessage +from django.core.mail import send_mail, EmailMessage import datautils import utils # testing task + + @app.task def add(x, y): return x + y # create workout + + @app.task -def handle_new_workout_from_file(r,f2, - workouttype='rower', - title='Workout', - makeprivate=False, - notes=''): - return new_workout_from_file(r,f2,workouttype, - title,makeprivate,notes) +def handle_new_workout_from_file(r, f2, + workouttype='rower', + title='Workout', + makeprivate=False, + notes=''): + return new_workout_from_file(r, f2, workouttype, + title, makeprivate, notes) # process and update workouts + + @app.task -def handle_updatedps(useremail,workoutids,debug=False): - for wid,f1 in workoutids: +def handle_updatedps(useremail, workoutids, debug=False): + for wid, f1 in workoutids: havedata = 1 try: rowdata = rdata(f1) except IOError: try: - rowdata = rdata(f1+'.csv') + rowdata = rdata(f1 + '.csv') except IOError: try: - rowdata = rdata(f1+'.gz') + rowdata = rdata(f1 + '.gz') except IOError: havedata = 0 if havedata: - update_strokedata(wid,rowdata.df,debug=debug) + update_strokedata(wid, rowdata.df, debug=debug) subject = "Rowsandall.com Your Distance per Stroke metric has been updated" message = "All your workouts now have Distance per Stroke" @@ -77,18 +83,20 @@ def handle_updatedps(useremail,workoutids,debug=False): [useremail]) res = email.send() - + return 1 # send email when a breakthrough workout is uploaded + + @app.task -def handle_sendemail_breakthrough(workoutid,useremail, - userfirstname,userlastname, - btvalues = pd.DataFrame().to_json()): +def handle_sendemail_breakthrough(workoutid, useremail, + userfirstname, userlastname, + btvalues=pd.DataFrame().to_json()): # send email with attachment subject = "A breakthrough workout on rowsandall.com" - message = "Dear "+userfirstname+",\n" + message = "Dear " + userfirstname + ",\n" message += "Congratulations! Your recent workout has been analyzed" message += " by Rowsandall.com and it appears your fitness," message += " as measured by Critical Power, has improved!" @@ -98,17 +106,17 @@ def handle_sendemail_breakthrough(workoutid,useremail, message += " http://analytics.rowsandall.com/2017/06/17/how-do-we-calculate-critical-power/ \n\n" message += "Link to the workout http://rowsandall.com/rowers/workout/" message += str(workoutid) - message +="/edit\n\n" - message +="To add the workout to your Ranking workouts and see the updated CP plot, click the following link:\n" + message += "/edit\n\n" + message += "To add the workout to your Ranking workouts and see the updated CP plot, click the following link:\n" message += "http://rowsandall.com/rowers/workout/" message += str(workoutid) message += "/updatecp\n\n" btvalues = pd.read_json(btvalues) - btvalues.sort_values('delta',axis=0,inplace=True) - + btvalues.sort_values('delta', axis=0, inplace=True) + if not btvalues.empty: - message += "These were the breakthrough values:\n" + message += "These were the breakthrough values:\n" for t in btvalues.itertuples(): delta = t.delta cpvalue = t.cpvalues @@ -116,23 +124,21 @@ def handle_sendemail_breakthrough(workoutid,useremail, message += "Time: {delta} seconds\n".format( delta=delta - ) + ) message += "New: {cpvalue:.0f} Watt\n".format( cpvalue=cpvalue - ) + ) message += "Old: {pwr:.0f} Watt\n\n".format( pwr=pwr - ) + ) - message += "To opt out of these email notifications, deselect the checkbox on your Profile page under Account Information.\n\n" message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [useremail]) - + 'Rowsandall ', + [useremail]) res = email.send() @@ -140,14 +146,16 @@ def handle_sendemail_breakthrough(workoutid,useremail, return 1 # send email when a breakthrough workout is uploaded + + @app.task -def handle_sendemail_hard(workoutid,useremail, - userfirstname,userlastname, - btvalues = pd.DataFrame().to_json()): +def handle_sendemail_hard(workoutid, useremail, + userfirstname, userlastname, + btvalues=pd.DataFrame().to_json()): # send email with attachment subject = "That was a pretty hard workout on rowsandall.com" - message = "Dear "+userfirstname+",\n" + message = "Dear " + userfirstname + ",\n" message += "Congratulations! Your recent workout has been analyzed" message += " by Rowsandall.com and it appears that it was pretty hard work." message += " You were working pretty close to your Critical Power\n\n" @@ -157,16 +165,15 @@ def handle_sendemail_hard(workoutid,useremail, message += " http://analytics.rowsandall.com/2017/06/17/how-do-we-calculate-critical-power/ \n\n" message += "Link to the workout http://rowsandall.com/rowers/workout/" message += str(workoutid) - message +="/edit\n\n" + message += "/edit\n\n" message += "To opt out of these email notifications, deselect the checkbox on your Profile page under Account Information.\n\n" message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [useremail]) - + 'Rowsandall ', + [useremail]) res = email.send() @@ -176,25 +183,25 @@ def handle_sendemail_hard(workoutid,useremail, # send email to me when an unrecognized file is uploaded @app.task -def handle_sendemail_unrecognized(unrecognizedfile,useremail): +def handle_sendemail_unrecognized(unrecognizedfile, useremail): # send email with attachment fullemail = 'roosendaalsander@gmail.com' subject = "Unrecognized file from Rowsandall.com" message = "Dear Sander,\n\n" message += "Please find attached a file that someone tried to upload to rowsandall.com. The file was not recognized as a valid file type.\n\n" - message += "User Email "+useremail+"\n\n" + message += "User Email " + useremail + "\n\n" message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) + try: email.attach_file(unrecognizedfile) except IOError: pass - + res = email.send() # remove tcx file @@ -202,38 +209,69 @@ def handle_sendemail_unrecognized(unrecognizedfile,useremail): os.remove(unrecognizedfile) except: pass - + return 1 - + +# send email to owner when an unrecognized file is uploaded +@app.task +def handle_sendemail_unrecognizedowner(useremail, userfirstname): + + # send email with attachment + fullemail = useremail + subject = "Unrecognized file from Rowsandall.com" + message = "Dear " + userfirstname + ",\n\n" + message += """ + The file you tried to send to rowsandall.com was not recognized by + our email processing system. You may have sent a file in a format + that is not supported. Sometimes, rowing apps make file format changes. + When that happens, it takes some time for rowsandall.comm to make + the necessary changes on our side and support the app again. + + The file has been sent to the developer at rowsandall.com for evaluation. + + + """ + + message += "Best Regards, the Rowsandall Team" + + email = EmailMessage(subject, message, + 'Rowsandall ', + [fullemail]) + + res = email.send() + + return 1 + + # Send email with TCX attachment @app.task -def handle_sendemailtcx(first_name,last_name,email,tcxfile): +def handle_sendemailtcx(first_name, last_name, email, tcxfile): # send email with attachment fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "File from Rowsandall.com" - message = "Dear "+first_name+",\n\n" + message = "Dear " + first_name + ",\n\n" message += "Please find attached the requested file for your workout.\n\n" message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) email.attach_file(tcxfile) - + res = email.send() # remove tcx file os.remove(tcxfile) return 1 + @app.task -def handle_zip_file(emailfrom,subject,file): +def handle_zip_file(emailfrom, subject, file): message = "... zip processing ... " - email = EmailMessage(subject,message, + email = EmailMessage(subject, message, emailfrom, ['workouts@rowsandall.com']) email.attach_file(file) @@ -242,64 +280,68 @@ def handle_zip_file(emailfrom,subject,file): return 1 # Send email with CSV attachment + + @app.task -def handle_sendemailcsv(first_name,last_name,email,csvfile): +def handle_sendemailcsv(first_name, last_name, email, csvfile): # send email with attachment fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "File from Rowsandall.com" - message = "Dear "+first_name+",\n\n" + message = "Dear " + first_name + ",\n\n" message += "Please find attached the requested file for your workout.\n\n" message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) if os.path.isfile(csvfile): email.attach_file(csvfile) else: csvfile2 = csvfile - with gzip.open(csvfile+'.gz','rb') as f_in, open(csvfile2,'wb') as f_out: + with gzip.open(csvfile + '.gz', 'rb') as f_in, open(csvfile2, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) - + email.attach_file(csvfile2) os.remove(csvfile2) - + res = email.send() return 1 # Calculate wind and stream corrections for OTW rowing + + @app.task -def handle_otwsetpower(f1,boattype,weightvalue, - first_name,last_name,email,workoutid,ps=[1,1,1,1], +def handle_otwsetpower(f1, boattype, weightvalue, + first_name, last_name, email, workoutid, ps=[ + 1, 1, 1, 1], ratio=1.0, debug=False): try: rowdata = rdata(f1) except IOError: try: - rowdata = rdata(f1+'.csv') + rowdata = rdata(f1 + '.csv') except IOError: - rowdata = rdata(f1+'.gz') - + rowdata = rdata(f1 + '.gz') + weightvalue = float(weightvalue) # do something with boat type boatfile = { - '1x':'static/rigging/1x.txt', - '2x':'static/rigging/2x.txt', - '2-':'static/rigging/2-.txt', - '4x':'static/rigging/4x.txt', - '4-':'static/rigging/4-.txt', - '8+':'static/rigging/8+.txt', - } + '1x': 'static/rigging/1x.txt', + '2x': 'static/rigging/2x.txt', + '2-': 'static/rigging/2-.txt', + '4x': 'static/rigging/4x.txt', + '4-': 'static/rigging/4-.txt', + '8+': 'static/rigging/8+.txt', + } try: - rg = rowingdata.getrigging(boatfile[boattype]) + rg = rowingdata.getrigging(boatfile[boattype]) except KeyError: - rg = rowingdata.getrigging('static/rigging/1x.txt') + rg = rowingdata.getrigging('static/rigging/1x.txt') # do calculation, but do not overwrite NK Empower Power data powermeasured = False @@ -310,117 +352,119 @@ def handle_otwsetpower(f1,boattype,weightvalue, except KeyError: pass - rowdata.otw_setpower_silent(skiprows=5,mc=weightvalue,rg=rg, + rowdata.otw_setpower_silent(skiprows=5, mc=weightvalue, rg=rg, powermeasured=powermeasured) # save data - rowdata.write_csv(f1,gzip=True) - update_strokedata(workoutid,rowdata.df,debug=debug) + rowdata.write_csv(f1, gzip=True) + update_strokedata(workoutid, rowdata.df, debug=debug) - totaltime = rowdata.df['TimeStamp (sec)'].max()-rowdata.df['TimeStamp (sec)'].min() + totaltime = rowdata.df['TimeStamp (sec)'].max( + ) - rowdata.df['TimeStamp (sec)'].min() try: - totaltime = totaltime+rowdata.df.ix[0,' ElapsedTime (sec)'] + totaltime = totaltime + rowdata.df.ix[0, ' ElapsedTime (sec)'] except KeyError: pass - df = getsmallrowdata_db(['power','workoutid','time'],ids=[workoutid]) + df = getsmallrowdata_db(['power', 'workoutid', 'time'], ids=[workoutid]) thesecs = totaltime - maxt = 1.05*thesecs + maxt = 1.05 * thesecs logarr = datautils.getlogarr(maxt) dfgrouped = df.groupby(['workoutid']) - delta,cpvalues,avgpower = datautils.getcp(dfgrouped,logarr) - - + delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) + #delta,cpvalues,avgpower = datautils.getsinglecp(rowdata.df) - res,btvalues,res2 = utils.isbreakthrough(delta,cpvalues,ps[0],ps[1],ps[2],ps[3],ratio) + res, btvalues, res2 = utils.isbreakthrough( + delta, cpvalues, ps[0], ps[1], ps[2], ps[3], ratio) if res: - handle_sendemail_breakthrough(workoutid,email, + handle_sendemail_breakthrough(workoutid, email, first_name, - last_name,btvalues=btvalues.to_json()) + last_name, btvalues=btvalues.to_json()) # send email fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "Your Rowsandall OTW calculations are ready" - message = "Dear "+first_name+",\n\n" + message = "Dear " + first_name + ",\n\n" message += "Your Rowsandall OTW calculations are ready.\n" message += "Thank you for using rowsandall.com.\n\n" message += "Rowsandall OTW calculations have not been fully implemented yet.\n" message += "We are now running an experimental version for debugging purposes. \n" message += "Your wind/stream corrected plot is available here: http://rowsandall.com/rowers/workout/" message += str(workoutid) - message +="/interactiveotwplot\n\n" + message += "/interactiveotwplot\n\n" message += "Please report any bugs/inconsistencies/unexpected results at rowsandall.slack.com or by reply to this email.\n\n" message += "Best Regards, The Rowsandall Physics Department." send_mail(subject, message, - 'Rowsandall Physics Department ', - [fullemail]) + 'Rowsandall Physics Department ', + [fullemail]) return 1 # This function generates all the static (PNG image) plots -@app.task -def handle_makeplot(f1,f2,t,hrdata,plotnr,imagename): - hrmax = hrdata['hrmax'] - hrut2 = hrdata['hrut2'] - hrut1 = hrdata['hrut1'] - hrat = hrdata['hrat'] - hrtr = hrdata['hrtr'] - hran = hrdata['hran'] + +@app.task +def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename): + + hrmax = hrdata['hrmax'] + hrut2 = hrdata['hrut2'] + hrut1 = hrdata['hrut1'] + hrat = hrdata['hrat'] + hrtr = hrdata['hrtr'] + hran = hrdata['hran'] ftp = hrdata['ftp'] powerzones = deserialize_list(hrdata['powerzones']) powerperc = np.array(deserialize_list(hrdata['powerperc'])).astype(int) - - rr = rowingdata.rower(hrmax=hrmax,hrut2=hrut2, - hrut1=hrut1,hrat=hrat, - hrtr=hrtr,hran=hran, - ftp=ftp,powerperc=powerperc, + + rr = rowingdata.rower(hrmax=hrmax, hrut2=hrut2, + hrut1=hrut1, hrat=hrat, + hrtr=hrtr, hran=hran, + ftp=ftp, powerperc=powerperc, powerzones=powerzones) try: - row = rdata(f2,rower=rr) + row = rdata(f2, rower=rr) except IOError: - row = rdata(f2+'.gz',rower=rr) - + row = rdata(f2 + '.gz', rower=rr) haspower = row.df[' Power (watts)'].mean() > 50 - + nr_rows = len(row.df) - if (plotnr in [1,2,4,5,8,11,9,12]) and (nr_rows > 1200): - bin = int(nr_rows/1200.) - df = row.df.groupby(lambda x:x/bin).mean() - row.df = df + if (plotnr in [1, 2, 4, 5, 8, 11, 9, 12]) and (nr_rows > 1200): + bin = int(nr_rows / 1200.) + df = row.df.groupby(lambda x: x / bin).mean() + row.df = df nr_rows = len(row.df) - if (plotnr==1): - fig1 = row.get_timeplot_erg(t) - elif (plotnr==2): - fig1 = row.get_metersplot_erg(t) - elif (plotnr==3): - fig1 = row.get_piechart(t) - elif (plotnr==4): + if (plotnr == 1): + fig1 = row.get_timeplot_erg(t) + elif (plotnr == 2): + fig1 = row.get_metersplot_erg(t) + elif (plotnr == 3): + fig1 = row.get_piechart(t) + elif (plotnr == 4): if haspower: fig1 = row.get_timeplot_otwempower(t) else: - fig1 = row.get_timeplot_otw(t) - elif (plotnr==5): + fig1 = row.get_timeplot_otw(t) + elif (plotnr == 5): if haspower: fig1 = row.get_metersplot_otwempower(t) else: - fig1 = row.get_metersplot_otw(t) - elif (plotnr==6): - fig1 = row.get_piechart(t) - elif (plotnr==7) or (plotnr==10): - fig1 = row.get_metersplot_erg2(t) - elif (plotnr==8) or (plotnr==11): - fig1 = row.get_timeplot_erg2(t) - elif (plotnr==9) or (plotnr==12): - fig1 = row.get_time_otwpower(t) - elif (plotnr==13) or (plotnr==16): - fig1 = row.get_power_piechart(t) + fig1 = row.get_metersplot_otw(t) + elif (plotnr == 6): + fig1 = row.get_piechart(t) + elif (plotnr == 7) or (plotnr == 10): + fig1 = row.get_metersplot_erg2(t) + elif (plotnr == 8) or (plotnr == 11): + fig1 = row.get_timeplot_erg2(t) + elif (plotnr == 9) or (plotnr == 12): + fig1 = row.get_time_otwpower(t) + elif (plotnr == 13) or (plotnr == 16): + fig1 = row.get_power_piechart(t) canvas = FigureCanvas(fig1) # plt.savefig('static/plots/'+imagename,format='png') - canvas.print_figure('static/plots/'+imagename) + canvas.print_figure('static/plots/' + imagename) # plt.imsave(fname='static/plots/'+imagename) plt.close(fig1) fig1.clf() @@ -430,12 +474,13 @@ def handle_makeplot(f1,f2,t,hrdata,plotnr,imagename): # Team related remote tasks + @app.task -def handle_sendemail_invite(email,name,code,teamname,manager): - fullemail = name+' <'+email+'>' - subject = 'Invitation to join team '+teamname - message = 'Dear '+name+',\n\n' - message += manager+' is inviting you to join his team '+teamname +def handle_sendemail_invite(email, name, code, teamname, manager): + fullemail = name + ' <' + email + '>' + subject = 'Invitation to join team ' + teamname + message = 'Dear ' + name + ',\n\n' + message += manager + ' is inviting you to join his team ' + teamname message += ' on rowsandall.com\n\n' message += 'By accepting the invite, you will have access to your' message += " team's workouts on rowsandall.com and your workouts will " @@ -444,117 +489,119 @@ def handle_sendemail_invite(email,name,code,teamname,manager): message += 'If you already have an account on rowsandall.com, you can login to the site and you will find the invitation here on the Teams page:\n' message += ' https://rowsandall.com/rowers/me/teams \n\n' message += 'You can also click the direct link: \n' - message += 'https://rowsandall.com/rowers/me/invitation/'+code+' \n\n' + message += 'https://rowsandall.com/rowers/me/invitation/' + code + ' \n\n' message += 'If you are not yet registered to rowsandall.com, ' message += 'you can register for free at https://rowsandall.com/rowers/register\n' message += 'After you set up your account, you can use the direct link: ' - message += 'https://rowsandall.com/rowers/me/invitation/'+code+' \n\n' + message += 'https://rowsandall.com/rowers/me/invitation/' + code + ' \n\n' message += 'You can also manually accept your team membership with the code.\n' message += 'You will need to do this if you registered under a different email address than this one.\n' - message += 'Code: '+code+'\n' + message += 'Code: ' + code + '\n' message += 'Link to manually accept your team membership: ' message += 'https://rowsandall.com/rowers/me/invitation\n\n' message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemailnewresponse(first_name,last_name, +def handle_sendemailnewresponse(first_name, last_name, email, commenter_first_name, commenter_last_name, comment, - workoutname,workoutid,commentid): - fullemail = first_name+' '+last_name+' <'+email+'>' - subject = 'New comment on workout '+workoutname - message = 'Dear '+first_name+',\n\n' - message += commenter_first_name+' '+commenter_last_name + workoutname, workoutid, commentid): + fullemail = first_name + ' ' + last_name + ' <' + email + '>' + subject = 'New comment on workout ' + workoutname + message = 'Dear ' + first_name + ',\n\n' + message += commenter_first_name + ' ' + commenter_last_name message += ' has written a new comment on the workout ' - message += workoutname+'\n\n' + message += workoutname + '\n\n' message += comment message += '\n\n' message += 'You can read the comment here:\n' - message += 'https://rowsandall.com/rowers/workout/'+str(workoutid)+'/comment' + message += 'https://rowsandall.com/rowers/workout/' + \ + str(workoutid) + '/comment' message += '\n\n' message += 'You are receiving this email because you are subscribed ' message += 'to comments on this workout. To unsubscribe, follow this link:\n' - message += 'https://rowsandall.com/rowers/workout/'+str(workoutid)+'/unsubscribe' + message += 'https://rowsandall.com/rowers/workout/' + \ + str(workoutid) + '/unsubscribe' email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 - + + @app.task def handle_sendemailnewcomment(first_name, - last_name, - email, + last_name, + email, commenter_first_name, commenter_last_name, - comment,workoutname, + comment, workoutname, workoutid): - fullemail = first_name+' '+last_name+' <'+email+'>' - subject = 'New comment on workout '+workoutname - message = 'Dear '+first_name+',\n\n' - message += commenter_first_name+' '+commenter_last_name + fullemail = first_name + ' ' + last_name + ' <' + email + '>' + subject = 'New comment on workout ' + workoutname + message = 'Dear ' + first_name + ',\n\n' + message += commenter_first_name + ' ' + commenter_last_name message += ' has written a new comment on your workout ' - message += workoutname+'\n\n' + message += workoutname + '\n\n' message += comment message += '\n\n' message += 'You can read the comment here:\n' - message += 'https://rowsandall.com/rowers/workout/'+str(workoutid)+'/comment' + message += 'https://rowsandall.com/rowers/workout/' + \ + str(workoutid) + '/comment' email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 - + @app.task -def handle_sendemail_request(email,name,code,teamname,requestor,id): - fullemail = name+' <'+email+'>' - subject = 'Request to join team '+teamname - message = 'Dear '+name+',\n\n' - message += requestor+' is requesting admission to your team '+teamname +def handle_sendemail_request(email, name, code, teamname, requestor, id): + fullemail = name + ' <' + email + '>' + subject = 'Request to join team ' + teamname + message = 'Dear ' + name + ',\n\n' + message += requestor + ' is requesting admission to your team ' + teamname message += ' on rowsandall.com\n\n' message += 'Click the direct link to accept: \n' - message += 'https://rowsandall.com/rowers/me/request/'+code+' \n\n' + message += 'https://rowsandall.com/rowers/me/request/' + code + ' \n\n' message += 'Click the following link to reject the request: \n' - message += 'https://rowsandall.com/rowers/me/request/'+str(id)+' \n\n' + message += 'https://rowsandall.com/rowers/me/request/' + str(id) + ' \n\n' message += 'You can find all pending requests on your team management page:\n' message += 'https://rowsandall.com/rowers/me/teams\n\n' message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemail_request_accept(email,name,teamname,managername): - fullemail = name+' <'+email+'>' - subject = 'Welcome to '+teamname - message = 'Dear '+name+',\n\n' +def handle_sendemail_request_accept(email, name, teamname, managername): + fullemail = name + ' <' + email + '>' + subject = 'Welcome to ' + teamname + message = 'Dear ' + name + ',\n\n' message += managername message += ' has accepted your request to be part of the team ' message += teamname @@ -562,19 +609,19 @@ def handle_sendemail_request_accept(email,name,teamname,managername): message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemail_request_reject(email,name,teamname,managername): - fullemail = name+' <'+email+'>' - subject = 'Your application to '+teamname+' was rejected' - message = 'Dear '+name+',\n\n' +def handle_sendemail_request_reject(email, name, teamname, managername): + fullemail = name + ' <' + email + '>' + subject = 'Your application to ' + teamname + ' was rejected' + message = 'Dear ' + name + ',\n\n' message += 'Unfortunately, ' message += managername message += ' has rejected your request to be part of the team ' @@ -583,19 +630,19 @@ def handle_sendemail_request_reject(email,name,teamname,managername): message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemail_member_dropped(email,name,teamname,managername): - fullemail = name+' <'+email+'>' - subject = 'You were removed from '+teamname - message = 'Dear '+name+',\n\n' +def handle_sendemail_member_dropped(email, name, teamname, managername): + fullemail = name + ' <' + email + '>' + subject = 'You were removed from ' + teamname + message = 'Dear ' + name + ',\n\n' message += 'Unfortunately, ' message += managername message += ' has removed you from the team ' @@ -604,42 +651,42 @@ def handle_sendemail_member_dropped(email,name,teamname,managername): message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemail_team_removed(email,name,teamname,managername): - fullemail = name+' <'+email+'>' - subject = 'Team '+teamname+' was deleted' - message = 'Dear '+name+',\n\n' +def handle_sendemail_team_removed(email, name, teamname, managername): + fullemail = name + ' <' + email + '>' + subject = 'Team ' + teamname + ' was deleted' + message = 'Dear ' + name + ',\n\n' message += managername message += ' has decided to delete the team ' message += teamname message += '\n\n' - message += 'The '+teamname+' tag has been removed from all your ' + message += 'The ' + teamname + ' tag has been removed from all your ' message += 'workouts on rowsandall.com.\n\n' message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemail_invite_reject(email,name,teamname,managername): - fullemail = managername+' <'+email+'>' - subject = 'Your invitation to '+name+' was rejected' - message = 'Dear '+managername+',\n\n' - message += 'Unfortunately, ' +def handle_sendemail_invite_reject(email, name, teamname, managername): + fullemail = managername + ' <' + email + '>' + subject = 'Your invitation to ' + name + ' was rejected' + message = 'Dear ' + managername + ',\n\n' + message += 'Unfortunately, ' message += name message += ' has rejected your invitation to be part of the team ' message += teamname @@ -647,33 +694,31 @@ def handle_sendemail_invite_reject(email,name,teamname,managername): message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 + @app.task -def handle_sendemail_invite_accept(email,name,teamname,managername): - fullemail = managername+' <'+email+'>' - subject = 'Your invitation to '+name+' was accepted' - message = 'Dear '+managername+',\n\n' - message += name+' has accepted your invitation to be part of the team '+teamname+'\n\n' +def handle_sendemail_invite_accept(email, name, teamname, managername): + fullemail = managername + ' <' + email + '>' + subject = 'Your invitation to ' + name + ' was accepted' + message = 'Dear ' + managername + ',\n\n' + message += name + ' has accepted your invitation to be part of the team ' + teamname + '\n\n' message += "Best Regards, the Rowsandall Team" email = EmailMessage(subject, message, - 'Rowsandall ', - [fullemail]) - + 'Rowsandall ', + [fullemail]) res = email.send() return 1 - # Another simple task for debugging purposes -def add2(x,y): - return x+y +def add2(x, y): + return x + y From b7e9fbaa458e74826d71301f43930b62aa25f376 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 23 Oct 2017 20:10:46 +0200 Subject: [PATCH 13/15] Pro metrics restricted to Pro users When I opened Favorite Flex CHarts to all, I overlooked that this was opening a back door to Basic users to get to view the Pro metrics. Corrected that now. When a Favorite Flex chart with Pro metrics is loaded, for Basic users, the values are replaced with time (x), pace (y1) and spm (y2) --- rowers/interactiveplots.py | 1 - rowers/views.py | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 12c0a1ec..60cad3cc 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -2046,7 +2046,6 @@ def interactive_flex_chart2(id=0,promember=0, plottype='line', workstrokesonly=False): - #rowdata,row = dataprep.getrowdata_db(id=id) columns = [xparam,yparam1,yparam2, 'ftime','distance','fpace', diff --git a/rowers/views.py b/rowers/views.py index 8b2c5b75..5ce1156b 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -6197,6 +6197,19 @@ def workout_flexchart3_view(request,*args,**kwargs): else: workstrokesonly = False + if not promember: + for name,d in rowingmetrics: + if d['type'] != 'basic': + if xparam == name: + xparam = 'time' + messages.info(request,'To use '+d['verbose_name']+', you have to be Pro member') + if yparam1 == name: + yparam1 = 'pace' + messages.info(request,'To use '+d['verbose_name']+', you have to be Pro member') + if yparam2 == name: + yparam2 = 'spm' + messages.info(request,'To use '+d['verbose_name']+', you have to be Pro member') + # create interactive plot try: script,div,js_resources,css_resources,workstrokesonly = interactive_flex_chart2(id,xparam=xparam,yparam1=yparam1, @@ -8539,10 +8552,10 @@ def rower_favoritecharts_view(request): if request.method == 'POST': favorites_formset = FavoriteChartFormSet(request.POST) - if favorites_formset.is_valid(): new_instances = [] for favorites_form in favorites_formset: + print 'mies' yparam1 = favorites_form.cleaned_data.get('yparam1') yparam2 = favorites_form.cleaned_data.get('yparam2') xparam = favorites_form.cleaned_data.get('xparam') @@ -8568,7 +8581,6 @@ def rower_favoritecharts_view(request): except IntegrityError: message = "something went wrong" messages.error(request,message) - else: favorites_formset = FavoriteChartFormSet(initial=favorites_data) From 7ce776dcc8003702d95d165d180ad4338e8e44c9 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 23 Oct 2017 20:41:56 +0200 Subject: [PATCH 14/15] fix of typo in metrics.py (default flex charts) --- rowers/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rowers/metrics.py b/rowers/metrics.py index b2986565..4e741b08 100644 --- a/rowers/metrics.py +++ b/rowers/metrics.py @@ -269,7 +269,7 @@ in case you recorded your heart rate during your workout""", heart rate versus time. """, }, { - 'yparam1':'strokeenergy', + 'yparam1':'driveenergy', 'yparam2':'hr', 'xparam':'time', 'plottype':'line', @@ -291,7 +291,7 @@ as you increase stroke rate. Typical values are > 10m for steady state dropping to 8m for race pace in the single.""", }, { - 'yparam1':'strokeenergy', + 'yparam1':'driveenergy', 'yparam2':'None', 'xparam':'spm', 'plottype':'line', From e636c9fad3361a94ddbb2b42f0e426eb833b3097 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 24 Oct 2017 08:00:56 +0200 Subject: [PATCH 15/15] landing page setting has verbose name --- rowers/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rowers/models.py b/rowers/models.py index fc40651a..54002477 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -284,7 +284,8 @@ class Rower(models.Model): workflowmiddlepanel = TemplateListField(default=defaultmiddle) defaultlandingpage = models.CharField(default='workout_edit_view', max_length=200, - choices=landingpages) + choices=landingpages, + verbose_name="Default Landing Page") # Access tokens c2token = models.CharField(default='',max_length=200,blank=True,null=True)