Merge branch 'release/v10.94'

rowers/dataprep.py

@@ -3,7 +3,6 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-
 # All the data preparation, data cleaning and data mangling should
 # be defined here
 from __future__ import unicode_literals, absolute_import

rowers/forms.py

@@ -395,6 +395,26 @@ class TeamUploadOptionsForm(forms.Form):
                                  initial='timeplot',
                                  label='Plot Type')
 
+    upload_to_C2 = forms.BooleanField(initial=False,required=False,
+                                      label='Export to Concept2 logbook')
+    upload_to_Strava = forms.BooleanField(initial=False,required=False,
+                                          label='Export to Strava')
+    upload_to_SportTracks = forms.BooleanField(initial=False,required=False,
+                                               label='Export to SportTracks')
+    upload_to_RunKeeper = forms.BooleanField(initial=False,required=False,
+                                             label='Export to RunKeeper')
+    upload_to_MapMyFitness = forms.BooleanField(initial=False,
+                                                required=False,
+                                                label='Export to MapMyFitness')
+    upload_to_TrainingPeaks = forms.BooleanField(initial=False,
+                                                 required=False,
+                                                 label='Export to TrainingPeaks')
+    # do_physics = forms.BooleanField(initial=False,required=False,label='Power Estimate (OTW)')
+    makeprivate = forms.BooleanField(initial=False,required=False,
+                                     label='Make Workout Private')
+
+
+
     class Meta:
         fields = ['make_plot','plottype']
 

rowers/management/commands/processemail.py

@@ -10,6 +10,9 @@ import re
 import time
 from time import strftime
 
+import requests
+import json
+
 import io
 
 from django.core.management.base import BaseCommand
@@ -56,6 +59,7 @@ def rdata(file_obj, rower=rrower()):
 def processattachment(rower, fileobj, title, uploadoptions,testing=False):
     try:
         filename = fileobj.name
+#        filename = os.path.abspath(fileobj.name)
     except AttributeError:
         filename = fileobj[6:]
         if testing:
@@ -85,9 +89,31 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
         therower = rower
 
 
-    workoutid = [
+    uploadoptions['secret'] = settings.UPLOAD_SERVICE_SECRET
-        make_new_workout_from_email(therower, filename, title,testing=testing)
+    uploadoptions['user'] = therower.user.id
-    ]
+    uploadoptions['file'] = 'media/'+filename
+    uploadoptions['title'] = title
+
+
+    url = "http://localhost:8000/rowers/workout/api/upload/"
+    if not testing:
+        response = requests.post(url,data=uploadoptions)
+        if response.status_code == 200:
+            response_json = response.json()
+            workoutid = [int(response_json['id'])]
+        else:
+            workoutid = [0]
+
+    # this is ugly and needs to be done better
+    if testing:
+        workoutid = [
+            make_new_workout_from_email(therower, filename, title,testing=testing)
+            ]
+        if workoutid[0] and uploadoptions and not 'error' in uploadoptions:
+            workout = Workout.objects.get(id=workoutid[0])
+            uploads.make_private(workout, uploadoptions)
+            uploads.set_workouttype(workout, uploadoptions)
+            uploads.do_sync(workout, uploadoptions)
 
 
     if 'raceid' in uploadoptions and workoutid[0] and rower.user.is_staff:
@@ -114,41 +140,12 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
             }
         )
 
-        if uploadoptions and not 'error' in uploadoptions:
+        if not testing:
-            workout = Workout.objects.get(id=workoutid[0])
+            if therower.getemailnotifications and not therower.emailbounced:
-            uploads.make_private(workout, uploadoptions)
+                email_sent = send_confirm(
-            uploads.set_workouttype(workout, uploadoptions)
+                    therower.user, title, link,
-            uploads.do_sync(workout, uploadoptions)
+                    uploadoptions
-        if 'make_plot' in uploadoptions:
+                )
-            plottype = uploadoptions['plottype']
-            workoutcsvfilename = workout.csvfilename[6:-4]
-            timestr = strftime("%Y%m%d-%H%M%S")
-            imagename = workoutcsvfilename + timestr + '.png'
-            result,jobid = uploads.make_plot(
-                workout.user, workout, workoutcsvfilename,
-                workout.csvfilename,
-                plottype, title,
-                imagename=imagename
-            )
-        try:
-            if workoutid and not testing:
-                if therower.getemailnotifications and not therower.emailbounced:
-                    email_sent = send_confirm(
-                        therower.user, title, link,
-                        uploadoptions
-                    )
-                    time.sleep(10)
-        except:
-            try:
-                time.sleep(10)
-                if workoutid:
-                    if therower.getemailnotifications and not therower.emailbounced:
-                        email_sent = send_confirm(
-                            therower.user, title, link,
-                            uploadoptions
-                        )
-            except:
-                pass
 
     return workoutid
 

rowers/mytypes.py

@@ -5,6 +5,9 @@ from __future__ import unicode_literals
 from six import iteritems
 import collections
 
+
+
+
 workouttypes_ordered = collections.OrderedDict({
     'water':'Standard Racing Shell',
     'rower':'Indoor Rower',
@@ -289,7 +292,7 @@ workoutsources = (
     ('ergstick','ergstick'),
     ('fit','fit'),
     ('unknown','unknown'))
-    
+
 boattypes = (
     ('1x', '1x (single)'),
     ('2x', '2x (double)'),

rowers/rows.py

@@ -7,6 +7,11 @@ import time
 import gzip
 import shutil
 import hashlib
+
+
+import uuid
+
+
 from django.core.exceptions import ValidationError
 
 def format_pace_tick(x,pos=None):
@@ -26,7 +31,7 @@ def format_time_tick(x,pos=None):
 def format_pace(x,pos=None):
     if isinf(x) or isnan(x):
         x=0
-        
+
     min=int(x/60)
     sec=(x-min*60.)
 
@@ -73,14 +78,14 @@ def must_be_csv(value):
     valid_extensions = ['.csv','.CSV']
     if not ext in valid_extensions:
         raise ValidationError(u'File not supported!')
-    
+
 def validate_kml(value):
     import os
     ext = os.path.splitext(value.name)[1]
     valid_extensions = ['.kml','.KML']
     if not ext in valid_extensions:
         raise ValidationError(u'File not supported!')
-    
+
 
 def handle_uploaded_image(i):
         from io import StringIO, BytesIO
@@ -92,8 +97,8 @@ def handle_uploaded_image(i):
                 image_str += chunk
 
         imagefile = BytesIO(image_str)
-        
+
-                
+
         image = Image.open(i)
 
         try:
@@ -105,7 +110,7 @@ def handle_uploaded_image(i):
         except (AttributeError, KeyError, IndexError):
                 # cases: image don't have getexif
                 exif = {'orientation':0}
-        
+
         if image.mode not in ("L", "RGB"):
                 image = image.convert("RGB")
 
@@ -128,18 +133,17 @@ def handle_uploaded_image(i):
 
         filename2 = os.path.join('static/plots/',filename)
         image.save(filename2,'JPEG')
-        
+
         return filename,filename2
-                
+
 
 def handle_uploaded_file(f):
     fname = f.name
-    timestr = time.strftime("%Y%m%d-%H%M%S")
+    timestr = uuid.uuid4().hex[:10]+'-'+time.strftime("%Y%m%d-%H%M%S")
     fname = timestr+'-'+fname
     fname2 = 'media/'+fname
     with open(fname2,'wb+') as destination:
         for chunk in f.chunks():
             destination.write(chunk)
-            
-    return fname,fname2
 
+    return fname,fname2

rowers/tests/test_emails.py

@@ -9,6 +9,7 @@ from .statements import *
 class EmailUpload(TestCase):
     def setUp(self):
         redis_connection.publish('tasks','KILL')
+        self.c = Client()
         u = User.objects.create_user('john',
                                      'sander@ds.ds',
                                      'koeinsloot')
@@ -50,6 +51,52 @@ workout run
                 except (IOError,FileNotFoundError,OSError):
                     pass
 
+    @patch('rowers.dataprep.create_engine')
+    @patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db)
+    def test_uploadapi(self,mocked_sqlalchemy,mocked_getsmallrowdata_db):
+        form_data = {
+            'title': 'test',
+            'workouttype':'rower',
+            'boattype': '1x',
+            'notes': 'aap noot mies',
+            'make_plot': False,
+            'upload_to_C2': False,
+            'plottype': 'timeplot',
+            'file': 'media/mailbox_attachments/colin3.csv',
+            'secret': settings.UPLOAD_SERVICE_SECRET,
+            'user': 1,
+        }
+
+        url = reverse('workout_upload_api')
+        response = self.c.post(url,form_data,HTTP_HOST='127.0.0.1:4533')
+        self.assertEqual(response.status_code,200)
+
+        # should also test if workout is created
+        w = Workout.objects.get(id=1)
+        self.assertEqual(w.name,'test')
+        self.assertEqual(w.notes,'aap noot mies')
+
+    @patch('rowers.dataprep.create_engine')
+    @patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db)
+    def test_uploadapi_credentials(self,mocked_sqlalchemy,mocked_getsmallrowdata_db):
+        form_data = {
+            'title': 'test',
+            'workouttype':'rower',
+            'boattype': '1x',
+            'notes': 'aap noot mies',
+            'make_plot': False,
+            'upload_to_C2': False,
+            'plottype': 'timeplot',
+            'file': 'media/mailbox_attachments/colin3.csv',
+            'secret': 'potjandorie2',
+            'user': 1,
+        }
+
+        url = reverse('workout_upload_api')
+        response = self.c.post(url,form_data)
+        self.assertEqual(response.status_code,403)
+
+
     @patch('rowers.dataprep.create_engine')
     @patch('rowers.polarstuff.get_polar_notifications')
     @patch('rowers.c2stuff.requests.get', side_effect=mocked_requests)

rowers/tests/test_uploads.py

@@ -245,7 +245,6 @@ class ViewTest(TestCase):
 
 
 
-
     @patch('rowers.dataprep.create_engine')
     @patch('rowers.dataprep.TCXParser')
     def test_upload_view_TCX_CN(self, mocked_sqlalchemy, mocked_tcx_parser):

rowers/urls.py

@@ -745,7 +745,7 @@ urlpatterns = [
     re_path(r'^courses/(?P<id>\d+)/map/$',views.course_map_view,name='course_map_view'),
     # URLS to be created
     re_path(r'^help/$',TemplateView.as_view(template_name='help.html'), name='help'),
-
+    re_path(r'^workout/api/upload/',views.workout_upload_api,name='workout_upload_api'),
     ]
 
 if settings.DEBUG:

rowers/views/workoutviews.py

@@ -3,12 +3,16 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import hashlib
+from shutil import copyfile
+
 from rowers.views.statements import *
 import rowers.teams as teams
 import rowers.mytypes as mytypes
 import numpy
 
 from urllib.parse import urlparse, parse_qs
+from json.decoder import JSONDecodeError
 
 def default(o):
     if isinstance(o, numpy.int64): return int(o)
@@ -4384,6 +4388,157 @@ def workout_toggle_ranking(request,id=0):
 
     return response
 
+# simple POST API for files on local (e.g. in mailbox)
+@csrf_exempt
+def workout_upload_api(request):
+    if request.method != 'POST':
+        message = {'status':'false','message':'this view cannot be accessed through GET'}
+        return JSONResponse(status=403,data=message)
+
+
+    # test if JSON
+    try:
+        json_data = json.loads(request.body)
+        secret = json_data['secret']
+        post_data = json_data
+    except (KeyError,JSONDecodeError):
+        post_data = request.POST
+
+    # only allow local host
+    hostt = request.get_host().split(':')
+    if hostt[0] not in ['localhost','127.0.0.1']:
+        message = {'status':'false','message':'permission denied'}
+        return JSONResponse(status=403,data=message)
+
+    # check credentials here
+    try:
+        secret = post_data['secret']
+    except KeyError:
+        message = {'status': 'false', 'message':'missing credentials'}
+        return JSONResponse(status=400,data=message)
+    if secret != settings.UPLOAD_SERVICE_SECRET:
+        message = {'status':'false','message':'invalid credentials'}
+        return JSONResponse(status=403,data=message)
+
+    form = DocumentsForm(post_data)
+    optionsform = TeamUploadOptionsForm(post_data)
+    rowerform = TeamInviteForm(post_data)
+    rowerform.fields.pop('email')
+    try:
+        fstr = post_data['file']
+        nn, ext = os.path.splitext(fstr)
+        if ext== '.gz':
+            nn, ext2 = os.path.splitext(nn)
+            ext = ext2+ext
+        f1 = uuid.uuid4().hex[:10]+'-'+time.strftime("%Y%m%d-%H%M%S")+ext
+        f2 = 'media/'+f1
+        copyfile(fstr,f2)
+    except KeyError:
+        message = {'status':'false','message':'no filename given'}
+        return JSONResponse(status=400,data=message)
+    except FileNotFoundError:
+        message = {'status':'false','message':'could not find file'}
+        return JSONResponse(status=400,data=message)
+
+    if form.is_valid():
+        t = form.cleaned_data['title']
+        boattype = form.cleaned_data['boattype']
+        workouttype = form.cleaned_data['workouttype']
+        if rowerform.is_valid():
+            u = rowerform.cleaned_data['user']
+            r = getrower(u)
+        elif 'useremail' in post_data:
+            us = User.objects.filter(email=post_data['useremail'])
+            if len(us):
+                u = us[0]
+                r = getrower(u)
+            else:
+                message = {'status':'false','message':'could not find user'}
+                return JSonResponse(status=400,data=message)
+        else:
+            message = {'status':'false','message':'invalid user'}
+            return JSonResponse(status=400,data=message)
+
+        notes = form.cleaned_data['notes']
+        if optionsform.is_valid():
+            make_plot = optionsform.cleaned_data['make_plot']
+            plottype = optionsform.cleaned_data['plottype']
+            upload_to_c2 = optionsform.cleaned_data['upload_to_C2']
+            upload_to_strava = optionsform.cleaned_data['upload_to_Strava']
+            upload_to_st = optionsform.cleaned_data['upload_to_SportTracks']
+            upload_to_rk = optionsform.cleaned_data['upload_to_RunKeeper']
+            upload_to_ua = optionsform.cleaned_data['upload_to_MapMyFitness']
+            upload_to_tp = optionsform.cleaned_data['upload_to_TrainingPeaks']
+            makeprivate = optionsform.cleaned_data['makeprivate']
+        else:
+            message = optionsform.errors
+            return JSonResponse(status=400,data=message)
+
+
+        id, message, f2 = dataprep.new_workout_from_file(
+            r,f2,
+            workouttype=workouttype,
+            workoutsource=None,
+            boattype=boattype,
+            makeprivate=makeprivate,
+            title = t,
+            notes=notes,
+        )
+
+        if id <= 0:
+            message = {'status':'false','message':'unable to process file'}
+            return JSonResponse(status=400,data=message)
+
+        w = Workout.objects.get(id=id)
+
+        if make_plot:
+            res, jobid = uploads.make_plot(r,w,f1,f2,plottype,t)
+
+        if upload_to_c2:
+            try:
+                message,id = c2stuff.workout_c2_upload(r.user,w)
+            except NoTokenError:
+                pass
+
+        if upload_to_strava:
+            try:
+                message,id = stravastuff.workout_strava_upload(r.user,w)
+            except NoTokenError:
+                pass
+
+        if upload_to_st:
+            try:
+                message,id = sporttrackstuff.workout_sporttracks_upload(r.user,w)
+            except NoTokenError:
+                pass
+
+        if upload_to_rk:
+            try:
+                message,id = runkeeperstuff.workout_runkeeper_upload(r.user,w)
+            except NoTokenError:
+                pass
+
+        if upload_to_ua:
+            try:
+                message,id = underarmourstuff.workout_ua_upload(r.user,w)
+            except NoTokenError:
+                pass
+
+        if upload_to_tp:
+            try:
+                message,id = tpstuff.workout_tp_upload(r.user,w)
+            except NoTokenError:
+                pass
+
+    else: # form invalid
+        message = form.errors
+        return JsonResponse(status=400,data=message)
+
+    message = {'status': 'true','id':w.id}
+    return JSONResponse(status=200,data=message)
+
+
+
 
 # This is the main view for processing uploaded files
 @login_required()

rowsandall_app/settings.py

@@ -247,6 +247,10 @@ LOGOUT_REDIRECT_URL = '/'
 # Update Cache with task progress password
 
 PROGRESS_CACHE_SECRET = CFG['progress_cache_secret']
+try:
+	UPLOAD_SERVICE_SECRET = CFG['upload_service_secret']
+except KeyError:
+	UPLOAD_SERVICE_SECRET = "FoYezZWLSyfAVimumpHEeYsJjsNCerxV"
 
 # Concept 2
 C2_CLIENT_ID = CFG['c2_client_id']
@@ -357,8 +361,13 @@ CACHE_MIDDLEWARE_SECONDS = 900
 
 EMAIL_BACKEND = 'django_ses.SESBackend'
 
-AWS_SES_REGION_NAME = 'eu-west-1'
+AWS_SES_REGION_NAME = CFG['aws_smtp']
-AWS_SES_REGION_ENDPOINT = 'email.eu-west-1.amazonaws.com'
+AWS_SES_REGION_ENDPOINT = CFG['aws_smtp']
+
+AWS_SES_ACCESS_KEY_ID = CFG['aws_access_key_id']
+AWS_SES_SECRET_ACCESS_KEY = CFG['aws_secret_access_key']
+AWS_ACCESS_KEY_ID  = CFG['aws_access_key_id']
+AWS_SECRET_ACCESS_KEY = CFG['aws_secret_access_key']
 
 EMAIL_HOST = CFG['aws_smtp']
 EMAIL_PORT = CFG['aws_port']