diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 3912e8c1..3843d4ec 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -3,7 +3,6 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals - # All the data preparation, data cleaning and data mangling should # be defined here from __future__ import unicode_literals, absolute_import diff --git a/rowers/forms.py b/rowers/forms.py index 530c842d..829bbd3a 100644 --- a/rowers/forms.py +++ b/rowers/forms.py @@ -395,6 +395,26 @@ class TeamUploadOptionsForm(forms.Form): initial='timeplot', label='Plot Type') + upload_to_C2 = forms.BooleanField(initial=False,required=False, + label='Export to Concept2 logbook') + upload_to_Strava = forms.BooleanField(initial=False,required=False, + label='Export to Strava') + upload_to_SportTracks = forms.BooleanField(initial=False,required=False, + label='Export to SportTracks') + upload_to_RunKeeper = forms.BooleanField(initial=False,required=False, + label='Export to RunKeeper') + upload_to_MapMyFitness = forms.BooleanField(initial=False, + required=False, + label='Export to MapMyFitness') + upload_to_TrainingPeaks = forms.BooleanField(initial=False, + required=False, + label='Export to TrainingPeaks') + # do_physics = forms.BooleanField(initial=False,required=False,label='Power Estimate (OTW)') + makeprivate = forms.BooleanField(initial=False,required=False, + label='Make Workout Private') + + + class Meta: fields = ['make_plot','plottype'] diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 8720b2cb..9f0f5c54 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -10,6 +10,9 @@ import re import time from time import strftime +import requests +import json + import io from django.core.management.base import BaseCommand @@ -56,6 +59,7 @@ def rdata(file_obj, rower=rrower()): def processattachment(rower, fileobj, title, uploadoptions,testing=False): try: filename = fileobj.name +# filename = os.path.abspath(fileobj.name) except AttributeError: filename = fileobj[6:] if testing: @@ -85,9 +89,31 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False): therower = rower - workoutid = [ - make_new_workout_from_email(therower, filename, title,testing=testing) - ] + uploadoptions['secret'] = settings.UPLOAD_SERVICE_SECRET + uploadoptions['user'] = therower.user.id + uploadoptions['file'] = 'media/'+filename + uploadoptions['title'] = title + + + url = "http://localhost:8000/rowers/workout/api/upload/" + if not testing: + response = requests.post(url,data=uploadoptions) + if response.status_code == 200: + response_json = response.json() + workoutid = [int(response_json['id'])] + else: + workoutid = [0] + + # this is ugly and needs to be done better + if testing: + workoutid = [ + make_new_workout_from_email(therower, filename, title,testing=testing) + ] + if workoutid[0] and uploadoptions and not 'error' in uploadoptions: + workout = Workout.objects.get(id=workoutid[0]) + uploads.make_private(workout, uploadoptions) + uploads.set_workouttype(workout, uploadoptions) + uploads.do_sync(workout, uploadoptions) if 'raceid' in uploadoptions and workoutid[0] and rower.user.is_staff: @@ -114,41 +140,12 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False): } ) - if uploadoptions and not 'error' in uploadoptions: - workout = Workout.objects.get(id=workoutid[0]) - uploads.make_private(workout, uploadoptions) - uploads.set_workouttype(workout, uploadoptions) - uploads.do_sync(workout, uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - workoutcsvfilename = workout.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = workoutcsvfilename + timestr + '.png' - result,jobid = uploads.make_plot( - workout.user, workout, workoutcsvfilename, - workout.csvfilename, - plottype, title, - imagename=imagename - ) - try: - if workoutid and not testing: - if therower.getemailnotifications and not therower.emailbounced: - email_sent = send_confirm( - therower.user, title, link, - uploadoptions - ) - time.sleep(10) - except: - try: - time.sleep(10) - if workoutid: - if therower.getemailnotifications and not therower.emailbounced: - email_sent = send_confirm( - therower.user, title, link, - uploadoptions - ) - except: - pass + if not testing: + if therower.getemailnotifications and not therower.emailbounced: + email_sent = send_confirm( + therower.user, title, link, + uploadoptions + ) return workoutid diff --git a/rowers/mytypes.py b/rowers/mytypes.py index 6a10e967..e8b7aa9b 100644 --- a/rowers/mytypes.py +++ b/rowers/mytypes.py @@ -5,6 +5,9 @@ from __future__ import unicode_literals from six import iteritems import collections + + + workouttypes_ordered = collections.OrderedDict({ 'water':'Standard Racing Shell', 'rower':'Indoor Rower', @@ -289,7 +292,7 @@ workoutsources = ( ('ergstick','ergstick'), ('fit','fit'), ('unknown','unknown')) - + boattypes = ( ('1x', '1x (single)'), ('2x', '2x (double)'), diff --git a/rowers/rows.py b/rowers/rows.py index 6393fa02..fa34cf00 100644 --- a/rowers/rows.py +++ b/rowers/rows.py @@ -7,6 +7,11 @@ import time import gzip import shutil import hashlib + + +import uuid + + from django.core.exceptions import ValidationError def format_pace_tick(x,pos=None): @@ -26,7 +31,7 @@ def format_time_tick(x,pos=None): def format_pace(x,pos=None): if isinf(x) or isnan(x): x=0 - + min=int(x/60) sec=(x-min*60.) @@ -73,14 +78,14 @@ def must_be_csv(value): valid_extensions = ['.csv','.CSV'] if not ext in valid_extensions: raise ValidationError(u'File not supported!') - + def validate_kml(value): import os ext = os.path.splitext(value.name)[1] valid_extensions = ['.kml','.KML'] if not ext in valid_extensions: raise ValidationError(u'File not supported!') - + def handle_uploaded_image(i): from io import StringIO, BytesIO @@ -92,8 +97,8 @@ def handle_uploaded_image(i): image_str += chunk imagefile = BytesIO(image_str) - - + + image = Image.open(i) try: @@ -105,7 +110,7 @@ def handle_uploaded_image(i): except (AttributeError, KeyError, IndexError): # cases: image don't have getexif exif = {'orientation':0} - + if image.mode not in ("L", "RGB"): image = image.convert("RGB") @@ -128,18 +133,17 @@ def handle_uploaded_image(i): filename2 = os.path.join('static/plots/',filename) image.save(filename2,'JPEG') - + return filename,filename2 - + def handle_uploaded_file(f): fname = f.name - timestr = time.strftime("%Y%m%d-%H%M%S") + timestr = uuid.uuid4().hex[:10]+'-'+time.strftime("%Y%m%d-%H%M%S") fname = timestr+'-'+fname fname2 = 'media/'+fname with open(fname2,'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) - - return fname,fname2 + return fname,fname2 diff --git a/rowers/tests/test_emails.py b/rowers/tests/test_emails.py index b65bcd9c..11149be0 100644 --- a/rowers/tests/test_emails.py +++ b/rowers/tests/test_emails.py @@ -9,6 +9,7 @@ from .statements import * class EmailUpload(TestCase): def setUp(self): redis_connection.publish('tasks','KILL') + self.c = Client() u = User.objects.create_user('john', 'sander@ds.ds', 'koeinsloot') @@ -50,6 +51,52 @@ workout run except (IOError,FileNotFoundError,OSError): pass + @patch('rowers.dataprep.create_engine') + @patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db) + def test_uploadapi(self,mocked_sqlalchemy,mocked_getsmallrowdata_db): + form_data = { + 'title': 'test', + 'workouttype':'rower', + 'boattype': '1x', + 'notes': 'aap noot mies', + 'make_plot': False, + 'upload_to_C2': False, + 'plottype': 'timeplot', + 'file': 'media/mailbox_attachments/colin3.csv', + 'secret': settings.UPLOAD_SERVICE_SECRET, + 'user': 1, + } + + url = reverse('workout_upload_api') + response = self.c.post(url,form_data,HTTP_HOST='127.0.0.1:4533') + self.assertEqual(response.status_code,200) + + # should also test if workout is created + w = Workout.objects.get(id=1) + self.assertEqual(w.name,'test') + self.assertEqual(w.notes,'aap noot mies') + + @patch('rowers.dataprep.create_engine') + @patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db) + def test_uploadapi_credentials(self,mocked_sqlalchemy,mocked_getsmallrowdata_db): + form_data = { + 'title': 'test', + 'workouttype':'rower', + 'boattype': '1x', + 'notes': 'aap noot mies', + 'make_plot': False, + 'upload_to_C2': False, + 'plottype': 'timeplot', + 'file': 'media/mailbox_attachments/colin3.csv', + 'secret': 'potjandorie2', + 'user': 1, + } + + url = reverse('workout_upload_api') + response = self.c.post(url,form_data) + self.assertEqual(response.status_code,403) + + @patch('rowers.dataprep.create_engine') @patch('rowers.polarstuff.get_polar_notifications') @patch('rowers.c2stuff.requests.get', side_effect=mocked_requests) diff --git a/rowers/tests/test_uploads.py b/rowers/tests/test_uploads.py index 22f9a4e5..21e03df9 100644 --- a/rowers/tests/test_uploads.py +++ b/rowers/tests/test_uploads.py @@ -245,7 +245,6 @@ class ViewTest(TestCase): - @patch('rowers.dataprep.create_engine') @patch('rowers.dataprep.TCXParser') def test_upload_view_TCX_CN(self, mocked_sqlalchemy, mocked_tcx_parser): diff --git a/rowers/urls.py b/rowers/urls.py index 00b83ac9..625ee831 100644 --- a/rowers/urls.py +++ b/rowers/urls.py @@ -745,7 +745,7 @@ urlpatterns = [ re_path(r'^courses/(?P\d+)/map/$',views.course_map_view,name='course_map_view'), # URLS to be created re_path(r'^help/$',TemplateView.as_view(template_name='help.html'), name='help'), - + re_path(r'^workout/api/upload/',views.workout_upload_api,name='workout_upload_api'), ] if settings.DEBUG: diff --git a/rowers/views/workoutviews.py b/rowers/views/workoutviews.py index 5c6cb869..daaf1406 100644 --- a/rowers/views/workoutviews.py +++ b/rowers/views/workoutviews.py @@ -3,12 +3,16 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +import hashlib +from shutil import copyfile + from rowers.views.statements import * import rowers.teams as teams import rowers.mytypes as mytypes import numpy from urllib.parse import urlparse, parse_qs +from json.decoder import JSONDecodeError def default(o): if isinstance(o, numpy.int64): return int(o) @@ -4384,6 +4388,157 @@ def workout_toggle_ranking(request,id=0): return response +# simple POST API for files on local (e.g. in mailbox) +@csrf_exempt +def workout_upload_api(request): + if request.method != 'POST': + message = {'status':'false','message':'this view cannot be accessed through GET'} + return JSONResponse(status=403,data=message) + + + # test if JSON + try: + json_data = json.loads(request.body) + secret = json_data['secret'] + post_data = json_data + except (KeyError,JSONDecodeError): + post_data = request.POST + + # only allow local host + hostt = request.get_host().split(':') + if hostt[0] not in ['localhost','127.0.0.1']: + message = {'status':'false','message':'permission denied'} + return JSONResponse(status=403,data=message) + + # check credentials here + try: + secret = post_data['secret'] + except KeyError: + message = {'status': 'false', 'message':'missing credentials'} + return JSONResponse(status=400,data=message) + if secret != settings.UPLOAD_SERVICE_SECRET: + message = {'status':'false','message':'invalid credentials'} + return JSONResponse(status=403,data=message) + + form = DocumentsForm(post_data) + optionsform = TeamUploadOptionsForm(post_data) + rowerform = TeamInviteForm(post_data) + rowerform.fields.pop('email') + try: + fstr = post_data['file'] + nn, ext = os.path.splitext(fstr) + if ext== '.gz': + nn, ext2 = os.path.splitext(nn) + ext = ext2+ext + f1 = uuid.uuid4().hex[:10]+'-'+time.strftime("%Y%m%d-%H%M%S")+ext + f2 = 'media/'+f1 + copyfile(fstr,f2) + except KeyError: + message = {'status':'false','message':'no filename given'} + return JSONResponse(status=400,data=message) + except FileNotFoundError: + message = {'status':'false','message':'could not find file'} + return JSONResponse(status=400,data=message) + + if form.is_valid(): + t = form.cleaned_data['title'] + boattype = form.cleaned_data['boattype'] + workouttype = form.cleaned_data['workouttype'] + if rowerform.is_valid(): + u = rowerform.cleaned_data['user'] + r = getrower(u) + elif 'useremail' in post_data: + us = User.objects.filter(email=post_data['useremail']) + if len(us): + u = us[0] + r = getrower(u) + else: + message = {'status':'false','message':'could not find user'} + return JSonResponse(status=400,data=message) + else: + message = {'status':'false','message':'invalid user'} + return JSonResponse(status=400,data=message) + + notes = form.cleaned_data['notes'] + if optionsform.is_valid(): + make_plot = optionsform.cleaned_data['make_plot'] + plottype = optionsform.cleaned_data['plottype'] + upload_to_c2 = optionsform.cleaned_data['upload_to_C2'] + upload_to_strava = optionsform.cleaned_data['upload_to_Strava'] + upload_to_st = optionsform.cleaned_data['upload_to_SportTracks'] + upload_to_rk = optionsform.cleaned_data['upload_to_RunKeeper'] + upload_to_ua = optionsform.cleaned_data['upload_to_MapMyFitness'] + upload_to_tp = optionsform.cleaned_data['upload_to_TrainingPeaks'] + makeprivate = optionsform.cleaned_data['makeprivate'] + else: + message = optionsform.errors + return JSonResponse(status=400,data=message) + + + id, message, f2 = dataprep.new_workout_from_file( + r,f2, + workouttype=workouttype, + workoutsource=None, + boattype=boattype, + makeprivate=makeprivate, + title = t, + notes=notes, + ) + + if id <= 0: + message = {'status':'false','message':'unable to process file'} + return JSonResponse(status=400,data=message) + + w = Workout.objects.get(id=id) + + if make_plot: + res, jobid = uploads.make_plot(r,w,f1,f2,plottype,t) + + if upload_to_c2: + try: + message,id = c2stuff.workout_c2_upload(r.user,w) + except NoTokenError: + pass + + if upload_to_strava: + try: + message,id = stravastuff.workout_strava_upload(r.user,w) + except NoTokenError: + pass + + if upload_to_st: + try: + message,id = sporttrackstuff.workout_sporttracks_upload(r.user,w) + except NoTokenError: + pass + + if upload_to_rk: + try: + message,id = runkeeperstuff.workout_runkeeper_upload(r.user,w) + except NoTokenError: + pass + + if upload_to_ua: + try: + message,id = underarmourstuff.workout_ua_upload(r.user,w) + except NoTokenError: + pass + + if upload_to_tp: + try: + message,id = tpstuff.workout_tp_upload(r.user,w) + except NoTokenError: + pass + + else: # form invalid + message = form.errors + return JsonResponse(status=400,data=message) + + message = {'status': 'true','id':w.id} + return JSONResponse(status=200,data=message) + + + # This is the main view for processing uploaded files @login_required() diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index c453669c..e87a4a78 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -247,6 +247,10 @@ LOGOUT_REDIRECT_URL = '/' # Update Cache with task progress password PROGRESS_CACHE_SECRET = CFG['progress_cache_secret'] +try: + UPLOAD_SERVICE_SECRET = CFG['upload_service_secret'] +except KeyError: + UPLOAD_SERVICE_SECRET = "FoYezZWLSyfAVimumpHEeYsJjsNCerxV" # Concept 2 C2_CLIENT_ID = CFG['c2_client_id'] @@ -357,8 +361,13 @@ CACHE_MIDDLEWARE_SECONDS = 900 EMAIL_BACKEND = 'django_ses.SESBackend' -AWS_SES_REGION_NAME = 'eu-west-1' -AWS_SES_REGION_ENDPOINT = 'email.eu-west-1.amazonaws.com' +AWS_SES_REGION_NAME = CFG['aws_smtp'] +AWS_SES_REGION_ENDPOINT = CFG['aws_smtp'] + +AWS_SES_ACCESS_KEY_ID = CFG['aws_access_key_id'] +AWS_SES_SECRET_ACCESS_KEY = CFG['aws_secret_access_key'] +AWS_ACCESS_KEY_ID = CFG['aws_access_key_id'] +AWS_SECRET_ACCESS_KEY = CFG['aws_secret_access_key'] EMAIL_HOST = CFG['aws_smtp'] EMAIL_PORT = CFG['aws_port']