sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

Merge branch 'release/v10.94'

Browse Source
This commit is contained in:
Sander Roosendaal
2020-02-16 10:14:43 +01:00
parent 6443dc8932 a6fd60a7cc
commit fdd4f61d0f
10 changed files with 288 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
rowers/dataprep.py
View File

@@ -3,7 +3,6 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
# All the data preparation, data cleaning and data mangling should
# be defined here
from __future__ import unicode_literals, absolute_import

20
rowers/forms.py
View File

@@ -395,6 +395,26 @@ class TeamUploadOptionsForm(forms.Form):
initial='timeplot',
label='Plot Type')
upload_to_C2 = forms.BooleanField(initial=False,required=False,
label='Export to Concept2 logbook')
upload_to_Strava = forms.BooleanField(initial=False,required=False,
label='Export to Strava')
upload_to_SportTracks = forms.BooleanField(initial=False,required=False,
label='Export to SportTracks')
upload_to_RunKeeper = forms.BooleanField(initial=False,required=False,
label='Export to RunKeeper')
upload_to_MapMyFitness = forms.BooleanField(initial=False,
required=False,
label='Export to MapMyFitness')
upload_to_TrainingPeaks = forms.BooleanField(initial=False,
required=False,
label='Export to TrainingPeaks')
# do_physics = forms.BooleanField(initial=False,required=False,label='Power Estimate (OTW)')
makeprivate = forms.BooleanField(initial=False,required=False,
label='Make Workout Private')
class Meta:
fields = ['make_plot','plottype']

73
rowers/management/commands/processemail.py
View File

@@ -10,6 +10,9 @@ import re
import time
from time import strftime
import requests
import json
import io
from django.core.management.base import BaseCommand
@@ -56,6 +59,7 @@ def rdata(file_obj, rower=rrower()):
def processattachment(rower, fileobj, title, uploadoptions,testing=False):
try:
filename = fileobj.name
# filename = os.path.abspath(fileobj.name)
except AttributeError:
filename = fileobj[6:]
if testing:
@@ -85,9 +89,31 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
therower = rower
workoutid = [
make_new_workout_from_email(therower, filename, title,testing=testing)
]
uploadoptions['secret'] = settings.UPLOAD_SERVICE_SECRET
uploadoptions['user'] = therower.user.id
uploadoptions['file'] = 'media/'+filename
uploadoptions['title'] = title
url = "http://localhost:8000/rowers/workout/api/upload/"
if not testing:
response = requests.post(url,data=uploadoptions)
if response.status_code == 200:
response_json = response.json()
workoutid = [int(response_json['id'])]
else:
workoutid = [0]
# this is ugly and needs to be done better
if testing:
workoutid = [
make_new_workout_from_email(therower, filename, title,testing=testing)
]
if workoutid[0] and uploadoptions and not 'error' in uploadoptions:
workout = Workout.objects.get(id=workoutid[0])
uploads.make_private(workout, uploadoptions)
uploads.set_workouttype(workout, uploadoptions)
uploads.do_sync(workout, uploadoptions)
if 'raceid' in uploadoptions and workoutid[0] and rower.user.is_staff:
@@ -114,41 +140,12 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
}
)
if uploadoptions and not 'error' in uploadoptions:
workout = Workout.objects.get(id=workoutid[0])
uploads.make_private(workout, uploadoptions)
uploads.set_workouttype(workout, uploadoptions)
uploads.do_sync(workout, uploadoptions)
if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype']
workoutcsvfilename = workout.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S")
imagename = workoutcsvfilename + timestr + '.png'
result,jobid = uploads.make_plot(
workout.user, workout, workoutcsvfilename,
workout.csvfilename,
plottype, title,
imagename=imagename
)
try:
if workoutid and not testing:
if therower.getemailnotifications and not therower.emailbounced:
email_sent = send_confirm(
therower.user, title, link,
uploadoptions
)
time.sleep(10)
except:
try:
time.sleep(10)
if workoutid:
if therower.getemailnotifications and not therower.emailbounced:
email_sent = send_confirm(
therower.user, title, link,
uploadoptions
)
except:
pass
if not testing:
if therower.getemailnotifications and not therower.emailbounced:
email_sent = send_confirm(
therower.user, title, link,
uploadoptions
)
return workoutid

5
rowers/mytypes.py
View File

@@ -5,6 +5,9 @@ from __future__ import unicode_literals
from six import iteritems
import collections
workouttypes_ordered = collections.OrderedDict({
'water':'Standard Racing Shell',
'rower':'Indoor Rower',
@@ -289,7 +292,7 @@ workoutsources = (
('ergstick','ergstick'),
('fit','fit'),
('unknown','unknown'))
boattypes = (
('1x', '1x (single)'),
('2x', '2x (double)'),

26
rowers/rows.py
View File

@@ -7,6 +7,11 @@ import time
import gzip
import shutil
import hashlib
import uuid
from django.core.exceptions import ValidationError
def format_pace_tick(x,pos=None):
@@ -26,7 +31,7 @@ def format_time_tick(x,pos=None):
def format_pace(x,pos=None):
if isinf(x) or isnan(x):
x=0
min=int(x/60)
sec=(x-min*60.)
@@ -73,14 +78,14 @@ def must_be_csv(value):
valid_extensions = ['.csv','.CSV']
if not ext in valid_extensions:
raise ValidationError(u'File not supported!')
def validate_kml(value):
import os
ext = os.path.splitext(value.name)[1]
valid_extensions = ['.kml','.KML']
if not ext in valid_extensions:
raise ValidationError(u'File not supported!')
def handle_uploaded_image(i):
from io import StringIO, BytesIO
@@ -92,8 +97,8 @@ def handle_uploaded_image(i):
image_str += chunk
imagefile = BytesIO(image_str)
image = Image.open(i)
try:
@@ -105,7 +110,7 @@ def handle_uploaded_image(i):
except (AttributeError, KeyError, IndexError):
# cases: image don't have getexif
exif = {'orientation':0}
if image.mode not in ("L", "RGB"):
image = image.convert("RGB")
@@ -128,18 +133,17 @@ def handle_uploaded_image(i):
filename2 = os.path.join('static/plots/',filename)
image.save(filename2,'JPEG')
return filename,filename2
def handle_uploaded_file(f):
fname = f.name
timestr = time.strftime("%Y%m%d-%H%M%S")
timestr = uuid.uuid4().hex[:10]+'-'+time.strftime("%Y%m%d-%H%M%S")
fname = timestr+'-'+fname
fname2 = 'media/'+fname
with open(fname2,'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
return fname,fname2
return fname,fname2

47
rowers/tests/test_emails.py
View File

@@ -9,6 +9,7 @@ from .statements import *
class EmailUpload(TestCase):
def setUp(self):
redis_connection.publish('tasks','KILL')
self.c = Client()
u = User.objects.create_user('john',
'sander@ds.ds',
'koeinsloot')
@@ -50,6 +51,52 @@ workout run
except (IOError,FileNotFoundError,OSError):
pass
@patch('rowers.dataprep.create_engine')
@patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db)
def test_uploadapi(self,mocked_sqlalchemy,mocked_getsmallrowdata_db):
form_data = {
'title': 'test',
'workouttype':'rower',
'boattype': '1x',
'notes': 'aap noot mies',
'make_plot': False,
'upload_to_C2': False,
'plottype': 'timeplot',
'file': 'media/mailbox_attachments/colin3.csv',
'secret': settings.UPLOAD_SERVICE_SECRET,
'user': 1,
}
url = reverse('workout_upload_api')
response = self.c.post(url,form_data,HTTP_HOST='127.0.0.1:4533')
self.assertEqual(response.status_code,200)
# should also test if workout is created
w = Workout.objects.get(id=1)
self.assertEqual(w.name,'test')
self.assertEqual(w.notes,'aap noot mies')
@patch('rowers.dataprep.create_engine')
@patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db)
def test_uploadapi_credentials(self,mocked_sqlalchemy,mocked_getsmallrowdata_db):
form_data = {
'title': 'test',
'workouttype':'rower',
'boattype': '1x',
'notes': 'aap noot mies',
'make_plot': False,
'upload_to_C2': False,
'plottype': 'timeplot',
'file': 'media/mailbox_attachments/colin3.csv',
'secret': 'potjandorie2',
'user': 1,
}
url = reverse('workout_upload_api')
response = self.c.post(url,form_data)
self.assertEqual(response.status_code,403)
@patch('rowers.dataprep.create_engine')
@patch('rowers.polarstuff.get_polar_notifications')
@patch('rowers.c2stuff.requests.get', side_effect=mocked_requests)

1
rowers/tests/test_uploads.py
View File

@@ -245,7 +245,6 @@ class ViewTest(TestCase):
@patch('rowers.dataprep.create_engine')
@patch('rowers.dataprep.TCXParser')
def test_upload_view_TCX_CN(self, mocked_sqlalchemy, mocked_tcx_parser):

2
rowers/urls.py
View File

@@ -745,7 +745,7 @@ urlpatterns = [
re_path(r'^courses/(?P<id>\d+)/map/$',views.course_map_view,name='course_map_view'),
# URLS to be created
re_path(r'^help/$',TemplateView.as_view(template_name='help.html'), name='help'),
re_path(r'^workout/api/upload/',views.workout_upload_api,name='workout_upload_api'),
]
if settings.DEBUG:

155
rowers/views/workoutviews.py
View File

@@ -3,12 +3,16 @@ from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import hashlib
from shutil import copyfile
from rowers.views.statements import *
import rowers.teams as teams
import rowers.mytypes as mytypes
import numpy
from urllib.parse import urlparse, parse_qs
from json.decoder import JSONDecodeError
def default(o):
if isinstance(o, numpy.int64): return int(o)
@@ -4384,6 +4388,157 @@ def workout_toggle_ranking(request,id=0):
return response
# simple POST API for files on local (e.g. in mailbox)
@csrf_exempt
def workout_upload_api(request):
if request.method != 'POST':
message = {'status':'false','message':'this view cannot be accessed through GET'}
return JSONResponse(status=403,data=message)
# test if JSON
try:
json_data = json.loads(request.body)
secret = json_data['secret']
post_data = json_data
except (KeyError,JSONDecodeError):
post_data = request.POST
# only allow local host
hostt = request.get_host().split(':')
if hostt[0] not in ['localhost','127.0.0.1']:
message = {'status':'false','message':'permission denied'}
return JSONResponse(status=403,data=message)
# check credentials here
try:
secret = post_data['secret']
except KeyError:
message = {'status': 'false', 'message':'missing credentials'}
return JSONResponse(status=400,data=message)
if secret != settings.UPLOAD_SERVICE_SECRET:
message = {'status':'false','message':'invalid credentials'}
return JSONResponse(status=403,data=message)
form = DocumentsForm(post_data)
optionsform = TeamUploadOptionsForm(post_data)
rowerform = TeamInviteForm(post_data)
rowerform.fields.pop('email')
try:
fstr = post_data['file']
nn, ext = os.path.splitext(fstr)
if ext== '.gz':
nn, ext2 = os.path.splitext(nn)
ext = ext2+ext
f1 = uuid.uuid4().hex[:10]+'-'+time.strftime("%Y%m%d-%H%M%S")+ext
f2 = 'media/'+f1
copyfile(fstr,f2)
except KeyError:
message = {'status':'false','message':'no filename given'}
return JSONResponse(status=400,data=message)
except FileNotFoundError:
message = {'status':'false','message':'could not find file'}
return JSONResponse(status=400,data=message)
if form.is_valid():
t = form.cleaned_data['title']
boattype = form.cleaned_data['boattype']
workouttype = form.cleaned_data['workouttype']
if rowerform.is_valid():
u = rowerform.cleaned_data['user']
r = getrower(u)
elif 'useremail' in post_data:
us = User.objects.filter(email=post_data['useremail'])
if len(us):
u = us[0]
r = getrower(u)
else:
message = {'status':'false','message':'could not find user'}
return JSonResponse(status=400,data=message)
else:
message = {'status':'false','message':'invalid user'}
return JSonResponse(status=400,data=message)
notes = form.cleaned_data['notes']
if optionsform.is_valid():
make_plot = optionsform.cleaned_data['make_plot']
plottype = optionsform.cleaned_data['plottype']
upload_to_c2 = optionsform.cleaned_data['upload_to_C2']
upload_to_strava = optionsform.cleaned_data['upload_to_Strava']
upload_to_st = optionsform.cleaned_data['upload_to_SportTracks']
upload_to_rk = optionsform.cleaned_data['upload_to_RunKeeper']
upload_to_ua = optionsform.cleaned_data['upload_to_MapMyFitness']
upload_to_tp = optionsform.cleaned_data['upload_to_TrainingPeaks']
makeprivate = optionsform.cleaned_data['makeprivate']
else:
message = optionsform.errors
return JSonResponse(status=400,data=message)
id, message, f2 = dataprep.new_workout_from_file(
r,f2,
workouttype=workouttype,
workoutsource=None,
boattype=boattype,
makeprivate=makeprivate,
title = t,
notes=notes,
)
if id <= 0:
message = {'status':'false','message':'unable to process file'}
return JSonResponse(status=400,data=message)
w = Workout.objects.get(id=id)
if make_plot:
res, jobid = uploads.make_plot(r,w,f1,f2,plottype,t)
if upload_to_c2:
try:
message,id = c2stuff.workout_c2_upload(r.user,w)
except NoTokenError:
pass
if upload_to_strava:
try:
message,id = stravastuff.workout_strava_upload(r.user,w)
except NoTokenError:
pass
if upload_to_st:
try:
message,id = sporttrackstuff.workout_sporttracks_upload(r.user,w)
except NoTokenError:
pass
if upload_to_rk:
try:
message,id = runkeeperstuff.workout_runkeeper_upload(r.user,w)
except NoTokenError:
pass
if upload_to_ua:
try:
message,id = underarmourstuff.workout_ua_upload(r.user,w)
except NoTokenError:
pass
if upload_to_tp:
try:
message,id = tpstuff.workout_tp_upload(r.user,w)
except NoTokenError:
pass
else: # form invalid
message = form.errors
return JsonResponse(status=400,data=message)
message = {'status': 'true','id':w.id}
return JSONResponse(status=200,data=message)
# This is the main view for processing uploaded files
@login_required()

13
rowsandall_app/settings.py
View File

@@ -247,6 +247,10 @@ LOGOUT_REDIRECT_URL = '/'
# Update Cache with task progress password
PROGRESS_CACHE_SECRET = CFG['progress_cache_secret']
try:
UPLOAD_SERVICE_SECRET = CFG['upload_service_secret']
except KeyError:
UPLOAD_SERVICE_SECRET = "FoYezZWLSyfAVimumpHEeYsJjsNCerxV"
# Concept 2
C2_CLIENT_ID = CFG['c2_client_id']
@@ -357,8 +361,13 @@ CACHE_MIDDLEWARE_SECONDS = 900
EMAIL_BACKEND = 'django_ses.SESBackend'
AWS_SES_REGION_NAME = 'eu-west-1'
AWS_SES_REGION_ENDPOINT = 'email.eu-west-1.amazonaws.com'
AWS_SES_REGION_NAME = CFG['aws_smtp']
AWS_SES_REGION_ENDPOINT = CFG['aws_smtp']
AWS_SES_ACCESS_KEY_ID = CFG['aws_access_key_id']
AWS_SES_SECRET_ACCESS_KEY = CFG['aws_secret_access_key']
AWS_ACCESS_KEY_ID = CFG['aws_access_key_id']
AWS_SECRET_ACCESS_KEY = CFG['aws_secret_access_key']
EMAIL_HOST = CFG['aws_smtp']
EMAIL_PORT = CFG['aws_port']