Private
Public Access
1
0

further tests in async tasks

This commit is contained in:
Sander Roosendaal
2021-01-15 07:18:43 +01:00
parent ef3fac5ec1
commit 8ebb55cfd7
5 changed files with 297 additions and 354 deletions

View File

@@ -9,7 +9,7 @@ from rowingdata import rowingdata as rrdata
from rowingdata import make_cumvalues
from rowingdata import rower as rrower
from rowingdata import main as rmain
from rowingdata import empower_bug_correction,get_empower_rigging
from rowingdata import empower_bug_correction,get_empower_rigging, get_file_type
from rowingdata.csvparsers import make_cumvalues_array
from time import strftime
from pandas import DataFrame,Series
@@ -32,6 +32,8 @@ from rowsandall_app.settings_dev import DATABASES as DEV_DATABASES
from rowsandall_app.settings_dev import use_sqlite
from rowers.utils import lbstoN
import pytz
from timezonefinder import TimezoneFinder
try:
@@ -322,167 +324,6 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename,
return data
# Processes painsled CSV file to database
def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
dosummary=True,title='Workout',
notes='',totaldist=0,totaltime=0,
workoutsource='unknown',
summary='',
makeprivate=False,
oarlength=2.89,inboard=0.88):
message = None
powerperc = 100*np.array([r.pw_ut2,
r.pw_ut1,
r.pw_at,
r.pw_tr,r.pw_an])/r.ftp
# make workout and put in database
rr = rrower(hrmax=r.max,hrut2=r.ut2,
hrut1=r.ut1,hrat=r.at,
hrtr=r.tr,hran=r.an,ftp=r.ftp,
powerperc=powerperc,powerzones=r.powerzones)
row = rdata(f2,rower=rr)
checks = row.check_consistency()
allchecks = 1
for key,value in checks.iteritems():
if not value:
allchecks = 0
if not allchecks:
#row.repair()
pass
if row == 0:
return (0,'Error: CSV data file not found')
if dosmooth:
# auto smoothing
pace = row.df[' Stroke500mPace (sec/500m)'].values
velo = 500./pace
f = row.df['TimeStamp (sec)'].diff().mean()
if f !=0:
windowsize = 2*(int(10./(f)))+1
else:
windowsize = 1
if not 'originalvelo' in row.df:
row.df['originalvelo'] = velo
if windowsize > 3 and windowsize<len(velo):
velo2 = savgol_filter(velo,windowsize,3)
else:
velo2 = velo
velo3 = pd.Series(velo2)
velo3 = velo3.replace([-np.inf,np.inf],np.nan)
velo3 = velo3.fillna(method='ffill')
pace2 = 500./abs(velo3)
row.df[' Stroke500mPace (sec/500m)'] = pace2
row.df = row.df.fillna(0)
row.write_csv(f2,gzip=True)
try:
os.remove(f2)
except:
pass
# recalculate power data
if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
try:
row.erg_recalculatepower()
row.write_csv(f2,gzip=True)
except:
pass
averagehr = row.df[' HRCur (bpm)'].mean()
maxhr = row.df[' HRCur (bpm)'].max()
if totaldist == 0:
totaldist = row.df['cum_dist'].max()
if totaltime == 0:
totaltime = row.df['TimeStamp (sec)'].max()-row.df['TimeStamp (sec)'].min()
totaltime = totaltime+row.df.loc[0,' ElapsedTime (sec)']
hours = int(totaltime/3600.)
if hours>23:
message = 'Warning: The workout duration was longer than 23 hours. '
hours = 23
minutes = int((totaltime - 3600.*hours)/60.)
if minutes>59:
minutes = 59
if not message:
message = 'Warning: there is something wrong with the workout duration'
seconds = int(totaltime - 3600.*hours - 60.*minutes)
if seconds > 59:
seconds = 59
if not message:
message = 'Warning: there is something wrong with the workout duration'
tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds))
if tenths > 9:
tenths = 9
if not message:
message = 'Warning: there is something wrong with the workout duration'
duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths)
if dosummary:
summary = row.summary()
summary += '\n'
summary += row.intervalstats()
workoutdate = row.rowdatetime.strftime('%Y-%m-%d')
workoutstarttime = row.rowdatetime.strftime('%H:%M:%S')
workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc)
if makeprivate:
privacy = 'private'
else:
privacy = 'visible'
# check for duplicate start times
ws = Workout.objects.filter(startdatetime=workoutstartdatetime,
user=r)
if (len(ws) != 0):
message = "Warning: This workout probably already exists in the database"
privacy = 'private'
w = Workout(user=r,name=title,date=workoutdate,
workouttype=workouttype,
workoutsource=workoutsource,
duration=duration,distance=totaldist,
weightcategory=r.weightcategory,
starttime=workoutstarttime,
csvfilename=f2,notes=notes,summary=summary,
maxhr=maxhr,averagehr=averagehr,
startdatetime=workoutstartdatetime,
inboard=inboard,oarlength=oarlength,
privacy=privacy)
w.save()
if privacy == 'visible':
ts = Team.objects.filter(rower=r)
for t in ts:
w.team.add(t)
# put stroke data in database
res = dataprep(row.df,id=w.id,bands=True,
barchart=True,otwpower=True,empower=True,inboard=inboard)
return (w.id,message)
def handle_nonpainsled(f2,fileformat,summary=''):
oarlength = 2.89
@@ -566,88 +407,6 @@ def handle_nonpainsled(f2,fileformat,summary=''):
return (f2,summary,oarlength,inboard)
# Create new workout from file and store it in the database
# This routine should be used everywhere in views.py and mailprocessing.py
# Currently there is code duplication
def new_workout_from_file(r,f2,
workouttype='rower',
title='Workout',
makeprivate=False,
notes=''):
message = None
fileformat = get_file_type(f2)
summary = ''
oarlength = 2.89
inboard = 0.88
if len(fileformat)==3 and fileformat[0]=='zip':
f_to_be_deleted = f2
with zipfile.ZipFile(f2) as z:
for fname in z.namelist():
f3 = z.extract(fname,path='media/')
id,message,f2 = new_workout_from_file(r,f3,
workouttype=workouttype,
makeprivate=makeprivate,
title = title,
notes='')
os.remove(f_to_be_deleted)
return id,message,f2
# Some people try to upload Concept2 logbook summaries
if fileformat == 'c2log':
os.remove(f2)
message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook."
return (0,message,f2)
if fileformat == 'nostrokes':
os.remove(f2)
message = "It looks like this file doesn't contain stroke data."
return (0,message,f2)
# Some people try to upload RowPro summary logs
if fileformat == 'rowprolog':
os.remove(f2)
message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log."
return (0,message,f2)
# Sometimes people try an unsupported file type.
# Send an email to info@rowsandall.com with the file attached
# for me to check if it is a bug, or a new file type
# worth supporting
if fileformat == 'unknown':
message = "We couldn't recognize the file type"
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2,
r.user.email)
else:
res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2,r.user.email)
return (0,message,f2)
# handle non-Painsled by converting it to painsled compatible CSV
if (fileformat != 'csv'):
try:
f2,summary,oarlength,inboard = handle_nonpainsled(f2,
fileformat,
summary=summary)
except:
errorstring = str(sys.exc_info()[0])
message = 'Something went wrong: '+errorstring
return (0,message,'')
dosummary = (fileformat != 'fit' and fileformat != 'speedcoach2')
id,message = save_workout_database(f2,r,
workouttype=workouttype,
makeprivate=makeprivate,
dosummary=dosummary,
summary=summary,
inboard=inboard,oarlength=oarlength,
title=title)
return (id,message,f2)
def delete_strokedata(id,debug=False):
dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id)

View File

@@ -69,7 +69,7 @@ from rowers import mytypes
from rowers.dataprepnodjango import (
update_strokedata, new_workout_from_file,
update_strokedata,
getsmallrowdata_db, updatecpdata_sql,update_c2id_sql,
update_workout_field_sql,
update_agegroup_db,fitnessmetric_to_sql,
@@ -771,15 +771,7 @@ def long_test_task2(self,aantal,**kwargs):
# create workout
@app.task
def handle_new_workout_from_file(r, f2,
workouttype='rower',
boattype='1x',
makeprivate=False,
notes='',debug=False):
return new_workout_from_file(r, f2, workouttype,
title, makeprivate, notes)
# process and update workouts

View File

@@ -58,6 +58,9 @@ redis_connection = StrictRedis()
from django_mailbox.models import Mailbox,MessageAttachment,Message
def mocked_send_template_email(*args,**kwargs):
return 1
def mocked_myqueue(*args, **kwargs):
class Job:
def __init__(self,*args, **kwargs):
@@ -915,3 +918,10 @@ def mocked_requests(*args, **kwargs):
return MockResponse(c2workoutdata,200)
return MockResponse(None,404)
class MockEmailMessage:
def __init__(*args,**kwargs):
pass
def send(self):
return 1

View File

@@ -0,0 +1,282 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from .statements import *
from rowers.mytypes import rowtypes
import pandas as pd
nu = datetime.datetime.now()
from rowers import tasks
# asynchronous tasks
class AsyncTaskTests(TestCase):
def setUp(self):
self.u = UserFactory()
self.r = Rower.objects.create(user=self.u,
birthdate=faker.profile()['birthdate'],
gdproptin=True,surveydone=True,
gdproptindate=timezone.now(),
rowerplan='coach')
self.c = Client()
self.user_workouts = WorkoutFactory.create_batch(5, user=self.r)
self.factory = RequestFactory()
self.password = faker.word()
self.u.set_password(self.password)
self.u.save()
result = get_random_file(filename='rowers/tests/testdata/onwater2.csv')
self.wwater = WorkoutFactory(user=self.r,
csvfilename=result['filename'],
starttime=result['starttime'],
startdatetime=result['startdatetime'],
duration=result['duration'],
distance=result['totaldist'],
workouttype = 'water',
)
def test_safetimedelta(self):
x = 5
y = tasks.safetimedelta(x)
self.assertEqual(datetime.timedelta(seconds=5),y)
x = np.nan
y = tasks.safetimedelta(x)
self.assertEqual(datetime.timedelta(seconds=0),y)
@patch('rowers.c2stuff.requests.post', side_effect=mocked_requests)
@patch('rowers.c2stuff.requests.get', side_effect=mocked_requests)
def test_c2_sync(self, mock_get, mock_post):
authorizationstring = str('Bearer aap')
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json'}
data = c2stuff.createc2workoutdata(self.wwater)
url = "https://log.concept2.com/api/users/%s/results" % (1)
res = tasks.handle_c2_sync(1,url,headers,data)
self.assertEqual(res,1)
# to do - mock the grpc channel and metrics_pb2
@patch('rowers.dataprep.create_engine')
def test_handle_calctrimp(self, mocked_sqlalchemy):
result = get_random_file()
res = tasks.handle_calctrimp(1,result['filename'],200,'male',160,90,52)
self.assertEqual(res,1)
@patch('rowers.tasks.EmailMessage',side_effect=MockEmailMessage)
def test_handle_updatedps(self,MockEmailMessage):
result = get_random_file()
filename = result['filename']
res = tasks.handle_updatedps('roosendaalsander@gmail.com',[(1,filename)])
self.assertEqual(res,1)
@patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email)
def test_handle_send_email_alert(self,mocked_send_template_email):
useremail = self.u.email
userfirstname = self.u.first_name
userlastname = self.u.last_name,
rowerfirstname = self.u.first_name,
alertname = 'Test Alert'
stats = {
'percentage': 45,
'workouts': 3,
'nr_strokes_qualifying': 23,
'nr_strokes': 5,
'median': 33.23,
'median_q': 32.121,
'startdate': '2020-01-01',
'enddate': '2020-23-01',
}
res = tasks.handle_send_email_alert(useremail,userfirstname,userlastname,
rowerfirstname,alertname,stats)
self.assertEqual(res,1)
@patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email)
def test_handle_send_email_transactions(self,mocked_send_template_email):
res = tasks.handle_send_email_transaction(self.u.username,self.u.email,23)
self.assertEqual(res,1)
res = tasks.handle_send_email_failed_cancel(self.u.first_name,self.u.email,self.u.username,23)
self.assertEqual(res,1)
res = tasks.handle_send_email_subscription_update(
self.u.first_name,self.u.email,'testplan',True,23,24,'2020-12-02','down')
self.assertEqual(res,1)
res = tasks.handle_send_email_subscription_update(
self.u.first_name,self.u.email,'testplan',True,23,24,'2020-12-02','up')
self.assertEqual(res,1)
res = tasks.handle_send_email_subscription_create(
self.u.username,self.u.email,'pro',True,24,24,'2020-12-02'
)
self.assertEqual(res,1)
res = tasks.handle_sendemail_expired(self.u.email,self.u.first_name,self.u.last_name,
'2020-12-12')
self.assertEqual(res,1)
@patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email)
def test_handle_raceemails(self,mocked_send_template_email):
u = self.u
useremail = u.email
username = u.first_name
racename = 'Hop'
raceid = 12
registeredname = 'Jaap'
res = tasks.handle_sendemail_raceregistration(useremail,username,registeredname,racename,raceid)
self.assertEqual(res,1)
result = get_random_file()
filename = result['filename']
res = tasks.handle_sendemail_coursefail(useremail,username,filename)
self.assertEqual(res,1)
res = tasks.handle_sendemail_optout(useremail,username,registeredname,racename,raceid)
self.assertEqual(res,1)
res = tasks.handle_sendemail_racesubmission(useremail,username,registeredname,racename,raceid)
self.assertEqual(res,1)
res = tasks.handle_send_disqualification_email(useremail,username,'daarom','omdat',racename)
self.assertEqual(res,1)
res = tasks.handle_send_withdraw_email(useremail,username,'daarom','message',racename)
self.assertEqual(res,1)
@patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email)
def test_handle_otheremails(self,mocked_send_template_email):
u = self.u
useremail = u.email
username = u.first_name
userfirstname = u.first_name
userlastname = u.last_name
filename = get_random_file()['filename']
btvalues = pd.DataFrame({
'delta':[3,1,3],
'cpvalues':[100,200,300],
'pwr':[100,200,300]
}).to_json()
res = tasks.handle_sendemail_breakthrough(1,useremail,userfirstname,userlastname,
btvalues=btvalues)
self.assertEqual(res,1)
res = tasks.handle_sendemail_hard(1,useremail,userfirstname,userlastname,
btvalues=btvalues)
self.assertEqual(res,1)
res = tasks.handle_sendemail_userdeleted(username,useremail)
self.assertEqual(res,1)
res = tasks.handle_sendemail_unrecognized(filename,useremail)
self.assertEqual(res,1)
res = tasks.handle_sendemail_unrecognizedowner(useremail,userfirstname)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemailics(userfirstname,userlastname,useremail,filename)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemailkml(userfirstname,userlastname,useremail,filename)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemailtcx(userfirstname,userlastname,useremail,filename)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemailsummary(userfirstname,userlastname,useremail,filename)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemailcsv(userfirstname,userlastname,useremail,filename)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemail_ical(userfirstname,userlastname,useremail,'url',filename)
self.assertEqual(res,1)
filename = get_random_file()['filename']
res = tasks.handle_sendemailfile(userfirstname,userlastname,useremail,filename)
self.assertEqual(res,1)
def test_sigdig(self):
x = 3.14159
y = tasks.sigdig(x,digits=2)
self.assertEqual(y,'3.1')
y = tasks.sigdig(x)
self.assertEqual(y,'3.14')
y = tasks.sigdig(x,digits=0)
self.assertEqual(y,'0')
y = tasks.sigdig(x,digits=1)
self.assertEqual(y,'3')
y = tasks.sigdig(x,digits=4)
self.assertEqual(y,'3.142')
@patch('rowers.sporttracksstuff.requests.post', side_effect=mocked_requests)
@patch('rowers.sporttracksstuff.requests.get', side_effect=mocked_requests)
def test_sporttracks_sync(self, mock_get, mock_post):
authorizationstring = str('Bearer aap')
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json'}
url = "https://api.sporttracks.mobi/api/v2/fitnessActivities.json"
data = sporttracksstuff.createsporttracksworkoutdata(self.wwater)
res = tasks.handle_sporttracks_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default))
self.assertEqual(res,1)
@patch('rowers.runkeeperstuff.requests.post', side_effect=mocked_requests)
@patch('rowers.runkeeperstuff.requests.get', side_effect=mocked_requests)
def test_runkeeper_sync(self, mock_get, mock_post):
authorizationstring = str('Bearer aap')
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/vnd.com.runkeeper.NewFitnessActivity+json',
'Content-Length':'nnn'}
url = "https://api.runkeeper.com/fitnessActivities"
data = runkeeperstuff.createrunkeeperworkoutdata(self.wwater)
res = tasks.handle_runkeeper_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default))
self.assertEqual(res,1)
@patch('rowers.c2stuff.requests.post',side_effect=mocked_requests)
@patch('rowers.c2stuff.requests.get',side_effect=mocked_requests)
def test_import_c2_strokedata(self, mock_get, mock_post):
c2token = 'aap'
c2id = 1212
workoutid = 1
starttimeunix = 121
csvfilename = self.wwater.csvfilename
res = tasks.handle_c2_import_stroke_data(c2token,c2id,workoutid,starttimeunix,csvfilename)
self.assertEqual(res,1)

View File

@@ -8,106 +8,6 @@ from rowers.mytypes import rowtypes
nu = datetime.datetime.now()
from rowers import tasks
# asynchronous tasks
class AsyncTaskTests(TestCase):
def setUp(self):
self.u = UserFactory()
self.r = Rower.objects.create(user=self.u,
birthdate=faker.profile()['birthdate'],
gdproptin=True,surveydone=True,
gdproptindate=timezone.now(),
rowerplan='coach')
self.c = Client()
self.user_workouts = WorkoutFactory.create_batch(5, user=self.r)
self.factory = RequestFactory()
self.password = faker.word()
self.u.set_password(self.password)
self.u.save()
result = get_random_file(filename='rowers/tests/testdata/onwater2.csv')
self.wwater = WorkoutFactory(user=self.r,
csvfilename=result['filename'],
starttime=result['starttime'],
startdatetime=result['startdatetime'],
duration=result['duration'],
distance=result['totaldist'],
workouttype = 'water',
)
def test_safetimedelta(self):
x = 5
y = tasks.safetimedelta(x)
self.assertEqual(datetime.timedelta(seconds=5),y)
x = np.nan
y = tasks.safetimedelta(x)
self.assertEqual(datetime.timedelta(seconds=0),y)
@patch('rowers.c2stuff.requests.post', side_effect=mocked_requests)
@patch('rowers.c2stuff.requests.get', side_effect=mocked_requests)
def test_c2_sync(self, mock_get, mock_post):
authorizationstring = str('Bearer aap')
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json'}
data = c2stuff.createc2workoutdata(self.wwater)
url = "https://log.concept2.com/api/users/%s/results" % (1)
res = tasks.handle_c2_sync(1,url,headers,data)
self.assertEqual(res,1)
@patch('rowers.sporttracksstuff.requests.post', side_effect=mocked_requests)
@patch('rowers.sporttracksstuff.requests.get', side_effect=mocked_requests)
def test_sporttracks_sync(self, mock_get, mock_post):
authorizationstring = str('Bearer aap')
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json'}
url = "https://api.sporttracks.mobi/api/v2/fitnessActivities.json"
data = sporttracksstuff.createsporttracksworkoutdata(self.wwater)
res = tasks.handle_sporttracks_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default))
self.assertEqual(res,1)
@patch('rowers.runkeeperstuff.requests.post', side_effect=mocked_requests)
@patch('rowers.runkeeperstuff.requests.get', side_effect=mocked_requests)
def test_runkeeper_sync(self, mock_get, mock_post):
authorizationstring = str('Bearer aap')
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/vnd.com.runkeeper.NewFitnessActivity+json',
'Content-Length':'nnn'}
url = "https://api.runkeeper.com/fitnessActivities"
data = runkeeperstuff.createrunkeeperworkoutdata(self.wwater)
res = tasks.handle_runkeeper_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default))
self.assertEqual(res,1)
@patch('rowers.c2stuff.requests.post',side_effect=mocked_requests)
@patch('rowers.c2stuff.requests.get',side_effect=mocked_requests)
def test_import_c2_strokedata(self, mock_get, mock_post):
c2token = 'aap'
c2id = 1212
workoutid = 1
starttimeunix = 121
csvfilename = self.wwater.csvfilename
res = tasks.handle_c2_import_stroke_data(c2token,c2id,workoutid,starttimeunix,csvfilename)
self.assertEqual(res,1)
# interactive plots
from rowers import interactiveplots