From 8ebb55cfd7bf3d47b623afef3c82767f40519248 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 15 Jan 2021 07:18:43 +0100 Subject: [PATCH] further tests in async tasks --- rowers/dataprepnodjango.py | 247 +-------------------------- rowers/tasks.py | 12 +- rowers/tests/mocks.py | 10 ++ rowers/tests/test_async_tasks.py | 282 +++++++++++++++++++++++++++++++ rowers/tests/test_unit_tests.py | 100 ----------- 5 files changed, 297 insertions(+), 354 deletions(-) create mode 100644 rowers/tests/test_async_tasks.py diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index a6ca3b5f..0dfcb5df 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -9,7 +9,7 @@ from rowingdata import rowingdata as rrdata from rowingdata import make_cumvalues from rowingdata import rower as rrower from rowingdata import main as rmain -from rowingdata import empower_bug_correction,get_empower_rigging +from rowingdata import empower_bug_correction,get_empower_rigging, get_file_type from rowingdata.csvparsers import make_cumvalues_array from time import strftime from pandas import DataFrame,Series @@ -32,6 +32,8 @@ from rowsandall_app.settings_dev import DATABASES as DEV_DATABASES from rowsandall_app.settings_dev import use_sqlite from rowers.utils import lbstoN +import pytz +from timezonefinder import TimezoneFinder try: @@ -322,167 +324,6 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, return data -# Processes painsled CSV file to database -def save_workout_database(f2,r,dosmooth=True,workouttype='rower', - dosummary=True,title='Workout', - notes='',totaldist=0,totaltime=0, - workoutsource='unknown', - summary='', - makeprivate=False, - oarlength=2.89,inboard=0.88): - message = None - powerperc = 100*np.array([r.pw_ut2, - r.pw_ut1, - r.pw_at, - r.pw_tr,r.pw_an])/r.ftp - - # make workout and put in database - rr = rrower(hrmax=r.max,hrut2=r.ut2, - hrut1=r.ut1,hrat=r.at, - hrtr=r.tr,hran=r.an,ftp=r.ftp, - powerperc=powerperc,powerzones=r.powerzones) - row = rdata(f2,rower=rr) - - checks = row.check_consistency() - allchecks = 1 - for key,value in checks.iteritems(): - if not value: - allchecks = 0 - - if not allchecks: - #row.repair() - pass - - - if row == 0: - return (0,'Error: CSV data file not found') - - if dosmooth: - # auto smoothing - pace = row.df[' Stroke500mPace (sec/500m)'].values - velo = 500./pace - - f = row.df['TimeStamp (sec)'].diff().mean() - if f !=0: - windowsize = 2*(int(10./(f)))+1 - else: - windowsize = 1 - if not 'originalvelo' in row.df: - row.df['originalvelo'] = velo - - if windowsize > 3 and windowsize23: - message = 'Warning: The workout duration was longer than 23 hours. ' - hours = 23 - - minutes = int((totaltime - 3600.*hours)/60.) - if minutes>59: - minutes = 59 - if not message: - message = 'Warning: there is something wrong with the workout duration' - - seconds = int(totaltime - 3600.*hours - 60.*minutes) - if seconds > 59: - seconds = 59 - if not message: - message = 'Warning: there is something wrong with the workout duration' - - tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds)) - if tenths > 9: - tenths = 9 - if not message: - message = 'Warning: there is something wrong with the workout duration' - - duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) - - if dosummary: - summary = row.summary() - summary += '\n' - summary += row.intervalstats() - - workoutdate = row.rowdatetime.strftime('%Y-%m-%d') - workoutstarttime = row.rowdatetime.strftime('%H:%M:%S') - workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc) - - if makeprivate: - privacy = 'private' - else: - privacy = 'visible' - - # check for duplicate start times - ws = Workout.objects.filter(startdatetime=workoutstartdatetime, - user=r) - if (len(ws) != 0): - message = "Warning: This workout probably already exists in the database" - privacy = 'private' - - - - w = Workout(user=r,name=title,date=workoutdate, - workouttype=workouttype, - workoutsource=workoutsource, - duration=duration,distance=totaldist, - weightcategory=r.weightcategory, - starttime=workoutstarttime, - csvfilename=f2,notes=notes,summary=summary, - maxhr=maxhr,averagehr=averagehr, - startdatetime=workoutstartdatetime, - inboard=inboard,oarlength=oarlength, - privacy=privacy) - - - w.save() - - if privacy == 'visible': - ts = Team.objects.filter(rower=r) - for t in ts: - w.team.add(t) - - # put stroke data in database - res = dataprep(row.df,id=w.id,bands=True, - barchart=True,otwpower=True,empower=True,inboard=inboard) - - return (w.id,message) def handle_nonpainsled(f2,fileformat,summary=''): oarlength = 2.89 @@ -566,88 +407,6 @@ def handle_nonpainsled(f2,fileformat,summary=''): return (f2,summary,oarlength,inboard) -# Create new workout from file and store it in the database -# This routine should be used everywhere in views.py and mailprocessing.py -# Currently there is code duplication -def new_workout_from_file(r,f2, - workouttype='rower', - title='Workout', - makeprivate=False, - notes=''): - message = None - fileformat = get_file_type(f2) - summary = '' - oarlength = 2.89 - inboard = 0.88 - if len(fileformat)==3 and fileformat[0]=='zip': - f_to_be_deleted = f2 - with zipfile.ZipFile(f2) as z: - for fname in z.namelist(): - f3 = z.extract(fname,path='media/') - id,message,f2 = new_workout_from_file(r,f3, - workouttype=workouttype, - makeprivate=makeprivate, - title = title, - notes='') - os.remove(f_to_be_deleted) - return id,message,f2 - - # Some people try to upload Concept2 logbook summaries - if fileformat == 'c2log': - os.remove(f2) - message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook." - return (0,message,f2) - - if fileformat == 'nostrokes': - os.remove(f2) - message = "It looks like this file doesn't contain stroke data." - return (0,message,f2) - - # Some people try to upload RowPro summary logs - if fileformat == 'rowprolog': - os.remove(f2) - message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log." - return (0,message,f2) - - # Sometimes people try an unsupported file type. - # Send an email to info@rowsandall.com with the file attached - # for me to check if it is a bug, or a new file type - # worth supporting - if fileformat == 'unknown': - message = "We couldn't recognize the file type" - if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, - r.user.email) - - else: - res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2,r.user.email) - return (0,message,f2) - - # handle non-Painsled by converting it to painsled compatible CSV - if (fileformat != 'csv'): - try: - f2,summary,oarlength,inboard = handle_nonpainsled(f2, - fileformat, - summary=summary) - except: - errorstring = str(sys.exc_info()[0]) - message = 'Something went wrong: '+errorstring - return (0,message,'') - - - - dosummary = (fileformat != 'fit' and fileformat != 'speedcoach2') - - id,message = save_workout_database(f2,r, - workouttype=workouttype, - makeprivate=makeprivate, - dosummary=dosummary, - summary=summary, - inboard=inboard,oarlength=oarlength, - title=title) - - return (id,message,f2) def delete_strokedata(id,debug=False): dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id) diff --git a/rowers/tasks.py b/rowers/tasks.py index be750f6f..c7d4384b 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -69,7 +69,7 @@ from rowers import mytypes from rowers.dataprepnodjango import ( - update_strokedata, new_workout_from_file, + update_strokedata, getsmallrowdata_db, updatecpdata_sql,update_c2id_sql, update_workout_field_sql, update_agegroup_db,fitnessmetric_to_sql, @@ -771,15 +771,7 @@ def long_test_task2(self,aantal,**kwargs): -# create workout -@app.task -def handle_new_workout_from_file(r, f2, - workouttype='rower', - boattype='1x', - makeprivate=False, - notes='',debug=False): - return new_workout_from_file(r, f2, workouttype, - title, makeprivate, notes) + # process and update workouts diff --git a/rowers/tests/mocks.py b/rowers/tests/mocks.py index 3021adb0..a1199487 100644 --- a/rowers/tests/mocks.py +++ b/rowers/tests/mocks.py @@ -58,6 +58,9 @@ redis_connection = StrictRedis() from django_mailbox.models import Mailbox,MessageAttachment,Message +def mocked_send_template_email(*args,**kwargs): + return 1 + def mocked_myqueue(*args, **kwargs): class Job: def __init__(self,*args, **kwargs): @@ -915,3 +918,10 @@ def mocked_requests(*args, **kwargs): return MockResponse(c2workoutdata,200) return MockResponse(None,404) + +class MockEmailMessage: + def __init__(*args,**kwargs): + pass + + def send(self): + return 1 diff --git a/rowers/tests/test_async_tasks.py b/rowers/tests/test_async_tasks.py new file mode 100644 index 00000000..c22c4dc0 --- /dev/null +++ b/rowers/tests/test_async_tasks.py @@ -0,0 +1,282 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from .statements import * +from rowers.mytypes import rowtypes +import pandas as pd + +nu = datetime.datetime.now() +from rowers import tasks + +# asynchronous tasks +class AsyncTaskTests(TestCase): + def setUp(self): + self.u = UserFactory() + + self.r = Rower.objects.create(user=self.u, + birthdate=faker.profile()['birthdate'], + gdproptin=True,surveydone=True, + gdproptindate=timezone.now(), + rowerplan='coach') + + self.c = Client() + self.user_workouts = WorkoutFactory.create_batch(5, user=self.r) + self.factory = RequestFactory() + self.password = faker.word() + self.u.set_password(self.password) + self.u.save() + + result = get_random_file(filename='rowers/tests/testdata/onwater2.csv') + + self.wwater = WorkoutFactory(user=self.r, + csvfilename=result['filename'], + starttime=result['starttime'], + startdatetime=result['startdatetime'], + duration=result['duration'], + distance=result['totaldist'], + workouttype = 'water', + ) + + def test_safetimedelta(self): + x = 5 + y = tasks.safetimedelta(x) + self.assertEqual(datetime.timedelta(seconds=5),y) + + x = np.nan + y = tasks.safetimedelta(x) + self.assertEqual(datetime.timedelta(seconds=0),y) + + @patch('rowers.c2stuff.requests.post', side_effect=mocked_requests) + @patch('rowers.c2stuff.requests.get', side_effect=mocked_requests) + def test_c2_sync(self, mock_get, mock_post): + authorizationstring = str('Bearer aap') + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json'} + + data = c2stuff.createc2workoutdata(self.wwater) + + url = "https://log.concept2.com/api/users/%s/results" % (1) + res = tasks.handle_c2_sync(1,url,headers,data) + self.assertEqual(res,1) + + # to do - mock the grpc channel and metrics_pb2 + @patch('rowers.dataprep.create_engine') + def test_handle_calctrimp(self, mocked_sqlalchemy): + result = get_random_file() + res = tasks.handle_calctrimp(1,result['filename'],200,'male',160,90,52) + self.assertEqual(res,1) + + @patch('rowers.tasks.EmailMessage',side_effect=MockEmailMessage) + def test_handle_updatedps(self,MockEmailMessage): + result = get_random_file() + filename = result['filename'] + res = tasks.handle_updatedps('roosendaalsander@gmail.com',[(1,filename)]) + + self.assertEqual(res,1) + + @patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email) + def test_handle_send_email_alert(self,mocked_send_template_email): + useremail = self.u.email + userfirstname = self.u.first_name + userlastname = self.u.last_name, + rowerfirstname = self.u.first_name, + alertname = 'Test Alert' + stats = { + 'percentage': 45, + 'workouts': 3, + 'nr_strokes_qualifying': 23, + 'nr_strokes': 5, + 'median': 33.23, + 'median_q': 32.121, + 'startdate': '2020-01-01', + 'enddate': '2020-23-01', + } + + res = tasks.handle_send_email_alert(useremail,userfirstname,userlastname, + rowerfirstname,alertname,stats) + + self.assertEqual(res,1) + + @patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email) + def test_handle_send_email_transactions(self,mocked_send_template_email): + res = tasks.handle_send_email_transaction(self.u.username,self.u.email,23) + self.assertEqual(res,1) + + res = tasks.handle_send_email_failed_cancel(self.u.first_name,self.u.email,self.u.username,23) + self.assertEqual(res,1) + + res = tasks.handle_send_email_subscription_update( + self.u.first_name,self.u.email,'testplan',True,23,24,'2020-12-02','down') + self.assertEqual(res,1) + + res = tasks.handle_send_email_subscription_update( + self.u.first_name,self.u.email,'testplan',True,23,24,'2020-12-02','up') + self.assertEqual(res,1) + + res = tasks.handle_send_email_subscription_create( + self.u.username,self.u.email,'pro',True,24,24,'2020-12-02' + ) + self.assertEqual(res,1) + + res = tasks.handle_sendemail_expired(self.u.email,self.u.first_name,self.u.last_name, + '2020-12-12') + self.assertEqual(res,1) + + @patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email) + def test_handle_raceemails(self,mocked_send_template_email): + u = self.u + useremail = u.email + username = u.first_name + racename = 'Hop' + raceid = 12 + registeredname = 'Jaap' + + res = tasks.handle_sendemail_raceregistration(useremail,username,registeredname,racename,raceid) + self.assertEqual(res,1) + + result = get_random_file() + filename = result['filename'] + + res = tasks.handle_sendemail_coursefail(useremail,username,filename) + self.assertEqual(res,1) + + res = tasks.handle_sendemail_optout(useremail,username,registeredname,racename,raceid) + self.assertEqual(res,1) + + res = tasks.handle_sendemail_racesubmission(useremail,username,registeredname,racename,raceid) + self.assertEqual(res,1) + + res = tasks.handle_send_disqualification_email(useremail,username,'daarom','omdat',racename) + self.assertEqual(res,1) + + res = tasks.handle_send_withdraw_email(useremail,username,'daarom','message',racename) + self.assertEqual(res,1) + + @patch('rowers.tasks.send_template_email',side_effect=mocked_send_template_email) + def test_handle_otheremails(self,mocked_send_template_email): + u = self.u + useremail = u.email + username = u.first_name + userfirstname = u.first_name + userlastname = u.last_name + filename = get_random_file()['filename'] + + btvalues = pd.DataFrame({ + 'delta':[3,1,3], + 'cpvalues':[100,200,300], + 'pwr':[100,200,300] + }).to_json() + + res = tasks.handle_sendemail_breakthrough(1,useremail,userfirstname,userlastname, + btvalues=btvalues) + + self.assertEqual(res,1) + + res = tasks.handle_sendemail_hard(1,useremail,userfirstname,userlastname, + btvalues=btvalues) + self.assertEqual(res,1) + + res = tasks.handle_sendemail_userdeleted(username,useremail) + self.assertEqual(res,1) + + res = tasks.handle_sendemail_unrecognized(filename,useremail) + self.assertEqual(res,1) + + res = tasks.handle_sendemail_unrecognizedowner(useremail,userfirstname) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemailics(userfirstname,userlastname,useremail,filename) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemailkml(userfirstname,userlastname,useremail,filename) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemailtcx(userfirstname,userlastname,useremail,filename) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemailsummary(userfirstname,userlastname,useremail,filename) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemailcsv(userfirstname,userlastname,useremail,filename) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemail_ical(userfirstname,userlastname,useremail,'url',filename) + self.assertEqual(res,1) + + filename = get_random_file()['filename'] + res = tasks.handle_sendemailfile(userfirstname,userlastname,useremail,filename) + self.assertEqual(res,1) + + + def test_sigdig(self): + x = 3.14159 + + y = tasks.sigdig(x,digits=2) + self.assertEqual(y,'3.1') + + y = tasks.sigdig(x) + self.assertEqual(y,'3.14') + + y = tasks.sigdig(x,digits=0) + self.assertEqual(y,'0') + + y = tasks.sigdig(x,digits=1) + self.assertEqual(y,'3') + + y = tasks.sigdig(x,digits=4) + self.assertEqual(y,'3.142') + + + @patch('rowers.sporttracksstuff.requests.post', side_effect=mocked_requests) + @patch('rowers.sporttracksstuff.requests.get', side_effect=mocked_requests) + def test_sporttracks_sync(self, mock_get, mock_post): + authorizationstring = str('Bearer aap') + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json'} + + url = "https://api.sporttracks.mobi/api/v2/fitnessActivities.json" + + data = sporttracksstuff.createsporttracksworkoutdata(self.wwater) + + + res = tasks.handle_sporttracks_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default)) + self.assertEqual(res,1) + + @patch('rowers.runkeeperstuff.requests.post', side_effect=mocked_requests) + @patch('rowers.runkeeperstuff.requests.get', side_effect=mocked_requests) + def test_runkeeper_sync(self, mock_get, mock_post): + authorizationstring = str('Bearer aap') + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/vnd.com.runkeeper.NewFitnessActivity+json', + 'Content-Length':'nnn'} + + url = "https://api.runkeeper.com/fitnessActivities" + + data = runkeeperstuff.createrunkeeperworkoutdata(self.wwater) + + + res = tasks.handle_runkeeper_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default)) + self.assertEqual(res,1) + + @patch('rowers.c2stuff.requests.post',side_effect=mocked_requests) + @patch('rowers.c2stuff.requests.get',side_effect=mocked_requests) + def test_import_c2_strokedata(self, mock_get, mock_post): + c2token = 'aap' + c2id = 1212 + workoutid = 1 + starttimeunix = 121 + csvfilename = self.wwater.csvfilename + + res = tasks.handle_c2_import_stroke_data(c2token,c2id,workoutid,starttimeunix,csvfilename) + self.assertEqual(res,1) diff --git a/rowers/tests/test_unit_tests.py b/rowers/tests/test_unit_tests.py index b99d4823..c1e2c9f3 100644 --- a/rowers/tests/test_unit_tests.py +++ b/rowers/tests/test_unit_tests.py @@ -8,106 +8,6 @@ from rowers.mytypes import rowtypes nu = datetime.datetime.now() -from rowers import tasks - -# asynchronous tasks -class AsyncTaskTests(TestCase): - def setUp(self): - self.u = UserFactory() - - self.r = Rower.objects.create(user=self.u, - birthdate=faker.profile()['birthdate'], - gdproptin=True,surveydone=True, - gdproptindate=timezone.now(), - rowerplan='coach') - - self.c = Client() - self.user_workouts = WorkoutFactory.create_batch(5, user=self.r) - self.factory = RequestFactory() - self.password = faker.word() - self.u.set_password(self.password) - self.u.save() - - result = get_random_file(filename='rowers/tests/testdata/onwater2.csv') - - self.wwater = WorkoutFactory(user=self.r, - csvfilename=result['filename'], - starttime=result['starttime'], - startdatetime=result['startdatetime'], - duration=result['duration'], - distance=result['totaldist'], - workouttype = 'water', - ) - - def test_safetimedelta(self): - x = 5 - y = tasks.safetimedelta(x) - self.assertEqual(datetime.timedelta(seconds=5),y) - - x = np.nan - y = tasks.safetimedelta(x) - self.assertEqual(datetime.timedelta(seconds=0),y) - - @patch('rowers.c2stuff.requests.post', side_effect=mocked_requests) - @patch('rowers.c2stuff.requests.get', side_effect=mocked_requests) - def test_c2_sync(self, mock_get, mock_post): - authorizationstring = str('Bearer aap') - headers = {'Authorization': authorizationstring, - 'user-agent': 'sanderroosendaal', - 'Content-Type': 'application/json'} - - data = c2stuff.createc2workoutdata(self.wwater) - - url = "https://log.concept2.com/api/users/%s/results" % (1) - res = tasks.handle_c2_sync(1,url,headers,data) - self.assertEqual(res,1) - - - - @patch('rowers.sporttracksstuff.requests.post', side_effect=mocked_requests) - @patch('rowers.sporttracksstuff.requests.get', side_effect=mocked_requests) - def test_sporttracks_sync(self, mock_get, mock_post): - authorizationstring = str('Bearer aap') - headers = {'Authorization': authorizationstring, - 'user-agent': 'sanderroosendaal', - 'Content-Type': 'application/json'} - - url = "https://api.sporttracks.mobi/api/v2/fitnessActivities.json" - - data = sporttracksstuff.createsporttracksworkoutdata(self.wwater) - - - res = tasks.handle_sporttracks_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default)) - self.assertEqual(res,1) - - @patch('rowers.runkeeperstuff.requests.post', side_effect=mocked_requests) - @patch('rowers.runkeeperstuff.requests.get', side_effect=mocked_requests) - def test_runkeeper_sync(self, mock_get, mock_post): - authorizationstring = str('Bearer aap') - headers = {'Authorization': authorizationstring, - 'user-agent': 'sanderroosendaal', - 'Content-Type': 'application/vnd.com.runkeeper.NewFitnessActivity+json', - 'Content-Length':'nnn'} - - url = "https://api.runkeeper.com/fitnessActivities" - - data = runkeeperstuff.createrunkeeperworkoutdata(self.wwater) - - - res = tasks.handle_runkeeper_sync(1,url,headers,json.dumps(data,default=sporttracksstuff.default)) - self.assertEqual(res,1) - - @patch('rowers.c2stuff.requests.post',side_effect=mocked_requests) - @patch('rowers.c2stuff.requests.get',side_effect=mocked_requests) - def test_import_c2_strokedata(self, mock_get, mock_post): - c2token = 'aap' - c2id = 1212 - workoutid = 1 - starttimeunix = 121 - csvfilename = self.wwater.csvfilename - - res = tasks.handle_c2_import_stroke_data(c2token,c2id,workoutid,starttimeunix,csvfilename) - self.assertEqual(res,1) # interactive plots from rowers import interactiveplots