From a9316d6f21c8333adb8adf22d5157ea65ea504e3 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 15:17:18 +0100 Subject: [PATCH 1/5] adding stroke count --- rowers/c2stuff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 4600d201..319a164f 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -574,6 +574,7 @@ def createc2workoutdata(w): "time": int(10*makeseconds(durationstr)), "weight_class": c2wc(w.weightcategory), "comments": w.notes, + "stroke_count": int(row.df.stroke_count), 'stroke_rate': int(row.df[' Cadence (stokes/min)'].mean()), 'drag_factor': int(row.dragfactor), "heart_rate": { From 3f0f8e69b007f73e332a13b24d4c133d5fae0977 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 15:19:30 +0100 Subject: [PATCH 2/5] changes --- rowers/c2stuff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 319a164f..3bb8949b 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -574,7 +574,7 @@ def createc2workoutdata(w): "time": int(10*makeseconds(durationstr)), "weight_class": c2wc(w.weightcategory), "comments": w.notes, - "stroke_count": int(row.df.stroke_count), + "stroke_count": int(row.stroke_count), 'stroke_rate': int(row.df[' Cadence (stokes/min)'].mean()), 'drag_factor': int(row.dragfactor), "heart_rate": { From 473f2b3cdf01d01ed7dbc660815589849d52fd1b Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 15:46:25 +0100 Subject: [PATCH 3/5] small bug fixes --- rowers/dataprepnodjango.py | 9 +- rowers/tasks.py | 181 +++++++++++++++++++------------------ 2 files changed, 98 insertions(+), 92 deletions(-) diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index d83682b9..9dd79427 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -166,7 +166,7 @@ def create_c2_stroke_data_db( try: spm = 60.*nr_strokes/totalseconds except ZeroDivisionError: - spm = 20*zeros(nr_strokes) + spm = 20*np.zeros(nr_strokes) step = totalseconds/float(nr_strokes) @@ -713,7 +713,7 @@ def getsmallrowdata_db(columns,ids=[],debug=False): columns = [c for c in columns if c != 'None'] if len(ids)>1: - for f in csvfilenames: + for id, f in zip(ids,csvfilenames): try: df = pd.read_parquet(f,columns=columns,engine='pyarrow') data.append(df) @@ -721,7 +721,10 @@ def getsmallrowdata_db(columns,ids=[],debug=False): pass - df = pd.concat(data,axis=0) + try: + df = pd.concat(data,axis=0) + except ValueError: + df = pd.DataFrame() else: df = pd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') diff --git a/rowers/tasks.py b/rowers/tasks.py index 4d22d545..ccc797aa 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -115,19 +115,19 @@ def add(x, y): return x + y - + @app.task def handle_c2_import_stroke_data(c2token, c2id,workoutid, starttimeunix, csvfilename,debug=True,**kwargs): - + if 'workouttype' in kwargs: workouttype = kwargs['workouttype'] else: workouttype = 'rower' - + authorizationstring = str('Bearer ' + c2token) headers = {'Authorization': authorizationstring, 'user-agent': 'sanderroosendaal', @@ -140,13 +140,13 @@ def handle_c2_import_stroke_data(c2token, strokedata,workoutid,starttimeunix, csvfilename,debug=debug,workouttype=workouttype ) - + return 1 else: url = "https://log.concept2.com/api/users/me/results/{id}".format(id=c2id) - + s = requests.get(url,headers=headers) - + if s.status_code == 200: workoutdata = s.json()['data'] distance = workoutdata['distance'] @@ -179,10 +179,10 @@ def handle_c2_import_stroke_data(c2token, workoutid,starttimeunix, csvfilename,debug=debug, ) - - + + return 1 - + return 0 return 0 @@ -191,7 +191,10 @@ def getagegrouprecord(age,sex='male',weightcategory='hwt', distance=2000,duration=None,indf=pd.DataFrame()): if not duration: - df = indf[indf['distance'] == distance] + try: + df = indf[indf['distance'] == distance] + except KeyError: + df = pd.DataFrame() else: duration = 60*int(duration) df = indf[indf['duration'] == duration] @@ -238,7 +241,7 @@ def polygon_to_path(polygon,debug=True): conn.close() engine.dispose() s = [] - + for point in points: s.append([point[1],point[2]]) @@ -270,7 +273,7 @@ def handle_check_race_course(self, try: row = rdata(csvfile=f1) except IOError: - try: + try: row = rdata(csvfile=f1 + '.csv') except IOError: try: @@ -278,7 +281,7 @@ def handle_check_race_course(self, except IOError: return 0 - + rowdata = row.df try: @@ -309,7 +312,7 @@ def handle_check_race_course(self, engine = create_engine(database_url_debug, echo=False) else: engine = create_engine(database_url, echo=False) - + # get polygons query = "SELECT id FROM rowers_geopolygon WHERE course_id = {courseid} ORDER BY order_in_course ASC".format( courseid=courseid @@ -342,7 +345,7 @@ def handle_check_race_course(self, coursetimeseconds, coursemeters, coursecompleted, - + ) = coursetime_paths(rowdata2,paths) ( coursetimefirst, @@ -378,7 +381,7 @@ def handle_check_race_course(self, coursemeters = records[records['coursetimeseconds'] == mintime]['coursemeters'].min() else: coursecompleted = False - + if coursecompleted: query = 'UPDATE rowers_virtualraceresult SET coursecompleted = 1, duration = "{duration}", distance = {distance}, workoutid = {workoutid} WHERE id={recordid}'.format( @@ -395,8 +398,8 @@ def handle_check_race_course(self, distance=int(coursemeters), workoutid=workoutid, ) - - + + with engine.connect() as conn, conn.begin(): result = conn.execute(query) @@ -430,7 +433,7 @@ def handle_getagegrouprecords(self, if sex == 'not specified': return 0 - + for distance in distances: worldclasspower = getagegrouprecord( age, @@ -477,22 +480,22 @@ def long_test_task(self,aantal,debug=False,job=None,session_key=None): return longtask.longtask(aantal,jobid=job.id,debug=debug, session_key=session_key) - + @app.task(bind=True) def long_test_task2(self,aantal,**kwargs): #debug=False,job=None,jobid='aap'): job = self.request job_id = job.id - + if 'jobkey' in kwargs: job_id = kwargs.pop('jobkey') - + kwargs['jobid'] = job_id - + return longtask.longtask2(aantal,**kwargs) - + # create workout @app.task @@ -520,7 +523,7 @@ def handle_update_empower(self, aantal = len(workoutdicts) counter = 0 - + for workoutdict in workoutdicts: wid = workoutdict['id'] inboard = workoutdict['inboard'] @@ -529,7 +532,7 @@ def handle_update_empower(self, f1 = workoutdict['filename'] # oarlength consistency checks will be done in view - + havedata = 1 try: rowdata = rdata(csvfile=f1) @@ -541,7 +544,7 @@ def handle_update_empower(self, rowdata = rdata(csvfile=f1 + '.gz') except IOError: havedata = 0 - + progressurl = SITE_URL siteurl = SITE_URL if debug: @@ -550,10 +553,10 @@ def handle_update_empower(self, secret = PROGRESS_CACHE_SECRET kwargs['job_id'] = job_id - + progressurl += "/rowers/record-progress/" progressurl += job_id - + if havedata: success = update_empower(wid, inboard, oarlength, boattype, rowdata.df, f1, debug=debug) @@ -561,7 +564,7 @@ def handle_update_empower(self, counter += 1 progress = 100.*float(counter)/float(aantal) - + post_data = { "secret":secret, "value":progress, @@ -601,7 +604,7 @@ def handle_calctrimp(id, engine = create_engine(database_url_debug, echo=False) else: engine = create_engine(database_url, echo=False) - + try: rowdata = rdata(csvfile=csvfilename) except IOError: @@ -621,7 +624,7 @@ def handle_calctrimp(id, return 0 df2 = df.copy() - + df2['time'] = df2[' ElapsedTime (sec)'] df2['time'] = df2['time'].apply( lambda x:safetimedelta(x) @@ -633,7 +636,7 @@ def handle_calctrimp(id, df2 = df2.rolling(30).mean() df2[' Power (watts)'] = df2[' Power (watts)'].abs() - + df2['pwr4'] = df2[' Power (watts)']**(4.0) # pwr4mean = wavg(df,'pwr4','deltat') pwr4mean = df2['pwr4'].mean() @@ -647,7 +650,7 @@ def handle_calctrimp(id, intensityfactor = normp/float(ftp) tss = 100.*((duration*normp*intensityfactor)/(3600.*ftp)) - + if sex == 'male': f = 1.92 else: @@ -665,7 +668,7 @@ def handle_calctrimp(id, hrtss = 100.*trimp/trimp1hr pp = 8.0 - + df['v4'] = df[' AverageBoatSpeed (m/s)']**(pp) v4mean = wavg(df,'v4','deltat') normv = v4mean**(1./pp) @@ -676,14 +679,14 @@ def handle_calctrimp(id, normw = w4mean**(1./pp) except KeyError: normw = 0 - + if not np.isfinite(normv): normv = 500./120. if not np.isfinite(normw): normw = 0 - + try: dum = int(tss) except ValueError: @@ -718,7 +721,7 @@ def handle_calctrimp(id, engine.dispose() return 1 - + @app.task def handle_updatedps(useremail, workoutids, debug=False,**kwargs): @@ -738,7 +741,7 @@ def handle_updatedps(useremail, workoutids, debug=False,**kwargs): if havedata: update_strokedata(wid, rowdata.df, debug=debug) - + subject = "Rowsandall.com Your Distance per Stroke metric has been updated" message = "All your workouts now have Distance per Stroke" @@ -766,7 +769,7 @@ def sigdig(value, digits = 3): # return integers as is if value % 1 == 0: return value - + places = digits - order - 1 if places > 0: fmtstr = "%%.%df" % (places) @@ -899,7 +902,7 @@ def handle_send_email_failed_cancel( d, **kwargs) return 1 - + @app.task def handle_send_email_subscription_update( @@ -932,7 +935,7 @@ def handle_send_email_subscription_update( template_name = 'subscription_update_email.html' notification_template_name = 'subscription_update_notification.html' subject = "Rowsandall Payment Confirmation" - + res = send_template_email(from_email,[useremail], subject, template_name, @@ -1199,7 +1202,7 @@ def handle_sendemail_userdeleted(name, email, debug=False, **kwargs): if not emailbounced: res = email.send() - + return 1 @@ -1232,7 +1235,7 @@ def handle_sendemail_unrecognized(unrecognizedfile, useremail, if not emailbounced: res = email.send() - + # remove tcx file try: @@ -1286,7 +1289,7 @@ def handle_sendemailics(first_name, last_name, email, icsfile, **kwargs): os.remove(icsfile) return 1 - + @app.task def handle_sendemailkml(first_name, last_name, email, kmlfile,**kwargs): @@ -1341,10 +1344,10 @@ def handle_zip_file(emailfrom, subject, file,**kwargs): debug = kwargs['debug'] except KeyError: debug = False - + if debug: print(message) - + email = EmailMessage(subject, message, emailfrom, ['workouts@rowsandall.com']) @@ -1354,8 +1357,8 @@ def handle_zip_file(emailfrom, subject, file,**kwargs): res = email.send() - - + + if debug: print("sent") time.sleep(60) @@ -1379,7 +1382,7 @@ def handle_sendemailsummary(first_name, last_name, email, csvfile, **kwargs): subject,'summarymail.html',d, attach_file=csvfile, **kwargs) - + try: os.remove(csvfile) except: @@ -1392,7 +1395,7 @@ def handle_sendemailsummary(first_name, last_name, email, csvfile, **kwargs): @app.task def handle_sendemailcsv(first_name, last_name, email, csvfile,**kwargs): - + # send email with attachment fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "File from Rowsandall.com" @@ -1407,7 +1410,7 @@ def handle_sendemailcsv(first_name, last_name, email, csvfile,**kwargs): res = send_template_email(from_email,[fullemail], subject,'csvemail.html',d, attach_file=csvfile,**kwargs) - + return 1 @@ -1427,7 +1430,7 @@ def handle_sendemail_ical(first_name, last_name, email, url, icsfile, **kwargs): progressurl = SITE_URL_DEV siteurl = SITE_URL_DEV - + d = {'first_name':first_name, 'siteurl':siteurl, 'url':url, @@ -1439,20 +1442,20 @@ def handle_sendemail_ical(first_name, last_name, email, url, icsfile, **kwargs): res = send_template_email(from_email,[fullemail], subject,'icsemail.html',d, attach_file=icsfile,**kwargs) - + try: os.remove(csvfile) except: pass - + return 1 - + @app.task def handle_sendemailfile(first_name, last_name, email, csvfile,**kwargs): - + # send email with attachment fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "File from Rowsandall.com" @@ -1467,7 +1470,7 @@ def handle_sendemailfile(first_name, last_name, email, csvfile,**kwargs): res = send_template_email(from_email,[fullemail], subject,'fileemail.html',d, attach_file=csvfile,**kwargs) - + if 'delete' in kwargs: dodelete = kwargs['delete'] @@ -1479,7 +1482,7 @@ def handle_sendemailfile(first_name, last_name, email, csvfile,**kwargs): os.remove(csvfile) except: pass - + return 1 # Calculate wind and stream corrections for OTW rowing @@ -1492,7 +1495,7 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, job = self.request job_id = job.id - + if 'jobkey' in kwargs: job_id = kwargs.pop('jobkey') if 'ps' in kwargs: @@ -1563,7 +1566,7 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, # determine cache file name physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue)) - + rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg, powermeasured=powermeasured, progressurl=progressurl, @@ -1638,7 +1641,7 @@ def handle_updateergcp(rower_id,workoutfilenames,debug=False,**kwargs): cpdata = rowingdata.cumcpdata(therows) cpdata.columns = cpdata.columns.str.lower() - + updatecpdata_sql(rower_id,cpdata['delta'],cpdata['cp'], table='ergcpdata',distance=cpdata['distance'], debug=debug) @@ -1659,7 +1662,7 @@ def cp_from_workoutids(workoutids,debug=False): maxt = 1.05*df['time'].max()/1000. logarr = datautils.getlogarr(maxt) - + delta,cpvalue,avgpower = datautils.getcp(dfgrouped,logarr) powerdf = pd.DataFrame({ @@ -1705,7 +1708,7 @@ def cp_from_workoutids(workoutids,debug=False): power2k = fitfunc(p1,t3) return powerfourmin,power2k,powerhour - + @app.task def handle_updatefitnessmetric(user_id,mode,workoutids,debug=False, @@ -1714,7 +1717,7 @@ def handle_updatefitnessmetric(user_id,mode,workoutids,debug=False, powerfourmin = -1 power2k = -1 powerhour = -1 - + mdict = { 'user_id': user_id, 'PowerFourMin': powerfourmin, @@ -1730,7 +1733,7 @@ def handle_updatefitnessmetric(user_id,mode,workoutids,debug=False, powerfourmin,power2k,powerhour = cp_from_workoutids(workoutids,debug=debug) if powerfourmin > 0 and power2k > 0 and powerhour > 0: - + mdict = { 'user_id': user_id, 'PowerFourMin': powerfourmin, @@ -1744,7 +1747,7 @@ def handle_updatefitnessmetric(user_id,mode,workoutids,debug=False, result = fitnessmetric_to_sql(mdict,debug=debug,doclean=True) else: result = 0 - + return result @app.task @@ -1760,13 +1763,13 @@ def handle_updatecp(rower_id,workoutids,debug=False,table='cpdata',**kwargs): logarr = datautils.getlogarr(maxt) - + delta,cpvalue,avgpower = datautils.getcp(dfgrouped,logarr) updatecpdata_sql(rower_id,delta,cpvalue,debug=debug,table=table) return 1 - + @app.task def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename, debug=False,**kwargs): @@ -1864,11 +1867,11 @@ def handle_sendemail_coachrequest(email,name,code,coachname, } form_email = 'Rowsandall ' - + res = send_template_email(from_email,[fullemail], subject,'coachrequestemail.html',d, **kwargs) - + return 1 @app.task @@ -1984,7 +1987,7 @@ def handle_sendemail_coacheerequest(email,name,code,coachname, res = send_template_email(from_email,[fullemail], subject,'coacheerequestemail.html',d, **kwargs) - + return 1 @app.task @@ -2004,13 +2007,13 @@ def handle_sendemail_invite(email, name, code, teamname, manager, 'teamname':teamname, 'siteurl':siteurl } - + from_email = 'Rowsandall ' - + res = send_template_email(from_email,[fullemail], subject,'teaminviteemail.html',d, **kwargs) - + return 1 @@ -2027,7 +2030,7 @@ def handle_sendemailnewresponse(first_name, last_name, subject = 'New comment on session ' + workoutname comment = u''+comment - + siteurl = SITE_URL if debug: siteurl = SITE_URL_DEV @@ -2035,12 +2038,12 @@ def handle_sendemailnewresponse(first_name, last_name, sessiontype = 'workout' if 'sessiontype' in kwargs: sessiontype=kwargs.pop('sessiontype') - + commentlink = '/rowers/workout/{workoutid}/comment/'.format( workoutid=encoder.encode_hex(workoutid)) if 'commentlink' in kwargs: commentlink = kwargs.pop('commentlink') - + d = { 'first_name':first_name, 'commenter_first_name':commenter_first_name, @@ -2056,7 +2059,7 @@ def handle_sendemailnewresponse(first_name, last_name, res = send_template_email(from_email, [fullemail], - subject,'teamresponseemail.html',d,**kwargs) + subject,'teamresponseemail.html',d,**kwargs) return 1 @@ -2072,13 +2075,13 @@ def handle_sendemailnewcomment(first_name, debug=False,**kwargs): - + fullemail = email from_email = 'Rowsandall ' subject = 'New comment on session ' + workoutname comment = u''+comment - + siteurl = SITE_URL if debug: siteurl = SITE_URL_DEV @@ -2086,7 +2089,7 @@ def handle_sendemailnewcomment(first_name, sessiontype = 'workout' if 'sessiontype' in kwargs: sessiontype=kwargs.pop('sessiontype') - + commentlink = '/rowers/workout/{workoutid}/comment/'.format( workoutid=encoder.encode_hex(workoutid)) if 'commentlink' in kwargs: @@ -2106,7 +2109,7 @@ def handle_sendemailnewcomment(first_name, res = send_template_email(from_email,[fullemail],subject, 'teamresponseemail.html',d,**kwargs) - + return 1 @@ -2148,9 +2151,9 @@ def handle_sendemail_request(email, name, code, teamname, requestor, id, 'id':id, 'first_name':name, } - + res = send_template_email(from_email,[fullemail],subject, - 'teamrequestemail.html',d,**kwargs) + 'teamrequestemail.html',d,**kwargs) return 1 @@ -2174,7 +2177,7 @@ def handle_sendemail_request_accept(email, name, teamname, managername, 'siteurl':siteurl, } res = send_template_email(from_email,[fullemail],subject, - 'teamwelcomeemail.html',d,**kwargs) + 'teamwelcomeemail.html',d,**kwargs) return 1 @@ -2198,7 +2201,7 @@ def handle_sendemail_request_reject(email, name, teamname, managername, 'siteurl':siteurl, } res = send_template_email(from_email,[fullemail],subject, - 'teamrejectemail.html',d,**kwargs) + 'teamrejectemail.html',d,**kwargs) return 1 @@ -2221,7 +2224,7 @@ def handle_sendemail_member_dropped(email, name, teamname, managername, 'siteurl':siteurl, } res = send_template_email(from_email,[fullemail],subject, - 'teamdropemail.html',d,**kwargs) + 'teamdropemail.html',d,**kwargs) return 1 @@ -2246,7 +2249,7 @@ def handle_sendemail_team_removed(email, name, teamname, managername, 'siteurl':siteurl, } res = send_template_email(from_email,[fullemail],subject, - 'teamremoveemail.html',d,**kwargs) + 'teamremoveemail.html',d,**kwargs) return 1 From 76b757f19bbd39e220c8cfbd6abe27ccb7687107 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 15:51:13 +0100 Subject: [PATCH 4/5] bug fix --- rowers/dataprep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 2c15a5a2..e62498b1 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -2007,9 +2007,9 @@ def read_df_sql(id): f = 'media/strokedata_{id}.parquet.gz'.format(id=id) df = pd.read_parquet(f) except OSError: - rowdata,row = getrowdata(id=ids[0]) + rowdata,row = getrowdata(id=id) if rowdata and len(rowdata.df): - data = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) + data = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) df = pd.read_parquet(f) df = df.fillna(value=0) From 4bf967195f1d44601f4051b64f3796e78153d048 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 16:01:10 +0100 Subject: [PATCH 5/5] corrected incorrect tabs --- rowers/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rowers/tasks.py b/rowers/tasks.py index ccc797aa..ab2c9261 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -130,8 +130,8 @@ def handle_c2_import_stroke_data(c2token, authorizationstring = str('Bearer ' + c2token) headers = {'Authorization': authorizationstring, - 'user-agent': 'sanderroosendaal', - 'Content-Type': 'application/json'} + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json'} url = "https://log.concept2.com/api/users/me/results/"+str(c2id)+"/strokes" s = requests.get(url,headers=headers) if s.status_code == 200: @@ -191,10 +191,10 @@ def getagegrouprecord(age,sex='male',weightcategory='hwt', distance=2000,duration=None,indf=pd.DataFrame()): if not duration: - try: - df = indf[indf['distance'] == distance] - except KeyError: - df = pd.DataFrame() + try: + df = indf[indf['distance'] == distance] + except KeyError: + df = pd.DataFrame() else: duration = 60*int(duration) df = indf[indf['duration'] == duration]