diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 5c71d85a..4600d201 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -120,7 +120,7 @@ def c2_open(user): if (timezone.now()>r.tokenexpirydate): res = rower_c2_token_refresh(user) if res == None: - raise NoTokenError("User has no token") + raise NoTokenError("User has no token") if res[0] != None: thetoken = res[0] else: @@ -156,7 +156,7 @@ def get_c2_workouts(rower): if not isprorower(rower): return 0 - + try: thetoken = c2_open(rower.user) except NoTokenError: @@ -181,7 +181,7 @@ def get_c2_workouts(rower): ] knownc2ids = uniqify(knownc2ids+tombstones) - + newids = [c2id for c2id in c2ids if not c2id in knownc2ids] for c2id in newids: @@ -267,7 +267,7 @@ def c2wc(weightclass): # Concept2 logbook sends over split data for each interval # We use it here to generate a custom summary -# Some users complained about small differences +# Some users complained about small differences def summaryfromsplitdata(splitdata,data,filename,sep='|'): totaldist = data['distance'] @@ -310,17 +310,17 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): restvelo = restdistance/resttime except (ZeroDivisionError,OverflowError): restvelo = 0 - + restpower = 2.8*restvelo**(3.0) try: avgdps = totaldist/data['stroke_count'] except (ZeroDivisionError,OverflowError,KeyError): avgdps = 0 - + from rowingdata import summarystring,workstring,interval_string - + sums = summarystring(totaldist,totaltime,avgpace,spm,avghr,maxhr, avgdps,avgpower,readFile=filename, separator=sep) @@ -336,16 +336,16 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): sums += '#-{sep}SDist{sep}-Split-{sep}-SPace-{sep}-Pwr-{sep}SPM-{sep}AvgHR{sep}MaxHR{sep}DPS-\n'.format( sep=sep ) - + intervalnr=0 sa = [] results = [] - + try: timebased = data['workout_type'] in ['FixedTimeSplits','FixedTimeInterval'] except KeyError: timebased = False - + for interval in splitdata: idist = interval['distance'] itime = interval['time']/10. @@ -373,7 +373,7 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): if timebased: iarr = [itime,'seconds','work'] resarr = [idist] - + if irest_time > 0: iarr += [irest_time,'seconds','rest'] try: @@ -390,7 +390,7 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): else: ivelo = 0 ipower = 0 - + sums += interval_string(intervalnr,idist,itime,ipace,ispm, iavghr,imaxhr,0,ipower,separator=sep) intervalnr+=1 @@ -398,14 +398,14 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): return sums,sa,results # Not used now. Could be used to add workout split data to Concept2 -# logbook but needs to be reviewed. +# logbook but needs to be reviewed. def createc2workoutdata_as_splits(w): filename = w.csvfilename row = rowingdata(csvfile=filename) # resize per minute df = row.df.groupby(lambda x:x/60).mean() - + averagehr = int(df[' HRCur (bpm)'].mean()) maxhr = int(df[' HRCur (bpm)'].max()) @@ -443,7 +443,7 @@ def createc2workoutdata_as_splits(w): wtype = w.workouttype if wtype in otwtypes: wtype = 'water' - + data = { "type": wtype, "date": w.startdatetime.isoformat(), @@ -467,7 +467,7 @@ def createc2workoutdata_as_splits(w): def createc2workoutdata(w): filename = w.csvfilename try: - row = rowingdata(filename) + row = rowingdata(csvfile=filename) except IOError: return 0 @@ -478,19 +478,45 @@ def createc2workoutdata(w): averagehr = 0 maxhr = 0 + # Calculate intervalstats + itime, idist, itype = row.intervalstats_values() + lapnames = row.df[' lapIdx'].unique() + nrintervals = len(itime) + if len(lapnames != nrintervals): + newlapnames = [] + for name in lapnames: + newlapnames += [name,name] + lapnames = newlapnames + intervaldata = [] + for i in range(nrintervals): + if itime[i]>0: + mask = (row.df[' lapIdx'] == lapnames[i]) & (row.df[' WorkoutState'] == itype[i]) + spmav = int(row.df[' Cadence (stokes/min)'][mask].mean().astype(int)) + hrav = int(row.df[' HRCur (bpm)'][mask].mean().astype(int)) + intervaldict = { + 'type': 'distance', + 'time': int(10*itime[i]), + 'distance': int(idist[i]), + 'heart_rate': { + 'average':hrav, + }, + 'stroke_rate': spmav, + } + intervaldata.append(intervaldict) + # adding diff, trying to see if this is valid t = 10*row.df.loc[:,'TimeStamp (sec)'].values-10*row.df.loc[:,'TimeStamp (sec)'].iloc[0] try: t[0] = t[1] except IndexError: pass - + d = 10*row.df.loc[:,' Horizontal (meters)'].values try: d[0] = d[1] except IndexError: pass - + p = abs(10*row.df.loc[:,' Stroke500mPace (sec/500m)'].values) p = np.clip(p,0,3600) if w.workouttype == 'bike': @@ -516,7 +542,7 @@ def createc2workoutdata(w): p = p.tolist() spm = spm.tolist() hr = hr.tolist() - + for i in range(len(t)): thisrecord = {"t":t[i], "d":d[i], @@ -542,19 +568,24 @@ def createc2workoutdata(w): data = { "type": mytypes.c2mapping[workouttype], "date": w.startdatetime.isoformat(), + "stroke_count": int(row.stroke_count), "timezone": w.timezone, "distance": int(w.distance), "time": int(10*makeseconds(durationstr)), "weight_class": c2wc(w.weightcategory), "comments": w.notes, + 'stroke_rate': int(row.df[' Cadence (stokes/min)'].mean()), + 'drag_factor': int(row.dragfactor), "heart_rate": { "average": averagehr, "max": maxhr, }, "stroke_data": stroke_data, + 'workout': { + 'splits': intervaldata, + } } - return data # Refresh Concept2 authorization token @@ -570,7 +601,7 @@ def do_refresh_token(refreshtoken): url = "https://log.concept2.com/oauth/access_token" s = Session() req = Request('POST',url, data=post_data, headers=headers) - + prepped = req.prepare() prepped.body+="&scope=" prepped.body+=scope @@ -672,7 +703,7 @@ def get_workout(user,c2id): url = "https://log.concept2.com/api/users/me/results/"+str(c2id) s = requests.get(url,headers=headers) - + data = s.json()['data'] splitdata = None @@ -694,7 +725,7 @@ def get_workout(user,c2id): strokedata = pd.DataFrame() else: strokedata = pd.DataFrame() - + return data,strokedata # Get stroke data belonging to C2 ID @@ -740,7 +771,7 @@ def get_c2_workout_list(user,page=1): return s - + # Get username, having access token. # Handy for checking if the API access is working def get_username(access_token): @@ -751,7 +782,7 @@ def get_username(access_token): import urllib url = "https://log.concept2.com/api/users/me" response = requests.get(url,headers=headers) - + me_json = response.json() @@ -776,14 +807,14 @@ def get_userid(access_token): response = requests.get(url,headers=headers) except: return 0 - + me_json = response.json() try: res = me_json['data']['id'] except KeyError: res = 0 - + return res # For debugging purposes @@ -799,7 +830,7 @@ def process_callback(request): return HttpResponse("got a user name: %s" % username) def default(o): - if isinstance(o, numpy.int64): return int(o) + if isinstance(o, numpy.int64): return int(o) raise TypeError # Uploading workout @@ -810,7 +841,7 @@ def workout_c2_upload(user,w): return "This workout type cannot be uploaded to Concept2",0 except KeyError: return "This workout type cannot be uploaded to Concept2",0 - + thetoken = c2_open(user) r = Rower.objects.get(user=user) @@ -825,7 +856,7 @@ def workout_c2_upload(user,w): if data == 0: return "Error: No data file. Contact info@rowsandall.com if the problem persists",0 - + authorizationstring = str('Bearer ' + r.c2token) headers = {'Authorization': authorizationstring, 'user-agent': 'sanderroosendaal', @@ -849,7 +880,7 @@ def workout_c2_upload(user,w): else: message = "Something went wrong in workout_c2_upload_view. Response code 200/201 but C2 sync failed: "+response.text c2id = 0 - + return message,c2id @@ -882,7 +913,7 @@ def add_workout_from_data(user,importid,data,strokedata, workouttype = mytypes.c2mappinginv[data['type']] except KeyError: workouttype = 'rower' - + if workouttype not in [x[0] for x in Workout.workouttypes]: workouttype = 'other' try: @@ -902,14 +933,14 @@ def add_workout_from_data(user,importid,data,strokedata, rowdatetime = iso8601.parse_date(data['start_date']) except ParseError: rowdatetime = iso8601.parse_date(data['date']) - - + + try: c2intervaltype = data['workout_type'] - + except KeyError: c2intervaltype = '' - + try: title = data['name'] except KeyError: @@ -951,7 +982,7 @@ def add_workout_from_data(user,importid,data,strokedata, spm = strokedata.loc[:,'spm'] except KeyError: spm = 0*dist2 - + try: hr = strokedata.loc[:,'hr'] except KeyError: @@ -966,7 +997,7 @@ def add_workout_from_data(user,importid,data,strokedata, velo = 1000./pace pace = 500./velo - + # save csv # Create data frame with all necessary data to write to csv df = pd.DataFrame({'TimeStamp (sec)':unixtime, @@ -988,9 +1019,9 @@ def add_workout_from_data(user,importid,data,strokedata, ' ElapsedTime (sec)':seconds }) - + df.sort_values(by='TimeStamp (sec)',ascending=True) - + timestr = strftime("%Y%m%d-%H%M%S") @@ -1024,6 +1055,6 @@ def add_workout_from_data(user,importid,data,strokedata, dosummary=True ) - + return id,message diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index c7e2871c..d83682b9 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -33,7 +33,7 @@ try: user = DATABASES['default']['USER'] except KeyError: user = '' -try: +try: password = DATABASES['default']['PASSWORD'] except KeyError: password = '' @@ -98,7 +98,7 @@ def niceformat(values): for v in values: formattedv = strfdelta(v) out.append(formattedv) - + return out def strfdelta(tdelta): @@ -114,7 +114,7 @@ def strfdelta(tdelta): seconds=seconds, tenths=tenths, ) - + return res def nicepaceformat(values): @@ -122,7 +122,7 @@ def nicepaceformat(values): for v in values: formattedv = strfdelta(v) out.append(formattedv) - + return out @@ -131,8 +131,8 @@ def timedeltaconv(x): dt = datetime.timedelta(seconds=x) else: dt = datetime.timedelta(seconds=350.) - - + + return dt def rdata(file,rower=rrower()): @@ -167,7 +167,7 @@ def create_c2_stroke_data_db( spm = 60.*nr_strokes/totalseconds except ZeroDivisionError: spm = 20*zeros(nr_strokes) - + step = totalseconds/float(nr_strokes) elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) @@ -186,7 +186,7 @@ def create_c2_stroke_data_db( else: power = 0 - + df = pd.DataFrame({ 'TimeStamp (sec)': unixtime, ' Horizontal (meters)': d, @@ -252,7 +252,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, spm = strokedata.loc[:,'spm'] except KeyError: spm = 0*dist2 - + try: hr = strokedata.loc[:,'hr'] except KeyError: @@ -268,7 +268,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, velo = 1000./pace - + # save csv # Create data frame with all necessary data to write to csv df = pd.DataFrame({'TimeStamp (sec)':unixtime, @@ -291,11 +291,11 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, 'cum_dist': dist2 }) - + df.sort_values(by='TimeStamp (sec)',ascending=True) - + timestr = strftime("%Y%m%d-%H%M%S") - + # Create CSV file name and save data to CSV file @@ -307,7 +307,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, data = dataprep(df,id=workoutid,bands=False,debug=debug) except: return 0 - + return data # Processes painsled CSV file to database @@ -341,7 +341,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', #row.repair() pass - + if row == 0: return (0,'Error: CSV data file not found') @@ -349,7 +349,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values velo = 500./pace - + f = row.df['TimeStamp (sec)'].diff().mean() if f !=0: windowsize = 2*(int(10./(f)))+1 @@ -366,9 +366,9 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', velo3 = pd.Series(velo2) velo3 = velo3.replace([-np.inf,np.inf],np.nan) velo3 = velo3.fillna(method='ffill') - + pace2 = 500./abs(velo3) - + row.df[' Stroke500mPace (sec/500m)'] = pace2 row.df = row.df.fillna(0) @@ -378,7 +378,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', os.remove(f2) except: pass - + # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: @@ -386,10 +386,10 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', row.write_csv(f2,gzip=True) except: pass - + averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() - + if totaldist == 0: totaldist = row.df['cum_dist'].max() if totaltime == 0: @@ -408,7 +408,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', minutes = 59 if not message: message = 'Warning: there is something wrong with the workout duration' - + seconds = int(totaltime - 3600.*hours - 60.*minutes) if seconds > 59: seconds = 59 @@ -420,7 +420,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', tenths = 9 if not message: message = 'Warning: there is something wrong with the workout duration' - + duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) if dosummary: @@ -444,7 +444,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', message = "Warning: This workout probably already exists in the database" privacy = 'private' - + w = Workout(user=r,name=title,date=workoutdate, workouttype=workouttype, @@ -481,7 +481,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle TCX if (fileformat == 'tcx'): row = TCXParser(f2) - + # handle Mystery if (fileformat == 'mystery'): row = MysteryParser(f2) @@ -489,7 +489,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle RowPerfect if (fileformat == 'rowperfect3'): row = RowPerfectParser(f2) - + # handle ErgData if (fileformat == 'ergdata'): row = ErgDataParser(f2) @@ -501,7 +501,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle Mike if (fileformat == 'bcmike'): row = BoatCoachAdvancedParser(f2) - + # handle BoatCoach OTW if (fileformat == 'boatcoachotw'): row = BoatCoachOTWParser(f2) @@ -518,7 +518,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): if (fileformat == 'speedcoach'): row = speedcoachParser(f2) - # handle speed coach GPS 2 + # handle speed coach GPS 2 if (fileformat == 'speedcoach2'): row = SpeedCoach2Parser(f2) try: @@ -531,7 +531,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle ErgStick if (fileformat == 'ergstick'): row = ErgStickParser(f2) - + # handle FIT if (fileformat == 'fit'): row = FITParser(f2) @@ -544,7 +544,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # should delete file f2 = f2[:-4]+'o.csv' row.write_csv(f2,gzip=True) - + #os.remove(f2) try: os.remove(f_to_be_deleted) @@ -573,7 +573,7 @@ def new_workout_from_file(r,f2, for fname in z.namelist(): f3 = z.extract(fname,path='media/') id,message,f2 = new_workout_from_file(r,f3, - workouttype=workouttype, + workouttype=workouttype, makeprivate=makeprivate, title = title, notes='') @@ -590,7 +590,7 @@ def new_workout_from_file(r,f2, os.remove(f2) message = "It looks like this file doesn't contain stroke data." return (0,message,f2) - + # Some people try to upload RowPro summary logs if fileformat == 'rowprolog': os.remove(f2) @@ -603,7 +603,7 @@ def new_workout_from_file(r,f2, # worth supporting if fileformat == 'unknown': message = "We couldn't recognize the file type" - if settings.DEBUG: + if settings.DEBUG: res = handle_sendemail_unrecognized.delay(f2, r.user.email) @@ -611,7 +611,7 @@ def new_workout_from_file(r,f2, res = queuehigh.enqueue(handle_sendemail_unrecognized, f2,r.user.email) return (0,message,f2) - + # handle non-Painsled by converting it to painsled compatible CSV if (fileformat != 'csv'): try: @@ -670,7 +670,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): success = False try: - df['power empower old'] = df[' Power (watts)'] + df['power empower old'] = df[' Power (watts)'] df[' Power (watts)'] = df[' Power (watts)'] * corr_factor df['driveenergy empower old'] = df['driveenergy'] df['driveenergy'] = df['driveenergy'] * corr_factor @@ -687,7 +687,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): if debug: print("not updated ",id) - + rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True, debug=debug) @@ -708,7 +708,7 @@ def testdata(time,distance,pace,spm): def getsmallrowdata_db(columns,ids=[],debug=False): - csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] data = [] columns = [c for c in columns if c != 'None'] @@ -720,14 +720,14 @@ def getsmallrowdata_db(columns,ids=[],debug=False): except OSError: pass - + df = pd.concat(data,axis=0) else: df = pd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') return df - + def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, doclean=False): # test if nan among values @@ -736,12 +736,12 @@ def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, m[key] = -1 if 'inf' in str(m[key]): m[key] = -1 - + if debug: engine = create_engine(database_url_debug, echo=False) else: engine = create_engine(database_url, echo=False) - + columns = ', '.join(m.keys()) if use_sqlite: placeholders = ", ".join(["?"] * len(m)) @@ -795,7 +795,7 @@ def read_cols_df_sql(ids,columns,debug=False): df = pd.concat(data,axis=0) return df - + def read_df_sql(id,debug=False): try: @@ -860,15 +860,15 @@ def delete_agegroup_db(age,sex,weightcategory,debug=False): print("Database locked") conn.close() engine.dispose() - + def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower, debug=False): - + delete_agegroup_db(age,sex,weightcategory,debug=debug) wcdurations = [None if type(y) is float and np.isnan(y) else y for y in wcdurations] wcpower = [None if type(y) is float and np.isnan(y) else y for y in wcpower] - + df = pd.DataFrame( { 'duration':wcdurations, @@ -893,7 +893,7 @@ def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower, conn.close() engine.dispose() - + def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),debug=False): deletecpdata_sql(rower_id,table=table,debug=debug) df = pd.DataFrame( @@ -918,8 +918,8 @@ def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),deb engine.dispose() - - + + def smalldataprep(therows,xparam,yparam1,yparam2): @@ -936,7 +936,7 @@ def smalldataprep(therows,xparam,yparam1,yparam2): try: rowdata = dataprep(rrdata(f1).df) - + rowdata = pd.DataFrame({xparam: rowdata[xparam], yparam1: rowdata[yparam1], yparam2: rowdata[yparam2], @@ -960,8 +960,8 @@ def smalldataprep(therows,xparam,yparam1,yparam2): pass return df - - + + def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, empower=True,debug=False,inboard=0.88,forceunit='lbs'): @@ -972,7 +972,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, if debug: print("dataprep",id) - + # rowdatadf.set_index([range(len(rowdatadf))],inplace=True) t = rowdatadf.loc[:,'TimeStamp (sec)'] t = pd.Series(t-rowdatadf.loc[:,'TimeStamp (sec)'].iloc[0]) @@ -985,7 +985,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, velo = rowdatadf.loc[:,' AverageBoatSpeed (m/s)'] except KeyError: velo = 500./p - + hr = rowdatadf.loc[:,' HRCur (bpm)'] spm = rowdatadf.loc[:,' Cadence (stokes/min)'] cumdist = rowdatadf.loc[:,'cum_dist'] @@ -997,12 +997,12 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, workoutstate = rowdatadf.loc[:,' WorkoutState'] except KeyError: workoutstate = 0*hr - + peakforce = rowdatadf.loc[:,' PeakDriveForce (lbs)'] forceratio = averageforce/peakforce forceratio = forceratio.fillna(value=0) - + try: drivetime = rowdatadf.loc[:,' DriveTime (ms)'] recoverytime = rowdatadf.loc[:,' StrokeRecoveryTime (ms)'] @@ -1033,7 +1033,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, except TypeError: t2 = 0*t - + p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) try: @@ -1042,7 +1042,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] except TypeError: drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] - + drivespeed = drivespeed.fillna(value=0) try: @@ -1100,7 +1100,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, tel = rowdatadf.loc[:,' ElapsedTime (sec)'] except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - + if empower: try: @@ -1132,7 +1132,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, else: driveenergy = data['driveenergy'] - + arclength = (inboard-0.05)*(np.radians(finish)-np.radians(catch)) if arclength.mean()>0: drivelength = arclength @@ -1151,7 +1151,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, totalangle = 0*t effectiveangle = 0*t - + if windowsize > 3 and windowsize