From e55c42f930dd757fa091cd1a2eb7e412c0ff02d1 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 24 Oct 2019 20:51:22 +0200 Subject: [PATCH 1/4] trivial changes white space --- rowers/dataprep.py | 106 ++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 1fb142d0..7de38d60 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -49,7 +49,7 @@ from rowingdata import ( SpeedCoach2Parser, FITParser, fitsummarydata, RitmoTimeParser,KinoMapParser, make_cumvalues,cumcpdata,ExcelTemplate, - summarydata, get_file_type, + summarydata, get_file_type, ) from rowingdata.csvparsers import HumonParser @@ -128,12 +128,12 @@ def polarization_index(df,rower): df.dropna(axis=0,inplace=True) df['dt'] = df['dt'].clip(upper=4,lower=0) - + masklow = (df['power']>0) & (df['power']=rower.pw_at) & (df['power']rower.pw_an) - + time_low_pw = df.loc[masklow,'dt'].sum() time_mid_pw = df.loc[maskmid,'dt'].sum() time_high_pw = df.loc[maskhigh,'dt'].sum() @@ -143,7 +143,7 @@ def polarization_index(df,rower): frac_high = time_high_pw/(time_low_pw+time_mid_pw+time_high_pw) index = math.log10(frac_high*100.*frac_low/frac_mid) - + return index @@ -157,7 +157,7 @@ def get_latlon(id): rowdata = rdata(w.csvfilename) if rowdata.df.empty: - return [pd.Series([]), pd.Series([])] + return [pd.Series([]), pd.Series([])] try: try: @@ -216,7 +216,7 @@ def workout_summary_to_df( trimps.append(workout_trimp(w)[0]) rscore = workout_rscore(w) rscores.append(int(rscore[0])) - + df = pd.DataFrame({ 'name':names, 'date':startdatetimes, @@ -285,7 +285,7 @@ def join_workouts(r,ids,title='Joined Workout', else: makeprivate = False - startdatetime = parent.startdatetime + startdatetime = parent.startdatetime else: oarlength = 2.89 inboard = 0.88 @@ -311,7 +311,7 @@ def join_workouts(r,ids,title='Joined Workout', workouttype = parent.workouttype notes = parent.notes summary = parent.summary - + files = [w.csvfilename for w in ws] row = rdata(files[0]) @@ -353,7 +353,7 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, ignoreadvanced=False): # clean data remove zeros and negative values - + # bring metrics which have negative values to positive domain if len(datadf)==0: return datadf @@ -377,7 +377,7 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, datadf['spm'] = datadf['spm'] + 1.0 except (KeyError,TypeError) as e: pass - + try: datadf = datadf.clip(lower=0) except TypeError: @@ -421,13 +421,13 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass - + try: mask = datadf['efficiency'] > 200. datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass - + try: mask = datadf['spm'] < 10 datadf.mask(mask,inplace=True) @@ -648,7 +648,7 @@ def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False): placeholders = ", ".join(["?"] * len(m)) query = "INSERT into %s ( %s ) Values (%s)" % (table, columns, placeholders) - + values = tuple(m[key] for key in m.keys()) with engine.connect() as conn, conn.begin(): result = conn.execute(query,values) @@ -684,8 +684,8 @@ def deletecpdata_sql(rower_id,table='cpdata'): conn.close() engine.dispose() - - + + def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=[]): deletecpdata_sql(rower_id) df = pd.DataFrame( @@ -836,7 +836,7 @@ def create_row_df(r,distance,duration,startdatetime,workouttype='rower', spm = 20. else: spm = avgspm - + step = totalseconds/float(nr_strokes) elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) @@ -867,7 +867,7 @@ def create_row_df(r,distance,duration,startdatetime,workouttype='rower', hr = avghr else: hr = 0 - + df = pd.DataFrame({ 'TimeStamp (sec)': unixtime, ' Horizontal (meters)': d, @@ -902,7 +902,7 @@ def create_row_df(r,distance,duration,startdatetime,workouttype='rower', return (id, message) from rowers.utils import totaltime_sec_to_string - + # Processes painsled CSV file to database def save_workout_database(f2, r, dosmooth=True, workouttype='rower', boattype='1x', @@ -934,7 +934,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', if row.df.empty: return (0, 'Error: CSV data file was empty') - + dtavg = row.df['TimeStamp (sec)'].diff().mean() if dtavg < 1: @@ -1025,7 +1025,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', if dosummary: summary = row.allstats() - + timezone_str = 'UTC' try: @@ -1094,7 +1094,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', ) ws2 = [] - + for ww in ws: t = ww.duration delta = datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) @@ -1146,7 +1146,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', rscore,normp = workout_rscore(w) trimp,hrtss = workout_trimp(w) - + isbreakthrough = False ishard = False if workouttype == 'water': @@ -1184,7 +1184,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', r.user.first_name, r.user.last_name, btvalues=btvalues.to_json()) - + # submit email task to send email about breakthrough workout if ishard: if r.getemailnotifications and not r.emailbounced: @@ -1194,7 +1194,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', r.user.first_name, r.user.last_name, btvalues=btvalues.to_json()) - + return (w.id, message) @@ -1266,7 +1266,7 @@ def handle_nonpainsled(f2, fileformat, summary=''): if not hasrecognized: return (0,'',0,0,'') - + f_to_be_deleted = f2 # should delete file f2 = f2[:-4] + 'o.csv' @@ -1322,7 +1322,7 @@ def new_workout_from_file(r, f2, f3 = f3[6:] a = MessageAttachment(message=msg,document=f3) a.save() - + return -1, message, f2 # Some people try to upload Concept2 logbook summaries @@ -1340,7 +1340,7 @@ def new_workout_from_file(r, f2, os.remove(f2) message = "KML files are not supported" return (0, message, f2) - + # Some people upload corrupted zip files if fileformat == 'notgzip': os.remove(f2) @@ -1371,7 +1371,7 @@ def new_workout_from_file(r, f2, handle_sendemail_unrecognized, f4, r.user.email) - + return (0, message, f2) if fileformat == 'att': # email attachment which can safely be ignored @@ -1393,7 +1393,7 @@ def new_workout_from_file(r, f2, if workoutsource is None: workoutsource = fileformat - + id, message = save_workout_database( f2, r, notes=notes, @@ -1657,7 +1657,7 @@ def getrowdata_db(id=0, doclean=False, convertnewtons=True, else: row = Workout.objects.get(id=id) - + if not data.empty and data['efficiency'].mean() == 0 and data['power'].mean() != 0 and checkefficiency == True: data = add_efficiency(id=id) @@ -1692,10 +1692,10 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True,comput df = pd.concat(data,axis=0) # df = dd.concat(data,axis=0) - + else: try: - df = pd.read_parquet(csvfilenames[0],columns=columns) + df = pd.read_parquet(csvfilenames[0],columns=columns) except OSError: rowdata,row = getrowdata(id=ids[0]) if rowdata and len(rowdata.df): @@ -1707,7 +1707,7 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True,comput # df = df.loc[:,~df.columns.duplicated()] - + if compute: data = df.copy() @@ -1717,7 +1717,7 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True,comput data.dropna(axis=1,how='all',inplace=True) data.dropna(axis=0,how='any',inplace=True) return data - + return df def getsmallrowdata_db_dask(columns, ids=[], doclean=True,workstrokesonly=True,compute=True): @@ -1744,10 +1744,10 @@ def getsmallrowdata_db_dask(columns, ids=[], doclean=True,workstrokesonly=True,c df = dd.concat(data,axis=0) # df = dd.concat(data,axis=0) - + else: try: - df = dd.read_parquet(csvfilenames[0],columns=columns) + df = dd.read_parquet(csvfilenames[0],columns=columns) except OSError: rowdata,row = getrowdata(id=ids[0]) if rowdata and len(rowdata.df): @@ -1759,7 +1759,7 @@ def getsmallrowdata_db_dask(columns, ids=[], doclean=True,workstrokesonly=True,c # df = df.loc[:,~df.columns.duplicated()] - + if compute: data = df.compute() @@ -1769,7 +1769,7 @@ def getsmallrowdata_db_dask(columns, ids=[], doclean=True,workstrokesonly=True,c data.dropna(axis=1,how='all',inplace=True) data.dropna(axis=0,how='any',inplace=True) return data - + return df def getsmallrowdata_db_old(columns, ids=[], doclean=True, workstrokesonly=True): @@ -1787,7 +1787,7 @@ def getsmallrowdata_db_old(columns, ids=[], doclean=True, workstrokesonly=True): f = row.df['TimeStamp (sec)'].diff().mean() except (AttributeError,KeyError) as e: f = 0 - + if f != 0 and not np.isnan(f): windowsize = 2 * (int(10. / (f))) + 1 else: @@ -1808,7 +1808,7 @@ def getsmallrowdata_db_old(columns, ids=[], doclean=True, workstrokesonly=True): except (KeyError, AttributeError): data[c] = 0 - + # convert newtons if doclean: @@ -1907,7 +1907,7 @@ def read_cols_df_sql(ids, columns, convertnewtons=True): data.append(df) df = pd.concat(data,axis=0) - + df = df.fillna(value=0) @@ -2215,9 +2215,9 @@ def add_efficiency(id=0): rowdata = remove_invalid_columns(rowdata) rowdata = rowdata.replace([-np.inf, np.inf], np.nan) rowdata = rowdata.fillna(method='ffill') - + delete_strokedata(id) - + if id != 0: rowdata['workoutid'] = id filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) @@ -2250,7 +2250,7 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, velo = rowdatadf.loc[:,' AverageBoatSpeed (m/s)'] except KeyError: velo = 500./p - + hr = rowdatadf.loc[:, ' HRCur (bpm)'] spm = rowdatadf.loc[:, ' Cadence (stokes/min)'] cumdist = rowdatadf.loc[:, 'cum_dist'] @@ -2316,7 +2316,7 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, powerhr = 60.*power/hr powerhr = powerhr.fillna(value=0) - + if driveenergy.mean() == 0 and driveenergy.std() == 0: driveenergy = 0*driveenergy+100 @@ -2521,7 +2521,7 @@ def workout_trimp(w): if w.trimp > 0: return w.trimp,w.hrtss - + r = w.user ftp = float(r.ftp) if w.workouttype in otwtypes: @@ -2557,13 +2557,13 @@ def workout_trimp(w): r.hrftp, r.max, r.rest) - + return 0,0 def workout_rscore(w): if w.rscore > 0: return w.rscore,w.normp - + r = w.user ftp = float(r.ftp) if w.workouttype in otwtypes: @@ -2574,7 +2574,7 @@ def workout_rscore(w): r.hrftp = int(hrftp) r.save() - + job = myqueue( queuehigh, @@ -2586,7 +2586,7 @@ def workout_rscore(w): r.hrftp, r.max, r.rest) - + return 0,0 def workout_normv(w,pp=4.0): @@ -2603,7 +2603,7 @@ def workout_normv(w,pp=4.0): r.hrftp = int(hrftp) r.save() - + job = myqueue( queuehigh, @@ -2615,7 +2615,5 @@ def workout_normv(w,pp=4.0): r.hrftp, r.max, r.rest) - - return 0,0 - + return 0,0 From 102efa2cf275e875cbd0673847019b1ca0fda8ca Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 07:59:07 +0100 Subject: [PATCH 2/4] fixed error in parquet --- rowers/dataprepnodjango.py | 126 ++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index c7e2871c..d83682b9 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -33,7 +33,7 @@ try: user = DATABASES['default']['USER'] except KeyError: user = '' -try: +try: password = DATABASES['default']['PASSWORD'] except KeyError: password = '' @@ -98,7 +98,7 @@ def niceformat(values): for v in values: formattedv = strfdelta(v) out.append(formattedv) - + return out def strfdelta(tdelta): @@ -114,7 +114,7 @@ def strfdelta(tdelta): seconds=seconds, tenths=tenths, ) - + return res def nicepaceformat(values): @@ -122,7 +122,7 @@ def nicepaceformat(values): for v in values: formattedv = strfdelta(v) out.append(formattedv) - + return out @@ -131,8 +131,8 @@ def timedeltaconv(x): dt = datetime.timedelta(seconds=x) else: dt = datetime.timedelta(seconds=350.) - - + + return dt def rdata(file,rower=rrower()): @@ -167,7 +167,7 @@ def create_c2_stroke_data_db( spm = 60.*nr_strokes/totalseconds except ZeroDivisionError: spm = 20*zeros(nr_strokes) - + step = totalseconds/float(nr_strokes) elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) @@ -186,7 +186,7 @@ def create_c2_stroke_data_db( else: power = 0 - + df = pd.DataFrame({ 'TimeStamp (sec)': unixtime, ' Horizontal (meters)': d, @@ -252,7 +252,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, spm = strokedata.loc[:,'spm'] except KeyError: spm = 0*dist2 - + try: hr = strokedata.loc[:,'hr'] except KeyError: @@ -268,7 +268,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, velo = 1000./pace - + # save csv # Create data frame with all necessary data to write to csv df = pd.DataFrame({'TimeStamp (sec)':unixtime, @@ -291,11 +291,11 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, 'cum_dist': dist2 }) - + df.sort_values(by='TimeStamp (sec)',ascending=True) - + timestr = strftime("%Y%m%d-%H%M%S") - + # Create CSV file name and save data to CSV file @@ -307,7 +307,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, data = dataprep(df,id=workoutid,bands=False,debug=debug) except: return 0 - + return data # Processes painsled CSV file to database @@ -341,7 +341,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', #row.repair() pass - + if row == 0: return (0,'Error: CSV data file not found') @@ -349,7 +349,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values velo = 500./pace - + f = row.df['TimeStamp (sec)'].diff().mean() if f !=0: windowsize = 2*(int(10./(f)))+1 @@ -366,9 +366,9 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', velo3 = pd.Series(velo2) velo3 = velo3.replace([-np.inf,np.inf],np.nan) velo3 = velo3.fillna(method='ffill') - + pace2 = 500./abs(velo3) - + row.df[' Stroke500mPace (sec/500m)'] = pace2 row.df = row.df.fillna(0) @@ -378,7 +378,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', os.remove(f2) except: pass - + # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: @@ -386,10 +386,10 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', row.write_csv(f2,gzip=True) except: pass - + averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() - + if totaldist == 0: totaldist = row.df['cum_dist'].max() if totaltime == 0: @@ -408,7 +408,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', minutes = 59 if not message: message = 'Warning: there is something wrong with the workout duration' - + seconds = int(totaltime - 3600.*hours - 60.*minutes) if seconds > 59: seconds = 59 @@ -420,7 +420,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', tenths = 9 if not message: message = 'Warning: there is something wrong with the workout duration' - + duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) if dosummary: @@ -444,7 +444,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', message = "Warning: This workout probably already exists in the database" privacy = 'private' - + w = Workout(user=r,name=title,date=workoutdate, workouttype=workouttype, @@ -481,7 +481,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle TCX if (fileformat == 'tcx'): row = TCXParser(f2) - + # handle Mystery if (fileformat == 'mystery'): row = MysteryParser(f2) @@ -489,7 +489,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle RowPerfect if (fileformat == 'rowperfect3'): row = RowPerfectParser(f2) - + # handle ErgData if (fileformat == 'ergdata'): row = ErgDataParser(f2) @@ -501,7 +501,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle Mike if (fileformat == 'bcmike'): row = BoatCoachAdvancedParser(f2) - + # handle BoatCoach OTW if (fileformat == 'boatcoachotw'): row = BoatCoachOTWParser(f2) @@ -518,7 +518,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): if (fileformat == 'speedcoach'): row = speedcoachParser(f2) - # handle speed coach GPS 2 + # handle speed coach GPS 2 if (fileformat == 'speedcoach2'): row = SpeedCoach2Parser(f2) try: @@ -531,7 +531,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle ErgStick if (fileformat == 'ergstick'): row = ErgStickParser(f2) - + # handle FIT if (fileformat == 'fit'): row = FITParser(f2) @@ -544,7 +544,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # should delete file f2 = f2[:-4]+'o.csv' row.write_csv(f2,gzip=True) - + #os.remove(f2) try: os.remove(f_to_be_deleted) @@ -573,7 +573,7 @@ def new_workout_from_file(r,f2, for fname in z.namelist(): f3 = z.extract(fname,path='media/') id,message,f2 = new_workout_from_file(r,f3, - workouttype=workouttype, + workouttype=workouttype, makeprivate=makeprivate, title = title, notes='') @@ -590,7 +590,7 @@ def new_workout_from_file(r,f2, os.remove(f2) message = "It looks like this file doesn't contain stroke data." return (0,message,f2) - + # Some people try to upload RowPro summary logs if fileformat == 'rowprolog': os.remove(f2) @@ -603,7 +603,7 @@ def new_workout_from_file(r,f2, # worth supporting if fileformat == 'unknown': message = "We couldn't recognize the file type" - if settings.DEBUG: + if settings.DEBUG: res = handle_sendemail_unrecognized.delay(f2, r.user.email) @@ -611,7 +611,7 @@ def new_workout_from_file(r,f2, res = queuehigh.enqueue(handle_sendemail_unrecognized, f2,r.user.email) return (0,message,f2) - + # handle non-Painsled by converting it to painsled compatible CSV if (fileformat != 'csv'): try: @@ -670,7 +670,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): success = False try: - df['power empower old'] = df[' Power (watts)'] + df['power empower old'] = df[' Power (watts)'] df[' Power (watts)'] = df[' Power (watts)'] * corr_factor df['driveenergy empower old'] = df['driveenergy'] df['driveenergy'] = df['driveenergy'] * corr_factor @@ -687,7 +687,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): if debug: print("not updated ",id) - + rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True, debug=debug) @@ -708,7 +708,7 @@ def testdata(time,distance,pace,spm): def getsmallrowdata_db(columns,ids=[],debug=False): - csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] data = [] columns = [c for c in columns if c != 'None'] @@ -720,14 +720,14 @@ def getsmallrowdata_db(columns,ids=[],debug=False): except OSError: pass - + df = pd.concat(data,axis=0) else: df = pd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') return df - + def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, doclean=False): # test if nan among values @@ -736,12 +736,12 @@ def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, m[key] = -1 if 'inf' in str(m[key]): m[key] = -1 - + if debug: engine = create_engine(database_url_debug, echo=False) else: engine = create_engine(database_url, echo=False) - + columns = ', '.join(m.keys()) if use_sqlite: placeholders = ", ".join(["?"] * len(m)) @@ -795,7 +795,7 @@ def read_cols_df_sql(ids,columns,debug=False): df = pd.concat(data,axis=0) return df - + def read_df_sql(id,debug=False): try: @@ -860,15 +860,15 @@ def delete_agegroup_db(age,sex,weightcategory,debug=False): print("Database locked") conn.close() engine.dispose() - + def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower, debug=False): - + delete_agegroup_db(age,sex,weightcategory,debug=debug) wcdurations = [None if type(y) is float and np.isnan(y) else y for y in wcdurations] wcpower = [None if type(y) is float and np.isnan(y) else y for y in wcpower] - + df = pd.DataFrame( { 'duration':wcdurations, @@ -893,7 +893,7 @@ def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower, conn.close() engine.dispose() - + def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),debug=False): deletecpdata_sql(rower_id,table=table,debug=debug) df = pd.DataFrame( @@ -918,8 +918,8 @@ def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),deb engine.dispose() - - + + def smalldataprep(therows,xparam,yparam1,yparam2): @@ -936,7 +936,7 @@ def smalldataprep(therows,xparam,yparam1,yparam2): try: rowdata = dataprep(rrdata(f1).df) - + rowdata = pd.DataFrame({xparam: rowdata[xparam], yparam1: rowdata[yparam1], yparam2: rowdata[yparam2], @@ -960,8 +960,8 @@ def smalldataprep(therows,xparam,yparam1,yparam2): pass return df - - + + def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, empower=True,debug=False,inboard=0.88,forceunit='lbs'): @@ -972,7 +972,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, if debug: print("dataprep",id) - + # rowdatadf.set_index([range(len(rowdatadf))],inplace=True) t = rowdatadf.loc[:,'TimeStamp (sec)'] t = pd.Series(t-rowdatadf.loc[:,'TimeStamp (sec)'].iloc[0]) @@ -985,7 +985,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, velo = rowdatadf.loc[:,' AverageBoatSpeed (m/s)'] except KeyError: velo = 500./p - + hr = rowdatadf.loc[:,' HRCur (bpm)'] spm = rowdatadf.loc[:,' Cadence (stokes/min)'] cumdist = rowdatadf.loc[:,'cum_dist'] @@ -997,12 +997,12 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, workoutstate = rowdatadf.loc[:,' WorkoutState'] except KeyError: workoutstate = 0*hr - + peakforce = rowdatadf.loc[:,' PeakDriveForce (lbs)'] forceratio = averageforce/peakforce forceratio = forceratio.fillna(value=0) - + try: drivetime = rowdatadf.loc[:,' DriveTime (ms)'] recoverytime = rowdatadf.loc[:,' StrokeRecoveryTime (ms)'] @@ -1033,7 +1033,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, except TypeError: t2 = 0*t - + p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) try: @@ -1042,7 +1042,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] except TypeError: drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] - + drivespeed = drivespeed.fillna(value=0) try: @@ -1100,7 +1100,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, tel = rowdatadf.loc[:,' ElapsedTime (sec)'] except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - + if empower: try: @@ -1132,7 +1132,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, else: driveenergy = data['driveenergy'] - + arclength = (inboard-0.05)*(np.radians(finish)-np.radians(catch)) if arclength.mean()>0: drivelength = arclength @@ -1151,7 +1151,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, totalangle = 0*t effectiveangle = 0*t - + if windowsize > 3 and windowsize Date: Thu, 31 Oct 2019 11:45:44 +0100 Subject: [PATCH 3/4] adding splits to C2 workout export --- rowers/c2stuff.py | 112 +++++++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 41 deletions(-) diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 5c71d85a..82f199f8 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -120,7 +120,7 @@ def c2_open(user): if (timezone.now()>r.tokenexpirydate): res = rower_c2_token_refresh(user) if res == None: - raise NoTokenError("User has no token") + raise NoTokenError("User has no token") if res[0] != None: thetoken = res[0] else: @@ -156,7 +156,7 @@ def get_c2_workouts(rower): if not isprorower(rower): return 0 - + try: thetoken = c2_open(rower.user) except NoTokenError: @@ -181,7 +181,7 @@ def get_c2_workouts(rower): ] knownc2ids = uniqify(knownc2ids+tombstones) - + newids = [c2id for c2id in c2ids if not c2id in knownc2ids] for c2id in newids: @@ -267,7 +267,7 @@ def c2wc(weightclass): # Concept2 logbook sends over split data for each interval # We use it here to generate a custom summary -# Some users complained about small differences +# Some users complained about small differences def summaryfromsplitdata(splitdata,data,filename,sep='|'): totaldist = data['distance'] @@ -310,17 +310,17 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): restvelo = restdistance/resttime except (ZeroDivisionError,OverflowError): restvelo = 0 - + restpower = 2.8*restvelo**(3.0) try: avgdps = totaldist/data['stroke_count'] except (ZeroDivisionError,OverflowError,KeyError): avgdps = 0 - + from rowingdata import summarystring,workstring,interval_string - + sums = summarystring(totaldist,totaltime,avgpace,spm,avghr,maxhr, avgdps,avgpower,readFile=filename, separator=sep) @@ -336,16 +336,16 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): sums += '#-{sep}SDist{sep}-Split-{sep}-SPace-{sep}-Pwr-{sep}SPM-{sep}AvgHR{sep}MaxHR{sep}DPS-\n'.format( sep=sep ) - + intervalnr=0 sa = [] results = [] - + try: timebased = data['workout_type'] in ['FixedTimeSplits','FixedTimeInterval'] except KeyError: timebased = False - + for interval in splitdata: idist = interval['distance'] itime = interval['time']/10. @@ -373,7 +373,7 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): if timebased: iarr = [itime,'seconds','work'] resarr = [idist] - + if irest_time > 0: iarr += [irest_time,'seconds','rest'] try: @@ -390,7 +390,7 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): else: ivelo = 0 ipower = 0 - + sums += interval_string(intervalnr,idist,itime,ipace,ispm, iavghr,imaxhr,0,ipower,separator=sep) intervalnr+=1 @@ -398,14 +398,14 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): return sums,sa,results # Not used now. Could be used to add workout split data to Concept2 -# logbook but needs to be reviewed. +# logbook but needs to be reviewed. def createc2workoutdata_as_splits(w): filename = w.csvfilename row = rowingdata(csvfile=filename) # resize per minute df = row.df.groupby(lambda x:x/60).mean() - + averagehr = int(df[' HRCur (bpm)'].mean()) maxhr = int(df[' HRCur (bpm)'].max()) @@ -443,7 +443,7 @@ def createc2workoutdata_as_splits(w): wtype = w.workouttype if wtype in otwtypes: wtype = 'water' - + data = { "type": wtype, "date": w.startdatetime.isoformat(), @@ -467,7 +467,7 @@ def createc2workoutdata_as_splits(w): def createc2workoutdata(w): filename = w.csvfilename try: - row = rowingdata(filename) + row = rowingdata(csvfile=filename) except IOError: return 0 @@ -478,19 +478,45 @@ def createc2workoutdata(w): averagehr = 0 maxhr = 0 + # Calculate intervalstats + itime, idist, itype = row.intervalstats_values() + lapnames = row.df[' lapIdx'].unique() + nrintervals = len(itime) + if len(lapnames != nrintervals): + newlapnames = [] + for name in lapnames: + newlapnames += [name,name] + lapnames = newlapnames + intervaldata = [] + for i in range(nrintervals): + if itime[i]>0: + mask = (row.df[' lapIdx'] == lapnames[i]) & (row.df[' WorkoutState'] == itype[i]) + spmav = int(row.df[' Cadence (stokes/min)'][mask].mean().astype(int)) + hrav = int(row.df[' HRCur (bpm)'][mask].mean().astype(int)) + intervaldict = { + 'type': 'distance', + 'time': int(10*itime[i]), + 'distance': int(idist[i]), + 'heart_rate': { + 'average':hrav, + }, + 'stroke_rate': spmav, + } + intervaldata.append(intervaldict) + # adding diff, trying to see if this is valid t = 10*row.df.loc[:,'TimeStamp (sec)'].values-10*row.df.loc[:,'TimeStamp (sec)'].iloc[0] try: t[0] = t[1] except IndexError: pass - + d = 10*row.df.loc[:,' Horizontal (meters)'].values try: d[0] = d[1] except IndexError: pass - + p = abs(10*row.df.loc[:,' Stroke500mPace (sec/500m)'].values) p = np.clip(p,0,3600) if w.workouttype == 'bike': @@ -516,7 +542,7 @@ def createc2workoutdata(w): p = p.tolist() spm = spm.tolist() hr = hr.tolist() - + for i in range(len(t)): thisrecord = {"t":t[i], "d":d[i], @@ -547,14 +573,18 @@ def createc2workoutdata(w): "time": int(10*makeseconds(durationstr)), "weight_class": c2wc(w.weightcategory), "comments": w.notes, + 'stroke_rate': int(row.df[' Cadence (stokes/min)'].mean()), + 'drag_factor': int(row.dragfactor), "heart_rate": { "average": averagehr, "max": maxhr, }, "stroke_data": stroke_data, + 'workout': { + 'splits': intervaldata, + } } - return data # Refresh Concept2 authorization token @@ -570,7 +600,7 @@ def do_refresh_token(refreshtoken): url = "https://log.concept2.com/oauth/access_token" s = Session() req = Request('POST',url, data=post_data, headers=headers) - + prepped = req.prepare() prepped.body+="&scope=" prepped.body+=scope @@ -672,7 +702,7 @@ def get_workout(user,c2id): url = "https://log.concept2.com/api/users/me/results/"+str(c2id) s = requests.get(url,headers=headers) - + data = s.json()['data'] splitdata = None @@ -694,7 +724,7 @@ def get_workout(user,c2id): strokedata = pd.DataFrame() else: strokedata = pd.DataFrame() - + return data,strokedata # Get stroke data belonging to C2 ID @@ -740,7 +770,7 @@ def get_c2_workout_list(user,page=1): return s - + # Get username, having access token. # Handy for checking if the API access is working def get_username(access_token): @@ -751,7 +781,7 @@ def get_username(access_token): import urllib url = "https://log.concept2.com/api/users/me" response = requests.get(url,headers=headers) - + me_json = response.json() @@ -776,14 +806,14 @@ def get_userid(access_token): response = requests.get(url,headers=headers) except: return 0 - + me_json = response.json() try: res = me_json['data']['id'] except KeyError: res = 0 - + return res # For debugging purposes @@ -799,7 +829,7 @@ def process_callback(request): return HttpResponse("got a user name: %s" % username) def default(o): - if isinstance(o, numpy.int64): return int(o) + if isinstance(o, numpy.int64): return int(o) raise TypeError # Uploading workout @@ -810,7 +840,7 @@ def workout_c2_upload(user,w): return "This workout type cannot be uploaded to Concept2",0 except KeyError: return "This workout type cannot be uploaded to Concept2",0 - + thetoken = c2_open(user) r = Rower.objects.get(user=user) @@ -825,7 +855,7 @@ def workout_c2_upload(user,w): if data == 0: return "Error: No data file. Contact info@rowsandall.com if the problem persists",0 - + authorizationstring = str('Bearer ' + r.c2token) headers = {'Authorization': authorizationstring, 'user-agent': 'sanderroosendaal', @@ -849,7 +879,7 @@ def workout_c2_upload(user,w): else: message = "Something went wrong in workout_c2_upload_view. Response code 200/201 but C2 sync failed: "+response.text c2id = 0 - + return message,c2id @@ -882,7 +912,7 @@ def add_workout_from_data(user,importid,data,strokedata, workouttype = mytypes.c2mappinginv[data['type']] except KeyError: workouttype = 'rower' - + if workouttype not in [x[0] for x in Workout.workouttypes]: workouttype = 'other' try: @@ -902,14 +932,14 @@ def add_workout_from_data(user,importid,data,strokedata, rowdatetime = iso8601.parse_date(data['start_date']) except ParseError: rowdatetime = iso8601.parse_date(data['date']) - - + + try: c2intervaltype = data['workout_type'] - + except KeyError: c2intervaltype = '' - + try: title = data['name'] except KeyError: @@ -951,7 +981,7 @@ def add_workout_from_data(user,importid,data,strokedata, spm = strokedata.loc[:,'spm'] except KeyError: spm = 0*dist2 - + try: hr = strokedata.loc[:,'hr'] except KeyError: @@ -966,7 +996,7 @@ def add_workout_from_data(user,importid,data,strokedata, velo = 1000./pace pace = 500./velo - + # save csv # Create data frame with all necessary data to write to csv df = pd.DataFrame({'TimeStamp (sec)':unixtime, @@ -988,9 +1018,9 @@ def add_workout_from_data(user,importid,data,strokedata, ' ElapsedTime (sec)':seconds }) - + df.sort_values(by='TimeStamp (sec)',ascending=True) - + timestr = strftime("%Y%m%d-%H%M%S") @@ -1024,6 +1054,6 @@ def add_workout_from_data(user,importid,data,strokedata, dosummary=True ) - + return id,message From 37024296c037df3e46588bce8d90bd5617c02967 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Oct 2019 15:09:05 +0100 Subject: [PATCH 4/4] solves #505 --- rowers/c2stuff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 82f199f8..4600d201 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -568,6 +568,7 @@ def createc2workoutdata(w): data = { "type": mytypes.c2mapping[workouttype], "date": w.startdatetime.isoformat(), + "stroke_count": int(row.stroke_count), "timezone": w.timezone, "distance": int(w.distance), "time": int(10*makeseconds(durationstr)),