diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index c7e2871c..d83682b9 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -33,7 +33,7 @@ try: user = DATABASES['default']['USER'] except KeyError: user = '' -try: +try: password = DATABASES['default']['PASSWORD'] except KeyError: password = '' @@ -98,7 +98,7 @@ def niceformat(values): for v in values: formattedv = strfdelta(v) out.append(formattedv) - + return out def strfdelta(tdelta): @@ -114,7 +114,7 @@ def strfdelta(tdelta): seconds=seconds, tenths=tenths, ) - + return res def nicepaceformat(values): @@ -122,7 +122,7 @@ def nicepaceformat(values): for v in values: formattedv = strfdelta(v) out.append(formattedv) - + return out @@ -131,8 +131,8 @@ def timedeltaconv(x): dt = datetime.timedelta(seconds=x) else: dt = datetime.timedelta(seconds=350.) - - + + return dt def rdata(file,rower=rrower()): @@ -167,7 +167,7 @@ def create_c2_stroke_data_db( spm = 60.*nr_strokes/totalseconds except ZeroDivisionError: spm = 20*zeros(nr_strokes) - + step = totalseconds/float(nr_strokes) elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) @@ -186,7 +186,7 @@ def create_c2_stroke_data_db( else: power = 0 - + df = pd.DataFrame({ 'TimeStamp (sec)': unixtime, ' Horizontal (meters)': d, @@ -252,7 +252,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, spm = strokedata.loc[:,'spm'] except KeyError: spm = 0*dist2 - + try: hr = strokedata.loc[:,'hr'] except KeyError: @@ -268,7 +268,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, velo = 1000./pace - + # save csv # Create data frame with all necessary data to write to csv df = pd.DataFrame({'TimeStamp (sec)':unixtime, @@ -291,11 +291,11 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, 'cum_dist': dist2 }) - + df.sort_values(by='TimeStamp (sec)',ascending=True) - + timestr = strftime("%Y%m%d-%H%M%S") - + # Create CSV file name and save data to CSV file @@ -307,7 +307,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename, data = dataprep(df,id=workoutid,bands=False,debug=debug) except: return 0 - + return data # Processes painsled CSV file to database @@ -341,7 +341,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', #row.repair() pass - + if row == 0: return (0,'Error: CSV data file not found') @@ -349,7 +349,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values velo = 500./pace - + f = row.df['TimeStamp (sec)'].diff().mean() if f !=0: windowsize = 2*(int(10./(f)))+1 @@ -366,9 +366,9 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', velo3 = pd.Series(velo2) velo3 = velo3.replace([-np.inf,np.inf],np.nan) velo3 = velo3.fillna(method='ffill') - + pace2 = 500./abs(velo3) - + row.df[' Stroke500mPace (sec/500m)'] = pace2 row.df = row.df.fillna(0) @@ -378,7 +378,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', os.remove(f2) except: pass - + # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: @@ -386,10 +386,10 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', row.write_csv(f2,gzip=True) except: pass - + averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() - + if totaldist == 0: totaldist = row.df['cum_dist'].max() if totaltime == 0: @@ -408,7 +408,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', minutes = 59 if not message: message = 'Warning: there is something wrong with the workout duration' - + seconds = int(totaltime - 3600.*hours - 60.*minutes) if seconds > 59: seconds = 59 @@ -420,7 +420,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', tenths = 9 if not message: message = 'Warning: there is something wrong with the workout duration' - + duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) if dosummary: @@ -444,7 +444,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', message = "Warning: This workout probably already exists in the database" privacy = 'private' - + w = Workout(user=r,name=title,date=workoutdate, workouttype=workouttype, @@ -481,7 +481,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle TCX if (fileformat == 'tcx'): row = TCXParser(f2) - + # handle Mystery if (fileformat == 'mystery'): row = MysteryParser(f2) @@ -489,7 +489,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle RowPerfect if (fileformat == 'rowperfect3'): row = RowPerfectParser(f2) - + # handle ErgData if (fileformat == 'ergdata'): row = ErgDataParser(f2) @@ -501,7 +501,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle Mike if (fileformat == 'bcmike'): row = BoatCoachAdvancedParser(f2) - + # handle BoatCoach OTW if (fileformat == 'boatcoachotw'): row = BoatCoachOTWParser(f2) @@ -518,7 +518,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): if (fileformat == 'speedcoach'): row = speedcoachParser(f2) - # handle speed coach GPS 2 + # handle speed coach GPS 2 if (fileformat == 'speedcoach2'): row = SpeedCoach2Parser(f2) try: @@ -531,7 +531,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # handle ErgStick if (fileformat == 'ergstick'): row = ErgStickParser(f2) - + # handle FIT if (fileformat == 'fit'): row = FITParser(f2) @@ -544,7 +544,7 @@ def handle_nonpainsled(f2,fileformat,summary=''): # should delete file f2 = f2[:-4]+'o.csv' row.write_csv(f2,gzip=True) - + #os.remove(f2) try: os.remove(f_to_be_deleted) @@ -573,7 +573,7 @@ def new_workout_from_file(r,f2, for fname in z.namelist(): f3 = z.extract(fname,path='media/') id,message,f2 = new_workout_from_file(r,f3, - workouttype=workouttype, + workouttype=workouttype, makeprivate=makeprivate, title = title, notes='') @@ -590,7 +590,7 @@ def new_workout_from_file(r,f2, os.remove(f2) message = "It looks like this file doesn't contain stroke data." return (0,message,f2) - + # Some people try to upload RowPro summary logs if fileformat == 'rowprolog': os.remove(f2) @@ -603,7 +603,7 @@ def new_workout_from_file(r,f2, # worth supporting if fileformat == 'unknown': message = "We couldn't recognize the file type" - if settings.DEBUG: + if settings.DEBUG: res = handle_sendemail_unrecognized.delay(f2, r.user.email) @@ -611,7 +611,7 @@ def new_workout_from_file(r,f2, res = queuehigh.enqueue(handle_sendemail_unrecognized, f2,r.user.email) return (0,message,f2) - + # handle non-Painsled by converting it to painsled compatible CSV if (fileformat != 'csv'): try: @@ -670,7 +670,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): success = False try: - df['power empower old'] = df[' Power (watts)'] + df['power empower old'] = df[' Power (watts)'] df[' Power (watts)'] = df[' Power (watts)'] * corr_factor df['driveenergy empower old'] = df['driveenergy'] df['driveenergy'] = df['driveenergy'] * corr_factor @@ -687,7 +687,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): if debug: print("not updated ",id) - + rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True, debug=debug) @@ -708,7 +708,7 @@ def testdata(time,distance,pace,spm): def getsmallrowdata_db(columns,ids=[],debug=False): - csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] data = [] columns = [c for c in columns if c != 'None'] @@ -720,14 +720,14 @@ def getsmallrowdata_db(columns,ids=[],debug=False): except OSError: pass - + df = pd.concat(data,axis=0) else: df = pd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') return df - + def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, doclean=False): # test if nan among values @@ -736,12 +736,12 @@ def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, m[key] = -1 if 'inf' in str(m[key]): m[key] = -1 - + if debug: engine = create_engine(database_url_debug, echo=False) else: engine = create_engine(database_url, echo=False) - + columns = ', '.join(m.keys()) if use_sqlite: placeholders = ", ".join(["?"] * len(m)) @@ -795,7 +795,7 @@ def read_cols_df_sql(ids,columns,debug=False): df = pd.concat(data,axis=0) return df - + def read_df_sql(id,debug=False): try: @@ -860,15 +860,15 @@ def delete_agegroup_db(age,sex,weightcategory,debug=False): print("Database locked") conn.close() engine.dispose() - + def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower, debug=False): - + delete_agegroup_db(age,sex,weightcategory,debug=debug) wcdurations = [None if type(y) is float and np.isnan(y) else y for y in wcdurations] wcpower = [None if type(y) is float and np.isnan(y) else y for y in wcpower] - + df = pd.DataFrame( { 'duration':wcdurations, @@ -893,7 +893,7 @@ def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower, conn.close() engine.dispose() - + def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),debug=False): deletecpdata_sql(rower_id,table=table,debug=debug) df = pd.DataFrame( @@ -918,8 +918,8 @@ def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),deb engine.dispose() - - + + def smalldataprep(therows,xparam,yparam1,yparam2): @@ -936,7 +936,7 @@ def smalldataprep(therows,xparam,yparam1,yparam2): try: rowdata = dataprep(rrdata(f1).df) - + rowdata = pd.DataFrame({xparam: rowdata[xparam], yparam1: rowdata[yparam1], yparam2: rowdata[yparam2], @@ -960,8 +960,8 @@ def smalldataprep(therows,xparam,yparam1,yparam2): pass return df - - + + def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, empower=True,debug=False,inboard=0.88,forceunit='lbs'): @@ -972,7 +972,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, if debug: print("dataprep",id) - + # rowdatadf.set_index([range(len(rowdatadf))],inplace=True) t = rowdatadf.loc[:,'TimeStamp (sec)'] t = pd.Series(t-rowdatadf.loc[:,'TimeStamp (sec)'].iloc[0]) @@ -985,7 +985,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, velo = rowdatadf.loc[:,' AverageBoatSpeed (m/s)'] except KeyError: velo = 500./p - + hr = rowdatadf.loc[:,' HRCur (bpm)'] spm = rowdatadf.loc[:,' Cadence (stokes/min)'] cumdist = rowdatadf.loc[:,'cum_dist'] @@ -997,12 +997,12 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, workoutstate = rowdatadf.loc[:,' WorkoutState'] except KeyError: workoutstate = 0*hr - + peakforce = rowdatadf.loc[:,' PeakDriveForce (lbs)'] forceratio = averageforce/peakforce forceratio = forceratio.fillna(value=0) - + try: drivetime = rowdatadf.loc[:,' DriveTime (ms)'] recoverytime = rowdatadf.loc[:,' StrokeRecoveryTime (ms)'] @@ -1033,7 +1033,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, except TypeError: t2 = 0*t - + p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) try: @@ -1042,7 +1042,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] except TypeError: drivespeed = 0.0*rowdatadf['TimeStamp (sec)'] - + drivespeed = drivespeed.fillna(value=0) try: @@ -1100,7 +1100,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, tel = rowdatadf.loc[:,' ElapsedTime (sec)'] except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - + if empower: try: @@ -1132,7 +1132,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, else: driveenergy = data['driveenergy'] - + arclength = (inboard-0.05)*(np.radians(finish)-np.radians(catch)) if arclength.mean()>0: drivelength = arclength @@ -1151,7 +1151,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, totalangle = 0*t effectiveangle = 0*t - + if windowsize > 3 and windowsize