From 7d6e778625635f3296054b8884e4ae139426fbaf Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 15 Sep 2017 15:31:13 +0200 Subject: [PATCH 1/5] dataprep partially done - continue on line 1400 --- rowers/dataprep.py | 85 ++++++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 223ad583..3e8687cf 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1173,16 +1173,25 @@ def getrowdata_db(id=0,doclean=False,convertnewtons=True): return data,row # Fetch a subset of the data from the DB -def getsmallrowdata_db(columns,ids=[],doclean=True,workstrokesonly=True, - convertnewtons=False): +def getsmallrowdata_db(columns,ids=[],doclean=True,workstrokesonly=True): prepmultipledata(ids) data = read_cols_df_sql(ids,columns) - if convertnewtons: - if 'peakforce' in columns: - data['peakforce'] = data['peakforce']*lbstoN - if 'averageforce' in columns: - data['averageforce'] = data['averageforce']*lbstoN + # convert newtons + + if 'peakforce' in columns: + funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + for id,u in funits: + if u=='lbs': + mask = data['workoutid']==id + data.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN + if 'averageforce' in columns: + funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + for id,u in funits: + if u=='lbs': + mask = data['workoutid']==id + data.loc[mask,'averageforce'] = data.loc[mask,'averageforce']*lbstoN + if doclean: data = clean_df_stats(data,ignorehr=True, @@ -1284,22 +1293,24 @@ def read_cols_df_sql(ids,columns,convertnewtons=True): df = df.fillna(value=0) - if convertnewtons: - try: - df['peakforce'] = df['peakforce']*lbstoN - except KeyError: - pass - - try: - df['averageforce'] = df['averageforce']*lbstoN - except KeyError: - pass + if 'peakforce' in columns: + funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + for id,u in funits: + if u=='lbs': + mask = data['workoutid']==id + df.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN + if 'averageforce' in columns: + funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + for id,u in funits: + if u=='lbs': + mask = data['workoutid']==id + df.loc[mask,'averageforce'] = data.loc[mask,'averageforce']*lbstoN engine.dispose() return df # Read stroke data from the DB for a Workout ID. Returns a pandas dataframe -def read_df_sql(id,convertnewtons=True): +def read_df_sql(id): engine = create_engine(database_url, echo=False) df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id}'.format( @@ -1307,7 +1318,10 @@ def read_df_sql(id,convertnewtons=True): engine.dispose() df = df.fillna(value=0) - if convertnewtons: + + funit = Workout.objects.get(id=id).forceunit + + if funit=='lbs': try: df['peakforce'] = df['peakforce']*lbstoN except KeyError: @@ -1322,7 +1336,7 @@ def read_df_sql(id,convertnewtons=True): # Get the necessary data from the strokedata table in the DB. # For the flex plot -def smalldataprep(therows,xparam,yparam1,yparam2,convertnewtons=True): +def smalldataprep(therows,xparam,yparam1,yparam2): df = pd.DataFrame() if yparam2 == 'None': yparam2 = 'power' @@ -1344,6 +1358,17 @@ def smalldataprep(therows,xparam,yparam1,yparam2,convertnewtons=True): 'spm': rowdata['spm'], } ) + if workout.forceunit == 'lbs': + try: + rowdata['peakforce'] *= lbstoN + except KeyError: + pass + + try: + rowdata['averageforce'] *= lbstoN + except KeyError: + pass + df = pd.concat([df,rowdata],ignore_index=True) except IOError: try: @@ -1355,20 +1380,20 @@ def smalldataprep(therows,xparam,yparam1,yparam2,convertnewtons=True): 'spm': rowdata['spm'], } ) + if workout.forceunit == 'lbs': + try: + rowdata['peakforce'] *= lbstoN + except KeyError: + pass + + try: + rowdata['averageforce'] *= lbstoN + except KeyError: + pass df = pd.concat([df,rowdata],ignore_index=True) except IOError: pass - if convertnewtons: - try: - df['peakforce'] = df['peakforce']*lbstoN - except KeyError: - pass - - try: - df['averageforce'] = df['averageforce']*lbstoN - except KeyError: - pass return df From 55049532e40ae9d1042c5645276825457be3137b Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 15 Sep 2017 15:34:30 +0200 Subject: [PATCH 2/5] bugfix --- rowers/dataprep.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 3e8687cf..93d26206 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1180,13 +1180,13 @@ def getsmallrowdata_db(columns,ids=[],doclean=True,workstrokesonly=True): # convert newtons if 'peakforce' in columns: - funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: if u=='lbs': mask = data['workoutid']==id data.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN if 'averageforce' in columns: - funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: if u=='lbs': mask = data['workoutid']==id @@ -1294,13 +1294,13 @@ def read_cols_df_sql(ids,columns,convertnewtons=True): df = df.fillna(value=0) if 'peakforce' in columns: - funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: if u=='lbs': mask = data['workoutid']==id df.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN if 'averageforce' in columns: - funits = ((w.id,w.forceunit) for Workout.objects.filter(id__in=ids)) + funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: if u=='lbs': mask = data['workoutid']==id From d38512a47c7f61eefc327e70121ec7a48997ae5a Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 17 Sep 2017 15:01:07 +0200 Subject: [PATCH 3/5] first pass - untested --- rowers/dataprep.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 93d26206..f5e7a775 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1202,7 +1202,7 @@ def getsmallrowdata_db(columns,ids=[],doclean=True,workstrokesonly=True): return data # Fetch both the workout and the workout stroke data (from CSV file) -def getrowdata(id=0,convertnewtons=True): +def getrowdata(id=0): # check if valid ID exists (workout exists) row = Workout.objects.get(id=id) @@ -1263,7 +1263,7 @@ def read_cols_df_sql(ids,columns,convertnewtons=True): if not c in axx: columns.remove(c) - columns = list(columns)+['distance','spm'] + columns = list(columns)+['distance','spm','workoutid'] columns = [x for x in columns if x != 'None'] columns = list(set(columns)) cls = '' @@ -1297,14 +1297,14 @@ def read_cols_df_sql(ids,columns,convertnewtons=True): funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: if u=='lbs': - mask = data['workoutid']==id + mask = df['workoutid']==id df.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN if 'averageforce' in columns: funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: if u=='lbs': - mask = data['workoutid']==id - df.loc[mask,'averageforce'] = data.loc[mask,'averageforce']*lbstoN + mask = df['workoutid']==id + df.loc[mask,'averageforce'] = df.loc[mask,'averageforce']*lbstoN engine.dispose() return df @@ -1399,6 +1399,9 @@ def smalldataprep(therows,xparam,yparam1,yparam2): # data fusion def datafusion(id1,id2,columns,offset): + workout1 = Workout.objects.get(id=id1) + workout2 = Workout.objects.get(id=id2) + df1,w1 = getrowdata_db(id=id1) df1 = df1.drop([#'cumdist', 'hr_ut2', @@ -1412,7 +1415,7 @@ def datafusion(id1,id2,columns,offset): 'workoutid', 'id'], 1,errors='ignore') - + # Add coordinates to DataFrame latitude,longitude = get_latlon(id1) @@ -1421,6 +1424,18 @@ def datafusion(id1,id2,columns,offset): df2 = getsmallrowdata_db(['time']+columns,ids=[id2],doclean=False) + + forceunit = 'N' + # set Force to same units + if 'averageforce' in columns or 'peakforce' in columns: + forceunit = workout2.forceunit + if workout1.forceunit == 'lbs' and workout2.forceunit == 'N': + df1['averageforce'] *= lbstoN + df1['peakforce'] *= lbstoN + if workout1.forceunit == 'N' and workout2.forceunit == 'lbs': + df1['averageforce'] /= lbstoN + df1['peakforce'] /= lbstoN + offsetmillisecs = offset.seconds*1000+offset.microseconds/1000. offsetmillisecs += offset.days*(3600*24*1000) df2['time'] = df2['time']+offsetmillisecs From 8ccc8dd3a01010b1ffab67663d0fa2ff8f01cd7e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 17 Sep 2017 15:40:52 +0200 Subject: [PATCH 4/5] tested split and flexchart --- rowers/dataprep.py | 30 ++++++++++++------------------ rowers/views.py | 6 +++--- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index f5e7a775..f7962e0e 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -433,6 +433,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', summary='', makeprivate=False, oarlength=2.89,inboard=0.88, + forceunit='lbs', consistencychecks=False): message = None powerperc = 100*np.array([r.pw_ut2, @@ -447,7 +448,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', powerperc=powerperc,powerzones=r.powerzones) row = rdata(f2,rower=rr) - dtavg = row.df['TimeStamp (sec)'].diff().mean() if dtavg < 1: @@ -640,6 +640,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', weightcategory=r.weightcategory, starttime=workoutstarttime, workoutsource=workoutsource, + forceunit=forceunit, csvfilename=f2,notes=notes,summary=summary, maxhr=maxhr,averagehr=averagehr, startdatetime=workoutstartdatetime, @@ -925,7 +926,8 @@ def split_workout(r,parent,splitsecond,splitmode): id,message = new_workout_from_df(r,data1, title=parent.name+' (1)', parent=parent, - setprivate=setprivate) + setprivate=setprivate, + forceunit='N') messages.append(message) ids.append(id) if 'keep second' in splitmode: @@ -938,12 +940,12 @@ def split_workout(r,parent,splitsecond,splitmode): setprivate = False dt = datetime.timedelta(seconds=splitsecond) - + id,message = new_workout_from_df(r,data2, title=parent.name+' (2)', parent=parent, setprivate=setprivate, - dt=dt) + dt=dt,forceunit='N') messages.append(message) ids.append(id) @@ -968,6 +970,7 @@ def new_workout_from_df(r,df, title='New Workout', parent=None, setprivate=False, + forceunit='lbs', dt=datetime.timedelta()): message = None @@ -1001,7 +1004,10 @@ def new_workout_from_df(r,df, timestr = strftime("%Y%m%d-%H%M%S") csvfilename ='media/df_'+timestr+'.csv' - + if forceunit == 'N': + # change to lbs for now + df['peakforce'] /= lbstoN + df['averageforce'] /= lbstoN df.rename(columns = columndict,inplace=True) @@ -1179,18 +1185,6 @@ def getsmallrowdata_db(columns,ids=[],doclean=True,workstrokesonly=True): # convert newtons - if 'peakforce' in columns: - funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) - for id,u in funits: - if u=='lbs': - mask = data['workoutid']==id - data.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN - if 'averageforce' in columns: - funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) - for id,u in funits: - if u=='lbs': - mask = data['workoutid']==id - data.loc[mask,'averageforce'] = data.loc[mask,'averageforce']*lbstoN if doclean: @@ -1298,7 +1292,7 @@ def read_cols_df_sql(ids,columns,convertnewtons=True): for id,u in funits: if u=='lbs': mask = df['workoutid']==id - df.loc[mask,'peakforce'] = data.loc[mask,'peakforce']*lbstoN + df.loc[mask,'peakforce'] = df.loc[mask,'peakforce']*lbstoN if 'averageforce' in columns: funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id,u in funits: diff --git a/rowers/views.py b/rowers/views.py index 20d7b897..234c11a1 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -5881,9 +5881,9 @@ def workout_flexchart3_view(request,*args,**kwargs): # create interactive plot try: script,div,js_resources,css_resources,workstrokesonly = interactive_flex_chart2(id,xparam=xparam,yparam1=yparam1, - yparam2=yparam2, - promember=promember,plottype=plottype, - workstrokesonly=workstrokesonly) + yparam2=yparam2, + promember=promember,plottype=plottype, + workstrokesonly=workstrokesonly) except ValueError: script,div = interactive_flex_chart2(id,xparam=xparam,yparam1=yparam1, yparam2=yparam2, From ffea92278e074ce09d99de404521d5f0698256d7 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 17 Sep 2017 15:58:36 +0200 Subject: [PATCH 5/5] added lbs/N force unit to model, use that --- rowers/dataprep.py | 16 ++-------------- rowers/views.py | 6 ++++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index f7962e0e..91ca3d6a 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1172,9 +1172,6 @@ def getrowdata_db(id=0,doclean=False,convertnewtons=True): if doclean: data = clean_df_stats(data,ignorehr=True) - # these two lines seem redundant ?? - #data['averageforce'] = data['averageforce'] - #data['peakforce'] = data['peakforce'] return data,row @@ -1420,16 +1417,7 @@ def datafusion(id1,id2,columns,offset): df2 = getsmallrowdata_db(['time']+columns,ids=[id2],doclean=False) forceunit = 'N' - # set Force to same units - if 'averageforce' in columns or 'peakforce' in columns: - forceunit = workout2.forceunit - if workout1.forceunit == 'lbs' and workout2.forceunit == 'N': - df1['averageforce'] *= lbstoN - df1['peakforce'] *= lbstoN - if workout1.forceunit == 'N' and workout2.forceunit == 'lbs': - df1['averageforce'] /= lbstoN - df1['peakforce'] /= lbstoN - + offsetmillisecs = offset.seconds*1000+offset.microseconds/1000. offsetmillisecs += offset.days*(3600*24*1000) df2['time'] = df2['time']+offsetmillisecs @@ -1460,7 +1448,7 @@ def datafusion(id1,id2,columns,offset): df['pace'] = df['pace']/1000. df['cum_dist'] = df['cumdist'] - return df + return df,forceunit def fix_newtons(id=0,limit=3000): # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) diff --git a/rowers/views.py b/rowers/views.py index 234c11a1..a640c5fa 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -8393,11 +8393,13 @@ def workout_fusion_view(request,id1=0,id2=1): timeoffset = -timeoffset # Create DataFrame - df = dataprep.datafusion(id1,id2,columns,timeoffset) + df,forceunit = dataprep.datafusion(id1,id2,columns,timeoffset) + idnew,message = dataprep.new_workout_from_df(r,df, title='Fused data', - parent=w1) + parent=w1, + forceunit=forceunit) if message != None: messages.error(request,message) else: