From e68995580285d703dd99a4a9de68f0bfba9456e7 Mon Sep 17 00:00:00 2001
From: Sander Roosendaal <roosendaalsander@gmail.com>
Date: Wed, 20 Sep 2017 11:18:42 +0200
Subject: [PATCH] added calcdps routine

---
 rowers/dataprep.py         |  86 ++++++++++++++------
 rowers/dataprepnodjango.py | 156 ++++++++++++++++++++++++++++---------
 rowers/tasks.py            |  28 ++++++-
 rowers/urls.py             |   1 +
 rowers/views.py            |  17 +++-
 5 files changed, 225 insertions(+), 63 deletions(-)

diff --git a/rowers/dataprep.py b/rowers/dataprep.py
index 6922c4cf..e4b29c24 100644
--- a/rowers/dataprep.py
+++ b/rowers/dataprep.py
@@ -95,6 +95,7 @@ columndict = {
     'wash':'wash',
     'slip':'slip',
     'workoutstate':' WorkoutState',
+    'cumdist':'cum_dist',
     }
     
 from scipy.signal import savgol_filter
@@ -1506,7 +1507,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
     hr = rowdatadf.ix[:,' HRCur (bpm)']
     spm = rowdatadf.ix[:,' Cadence (stokes/min)']
     cumdist = rowdatadf.ix[:,'cum_dist']
-
     power = rowdatadf.ix[:,' Power (watts)']
     averageforce = rowdatadf.ix[:,' AverageDriveForce (lbs)']
     drivelength = rowdatadf.ix[:,' DriveLength (meters)']
@@ -1605,6 +1605,11 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
 	data['hr_max'] = rowdatadf.ix[:,'hr_max']
 	data['hr_bottom'] = 0.0*data['hr']
 
+    try:
+        tel = rowdatadf.ix[:,' ElapsedTime (sec)']
+    except KeyError:
+        rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)']
+
     if barchart:
 	# time increments for bar chart
 	time_increments = rowdatadf.ix[:,' ElapsedTime (sec)'].diff()
@@ -1656,18 +1661,50 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
         except KeyError:
             slip = 0*power
         
-        totalangle = finish-catch
-        effectiveangle = finish-wash-catch-slip
+        try:
+            totalangle = finish-catch
+            effectiveangle = finish-wash-catch-slip
+        except ValueError:
+            totalangle = 0*power
+            effectiveangle = 0*power
+
         if windowsize > 3 and windowsize<len(slip):
-            wash = savgol_filter(wash,windowsize,3)
-            slip = savgol_filter(slip,windowsize,3)
-            catch = savgol_filter(catch,windowsize,3)
-            finish = savgol_filter(finish,windowsize,3)
-            peakforceangle = savgol_filter(peakforceangle,windowsize,3)
-            driveenergy = savgol_filter(driveenergy,windowsize,3)
-            drivelength = savgol_filter(drivelength,windowsize,3)
-            totalangle = savgol_filter(totalangle,windowsize,3)
-            effectiveangle = savgol_filter(effectiveangle,windowsize,3)
+            try:
+                wash = savgol_filter(wash,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                slip = savgol_filter(slip,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                catch = savgol_filter(catch,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                finish = savgol_filter(finish,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                peakforceangle = savgol_filter(peakforceangle,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                driveenergy = savgol_filter(driveenergy,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                drivelength = savgol_filter(drivelength,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                totalangle = savgol_filter(totalangle,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                effectiveangle = savgol_filter(effectiveangle,windowsize,3)
+            except TypeError:
+                pass
 
         velo = 500./p
 
@@ -1678,17 +1715,20 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
         efficiency = efficiency.replace([-np.inf,np.inf],np.nan)
         efficiency.fillna(method='ffill')
             
-        data['wash'] = wash
-        data['catch'] = catch
-        data['slip'] = slip
-        data['finish'] = finish
-        data['peakforceangle'] = peakforceangle
-        data['driveenergy'] = driveenergy
-        data['drivelength'] = drivelength
-        data['totalangle'] = totalangle
-        data['effectiveangle'] = effectiveangle
-        data['efficiency'] = efficiency
-        
+        try:
+            data['wash'] = wash
+            data['catch'] = catch
+            data['slip'] = slip
+            data['finish'] = finish
+            data['peakforceangle'] = peakforceangle
+            data['driveenergy'] = driveenergy
+            data['drivelength'] = drivelength
+            data['totalangle'] = totalangle
+            data['effectiveangle'] = effectiveangle
+            data['efficiency'] = efficiency
+        except ValueError:
+            pass
+            
     if otwpower:
 	try:
 	    nowindpace = rowdatadf.ix[:,'nowindpace']
diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py
index d8512ce7..05bb28be 100644
--- a/rowers/dataprepnodjango.py
+++ b/rowers/dataprepnodjango.py
@@ -15,7 +15,7 @@ from sqlalchemy import create_engine
 import sqlalchemy as sa
 
 from rowsandall_app.settings import DATABASES
-#from rowsandall_app.settings_dev import DATABASES
+from rowsandall_app.settings_dev import DATABASES as DEV_DATABASES
 
 from utils import lbstoN
 
@@ -50,8 +50,9 @@ database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format(
     port=port,
     )
 
+database_name_dev = DEV_DATABASES['default']['NAME']
 
-database_url_debug = 'sqlite:///'+database_name
+database_url_debug = 'sqlite:///'+database_name_dev
 
 # mapping the DB column names to the CSV file column names
 columndict = {
@@ -70,6 +71,7 @@ columndict = {
     'wash':'wash',
     'slip':'wash',
     'workoutstate':' WorkoutState',
+    'cumdist':'cum_dist',
     }
 
 from scipy.signal import savgol_filter
@@ -471,8 +473,12 @@ def delete_strokedata(id,debug=True):
 
 def update_strokedata(id,df,debug=True):
     delete_strokedata(id)
+    if debug:
+        print "updating ",id
     rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True,
                        debug=debug)
+
+    return rowdata
     
 def testdata(time,distance,pace,spm):
     t1 = np.issubdtype(time,np.number)
@@ -529,11 +535,15 @@ def read_cols_df_sql(ids,columns,debug=True):
 def read_df_sql(id,debug=True):
     if debug:
         engine = create_engine(database_url_debug, echo=False)
+        print "read_df",id
+        print database_url_debug
     else:
         engine = create_engine(database_url, echo=False)
 
-    df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id}'.format(
-        id=id)), engine)
+    df = pd.read_sql_query(sa.text(
+        'SELECT * FROM strokedata WHERE workoutid={id}'.format(
+            id=id
+        )), engine)
 
     engine.dispose()
     return df
@@ -583,10 +593,15 @@ def smalldataprep(therows,xparam,yparam1,yparam2):
         
         
 def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
-             empower=True,debug=True,inboard=0.88):
+             empower=True,debug=True,inboard=0.88,forceunit='lbs'):
 
     if rowdatadf.empty:
+        if debug:
+            print "empty"
         return 0
+
+    if debug:
+        print "dataprep",id
     
     rowdatadf.set_index([range(len(rowdatadf))],inplace=True)
     t = rowdatadf.ix[:,'TimeStamp (sec)']
@@ -613,6 +628,14 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
     forceratio = averageforce/peakforce
     forceratio = forceratio.fillna(value=0)
     
+    try:
+        drivetime = rowdatadf.ix[:,' DriveTime (ms)']
+        recoverytime = rowdatadf.ix[:,' StrokeRecoveryTime (ms)']
+        rhythm = 100.*drivetime/(recoverytime+drivetime)
+        rhythm = rhythm.fillna(value=0)
+    except:
+        rhythm = 0.0*forceratio
+
     f = rowdatadf['TimeStamp (sec)'].diff().mean()
     if f != 0:
         windowsize = 2*(int(10./(f)))+1
@@ -635,12 +658,30 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
     
     p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x))
 
-
-    drivespeed = drivelength/rowdatadf[' DriveTime (ms)']*1.0e3
+    try:
+        drivespeed = drivelength/rowdatadf[' DriveTime (ms)']*1.0e3
+    except KeyError:
+        drivespeed = 0.0*rowdatadf['TimeStamp (sec)']
+    except TypeError:
+        drivespeed = 0.0*rowdatadf['TimeStamp (sec)']
+        
     drivespeed = drivespeed.fillna(value=0)
-    driveenergy = drivelength*averageforce*lbstoN
+
+    try:
+        driveenergy = rowdatadf['driveenergy']
+    except KeyError:
+        if forceunit == 'lbs':
+            driveenergy = drivelength*averageforce*lbstoN
+        else:
+            drivenergy = drivelength*averageforce
+
     distance = rowdatadf.ix[:,'cum_dist']
 
+    velo = 500./p
+
+    distanceperstroke = 60.*velo/spm
+    if debug:
+        print distanceperstroke.mean()
 
 
 
@@ -654,6 +695,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
 	    ftime = niceformat(t2),
 	    fpace = nicepaceformat(p2),
 	    driveenergy=driveenergy,
+            distanceperstroke=distanceperstroke,
 	    power=power,
             workoutstate=workoutstate,
 	    averageforce=averageforce,
@@ -675,6 +717,12 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
 	data['hr_max'] = rowdatadf.ix[:,'hr_max']
 	data['hr_bottom'] = 0.0*data['hr']
 
+
+    try:
+        tel = rowdatadf.ix[:,' ElapsedTime (sec)']
+    except KeyError:
+        rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)']
+        
     if barchart:
 	# time increments for bar chart
 	time_increments = rowdatadf.ix[:,' ElapsedTime (sec)'].diff()
@@ -688,22 +736,22 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
         try:
             wash = rowdatadf.ix[:,'wash']
         except KeyError:
-            wash = 0*power
+            wash = 0*t
 
         try:
             catch = rowdatadf.ix[:,'catch']
         except KeyError:
-            catch = 0*power
+            catch = 0*t
 
         try:
             finish = rowdatadf.ix[:,'finish']
         except KeyError:
-            finish = 0*power
+            finish = 0*t
 
         try:
             peakforceangle = rowdatadf.ix[:,'peakforceangle']
         except KeyError:
-            peakforceangle = 0*power
+            peakforceangle = 0*t
 
 
         if data['driveenergy'].mean() == 0:
@@ -724,20 +772,53 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
         try:
             slip = rowdatadf.ix[:,'slip']
         except KeyError:
-            slip = 0*power
-        
-        totalangle = finish-catch
-        effectiveangle = finish-wash-catch-slip
+            slip = 0*t
+
+        try:
+            totalangle = finish-catch
+            effectiveangle = finish-wash-catch-slip
+        except ValueError:
+            totalangle = 0*t
+            effectiveangle = 0*t
+
+            
         if windowsize > 3 and windowsize<len(slip):
-            wash = savgol_filter(wash,windowsize,3)
-            slip = savgol_filter(slip,windowsize,3)
-            catch = savgol_filter(catch,windowsize,3)
-            finish = savgol_filter(finish,windowsize,3)
-            peakforceangle = savgol_filter(peakforceangle,windowsize,3)
-            driveenergy = savgol_filter(driveenergy,windowsize,3)
-            drivelength = savgol_filter(drivelength,windowsize,3)
-            totalangle = savgol_filter(totalangle,windowsize,3)
-            effectiveangle = savgol_filter(effectiveangle,windowsize,3)
+            try:
+                wash = savgol_filter(wash,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                slip = savgol_filter(slip,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                catch = savgol_filter(catch,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                finish = savgol_filter(finish,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                peakforceangle = savgol_filter(peakforceangle,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                driveenergy = savgol_filter(driveenergy,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                drivelength = savgol_filter(drivelength,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                totalangle = savgol_filter(totalangle,windowsize,3)
+            except TypeError:
+                pass
+            try:
+                effectiveangle = savgol_filter(effectiveangle,windowsize,3)
+            except TypeError:
+                pass
 
         velo = 500./p
 
@@ -746,18 +827,21 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
 
         efficiency = efficiency.replace([-np.inf,np.inf],np.nan)
         efficiency.fillna(method='ffill')
+
+        try:
+            data['wash'] = wash
+            data['catch'] = catch
+            data['slip'] = slip
+            data['finish'] = finish
+            data['peakforceangle'] = peakforceangle
+            data['driveenergy'] = driveenergy
+            data['drivelength'] = drivelength
+            data['totalangle'] = totalangle
+            data['effectiveangle'] = effectiveangle
+            data['efficiency'] = efficiency
+        except ValueError:
+            pass
             
-        data['wash'] = wash
-        data['catch'] = catch
-        data['slip'] = slip
-        data['finish'] = finish
-        data['peakforceangle'] = peakforceangle
-        data['driveenergy'] = driveenergy
-        data['drivelength'] = drivelength
-        data['totalangle'] = totalangle
-        data['effectiveangle'] = effectiveangle
-        data['efficiency'] = efficiency
-        
     if otwpower:
 	try:
 	    nowindpace = rowdatadf.ix[:,'nowindpace']
diff --git a/rowers/tasks.py b/rowers/tasks.py
index 8960a03b..28496b79 100644
--- a/rowers/tasks.py
+++ b/rowers/tasks.py
@@ -24,9 +24,10 @@ from django_rq import job
 
 from utils import serialize_list,deserialize_list
 
-from rowers.dataprepnodjango import update_strokedata
-from rowers.dataprepnodjango import new_workout_from_file
-from rowers.dataprepnodjango import getsmallrowdata_db
+from rowers.dataprepnodjango import (
+    update_strokedata, new_workout_from_file,
+    getsmallrowdata_db,read_df_sql,columndict,
+) 
 
 from django.core.mail import send_mail, BadHeaderError,EmailMessage
 
@@ -49,6 +50,27 @@ def handle_new_workout_from_file(r,f2,
     return new_workout_from_file(r,f2,workouttype,
                                  title,makeprivate,notes)
 
+# process and update workouts
+@app.task
+def handle_updatedps(useremail,workoutids,debug=False):
+    for wid,f1 in workoutids:
+        havedata = 1
+        try:
+            rowdata = rdata(f1)
+        except IOError:
+            try:
+                rowdata = rdata(f1+'.csv')
+            except IOError:
+                try:
+                    rowdata = rdata(f1+'.gz')
+                except IOError:
+                    havedata = 0
+
+        if havedata:
+            update_strokedata(wid,rowdata.df,debug=debug)
+
+    return 1
+
 # send email when a breakthrough workout is uploaded
 @app.task
 def handle_sendemail_breakthrough(workoutid,useremail,
diff --git a/rowers/urls.py b/rowers/urls.py
index e249ef0a..3f1ed3c6 100644
--- a/rowers/urls.py
+++ b/rowers/urls.py
@@ -254,6 +254,7 @@ urlpatterns = [
     url(r'^user-multiflex/$',views.multiflex_view),
     url(r'^user-multiflex$',views.multiflex_view),
     url(r'^me/teams/$',views.rower_teams_view),
+    url(r'^me/calcdps/$',views.rower_calcdps_view),
     url(r'^me/exportsettings/$',views.rower_exportsettings_view),
     url(r'^team/(?P<id>\d+)/$',views.team_view),
     url(r'^team/(?P<id>\d+)/memberstats$',views.team_members_stats_view),
diff --git a/rowers/views.py b/rowers/views.py
index a640c5fa..f154a300 100644
--- a/rowers/views.py
+++ b/rowers/views.py
@@ -93,7 +93,7 @@ from rowers.rows import handle_uploaded_file
 from rowers.tasks import handle_makeplot,handle_otwsetpower,handle_sendemailtcx,handle_sendemailcsv
 from rowers.tasks import (
     handle_sendemail_unrecognized,handle_sendemailnewcomment,
-    handle_sendemailnewresponse
+    handle_sendemailnewresponse, handle_updatedps
     )
 
 from scipy.signal import savgol_filter
@@ -9266,6 +9266,21 @@ def team_leaveconfirm_view(request,id=0):
                       'teams':get_my_teams(request.user),
                       })
 
+@login_required()
+def rower_calcdps_view(request):
+    r = getrower(request.user)
+
+    ws = [(w.id,w.csvfilename) for w in Workout.objects.filter(user=r)]
+    if settings.DEBUG:
+        res = handle_updatedps.delay(r.user.email,ws,debug=True)
+    else:
+        queue.enqueue(handle_updatedps,r.user.email,ws,debug=False) 
+
+    messages.info(request,"Your workouts are being updated in the background. You will receive email when this is done.")
+
+    url = reverse(workouts_view)
+    return HttpResponseRedirect(url)
+
 @login_required()
 def team_leave_view(request,id=0):
     r = getrower(request.user)