diff --git a/rowers/dataprep.py b/rowers/dataprep.py index ad1d5ccb..da4a7f7b 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -344,8 +344,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, wash = rowdatadf.ix[:,'wash'] catch = rowdatadf.ix[:,'catch'] finish = rowdatadf.ix[:,'finish'] - peakforce = rowdatadf.ix[:'peakforce'] - averageforce = rowdatadf.ix[:'averageforce'] peakforceangle = rowdatadf.ix[:,'peakforceangle'] driveenergy = rowdatadf.ix[:,'driveenergy'] drivelength = driveenergy/(averageforce*4.44822) @@ -355,8 +353,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, slip = savgol_filter(slip,windowsize,3) catch = savgol_filter(catch,windowsize,3) finish = savgol_filter(finish,windowsize,3) - peakforce = savgol_filter(peakforce,windowsize,3) - averageforce = savgol_filter(averageforce,windowsize,3) peakforceangle = savgol_filter(peakforceangle,windowsize,3) driveenergy = savgol_filter(driveenergy,windowsize,3) drivelength = savgol_filter(drivelength,windowsize,3) @@ -366,6 +362,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, data['finish'] = finish data['peakforceangle'] = peakforceangle data['driveenergy'] = driveenergy + data['drivelength'] = drivelength data['peakforce'] = peakforce data['averageforce'] = averageforce except KeyError: diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py new file mode 100644 index 00000000..b03f75f1 --- /dev/null +++ b/rowers/dataprepnodjango.py @@ -0,0 +1,389 @@ +from rowingdata import rowingdata as rrdata + +from rowingdata import rower as rrower +from rowingdata import main as rmain + +from pandas import DataFrame,Series + +import pandas as pd +import numpy as np +import itertools + +from sqlalchemy import create_engine +import sqlalchemy as sa + +from rowsandall_app.settings import DATABASES + +user = DATABASES['default']['USER'] +password = DATABASES['default']['PASSWORD'] +database_name = DATABASES['default']['NAME'] +host = DATABASES['default']['HOST'] +port = DATABASES['default']['PORT'] + +database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format( + user=user, + password=password, + database_name=database_name, + host=host, + port=port, + ) + + +database_url_debug = 'sqlite:///'+database_name + + +from scipy.signal import savgol_filter + +import datetime + +def niceformat(values): + out = [] + for v in values: + formattedv = strfdelta(v) + out.append(formattedv) + + return out + +def strfdelta(tdelta): + try: + minutes,seconds = divmod(tdelta.seconds,60) + tenths = int(tdelta.microseconds/1e5) + except AttributeError: + minutes,seconds = divmod(tdelta.view(np.int64),60e9) + seconds,rest = divmod(seconds,1e9) + tenths = int(rest/1e8) + res = "{minutes:0>2}:{seconds:0>2}.{tenths:0>1}".format( + minutes=minutes, + seconds=seconds, + tenths=tenths, + ) + + return res + +def nicepaceformat(values): + out = [] + for v in values: + formattedv = strfdelta(v) + out.append(formattedv) + + + return out + +def timedeltaconv(x): + if not np.isnan(x): + dt = datetime.timedelta(seconds=x) + else: + dt = datetime.timedelta(seconds=350.) + + + return dt + +def rdata(file,rower=rrower()): + try: + res = rrdata(file,rower=rower) + except IOError: + res = 0 + + return res + +def delete_strokedata(id,debug=True): + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( + id=id, + )) + with engine.connect() as conn, conn.begin(): + try: + result = conn.execute(query) + except: + print "Database Locked" + conn.close() + engine.dispose() + +def update_strokedata(id,df,debug=True): + delete_strokedata(id) + rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True, + debug=debug) + +def testdata(time,distance,pace,spm): + t1 = np.issubdtype(time,np.number) + t2 = np.issubdtype(distance,np.number) + t3 = np.issubdtype(pace,np.number) + t4 = np.issubdtype(spm,np.number) + + return t1 and t2 and t3 and t4 + + +def getsmallrowdata_db(columns,ids=[]): + prepmultipledata(ids) + data = read_cols_df_sql(ids,columns) + + return data + + +def prepmultipledata(ids,verbose=False,debug=True): + query = sa.text('SELECT DISTINCT workoutid FROM strokedata') + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + + + with engine.connect() as conn, conn.begin(): + res = conn.execute(query) + res = list(itertools.chain.from_iterable(res.fetchall())) + conn.close() + engine.dispose() + + res = list(set(ids)-set(res)) + for id in res: + rowdata,row = getrowdata(id=id) + if verbose: + print id + if rowdata: + data = dataprep(rowdata.df,id=id,bands=True,barchart=True,otwpower=True) + return res + +def read_cols_df_sql(ids,columns,debug=True): + columns = list(columns)+['distance','spm'] + columns = [x for x in columns if x != 'None'] + columns = list(set(columns)) + cls = '' + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + + + for column in columns: + cls += column+', ' + cls = cls[:-2] + if len(ids) == 0: + query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid=0'.format( + columns = cls, + )) + elif len(ids) == 1: + query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid={id}'.format( + id = ids[0], + columns = cls, + )) + else: + query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid IN {ids}'.format( + columns = cls, + ids = tuple(ids), + )) + + df = pd.read_sql_query(query,engine) + engine.dispose() + return df + + +def read_df_sql(id,debug=True): + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + + df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id}'.format( + id=id)), engine) + + engine.dispose() + return df + + + + + +def smalldataprep(therows,xparam,yparam1,yparam2): + df = pd.DataFrame() + if yparam2 == 'None': + yparam2 = 'power' + df[xparam] = [] + df[yparam1] = [] + df[yparam2] = [] + df['distance'] = [] + df['spm'] = [] + for workout in therows: + f1 = workout.csvfilename + + try: + rowdata = dataprep(rrdata(f1).df) + + rowdata = pd.DataFrame({xparam: rowdata[xparam], + yparam1: rowdata[yparam1], + yparam2: rowdata[yparam2], + 'distance': rowdata['distance'], + 'spm': rowdata['spm'], + } + ) + df = pd.concat([df,rowdata],ignore_index=True) + except IOError: + pass + + return df + + +def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, + empower=True,debug=True): + rowdatadf.set_index([range(len(rowdatadf))],inplace=True) + t = rowdatadf.ix[:,'TimeStamp (sec)'] + t = pd.Series(t-rowdatadf.ix[0,'TimeStamp (sec)']) + + row_index = rowdatadf.ix[:,' Stroke500mPace (sec/500m)'] > 3000 + rowdatadf.loc[row_index,' Stroke500mPace (sec/500m)'] = 3000. + + p = rowdatadf.ix[:,' Stroke500mPace (sec/500m)'] + hr = rowdatadf.ix[:,' HRCur (bpm)'] + spm = rowdatadf.ix[:,' Cadence (stokes/min)'] + cumdist = rowdatadf.ix[:,'cum_dist'] + + power = rowdatadf.ix[:,' Power (watts)'] + averageforce = rowdatadf.ix[:,' AverageDriveForce (lbs)'] + drivelength = rowdatadf.ix[:,' DriveLength (meters)'] + try: + workoutstate = rowdatadf.ix[:,' WorkoutState'] + except KeyError: + workoutstate = 0*hr + + peakforce = rowdatadf.ix[:,' PeakDriveForce (lbs)'] + + forceratio = averageforce/peakforce + forceratio = forceratio.fillna(value=0) + + f = rowdatadf['TimeStamp (sec)'].diff().mean() + windowsize = 2*(int(10./(f)))+1 + if windowsize <= 3: + windowsize = 5 + + if windowsize > 3: + spm = savgol_filter(spm,windowsize,3) + hr = savgol_filter(hr,windowsize,3) + drivelength = savgol_filter(drivelength,windowsize,3) + forceratio = savgol_filter(forceratio,windowsize,3) + + try: + t2 = t.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) + except TypeError: + t2 = 0*t + + + p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) + + + drivespeed = drivelength/rowdatadf[' DriveTime (ms)']*1.0e3 + drivespeed = drivespeed.fillna(value=0) + driveenergy = drivelength*averageforce*4.44822 + distance = rowdatadf.ix[:,'cum_dist'] + + + + + data = DataFrame( + dict( + time = t*1e3, + hr = hr, + pace = p*1e3, + spm = spm, + cumdist = cumdist, + ftime = niceformat(t2), + fpace = nicepaceformat(p2), + driveenergy=driveenergy, + power=power, + workoutstate=workoutstate, + averageforce=averageforce, + drivelength=drivelength, + peakforce=peakforce, + forceratio=forceratio, + distance=distance, + drivespeed=drivespeed, + ) + ) + + if bands: + # HR bands + data['hr_ut2'] = rowdatadf.ix[:,'hr_ut2'] + data['hr_ut1'] = rowdatadf.ix[:,'hr_ut1'] + data['hr_at'] = rowdatadf.ix[:,'hr_at'] + data['hr_tr'] = rowdatadf.ix[:,'hr_tr'] + data['hr_an'] = rowdatadf.ix[:,'hr_an'] + data['hr_max'] = rowdatadf.ix[:,'hr_max'] + data['hr_bottom'] = 0.0*data['hr'] + + if barchart: + # time increments for bar chart + time_increments = rowdatadf.ix[:,' ElapsedTime (sec)'].diff() + time_increments[0] = time_increments[1] + time_increments = 0.5*time_increments+0.5*np.abs(time_increments) + x_right = (t2+time_increments.apply(lambda x:timedeltaconv(x))) + + data['x_right'] = x_right + + if empower: + try: + wash = rowdatadf.ix[:,'wash'] + catch = rowdatadf.ix[:,'catch'] + finish = rowdatadf.ix[:,'finish'] + peakforceangle = rowdatadf.ix[:,'peakforceangle'] + driveenergy = rowdatadf.ix[:,'driveenergy'] + drivelength = driveenergy/(averageforce*4.44822) + slip = rowdatadf.ix[:,'slip'] + if windowsize > 3: + wash = savgol_filter(wash,windowsize,3) + slip = savgol_filter(slip,windowsize,3) + catch = savgol_filter(catch,windowsize,3) + finish = savgol_filter(finish,windowsize,3) + peakforceangle = savgol_filter(peakforceangle,windowsize,3) + driveenergy = savgol_filter(driveenergy,windowsize,3) + drivelength = savgol_filter(drivelength,windowsize,3) + data['wash'] = wash + data['catch'] = catch + data['slip'] = slip + data['finish'] = finish + data['peakforceangle'] = peakforceangle + data['driveenergy'] = driveenergy + data['drivelength'] = drivelength + data['peakforce'] = peakforce + data['averageforce'] = averageforce + except KeyError: + pass + + if otwpower: + try: + nowindpace = rowdatadf.ix[:,'nowindpace'] + except KeyError: + nowindpace = p + try: + equivergpower = rowdatadf.ix[:,'equivergpower'] + except KeyError: + equivergpower = 0*p+50. + + #nowindpace = nowindpace.apply(lambda x: timedeltaconv(x)) + ergvelo = (equivergpower/2.8)**(1./3.) + + ergpace = 500./ergvelo + ergpace[ergpace == np.inf] = 240. + #ergpace = ergpace.apply(lambda x: timedeltaconv(x)) + + data['ergpace'] = ergpace + data['nowindpace'] = nowindpace + data['equivergpower'] = equivergpower + data['fergpace'] = nicepaceformat(ergpace) + data['fnowindpace'] = nicepaceformat(nowindpace) + + data = data.replace([-np.inf,np.inf],np.nan) + data = data.fillna(method='ffill') + + # write data if id given + if id != 0: + data['workoutid'] = id + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + with engine.connect() as conn, conn.begin(): + data.to_sql('strokedata',engine,if_exists='append',index=False) + conn.close() + engine.dispose() + return data diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index c476e051..2be3c390 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -875,6 +875,7 @@ def interactive_flex_chart2(id=0,promember=0, axlabels = { 'time': 'Time', 'distance': 'Distance (m)', + 'cumdist': 'Distance (m)', 'hr': 'Heart Rate (bpm)', 'spm': 'Stroke Rate (spm)', 'pace': 'Pace (/500m)', @@ -962,7 +963,7 @@ def interactive_flex_chart2(id=0,promember=0, if xparam=='time': xaxmax = tseconds.max() xaxmin = tseconds.min() - elif xparam=='distance': + elif xparam=='distance' or xparam=='cumdist': xaxmax = rowdata['x1'].max() xaxmin = rowdata['x1'].min() else: @@ -1043,7 +1044,7 @@ def interactive_flex_chart2(id=0,promember=0, text_color='green', ) - if (xparam != 'time') and (xparam != 'distance'): + if (xparam != 'time') and (xparam != 'distance') and (xparam != 'cumdist'): plot.add_layout(x1means) plot.add_layout(xlabel) @@ -1080,7 +1081,7 @@ def interactive_flex_chart2(id=0,promember=0, yrange1 = Range1d(start=yaxminima[yparam1],end=yaxmaxima[yparam1]) plot.y_range = yrange1 - if (xparam != 'time') and (xparam != 'distance'): + if (xparam != 'time') and (xparam != 'distance') and (xparam != 'cumdist'): xrange1 = Range1d(start=yaxminima[xparam],end=yaxmaxima[xparam]) plot.x_range = xrange1 diff --git a/rowers/tasks.py b/rowers/tasks.py index e9606cea..40bf0a5b 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -15,6 +15,8 @@ from matplotlib import figure import stravalib +from rowers.dataprepnodjango import update_strokedata + from django.core.mail import send_mail, BadHeaderError,EmailMessage @@ -93,7 +95,9 @@ def handle_sendemailcsv(first_name,last_name,email,csvfile): return 1 @app.task -def handle_otwsetpower(f1,boattype,weightvalue,first_name,last_name,email,workoutid): +def handle_otwsetpower(f1,boattype,weightvalue, + first_name,last_name,email,workoutid, + debug=False): rowdata = rdata(f1) weightvalue = float(weightvalue) @@ -112,11 +116,19 @@ def handle_otwsetpower(f1,boattype,weightvalue,first_name,last_name,email,workou rg = rowingdata.getrigging('static/rigging/1x.txt') # do calculation - rowdata.otw_setpower_silent(skiprows=5,mc=weightvalue,rg=rg) + powermeasured = False + try: + w = rowdata.df['wash'] + powermeasured = True + except KeyError: + pass + + rowdata.otw_setpower_silent(skiprows=5,mc=weightvalue,rg=rg, + powermeasured=powermeasured) # save data rowdata.write_csv(f1) - dataprep.update_strokedata(workoutid,rowdata.df) + update_strokedata(workoutid,rowdata.df,debug=debug) # send email fullemail = first_name + " " + last_name + " " + "<" + email + ">" @@ -128,7 +140,7 @@ def handle_otwsetpower(f1,boattype,weightvalue,first_name,last_name,email,workou message += "Rowsandall OTW calculations have not been fully implemented yet.\n" message += "We are now running an experimental version for debugging purposes. \n" message += "Your wind/stream corrected plot is available here: http://rowsandall.com/rowers/workout/" - message += workoutid + message += str(workoutid) message +="/interactiveotwplot\n\n" # message += "This functionality will be available soon, though.\n\n" message += "Please report any bugs/inconsistencies/unexpected results at rowsandall.slack.com or by reply to this email.\n\n" diff --git a/rowsanda_107501 b/rowsanda_107501 new file mode 100644 index 00000000..5ccda6a8 Binary files /dev/null and b/rowsanda_107501 differ