Private
Public Access
1
0
Files
rowsandall/rowers/dataprepnodjango.py
2017-08-07 09:09:38 +02:00

761 lines
22 KiB
Python

# This is Data prep used for testing purposes (no Django environment)
# Uses the debug SQLite database for stroke data
from rowingdata import rowingdata as rrdata
from rowingdata import rower as rrower
from rowingdata import main as rmain
from pandas import DataFrame,Series
import pandas as pd
import numpy as np
import itertools
from sqlalchemy import create_engine
import sqlalchemy as sa
from rowsandall_app.settings import DATABASES
#from rowsandall_app.settings_dev import DATABASES
from utils import lbstoN
try:
user = DATABASES['default']['USER']
except KeyError:
user = ''
try:
password = DATABASES['default']['PASSWORD']
except KeyError:
password = ''
try:
database_name = DATABASES['default']['NAME']
except KeyError:
database_name = ''
try:
host = DATABASES['default']['HOST']
except KeyError:
host = ''
try:
port = DATABASES['default']['PORT']
except KeyError:
port = ''
database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format(
user=user,
password=password,
database_name=database_name,
host=host,
port=port,
)
database_url_debug = 'sqlite:///'+database_name
# mapping the DB column names to the CSV file column names
columndict = {
'time':'TimeStamp (sec)',
'hr':' HRCur (bpm)',
'pace':' Stroke500mPace (sec/500m)',
'spm':' Cadence (stokes/min)',
'power':' Power (watts)',
'averageforce':' AverageDriveForce (lbs)',
'drivelength':' DriveLength (meters)',
'peakforce':' PeakDriveForce (lbs)',
'distance':' Horizontal (meters)',
'catch':'catch',
'finish':'finish',
'peakforceangle':'peakforceangle',
'wash':'wash',
'slip':'wash',
'workoutstate':' WorkoutState',
}
from scipy.signal import savgol_filter
import datetime
def niceformat(values):
out = []
for v in values:
formattedv = strfdelta(v)
out.append(formattedv)
return out
def strfdelta(tdelta):
try:
minutes,seconds = divmod(tdelta.seconds,60)
tenths = int(tdelta.microseconds/1e5)
except AttributeError:
minutes,seconds = divmod(tdelta.view(np.int64),60e9)
seconds,rest = divmod(seconds,1e9)
tenths = int(rest/1e8)
res = "{minutes:0>2}:{seconds:0>2}.{tenths:0>1}".format(
minutes=minutes,
seconds=seconds,
tenths=tenths,
)
return res
def nicepaceformat(values):
out = []
for v in values:
formattedv = strfdelta(v)
out.append(formattedv)
return out
def timedeltaconv(x):
if not np.isnan(x):
dt = datetime.timedelta(seconds=x)
else:
dt = datetime.timedelta(seconds=350.)
return dt
def rdata(file,rower=rrower()):
try:
res = rrdata(file,rower=rower)
except IOError:
try:
res = rrdata(file+'.gz',rower=rower)
except IOError:
res = 0
return res
# Processes painsled CSV file to database
def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
dosummary=True,title='Workout',
notes='',totaldist=0,totaltime=0,
summary='',
makeprivate=False,
oarlength=2.89,inboard=0.88):
message = None
powerperc = 100*np.array([r.pw_ut2,
r.pw_ut1,
r.pw_at,
r.pw_tr,r.pw_an])/r.ftp
# make workout and put in database
rr = rrower(hrmax=r.max,hrut2=r.ut2,
hrut1=r.ut1,hrat=r.at,
hrtr=r.tr,hran=r.an,ftp=r.ftp,
powerperc=powerperc,powerzones=r.powerzones)
row = rdata(f2,rower=rr)
checks = row.check_consistency()
allchecks = 1
for key,value in checks.iteritems():
if not value:
allchecks = 0
if not allchecks:
#row.repair()
pass
if row == 0:
return (0,'Error: CSV data file not found')
if dosmooth:
# auto smoothing
pace = row.df[' Stroke500mPace (sec/500m)'].values
velo = 500./pace
f = row.df['TimeStamp (sec)'].diff().mean()
if f !=0:
windowsize = 2*(int(10./(f)))+1
else:
windowsize = 1
if not 'originalvelo' in row.df:
row.df['originalvelo'] = velo
if windowsize > 3 and windowsize<len(velo):
velo2 = savgol_filter(velo,windowsize,3)
else:
velo2 = velo
velo3 = pd.Series(velo2)
velo3 = velo3.replace([-np.inf,np.inf],np.nan)
velo3 = velo3.fillna(method='ffill')
pace2 = 500./abs(velo3)
row.df[' Stroke500mPace (sec/500m)'] = pace2
row.df = row.df.fillna(0)
row.write_csv(f2,gzip=True)
try:
os.remove(f2)
except:
pass
# recalculate power data
if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
try:
row.erg_recalculatepower()
row.write_csv(f2,gzip=True)
except:
pass
averagehr = row.df[' HRCur (bpm)'].mean()
maxhr = row.df[' HRCur (bpm)'].max()
if totaldist == 0:
totaldist = row.df['cum_dist'].max()
if totaltime == 0:
totaltime = row.df['TimeStamp (sec)'].max()-row.df['TimeStamp (sec)'].min()
totaltime = totaltime+row.df.ix[0,' ElapsedTime (sec)']
hours = int(totaltime/3600.)
if hours>23:
message = 'Warning: The workout duration was longer than 23 hours. '
hours = 23
minutes = int((totaltime - 3600.*hours)/60.)
if minutes>59:
minutes = 59
if not message:
message = 'Warning: there is something wrong with the workout duration'
seconds = int(totaltime - 3600.*hours - 60.*minutes)
if seconds > 59:
seconds = 59
if not message:
message = 'Warning: there is something wrong with the workout duration'
tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds))
if tenths > 9:
tenths = 9
if not message:
message = 'Warning: there is something wrong with the workout duration'
duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths)
if dosummary:
summary = row.summary()
summary += '\n'
summary += row.intervalstats()
workoutdate = row.rowdatetime.strftime('%Y-%m-%d')
workoutstarttime = row.rowdatetime.strftime('%H:%M:%S')
workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc)
if makeprivate:
privacy = 'private'
else:
privacy = 'visible'
# check for duplicate start times
ws = Workout.objects.filter(startdatetime=workoutstartdatetime,
user=r)
if (len(ws) != 0):
message = "Warning: This workout probably already exists in the database"
privacy = 'private'
w = Workout(user=r,name=title,date=workoutdate,
workouttype=workouttype,
duration=duration,distance=totaldist,
weightcategory=r.weightcategory,
starttime=workoutstarttime,
csvfilename=f2,notes=notes,summary=summary,
maxhr=maxhr,averagehr=averagehr,
startdatetime=workoutstartdatetime,
inboard=inboard,oarlength=oarlength,
privacy=privacy)
w.save()
if privacy == 'visible':
ts = Team.objects.filter(rower=r)
for t in ts:
w.team.add(t)
# put stroke data in database
res = dataprep(row.df,id=w.id,bands=True,
barchart=True,otwpower=True,empower=True,inboard=inboard)
return (w.id,message)
def handle_nonpainsled(f2,fileformat,summary=''):
oarlength = 2.89
inboard = 0.88
# handle RowPro:
if (fileformat == 'rp'):
row = RowProParser(f2)
# handle TCX
if (fileformat == 'tcx'):
row = TCXParser(f2)
# handle Mystery
if (fileformat == 'mystery'):
row = MysteryParser(f2)
# handle TCX no HR
if (fileformat == 'tcxnohr'):
row = TCXParserNoHR(f2)
# handle RowPerfect
if (fileformat == 'rowperfect3'):
row = RowPerfectParser(f2)
# handle ErgData
if (fileformat == 'ergdata'):
row = ErgDataParser(f2)
# handle Mike
if (fileformat == 'bcmike'):
row = BoatCoachAdvancedParser(f2)
# handle BoatCoach
if (fileformat == 'boatcoach'):
row = BoatCoachParser(f2)
# handle painsled desktop
if (fileformat == 'painsleddesktop'):
row = painsledDesktopParser(f2)
# handle speed coach GPS
if (fileformat == 'speedcoach'):
row = speedcoachParser(f2)
# handle speed coach GPS 2
if (fileformat == 'speedcoach2'):
row = SpeedCoach2Parser(f2)
try:
oarlength,inboard = get_empower_rigging(f2)
summary = row.allstats()
except:
pass
# handle ErgStick
if (fileformat == 'ergstick'):
row = ErgStickParser(f2)
# handle FIT
if (fileformat == 'fit'):
row = FITParser(f2)
s = fitsummarydata(f2)
s.setsummary()
summary = s.summarytext
f_to_be_deleted = f2
# should delete file
f2 = f2[:-4]+'o.csv'
row.write_csv(f2,gzip=True)
#os.remove(f2)
try:
os.remove(f_to_be_deleted)
except:
os.remove(f_to_be_deleted+'.gz')
return (f2,summary,oarlength,inboard)
# Create new workout from file and store it in the database
# This routine should be used everywhere in views.py and mailprocessing.py
# Currently there is code duplication
def new_workout_from_file(r,f2,
workouttype='rower',
title='Workout',
makeprivate=False,
notes=''):
message = None
fileformat = get_file_type(f2)
summary = ''
oarlength = 2.89
inboard = 0.88
if len(fileformat)==3 and fileformat[0]=='zip':
f_to_be_deleted = f2
with zipfile.ZipFile(f2) as z:
for fname in z.namelist():
f3 = z.extract(fname,path='media/')
id,message,f2 = new_workout_from_file(r,f3,
workouttype=workouttype,
makeprivate=makeprivate,
title = title,
notes='')
os.remove(f_to_be_deleted)
return id,message,f2
# Some people try to upload Concept2 logbook summaries
if fileformat == 'c2log':
os.remove(f2)
message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook."
return (0,message,f2)
if fileformat == 'nostrokes':
os.remove(f2)
message = "It looks like this file doesn't contain stroke data."
return (0,message,f2)
# Some people try to upload RowPro summary logs
if fileformat == 'rowprolog':
os.remove(f2)
message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log."
return (0,message,f2)
# Sometimes people try an unsupported file type.
# Send an email to info@rowsandall.com with the file attached
# for me to check if it is a bug, or a new file type
# worth supporting
if fileformat == 'unknown':
message = "We couldn't recognize the file type"
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2,
r.user.email)
else:
res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2,r.user.email)
return (0,message,f2)
# handle non-Painsled by converting it to painsled compatible CSV
if (fileformat != 'csv'):
try:
f2,summary,oarlength,inboard = handle_nonpainsled(f2,
fileformat,
summary=summary)
except:
errorstring = str(sys.exc_info()[0])
message = 'Something went wrong: '+errorstring
return (0,message,'')
dosummary = (fileformat != 'fit')
id,message = save_workout_database(f2,r,
workouttype=workouttype,
makeprivate=makeprivate,
dosummary=dosummary,
summary=summary,
inboard=inboard,oarlength=oarlength,
title=title)
return (id,message,f2)
def delete_strokedata(id,debug=True):
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format(
id=id,
))
with engine.connect() as conn, conn.begin():
try:
result = conn.execute(query)
except:
print "Database Locked"
conn.close()
engine.dispose()
def update_strokedata(id,df,debug=True):
delete_strokedata(id)
rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True,
debug=debug)
def testdata(time,distance,pace,spm):
t1 = np.issubdtype(time,np.number)
t2 = np.issubdtype(distance,np.number)
t3 = np.issubdtype(pace,np.number)
t4 = np.issubdtype(spm,np.number)
return t1 and t2 and t3 and t4
def getsmallrowdata_db(columns,ids=[]):
data = read_cols_df_sql(ids,columns)
return data
def read_cols_df_sql(ids,columns,debug=True):
columns = list(columns)+['distance','spm']
columns = [x for x in columns if x != 'None']
columns = list(set(columns))
cls = ''
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
for column in columns:
cls += column+', '
cls = cls[:-2]
if len(ids) == 0:
query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid=0'.format(
columns = cls,
))
elif len(ids) == 1:
query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid={id}'.format(
id = ids[0],
columns = cls,
))
else:
query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid IN {ids}'.format(
columns = cls,
ids = tuple(ids),
))
df = pd.read_sql_query(query,engine)
engine.dispose()
return df
def read_df_sql(id,debug=True):
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id}'.format(
id=id)), engine)
engine.dispose()
return df
def smalldataprep(therows,xparam,yparam1,yparam2):
df = pd.DataFrame()
if yparam2 == 'None':
yparam2 = 'power'
df[xparam] = []
df[yparam1] = []
df[yparam2] = []
df['distance'] = []
df['spm'] = []
for workout in therows:
f1 = workout.csvfilename
try:
rowdata = dataprep(rrdata(f1).df)
rowdata = pd.DataFrame({xparam: rowdata[xparam],
yparam1: rowdata[yparam1],
yparam2: rowdata[yparam2],
'distance': rowdata['distance'],
'spm': rowdata['spm'],
}
)
df = pd.concat([df,rowdata],ignore_index=True)
except IOError:
try:
rowdata = dataprep(rrdata(f1+'.gz').df)
rowdata = pd.DataFrame({xparam: rowdata[xparam],
yparam1: rowdata[yparam1],
yparam2: rowdata[yparam2],
'distance': rowdata['distance'],
'spm': rowdata['spm'],
}
)
df = pd.concat([df,rowdata],ignore_index=True)
except IOError:
pass
return df
def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
empower=True,debug=True):
if rowdatadf.empty:
return 0
rowdatadf.set_index([range(len(rowdatadf))],inplace=True)
t = rowdatadf.ix[:,'TimeStamp (sec)']
t = pd.Series(t-rowdatadf.ix[0,'TimeStamp (sec)'])
row_index = rowdatadf.ix[:,' Stroke500mPace (sec/500m)'] > 3000
rowdatadf.loc[row_index,' Stroke500mPace (sec/500m)'] = 3000.
p = rowdatadf.ix[:,' Stroke500mPace (sec/500m)']
hr = rowdatadf.ix[:,' HRCur (bpm)']
spm = rowdatadf.ix[:,' Cadence (stokes/min)']
cumdist = rowdatadf.ix[:,'cum_dist']
power = rowdatadf.ix[:,' Power (watts)']
averageforce = rowdatadf.ix[:,' AverageDriveForce (lbs)']
drivelength = rowdatadf.ix[:,' DriveLength (meters)']
try:
workoutstate = rowdatadf.ix[:,' WorkoutState']
except KeyError:
workoutstate = 0*hr
peakforce = rowdatadf.ix[:,' PeakDriveForce (lbs)']
forceratio = averageforce/peakforce
forceratio = forceratio.fillna(value=0)
f = rowdatadf['TimeStamp (sec)'].diff().mean()
if f != 0:
windowsize = 2*(int(10./(f)))+1
else:
windowsize = 1
if windowsize <= 3:
windowsize = 5
if windowsize > 3 and windowsize<len(hr):
spm = savgol_filter(spm,windowsize,3)
hr = savgol_filter(hr,windowsize,3)
drivelength = savgol_filter(drivelength,windowsize,3)
forceratio = savgol_filter(forceratio,windowsize,3)
try:
t2 = t.fillna(method='ffill').apply(lambda x: timedeltaconv(x))
except TypeError:
t2 = 0*t
p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x))
drivespeed = drivelength/rowdatadf[' DriveTime (ms)']*1.0e3
drivespeed = drivespeed.fillna(value=0)
driveenergy = drivelength*averageforce*lbstoN
distance = rowdatadf.ix[:,'cum_dist']
data = DataFrame(
dict(
time = t*1e3,
hr = hr,
pace = p*1e3,
spm = spm,
cumdist = cumdist,
ftime = niceformat(t2),
fpace = nicepaceformat(p2),
driveenergy=driveenergy,
power=power,
workoutstate=workoutstate,
averageforce=averageforce,
drivelength=drivelength,
peakforce=peakforce,
forceratio=forceratio,
distance=distance,
drivespeed=drivespeed,
)
)
if bands:
# HR bands
data['hr_ut2'] = rowdatadf.ix[:,'hr_ut2']
data['hr_ut1'] = rowdatadf.ix[:,'hr_ut1']
data['hr_at'] = rowdatadf.ix[:,'hr_at']
data['hr_tr'] = rowdatadf.ix[:,'hr_tr']
data['hr_an'] = rowdatadf.ix[:,'hr_an']
data['hr_max'] = rowdatadf.ix[:,'hr_max']
data['hr_bottom'] = 0.0*data['hr']
if barchart:
# time increments for bar chart
time_increments = rowdatadf.ix[:,' ElapsedTime (sec)'].diff()
time_increments[0] = time_increments[1]
time_increments = 0.5*time_increments+0.5*np.abs(time_increments)
x_right = (t2+time_increments.apply(lambda x:timedeltaconv(x)))
data['x_right'] = x_right
if empower:
try:
wash = rowdatadf.ix[:,'wash']
catch = rowdatadf.ix[:,'catch']
finish = rowdatadf.ix[:,'finish']
peakforceangle = rowdatadf.ix[:,'peakforceangle']
driveenergy = rowdatadf.ix[:,'driveenergy']
drivelength = driveenergy/(averageforce*4.44822)
slip = rowdatadf.ix[:,'slip']
if windowsize > 3:
wash = savgol_filter(wash,windowsize,3)
slip = savgol_filter(slip,windowsize,3)
catch = savgol_filter(catch,windowsize,3)
finish = savgol_filter(finish,windowsize,3)
peakforceangle = savgol_filter(peakforceangle,windowsize,3)
driveenergy = savgol_filter(driveenergy,windowsize,3)
drivelength = savgol_filter(drivelength,windowsize,3)
data['wash'] = wash
data['catch'] = catch
data['slip'] = slip
data['finish'] = finish
data['peakforceangle'] = peakforceangle
data['driveenergy'] = driveenergy
data['drivelength'] = drivelength
data['peakforce'] = peakforce
data['averageforce'] = averageforce
except KeyError:
pass
if otwpower:
try:
nowindpace = rowdatadf.ix[:,'nowindpace']
except KeyError:
nowindpace = p
try:
equivergpower = rowdatadf.ix[:,'equivergpower']
except KeyError:
equivergpower = 0*p+50.
nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x))
ergvelo = (equivergpower/2.8)**(1./3.)
ergpace = 500./ergvelo
ergpace[ergpace == np.inf] = 240.
ergpace2 = ergpace.apply(lambda x: timedeltaconv(x))
velo = 500./p
ergpw = 2.8*velo**3
efficiency = 100.*ergpw/power
efficiency = efficiency.replace([-np.inf,np.inf],np.nan)
efficiency.fillna(method='ffill')
data['ergpace'] = ergpace*1e3
data['nowindpace'] = nowindpace*1e3
data['equivergpower'] = equivergpower
data['fergpace'] = nicepaceformat(ergpace2)
data['fnowindpace'] = nicepaceformat(nowindpace2)
data['efficiency'] = efficiency
data = data.replace([-np.inf,np.inf],np.nan)
data = data.fillna(method='ffill')
# write data if id given
if id != 0:
data['workoutid'] = id
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
with engine.connect() as conn, conn.begin():
data.to_sql('strokedata',engine,if_exists='append',index=False)
conn.close()
engine.dispose()
return data