sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

cleaning up

Browse Source
This commit is contained in:
Sander Roosendaal
2017-10-22 11:33:18 +02:00
parent f0bfb810a4
commit dd05cf2d1b
4 changed files with 678 additions and 687 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
rowers/dataprep.py
View File

@@ -61,7 +61,6 @@ queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('default') queuehigh = django_rq.get_queue('default')
user = settings.DATABASES['default']['USER'] user = settings.DATABASES['default']['USER']
password = settings.DATABASES['default']['PASSWORD'] password = settings.DATABASES['default']['PASSWORD']
database_name = settings.DATABASES['default']['NAME'] database_name = settings.DATABASES['default']['NAME']
@@ -106,6 +105,7 @@ from scipy.signal import savgol_filter
import datetime import datetime
def get_latlon(id): def get_latlon(id):
try: try:
w = Workout.objects.get(id=id) w = Workout.objects.get(id=id)
@@ -126,6 +126,7 @@ def get_latlon(id):
return [pd.Series([]), pd.Series([])] return [pd.Series([]), pd.Series([])]
def get_workouts(ids, userid): def get_workouts(ids, userid):
goodids = [] goodids = []
for id in ids: for id in ids:
@@ -135,6 +136,7 @@ def get_workouts(ids,userid):
return [Workout.objects.get(id=id) for id in goodids] return [Workout.objects.get(id=id) for id in goodids]
def filter_df(datadf, fieldname, value, largerthan=True): def filter_df(datadf, fieldname, value, largerthan=True):
try: try:
@@ -142,7 +144,6 @@ def filter_df(datadf,fieldname,value,largerthan=True):
except KeyError: except KeyError:
return datadf return datadf
if largerthan: if largerthan:
mask = datadf[fieldname] < value mask = datadf[fieldname] < value
else: else:
@@ -150,7 +151,6 @@ def filter_df(datadf,fieldname,value,largerthan=True):
datadf.loc[mask, fieldname] = np.nan datadf.loc[mask, fieldname] = np.nan
return datadf return datadf
@@ -185,7 +185,6 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True,
datadf = datadf.clip(lower=0) datadf = datadf.clip(lower=0)
datadf.replace(to_replace=0, value=np.nan, inplace=True) datadf.replace(to_replace=0, value=np.nan, inplace=True)
# return from positive domain to negative # return from positive domain to negative
try: try:
datadf['catch'] = -datadf['catch'] datadf['catch'] = -datadf['catch']
@@ -216,7 +215,6 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True,
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['pace'] / 1000. > 300. mask = datadf['pace'] / 1000. > 300.
datadf.loc[mask, 'pace'] = np.nan datadf.loc[mask, 'pace'] = np.nan
@@ -241,14 +239,12 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True,
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['wash'] < 1 mask = datadf['wash'] < 1
datadf.loc[mask, 'wash'] = np.nan datadf.loc[mask, 'wash'] = np.nan
except KeyError: except KeyError:
pass pass
if not ignoreadvanced: if not ignoreadvanced:
try: try:
mask = datadf['rhythm'] < 5 mask = datadf['rhythm'] < 5
@@ -256,79 +252,66 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True,
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['rhythm'] > 70 mask = datadf['rhythm'] > 70
datadf.loc[mask, 'rhythm'] = np.nan datadf.loc[mask, 'rhythm'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['power'] < 20 mask = datadf['power'] < 20
datadf.loc[mask, 'power'] = np.nan datadf.loc[mask, 'power'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['drivelength'] < 0.5 mask = datadf['drivelength'] < 0.5
datadf.loc[mask, 'drivelength'] = np.nan datadf.loc[mask, 'drivelength'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['forceratio'] < 0.2 mask = datadf['forceratio'] < 0.2
datadf.loc[mask, 'forceratio'] = np.nan datadf.loc[mask, 'forceratio'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['forceratio'] > 1.0 mask = datadf['forceratio'] > 1.0
datadf.loc[mask, 'forceratio'] = np.nan datadf.loc[mask, 'forceratio'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['drivespeed'] < 0.5 mask = datadf['drivespeed'] < 0.5
datadf.loc[mask, 'drivespeed'] = np.nan datadf.loc[mask, 'drivespeed'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['drivespeed'] > 4 mask = datadf['drivespeed'] > 4
datadf.loc[mask, 'drivespeed'] = np.nan datadf.loc[mask, 'drivespeed'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['driveenergy'] > 2000 mask = datadf['driveenergy'] > 2000
datadf.loc[mask, 'driveenergy'] = np.nan datadf.loc[mask, 'driveenergy'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['driveenergy'] < 100 mask = datadf['driveenergy'] < 100
datadf.loc[mask, 'driveenergy'] = np.nan datadf.loc[mask, 'driveenergy'] = np.nan
except KeyError: except KeyError:
pass pass
try: try:
mask = datadf['catch'] > -30. mask = datadf['catch'] > -30.
datadf.loc[mask, 'catch'] = np.nan datadf.loc[mask, 'catch'] = np.nan
except KeyError: except KeyError:
pass pass
workoutstateswork = [1, 4, 5, 8, 9, 6, 7] workoutstateswork = [1, 4, 5, 8, 9, 6, 7]
workoutstatesrest = [3] workoutstatesrest = [3]
workoutstatetransition = [0, 2, 10, 11, 12, 13] workoutstatetransition = [0, 2, 10, 11, 12, 13]
@@ -341,6 +324,7 @@ def clean_df_stats(datadf,workstrokesonly=True,ignorehr=True,
return datadf return datadf
def getstatsfields(): def getstatsfields():
# Get field names and remove those that are not useful in stats # Get field names and remove those that are not useful in stats
fields = StrokeData._meta.get_fields() fields = StrokeData._meta.get_fields()
@@ -385,6 +369,8 @@ def niceformat(values):
return out return out
# A nice printable format for time delta values # A nice printable format for time delta values
def strfdelta(tdelta): def strfdelta(tdelta):
try: try:
minutes, seconds = divmod(tdelta.seconds, 60) minutes, seconds = divmod(tdelta.seconds, 60)
@@ -402,25 +388,28 @@ def strfdelta(tdelta):
return res return res
# A nice printable format for pace values # A nice printable format for pace values
def nicepaceformat(values): def nicepaceformat(values):
out = [] out = []
for v in values: for v in values:
formattedv = strfdelta(v) formattedv = strfdelta(v)
out.append(formattedv) out.append(formattedv)
return out return out
# Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace # Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace
def timedeltaconv(x): def timedeltaconv(x):
if np.isfinite(x) and x != 0 and x > 0 and x < 175000: if np.isfinite(x) and x != 0 and x > 0 and x < 175000:
dt = datetime.timedelta(seconds=x) dt = datetime.timedelta(seconds=x)
else: else:
dt = datetime.timedelta(seconds=350.) dt = datetime.timedelta(seconds=350.)
return dt return dt
def paceformatsecs(values): def paceformatsecs(values):
out = [] out = []
for v in values: for v in values:
@@ -431,6 +420,8 @@ def paceformatsecs(values):
return out return out
# Processes painsled CSV file to database # Processes painsled CSV file to database
def save_workout_database(f2, r, dosmooth=True, workouttype='rower', def save_workout_database(f2, r, dosmooth=True, workouttype='rower',
dosummary=True, title='Workout', dosummary=True, title='Workout',
workoutsource='unknown', workoutsource='unknown',
@@ -470,7 +461,8 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
if not value: if not value:
allchecks = 0 allchecks = 0
if consistencychecks: if consistencychecks:
a_messages.error(r.user,'Failed consistency check: '+key+', autocorrected') a_messages.error(
r.user, 'Failed consistency check: ' + key + ', autocorrected')
else: else:
pass pass
# a_messages.error(r.user,'Failed consistency check: '+key+', not corrected') # a_messages.error(r.user,'Failed consistency check: '+key+', not corrected')
@@ -481,7 +473,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
# row.repair() # row.repair()
pass pass
if row == 0: if row == 0:
return (0, 'Error: CSV data file not found') return (0, 'Error: CSV data file not found')
@@ -533,7 +524,8 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
if totaldist == 0: if totaldist == 0:
totaldist = row.df['cum_dist'].max() totaldist = row.df['cum_dist'].max()
if totaltime == 0: if totaltime == 0:
totaltime = row.df['TimeStamp (sec)'].max()-row.df['TimeStamp (sec)'].min() totaltime = row.df['TimeStamp (sec)'].max(
) - row.df['TimeStamp (sec)'].min()
try: try:
totaltime = totaltime + row.df.ix[0, ' ElapsedTime (sec)'] totaltime = totaltime + row.df.ix[0, ' ElapsedTime (sec)']
except KeyError: except KeyError:
@@ -573,7 +565,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
#summary += '\n' #summary += '\n'
#summary += row.intervalstats() #summary += row.intervalstats()
#workoutstartdatetime = row.rowdatetime #workoutstartdatetime = row.rowdatetime
timezone_str = 'UTC' timezone_str = 'UTC'
try: try:
@@ -581,8 +572,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
except ValueError: except ValueError:
workoutstartdatetime = row.rowdatetime workoutstartdatetime = row.rowdatetime
try: try:
latavg = row.df[' latitude'].mean() latavg = row.df[' latitude'].mean()
lonavg = row.df[' longitude'].mean() lonavg = row.df[' longitude'].mean()
@@ -608,8 +597,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
except KeyError: except KeyError:
timezone_str = r.defaulttimezone timezone_str = r.defaulttimezone
workoutdate = workoutstartdatetime.astimezone( workoutdate = workoutstartdatetime.astimezone(
pytz.timezone(timezone_str) pytz.timezone(timezone_str)
).strftime('%Y-%m-%d') ).strftime('%Y-%m-%d')
@@ -636,9 +623,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
message = "Warning: This workout probably already exists in the database" message = "Warning: This workout probably already exists in the database"
privacy = 'hidden' privacy = 'hidden'
w = Workout(user=r, name=title, date=workoutdate, w = Workout(user=r, name=title, date=workoutdate,
workouttype=workouttype, workouttype=workouttype,
duration=duration, distance=totaldist, duration=duration, distance=totaldist,
@@ -653,7 +637,6 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
timezone=timezone_str, timezone=timezone_str,
privacy=privacy) privacy=privacy)
w.save() w.save()
isbreakthrough = False isbreakthrough = False
@@ -668,7 +651,8 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
dfgrouped = df.groupby(['workoutid']) dfgrouped = df.groupby(['workoutid'])
delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr)
res,btvalues,res2 = utils.isbreakthrough(delta,cpvalues,r.p0,r.p1,r.p2,r.p3,r.cpratio) res, btvalues, res2 = utils.isbreakthrough(
delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio)
else: else:
res = 0 res = 0
res2 = 0 res2 = 0
@@ -680,7 +664,8 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
# submit email task to send email about breakthrough workout # submit email task to send email about breakthrough workout
if isbreakthrough: if isbreakthrough:
a_messages.info(r.user,'It looks like you have a new breakthrough workout') a_messages.info(
r.user, 'It looks like you have a new breakthrough workout')
if settings.DEBUG and r.getemailnotifications: if settings.DEBUG and r.getemailnotifications:
res = handle_sendemail_breakthrough.delay(w.id, r.user.email, res = handle_sendemail_breakthrough.delay(w.id, r.user.email,
r.user.first_name, r.user.first_name,
@@ -730,6 +715,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
return (w.id, message) return (w.id, message)
def handle_nonpainsled(f2, fileformat, summary=''): def handle_nonpainsled(f2, fileformat, summary=''):
oarlength = 2.89 oarlength = 2.89
inboard = 0.88 inboard = 0.88
@@ -752,7 +738,6 @@ def handle_nonpainsled(f2,fileformat,summary=''):
if (fileformat == 'ergdata'): if (fileformat == 'ergdata'):
row = ErgDataParser(f2) row = ErgDataParser(f2)
# handle CoxMate # handle CoxMate
if (fileformat == 'coxmate'): if (fileformat == 'coxmate'):
row = CoxMateParser(f2) row = CoxMateParser(f2)
@@ -786,7 +771,6 @@ def handle_nonpainsled(f2,fileformat,summary=''):
except: except:
pass pass
# handle ErgStick # handle ErgStick
if (fileformat == 'ergstick'): if (fileformat == 'ergstick'):
row = ErgStickParser(f2) row = ErgStickParser(f2)
@@ -801,7 +785,6 @@ def handle_nonpainsled(f2,fileformat,summary=''):
except: except:
pass pass
f_to_be_deleted = f2 f_to_be_deleted = f2
# should delete file # should delete file
f2 = f2[:-4] + 'o.csv' f2 = f2[:-4] + 'o.csv'
@@ -818,6 +801,8 @@ def handle_nonpainsled(f2,fileformat,summary=''):
# Create new workout from file and store it in the database # Create new workout from file and store it in the database
# This routine should be used everywhere in views.py and mailprocessing.py # This routine should be used everywhere in views.py and mailprocessing.py
# Currently there is code duplication # Currently there is code duplication
def new_workout_from_file(r, f2, def new_workout_from_file(r, f2,
workouttype='rower', workouttype='rower',
title='Workout', title='Workout',
@@ -869,7 +854,6 @@ def new_workout_from_file(r,f2,
message = "Rowsandall could not process this file. The extension is supported but the file seems corrupt. Contact info@rowsandall.com if you think this is incorrect." message = "Rowsandall could not process this file. The extension is supported but the file seems corrupt. Contact info@rowsandall.com if you think this is incorrect."
return (0, message, f2) return (0, message, f2)
# Some people try to upload RowPro summary logs # Some people try to upload RowPro summary logs
if fileformat == 'rowprolog': if fileformat == 'rowprolog':
os.remove(f2) os.remove(f2)
@@ -902,8 +886,6 @@ def new_workout_from_file(r,f2,
message = 'Something went wrong: ' + errorstring message = 'Something went wrong: ' + errorstring
return (0, message, '') return (0, message, '')
dosummary = (fileformat != 'fit') dosummary = (fileformat != 'fit')
id, message = save_workout_database(f2, r, id, message = save_workout_database(f2, r,
workouttype=workouttype, workouttype=workouttype,
@@ -916,6 +898,7 @@ def new_workout_from_file(r,f2,
return (id, message, f2) return (id, message, f2)
def split_workout(r, parent, splitsecond, splitmode): def split_workout(r, parent, splitsecond, splitmode):
data, row = getrowdata_db(id=parent.id) data, row = getrowdata_db(id=parent.id)
latitude, longitude = get_latlon(parent.id) latitude, longitude = get_latlon(parent.id)
@@ -938,7 +921,6 @@ def split_workout(r,parent,splitsecond,splitmode):
data1['time'] = data1.index data1['time'] = data1.index
data1.reset_index(drop=True, inplace=True) data1.reset_index(drop=True, inplace=True)
data2 = data2.sort_values(['time']) data2 = data2.sort_values(['time'])
data2 = data2.interpolate(method='linear', axis=0, limit_direction='both', data2 = data2.interpolate(method='linear', axis=0, limit_direction='both',
limit=10) limit=10)
@@ -952,7 +934,6 @@ def split_workout(r,parent,splitsecond,splitmode):
data1['pace'] = data1['pace'] / 1000. data1['pace'] = data1['pace'] / 1000.
data2['pace'] = data2['pace'] / 1000. data2['pace'] = data2['pace'] / 1000.
data1.drop_duplicates(subset='time', inplace=True) data1.drop_duplicates(subset='time', inplace=True)
data2.drop_duplicates(subset='time', inplace=True) data2.drop_duplicates(subset='time', inplace=True)
@@ -991,7 +972,6 @@ def split_workout(r,parent,splitsecond,splitmode):
messages.append(message) messages.append(message)
ids.append(id) ids.append(id)
if not 'keep original' in splitmode: if not 'keep original' in splitmode:
if 'keep second' in splitmode or 'keep first' in splitmode: if 'keep second' in splitmode or 'keep first' in splitmode:
parent.delete() parent.delete()
@@ -1008,6 +988,8 @@ def split_workout(r,parent,splitsecond,splitmode):
# Create new workout from data frame and store it in the database # Create new workout from data frame and store it in the database
# This routine should be used everywhere in views.py and mailprocessing.py # This routine should be used everywhere in views.py and mailprocessing.py
# Currently there is code duplication # Currently there is code duplication
def new_workout_from_df(r, df, def new_workout_from_df(r, df,
title='New Workout', title='New Workout',
parent=None, parent=None,
@@ -1042,7 +1024,6 @@ def new_workout_from_df(r,df,
if setprivate: if setprivate:
makeprivate = True makeprivate = True
timestr = strftime("%Y%m%d-%H%M%S") timestr = strftime("%Y%m%d-%H%M%S")
csvfilename = 'media/df_' + timestr + '.csv' csvfilename = 'media/df_' + timestr + '.csv'
@@ -1075,11 +1056,9 @@ def new_workout_from_df(r,df,
dosmooth=False, dosmooth=False,
consistencychecks=False) consistencychecks=False)
return (id, message) return (id, message)
# Compare the data from the CSV file and the database # Compare the data from the CSV file and the database
# Currently only calculates number of strokes. To be expanded with # Currently only calculates number of strokes. To be expanded with
# more elaborate testing if needed # more elaborate testing if needed
@@ -1111,6 +1090,8 @@ def compare_data(id):
# Repair data for workouts where the CSV file is lost (or the DB entries # Repair data for workouts where the CSV file is lost (or the DB entries
# don't exist) # don't exist)
def repair_data(verbose=False): def repair_data(verbose=False):
ws = Workout.objects.all() ws = Workout.objects.all()
for w in ws: for w in ws:
@@ -1153,6 +1134,8 @@ def repair_data(verbose=False):
pass pass
# A wrapper around the rowingdata class, with some error catching # A wrapper around the rowingdata class, with some error catching
def rdata(file, rower=rrower()): def rdata(file, rower=rrower()):
try: try:
res = rrdata(csvfile=file, rower=rower) res = rrdata(csvfile=file, rower=rower)
@@ -1167,6 +1150,8 @@ def rdata(file,rower=rrower()):
return res return res
# Remove all stroke data for workout ID from database # Remove all stroke data for workout ID from database
def delete_strokedata(id): def delete_strokedata(id):
engine = create_engine(database_url, echo=False) engine = create_engine(database_url, echo=False)
query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format(
@@ -1181,11 +1166,15 @@ def delete_strokedata(id):
engine.dispose() engine.dispose()
# Replace stroke data in DB with data from CSV file # Replace stroke data in DB with data from CSV file
def update_strokedata(id, df): def update_strokedata(id, df):
delete_strokedata(id) delete_strokedata(id)
rowdata = dataprep(df, id=id, bands=True, barchart=True, otwpower=True) rowdata = dataprep(df, id=id, bands=True, barchart=True, otwpower=True)
# Test that all data are of a numerical time # Test that all data are of a numerical time
def testdata(time, distance, pace, spm): def testdata(time, distance, pace, spm):
t1 = np.issubdtype(time, np.number) t1 = np.issubdtype(time, np.number)
t2 = np.issubdtype(distance, np.number) t2 = np.issubdtype(distance, np.number)
@@ -1196,6 +1185,8 @@ def testdata(time,distance,pace,spm):
# Get data from DB for one workout (fetches all data). If data # Get data from DB for one workout (fetches all data). If data
# is not in DB, read from CSV file (and create DB entry) # is not in DB, read from CSV file (and create DB entry)
def getrowdata_db(id=0, doclean=False, convertnewtons=True): def getrowdata_db(id=0, doclean=False, convertnewtons=True):
data = read_df_sql(id) data = read_df_sql(id)
data['x_right'] = data['x_right'] / 1.0e6 data['x_right'] = data['x_right'] / 1.0e6
@@ -1203,7 +1194,8 @@ def getrowdata_db(id=0,doclean=False,convertnewtons=True):
if data.empty: if data.empty:
rowdata, row = getrowdata(id=id) rowdata, row = getrowdata(id=id)
if rowdata: if rowdata:
data = dataprep(rowdata.df,id=id,bands=True,barchart=True,otwpower=True) data = dataprep(rowdata.df, id=id, bands=True,
barchart=True, otwpower=True)
else: else:
data = pd.DataFrame() # returning empty dataframe data = pd.DataFrame() # returning empty dataframe
else: else:
@@ -1215,27 +1207,26 @@ def getrowdata_db(id=0,doclean=False,convertnewtons=True):
if doclean: if doclean:
data = clean_df_stats(data, ignorehr=True) data = clean_df_stats(data, ignorehr=True)
return data, row return data, row
# Fetch a subset of the data from the DB # Fetch a subset of the data from the DB
def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True): def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True):
prepmultipledata(ids) prepmultipledata(ids)
data = read_cols_df_sql(ids, columns) data = read_cols_df_sql(ids, columns)
# convert newtons # convert newtons
if doclean: if doclean:
data = clean_df_stats(data, ignorehr=True, data = clean_df_stats(data, ignorehr=True,
workstrokesonly=workstrokesonly) workstrokesonly=workstrokesonly)
return data return data
# Fetch both the workout and the workout stroke data (from CSV file) # Fetch both the workout and the workout stroke data (from CSV file)
def getrowdata(id=0): def getrowdata(id=0):
# check if valid ID exists (workout exists) # check if valid ID exists (workout exists)
@@ -1262,6 +1253,8 @@ def getrowdata(id=0):
# In theory, this should never yield any work, but it's a good # In theory, this should never yield any work, but it's a good
# safety net for programming errors elsewhere in the app # safety net for programming errors elsewhere in the app
# Also used heavily when I moved from CSV file only to CSV+Stroke data # Also used heavily when I moved from CSV file only to CSV+Stroke data
def prepmultipledata(ids, verbose=False): def prepmultipledata(ids, verbose=False):
query = sa.text('SELECT DISTINCT workoutid FROM strokedata') query = sa.text('SELECT DISTINCT workoutid FROM strokedata')
engine = create_engine(database_url, echo=False) engine = create_engine(database_url, echo=False)
@@ -1283,11 +1276,14 @@ def prepmultipledata(ids,verbose=False):
if verbose: if verbose:
print id print id
if rowdata and len(rowdata.df): if rowdata and len(rowdata.df):
data = dataprep(rowdata.df,id=id,bands=True,barchart=True,otwpower=True) data = dataprep(rowdata.df, id=id, bands=True,
barchart=True, otwpower=True)
return res return res
# Read a set of columns for a set of workout ids, returns data as a # Read a set of columns for a set of workout ids, returns data as a
# pandas dataframe # pandas dataframe
def read_cols_df_sql(ids, columns, convertnewtons=True): def read_cols_df_sql(ids, columns, convertnewtons=True):
# drop columns that are not in offical list # drop columns that are not in offical list
# axx = [ax[0] for ax in axes] # axx = [ax[0] for ax in axes]
@@ -1321,29 +1317,33 @@ def read_cols_df_sql(ids,columns,convertnewtons=True):
ids=tuple(ids), ids=tuple(ids),
)) ))
connection = engine.raw_connection() connection = engine.raw_connection()
df = pd.read_sql_query(query, engine) df = pd.read_sql_query(query, engine)
df = df.fillna(value=0) df = df.fillna(value=0)
if 'peakforce' in columns: if 'peakforce' in columns:
funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) funits = ((w.id, w.forceunit)
for w in Workout.objects.filter(id__in=ids))
for id, u in funits: for id, u in funits:
if u == 'lbs': if u == 'lbs':
mask = df['workoutid'] == id mask = df['workoutid'] == id
df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN
if 'averageforce' in columns: if 'averageforce' in columns:
funits = ((w.id,w.forceunit) for w in Workout.objects.filter(id__in=ids)) funits = ((w.id, w.forceunit)
for w in Workout.objects.filter(id__in=ids))
for id, u in funits: for id, u in funits:
if u == 'lbs': if u == 'lbs':
mask = df['workoutid'] == id mask = df['workoutid'] == id
df.loc[mask,'averageforce'] = df.loc[mask,'averageforce']*lbstoN df.loc[mask, 'averageforce'] = df.loc[mask,
'averageforce'] * lbstoN
engine.dispose() engine.dispose()
return df return df
# Read stroke data from the DB for a Workout ID. Returns a pandas dataframe # Read stroke data from the DB for a Workout ID. Returns a pandas dataframe
def read_df_sql(id): def read_df_sql(id):
engine = create_engine(database_url, echo=False) engine = create_engine(database_url, echo=False)
@@ -1370,6 +1370,8 @@ def read_df_sql(id):
# Get the necessary data from the strokedata table in the DB. # Get the necessary data from the strokedata table in the DB.
# For the flex plot # For the flex plot
def smalldataprep(therows, xparam, yparam1, yparam2): def smalldataprep(therows, xparam, yparam1, yparam2):
df = pd.DataFrame() df = pd.DataFrame()
if yparam2 == 'None': if yparam2 == 'None':
@@ -1428,10 +1430,11 @@ def smalldataprep(therows,xparam,yparam1,yparam2):
except IOError: except IOError:
pass pass
return df return df
# data fusion # data fusion
def datafusion(id1, id2, columns, offset): def datafusion(id1, id2, columns, offset):
workout1 = Workout.objects.get(id=id1) workout1 = Workout.objects.get(id=id1)
workout2 = Workout.objects.get(id=id2) workout2 = Workout.objects.get(id=id2)
@@ -1456,7 +1459,6 @@ def datafusion(id1,id2,columns,offset):
df1[' latitude'] = latitude df1[' latitude'] = latitude
df1[' longitude'] = longitude df1[' longitude'] = longitude
df2 = getsmallrowdata_db(['time'] + columns, ids=[id2], doclean=False) df2 = getsmallrowdata_db(['time'] + columns, ids=[id2], doclean=False)
forceunit = 'N' forceunit = 'N'
@@ -1465,13 +1467,11 @@ def datafusion(id1,id2,columns,offset):
offsetmillisecs += offset.days * (3600 * 24 * 1000) offsetmillisecs += offset.days * (3600 * 24 * 1000)
df2['time'] = df2['time'] + offsetmillisecs df2['time'] = df2['time'] + offsetmillisecs
keep1 = {c: c for c in set(df1.columns)} keep1 = {c: c for c in set(df1.columns)}
for c in columns: for c in columns:
keep1.pop(c) keep1.pop(c)
for c in df1.columns: for c in df1.columns:
if not c in keep1: if not c in keep1:
df1 = df1.drop(c, 1, errors='ignore') df1 = df1.drop(c, 1, errors='ignore')
@@ -1493,6 +1493,7 @@ def datafusion(id1,id2,columns,offset):
return df, forceunit return df, forceunit
def fix_newtons(id=0, limit=3000): def fix_newtons(id=0, limit=3000):
# rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False)
rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False) rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False)
@@ -1508,6 +1509,7 @@ def fix_newtons(id=0,limit=3000):
except KeyError: except KeyError:
pass pass
def add_efficiency(id=0): def add_efficiency(id=0):
rowdata, row = getrowdata_db(id=id, doclean=False, convertnewtons=False) rowdata, row = getrowdata_db(id=id, doclean=False, convertnewtons=False)
power = rowdata['power'] power = rowdata['power']
@@ -1524,7 +1526,8 @@ def add_efficiency(id=0):
rowdata['workoutid'] = id rowdata['workoutid'] = id
engine = create_engine(database_url, echo=False) engine = create_engine(database_url, echo=False)
with engine.connect() as conn, conn.begin(): with engine.connect() as conn, conn.begin():
rowdata.to_sql('strokedata',engine,if_exists='append',index=False) rowdata.to_sql('strokedata', engine,
if_exists='append', index=False)
conn.close() conn.close()
engine.dispose() engine.dispose()
return rowdata return rowdata
@@ -1533,6 +1536,8 @@ def add_efficiency(id=0):
# it reindexes, sorts, filters, and smooths the data, then # it reindexes, sorts, filters, and smooths the data, then
# saves it to the stroke_data table in the database # saves it to the stroke_data table in the database
# Takes a rowingdata object's DataFrame as input # Takes a rowingdata object's DataFrame as input
def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True,
empower=True, inboard=0.88, forceunit='lbs'): empower=True, inboard=0.88, forceunit='lbs'):
if rowdatadf.empty: if rowdatadf.empty:
@@ -1589,7 +1594,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
except TypeError: except TypeError:
t2 = 0 * t t2 = 0 * t
p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x))
try: try:
@@ -1597,7 +1601,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
except TypeError: except TypeError:
drivespeed = 0.0 * rowdatadf['TimeStamp (sec)'] drivespeed = 0.0 * rowdatadf['TimeStamp (sec)']
drivespeed = drivespeed.fillna(value=0) drivespeed = drivespeed.fillna(value=0)
try: try:
@@ -1613,7 +1616,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
distanceperstroke = 60. * velo / spm distanceperstroke = 60. * velo / spm
data = DataFrame( data = DataFrame(
dict( dict(
time=t * 1e3, time=t * 1e3,
@@ -1682,7 +1684,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
except KeyError: except KeyError:
peakforceangle = 0 * power peakforceangle = 0 * power
if data['driveenergy'].mean() == 0: if data['driveenergy'].mean() == 0:
try: try:
driveenergy = rowdatadf.ix[:, 'driveenergy'] driveenergy = rowdatadf.ix[:, 'driveenergy']
@@ -1691,7 +1692,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
else: else:
driveenergy = data['driveenergy'] driveenergy = data['driveenergy']
arclength = (inboard - 0.05) * (np.radians(finish) - np.radians(catch)) arclength = (inboard - 0.05) * (np.radians(finish) - np.radians(catch))
if arclength.mean() > 0: if arclength.mean() > 0:
drivelength = arclength drivelength = arclength
@@ -1750,7 +1750,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
velo = 500. / p velo = 500. / p
ergpw = 2.8 * velo**3 ergpw = 2.8 * velo**3
efficiency = 100. * ergpw / power efficiency = 100. * ergpw / power

3
rowers/forms.py
View File

@@ -182,7 +182,8 @@ class UploadOptionsForm(forms.Form):
label='Make Workout Private') label='Make Workout Private')
landingpage = forms.ChoiceField(choices=landingpages, landingpage = forms.ChoiceField(choices=landingpages,
initial='workout_edit_view') initial='workout_edit_view',
label='Landing Page')
class Meta: class Meta:
fields = ['make_plot','plottype','upload_toc2','makeprivate'] fields = ['make_plot','plottype','upload_toc2','makeprivate']

17
rowers/templates/document_form.html
View File

@@ -38,23 +38,10 @@
You can select one static plot to be generated immediately for You can select one static plot to be generated immediately for
this workout. You can select to export to major fitness this workout. You can select to export to major fitness
platforms automatically. platforms automatically.
If you check "make private", this workout will not be visible to your followers and will not show up in your teams' workouts list. If you check "make private", this workout will not be visible to your followers and will not show up in your teams' workouts list. With the Landing Page option, you can select to which (workout related) page you will be
taken after a successfull upload.
</p> </p>
<p>
Valid file types are:
<ul>
<li>Painsled iOS Stroke Export (CSV)</li>
<li>Painsled desktop version Stroke Export (CSV)</li>
<li>A TCX file with location data (lat,long) - with or without Heart Rate value, for example from RiM or CrewNerd</li>
<li>RowPro CSV export</li>
<li>SpeedCoach GPS and SpeedCoach GPS 2 CSV export</li>
<li>ErgData CSV export</li>
<li>ErgStick CSV export</li>
<li>BoatCoach CSV export</li>
<li>A FIT file with location data (experimental)</li>
</ul>
</p>
</div> </div>

30
rowers/views.py
View File

@@ -7627,34 +7627,39 @@ def workout_upload_view(request,
plottype = 'timeplot' plottype = 'timeplot'
try: try:
upload_toc2 = uploadoptions['upload_to_C2'] landingpage = uploadoptions['landingpage']
except KeyError: except KeyError:
upload_toc2 = False landingpage = r.defaultlandingpage
try: try:
upload_tostrava = uploadoptions['upload_to_Strava'] upload_to_c2 = uploadoptions['upload_to_C2']
except KeyError: except KeyError:
upload_tostrava = False upload_to_c2 = False
try: try:
upload_tost = uploadoptions['upload_to_SportTracks'] upload_to_strava = uploadoptions['upload_to_Strava']
except KeyError: except KeyError:
upload_tost = False upload_to_strava = False
try: try:
upload_tork = uploadoptions['upload_to_RunKeeper'] upload_to_st = uploadoptions['upload_to_SportTracks']
except KeyError: except KeyError:
upload_tork = False upload_to_st = False
try: try:
upload_toua = uploadoptions['upload_to_MapMyFitness'] upload_to_rk = uploadoptions['upload_to_RunKeeper']
except KeyError: except KeyError:
upload_toua = False upload_to_rk = False
try: try:
upload_totp = uploadoptions['upload_to_TrainingPeaks'] upload_to_ua = uploadoptions['upload_to_MapMyFitness']
except KeyError: except KeyError:
upload_totp = False upload_to_ua = False
try:
upload_to_tp = uploadoptions['upload_to_TrainingPeaks']
except KeyError:
upload_to_tp = False
if request.method == 'POST': if request.method == 'POST':
form = DocumentsForm(request.POST,request.FILES) form = DocumentsForm(request.POST,request.FILES)
@@ -7832,7 +7837,6 @@ def workout_upload_view(request,
return response return response
else: else:
form = DocumentsForm(initial=docformoptions) form = DocumentsForm(initial=docformoptions)
print uploadoptions
optionsform = UploadOptionsForm(initial=uploadoptions) optionsform = UploadOptionsForm(initial=uploadoptions)
return render(request, 'document_form.html', return render(request, 'document_form.html',
{'form':form, {'form':form,