sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

more autopep

Browse Source
This commit is contained in:
Sander Roosendaal
2022-03-10 15:46:33 +01:00
parent 81b035d6e8
commit 8070a68931
1 changed files with 106 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
rowers/dataprep.py
View File

@@ -31,7 +31,6 @@ from zipfile import BadZipFile
import zipfile import zipfile
import os import os
from rowers.models import strokedatafields from rowers.models import strokedatafields
from rowingdata.csvparsers import HumonParser
from rowingdata import ( from rowingdata import (
KinoMapParser, KinoMapParser,
@@ -56,8 +55,7 @@ from rowingdata import (
ETHParser, ETHParser,
NKLiNKLogbookParser, NKLiNKLogbookParser,
HeroParser, HeroParser,
SmartRowParser, SmartRowParser,)
)
# All the data preparation, data cleaning and data mangling should # All the data preparation, data cleaning and data mangling should
# be defined here # be defined here
@@ -111,13 +109,8 @@ import arrow
thetimezone = get_current_timezone() thetimezone = get_current_timezone()
#allowedcolumns = [item[0] for item in rowingmetrics]
allowedcolumns = [key for key, value in strokedatafields.items()] allowedcolumns = [key for key, value in strokedatafields.items()]
#from async_messages import messages as a_messages
queue = django_rq.get_queue('default') queue = django_rq.get_queue('default')
queuelow = django_rq.get_queue('low') queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('default') queuehigh = django_rq.get_queue('default')
@@ -169,8 +162,6 @@ def get_video_data(w, groups=['basic'], mode='water'):
p = nicepaceformat(p) p = nicepaceformat(p)
df2['pace'] = p df2['pace'] = p
#mask = df2['time'] < delay
#df2 = df2.mask(mask).dropna()
df2['time'] = (df2['time']-df2['time'].min()) df2['time'] = (df2['time']-df2['time'].min())
df2 = df2.round(decimals=2) df2 = df2.round(decimals=2)
@@ -190,8 +181,6 @@ def get_video_data(w, groups=['basic'], mode='water'):
coordinates.set_index(pd.to_timedelta( coordinates.set_index(pd.to_timedelta(
coordinates['time'], unit='s'), inplace=True) coordinates['time'], unit='s'), inplace=True)
coordinates = coordinates.resample('1s').mean().interpolate() coordinates = coordinates.resample('1s').mean().interpolate()
#mask = coordinates['time'] < delay
#coordinates = coordinates.mask(mask).dropna()
coordinates['time'] = coordinates['time']-coordinates['time'].min() coordinates['time'] = coordinates['time']-coordinates['time'].min()
latitude = coordinates['latitude'] latitude = coordinates['latitude']
longitude = coordinates['longitude'] longitude = coordinates['longitude']
@@ -295,11 +284,11 @@ def get_latlon_time(id):
try: try:
try: try:
latitude = rowdata.df.loc[:, ' latitude'] _ = rowdata.df.loc[:, ' latitude']
longitude = rowdata.df.loc[:, ' longitude'] _ = rowdata.df.loc[:, ' longitude']
except KeyError: # pragma: no cover except KeyError: # pragma: no cover
latitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] rowdata.df['latitude'] = 0 * rowdata.df.loc[:, 'TimeStamp (sec)']
longitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] rowdata.df['longitude'] = 0 * rowdata.df.loc[:, 'TimeStamp (sec)']
except AttributeError: # pragma: no cover except AttributeError: # pragma: no cover
return pd.DataFrame() return pd.DataFrame()
@@ -438,7 +427,7 @@ def get_workouts(ids, userid): # pragma: no cover
def filter_df(datadf, fieldname, value, largerthan=True): def filter_df(datadf, fieldname, value, largerthan=True):
try: try:
x = datadf[fieldname] _ = datadf[fieldname]
except KeyError: except KeyError:
return datadf return datadf
@@ -486,9 +475,9 @@ def join_workouts(r, ids, title='Joined Workout',
makeprivate = False makeprivate = False
startdatetime = timezone.now() startdatetime = timezone.now()
if setprivate == True and makeprivate == False: # pragma: no cover if setprivate is True and makeprivate is False: # pragma: no cover
makeprivate = True makeprivate = True
elif setprivate == False and makeprivate == True: # pragma: no cover elif setprivate is False and makeprivate is True: # pragma: no cover
makeprivate = False makeprivate = False
# reorder in chronological order # reorder in chronological order
@@ -501,6 +490,11 @@ def join_workouts(r, ids, title='Joined Workout',
workouttype = parent.workouttype workouttype = parent.workouttype
notes = parent.notes notes = parent.notes
summary = parent.summary summary = parent.summary
if parent.privacy == 'hidden':
makeprivate = True
else:
makeprivate = False
startdatetime = parent.startdatetime
files = [w.csvfilename for w in ws] files = [w.csvfilename for w in ws]
@@ -524,7 +518,9 @@ def join_workouts(r, ids, title='Joined Workout',
notes=notes, notes=notes,
oarlength=oarlength, oarlength=oarlength,
inboard=inboard, inboard=inboard,
startdatetime=startdatetime,
makeprivate=makeprivate, makeprivate=makeprivate,
summary=summary,
dosmooth=False, dosmooth=False,
consistencychecks=False) consistencychecks=False)
@@ -585,12 +581,12 @@ def resample(id, r, parent, overwrite='copy'):
row.write_csv(parent.csvfilename, gzip=True) row.write_csv(parent.csvfilename, gzip=True)
res = dataprep(row.df, id=parent.id, bands=True, barchart=True, _ = dataprep(row.df, id=parent.id, bands=True, barchart=True,
otwpower=True, empower=True, inboard=parent.inboard) otwpower=True, empower=True, inboard=parent.inboard)
isbreakthrough, ishard = checkbreakthrough(parent, r) isbreakthrough, ishard = checkbreakthrough(parent, r)
marker = check_marker(parent) _ = check_marker(parent)
result = update_wps(r, mytypes.otwtypes) _ = update_wps(r, mytypes.otwtypes)
result = update_wps(r, mytypes.otetypes) _ = update_wps(r, mytypes.otetypes)
tss, normp = workout_rscore(parent) tss, normp = workout_rscore(parent)
goldmedalstandard, goldmedalseconds = workout_goldmedalstandard(parent) goldmedalstandard, goldmedalseconds = workout_goldmedalstandard(parent)
@@ -607,7 +603,7 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
# clean data remove zeros and negative values # clean data remove zeros and negative values
try: try:
workoutids = datadf['workoutid'].unique() _ = datadf['workoutid'].unique()
except KeyError: except KeyError:
datadf['workoutid'] = 0 datadf['workoutid'] = 0
@@ -638,13 +634,13 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
# protect 0 spm values from being nulled # protect 0 spm values from being nulled
try: try:
datadf['spm'] = datadf['spm'] + 1.0 datadf['spm'] = datadf['spm'] + 1.0
except (KeyError, TypeError) as e: except (KeyError, TypeError):
pass pass
# protect 0 workoutstate values from being nulled # protect 0 workoutstate values from being nulled
try: try:
datadf['workoutstate'] = datadf['workoutstate'] + 1 datadf['workoutstate'] = datadf['workoutstate'] + 1
except (KeyError, TypeError) as e: except (KeyError, TypeError):
pass pass
try: try:
@@ -676,13 +672,13 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
# bring spm back to real values # bring spm back to real values
try: try:
datadf['spm'] = datadf['spm'] - 1 datadf['spm'] = datadf['spm'] - 1
except (TypeError, KeyError) as e: except (TypeError, KeyError):
pass pass
# bring workoutstate back to real values # bring workoutstate back to real values
try: try:
datadf['workoutstate'] = datadf['workoutstate'] - 1 datadf['workoutstate'] = datadf['workoutstate'] - 1
except (TypeError, KeyError) as e: except (TypeError, KeyError):
pass pass
# return from positive domain to negative # return from positive domain to negative
@@ -840,11 +836,11 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
except (KeyError, TypeError): except (KeyError, TypeError):
pass pass
workoutstateswork = [1, 4, 5, 8, 9, 6, 7] # workoutstateswork = [1, 4, 5, 8, 9, 6, 7]
workoutstatesrest = [3] workoutstatesrest = [3]
workoutstatetransition = [0, 2, 10, 11, 12, 13] # workoutstatetransition = [0, 2, 10, 11, 12, 13]
if workstrokesonly == 'True' or workstrokesonly == True: if workstrokesonly == 'True' or workstrokesonly is True:
try: try:
datadf = datadf[~datadf['workoutstate'].isin(workoutstatesrest)] datadf = datadf[~datadf['workoutstate'].isin(workoutstatesrest)]
except: except:
@@ -872,10 +868,10 @@ def getpartofday(row, r):
timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) timezone_str = tf.timezone_at(lng=lonavg, lat=latavg)
except (ValueError, OverflowError): # pragma: no cover except (ValueError, OverflowError): # pragma: no cover
timezone_str = 'UTC' timezone_str = 'UTC'
if timezone_str == None: # pragma: no cover if timezone_str is None: # pragma: no cover
timezone_str = tf.closest_timezone_at(lng=lonavg, timezone_str = tf.closest_timezone_at(lng=lonavg,
lat=latavg) lat=latavg)
if timezone_str == None: if timezone_str is None:
timezone_str = r.defaulttimezone timezone_str = r.defaulttimezone
try: try:
workoutstartdatetime = pytz.timezone(timezone_str).localize( workoutstartdatetime = pytz.timezone(timezone_str).localize(
@@ -1008,7 +1004,7 @@ def update_c2id_sql(id, c2id):
table, c2id, id) table, c2id, id)
with engine.connect() as conn, conn.begin(): with engine.connect() as conn, conn.begin():
result = conn.execute(query) _ = conn.execute(query)
conn.close() conn.close()
engine.dispose() engine.dispose()
@@ -1022,7 +1018,8 @@ def getcpdata_sql(rower_id, table='cpdata'):
rower_id=rower_id, rower_id=rower_id,
table=table, table=table,
)) ))
connection = engine.raw_connection()
_ = engine.raw_connection()
df = pd.read_sql_query(query, engine) df = pd.read_sql_query(query, engine)
return df return df
@@ -1036,7 +1033,7 @@ def deletecpdata_sql(rower_id, table='cpdata'): # pragma: no cover
)) ))
with engine.connect() as conn, conn.begin(): with engine.connect() as conn, conn.begin():
try: try:
result = conn.execute(query) _ = conn.execute(query)
except: except:
print("Database locked") print("Database locked")
conn.close() conn.close()
@@ -1064,11 +1061,10 @@ def updatecpdata_sql(rower_id, delta, cp, table='cpdata', distance=[]): # pragm
def fetchcperg(rower, theworkouts): def fetchcperg(rower, theworkouts):
theids = [int(w.id) for w in theworkouts]
thefilenames = [w.csvfilename for w in theworkouts] thefilenames = [w.csvfilename for w in theworkouts]
cpdf = getcpdata_sql(rower.id, table='ergcpdata') cpdf = getcpdata_sql(rower.id, table='ergcpdata')
job = myqueue( _ = myqueue(
queuelow, queuelow,
handle_updateergcp, handle_updateergcp,
rower.id, rower.id,
@@ -1138,7 +1134,7 @@ def check_marker(workout):
theid = df.loc[indexmax, 'id'] theid = df.loc[indexmax, 'id']
wmax = Workout.objects.get(id=theid) wmax = Workout.objects.get(id=theid)
gms_max = wmax.goldmedalstandard # gms_max = wmax.goldmedalstandard
# check if equal, bigger, or smaller than previous # check if equal, bigger, or smaller than previous
if not wmax.rankingpiece: if not wmax.rankingpiece:
@@ -1210,16 +1206,17 @@ def calculate_goldmedalstandard(rower, workout, recurrance=True):
) )
) )
jsondf = df2.to_json() jsondf = df2.to_json()
job = myqueue(queuelow, handle_getagegrouprecords, _ = myqueue(queuelow, handle_getagegrouprecords,
jsondf, distances, durations, age, rower.sex, rower.weightcategory) jsondf, distances, durations, age, rower.sex, rower.weightcategory)
wcpower = pd.Series(wcpower, dtype='float') wcpower = pd.Series(wcpower, dtype='float')
wcdurations = pd.Series(wcdurations, dtype='float') wcdurations = pd.Series(wcdurations, dtype='float')
def fitfunc(pars, x): return pars[0] / \ def fitfunc(pars, x):
(1+(x/pars[2])) + pars[1]/(1+(x/pars[3])) return pars[0] / (1+(x/pars[2])) + pars[1]/(1+(x/pars[3]))
def errfunc(pars, x, y): return fitfunc(pars, x)-y def errfunc(pars, x, y):
return fitfunc(pars, x)-y
if len(wcdurations) >= 4: # pragma: no cover if len(wcdurations) >= 4: # pragma: no cover
p1wc, success = optimize.leastsq( p1wc, success = optimize.leastsq(
@@ -1227,7 +1224,7 @@ def calculate_goldmedalstandard(rower, workout, recurrance=True):
else: else:
factor = fitfunc(p0, wcdurations.mean()/wcpower.mean()) factor = fitfunc(p0, wcdurations.mean()/wcpower.mean())
p1wc = [p0[0]/factor, p0[1]/factor, p0[2], p0[3]] p1wc = [p0[0]/factor, p0[1]/factor, p0[2], p0[3]]
success = 0
return 0, 0 return 0, 0
times = df['delta'] times = df['delta']
@@ -1292,7 +1289,7 @@ def setcp(workout, background=False, recurrance=True):
return pd.DataFrame(), pd.Series(dtype='float'), pd.Series(dtype='float') return pd.DataFrame(), pd.Series(dtype='float'), pd.Series(dtype='float')
if background: # pragma: no cover if background: # pragma: no cover
job = myqueue(queuelow, handle_setcp, strokesdf, filename, workout.id) _ = myqueue(queuelow, handle_setcp, strokesdf, filename, workout.id)
return pd.DataFrame({'delta': [], 'cp': []}), pd.Series(dtype='float'), pd.Series(dtype='float') return pd.DataFrame({'delta': [], 'cp': []}), pd.Series(dtype='float'), pd.Series(dtype='float')
if not strokesdf.empty: if not strokesdf.empty:
@@ -1339,7 +1336,7 @@ def update_wps(r, types, mode='water', asynchron=True):
ids = [w.id for w in workouts] ids = [w.id for w in workouts]
if asynchron: if asynchron:
job = myqueue( _ = myqueue(
queue, queue,
handle_update_wps, handle_update_wps,
r.id, r.id,
@@ -1441,11 +1438,11 @@ def fetchcp(rower, theworkouts, table='cpdata'): # pragma: no cover
if not cpdf.empty: if not cpdf.empty:
return cpdf['delta'], cpdf['cp'], avgpower2 return cpdf['delta'], cpdf['cp'], avgpower2
else: else:
job = myqueue(queuelow, _ = myqueue(queuelow,
handle_updatecp, handle_updatecp,
rower.id, rower.id,
theids, theids,
table=table) table=table)
return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2
@@ -1489,11 +1486,11 @@ def create_row_df(r, distance, duration, startdatetime, workouttype='rower',
else: else:
spm = avgspm spm = avgspm
step = totalseconds/float(nr_strokes) # step = totalseconds/float(nr_strokes)
elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1))
dstep = distance/float(nr_strokes) # dstep = distance/float(nr_strokes)
d = np.arange(nr_strokes)*distance/(float(nr_strokes-1)) d = np.arange(nr_strokes)*distance/(float(nr_strokes-1))
@@ -1564,12 +1561,12 @@ def checkbreakthrough(w, r):
if workouttype in otwtypes: if workouttype in otwtypes:
res, btvalues, res2 = utils.isbreakthrough( res, btvalues, res2 = utils.isbreakthrough(
delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio)
success = update_rolling_cp(r, otwtypes, 'water') _ = update_rolling_cp(r, otwtypes, 'water')
elif workouttype in otetypes: elif workouttype in otetypes:
res, btvalues, res2 = utils.isbreakthrough( res, btvalues, res2 = utils.isbreakthrough(
delta, cpvalues, r.ep0, r.ep1, r.ep2, r.ep3, r.ecpratio) delta, cpvalues, r.ep0, r.ep1, r.ep2, r.ep3, r.ecpratio)
success = update_rolling_cp(r, otetypes, 'erg') _ = update_rolling_cp(r, otetypes, 'erg')
else: # pragma: no cover else: # pragma: no cover
res = 0 res = 0
res2 = 0 res2 = 0
@@ -1584,12 +1581,12 @@ def checkbreakthrough(w, r):
w.rankingpiece = True w.rankingpiece = True
w.save() w.save()
if r.getemailnotifications and not r.emailbounced: # pragma: no cover if r.getemailnotifications and not r.emailbounced: # pragma: no cover
job = myqueue(queuehigh, handle_sendemail_breakthrough, _ = myqueue(queuehigh, handle_sendemail_breakthrough,
w.id, w.id,
r.user.email, r.user.email,
r.user.first_name, r.user.first_name,
r.user.last_name, r.user.last_name,
btvalues=btvalues.to_json()) btvalues=btvalues.to_json())
# submit email task to send email about breakthrough workout # submit email task to send email about breakthrough workout
if ishard: # pragma: no cover if ishard: # pragma: no cover
@@ -1597,12 +1594,12 @@ def checkbreakthrough(w, r):
w.rankingpiece = True w.rankingpiece = True
w.save() w.save()
if r.getemailnotifications and not r.emailbounced: if r.getemailnotifications and not r.emailbounced:
job = myqueue(queuehigh, handle_sendemail_hard, _ = myqueue(queuehigh, handle_sendemail_hard,
w.id, w.id,
r.user.email, r.user.email,
r.user.first_name, r.user.first_name,
r.user.last_name, r.user.last_name,
btvalues=btvalues.to_json()) btvalues=btvalues.to_json())
return isbreakthrough, ishard return isbreakthrough, ishard
@@ -1624,7 +1621,6 @@ def checkduplicates(r, workoutdate, workoutstartdatetime, workoutenddatetime):
ws2.append(ww) ws2.append(ww)
if (len(ws2) != 0): if (len(ws2) != 0):
message = "Warning: This workout overlaps with an existing one and was marked as a duplicate"
duplicate = True duplicate = True
return duplicate return duplicate
@@ -1724,7 +1720,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower',
windowsize = 2 * (int(10. / (f))) + 1 windowsize = 2 * (int(10. / (f))) + 1
else: # pragma: no cover else: # pragma: no cover
windowsize = 1 windowsize = 1
if not 'originalvelo' in row.df: if 'originalvelo' not in row.df:
row.df['originalvelo'] = velo row.df['originalvelo'] = velo
if windowsize > 3 and windowsize < len(velo): if windowsize > 3 and windowsize < len(velo):
@@ -1861,16 +1857,16 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower',
w.team.add(t) w.team.add(t)
# put stroke data in database # put stroke data in database
res = dataprep(row.df, id=w.id, bands=True, _ = dataprep(row.df, id=w.id, bands=True,
barchart=True, otwpower=True, empower=True, inboard=inboard) barchart=True, otwpower=True, empower=True, inboard=inboard)
isbreakthrough, ishard = checkbreakthrough(w, r) isbreakthrough, ishard = checkbreakthrough(w, r)
marker = check_marker(w) _ = check_marker(w)
result = update_wps(r, mytypes.otwtypes) _ = update_wps(r, mytypes.otwtypes)
result = update_wps(r, mytypes.otetypes) _ = update_wps(r, mytypes.otetypes)
job = myqueue(queuehigh, handle_calctrimp, w.id, f2, _ = myqueue(queuehigh, handle_calctrimp, w.id, f2,
r.ftp, r.sex, r.hrftp, r.max, r.rest) r.ftp, r.sex, r.hrftp, r.max, r.rest)
return (w.id, message) return (w.id, message)
@@ -1914,7 +1910,7 @@ def get_startdate_time_zone(r, row, startdatetime=None):
startdatetime = row.rowdatetime startdatetime = row.rowdatetime
try: try:
tz = startdatetime.tzinfo _ = startdatetime.tzinfo
except AttributeError: # pragma: no cover except AttributeError: # pragma: no cover
startdatetime = row.rowdatetime startdatetime = row.rowdatetime
@@ -1987,7 +1983,6 @@ def parsenonpainsled(fileformat, f2, summary, startdatetime='', empowerfirmware=
s = 'Parsenonpainsled, start date time = {startdatetime}'.format( s = 'Parsenonpainsled, start date time = {startdatetime}'.format(
startdatetime=startdatetime, startdatetime=startdatetime,
#rowdatetime = row.rowdatetime
) )
dologging('debuglog.log', s) dologging('debuglog.log', s)
@@ -2158,11 +2153,10 @@ def new_workout_from_file(r, f2,
uploadoptions['file'] = datafile uploadoptions['file'] = datafile
url = settings.UPLOAD_SERVICE_URL url = settings.UPLOAD_SERVICE_URL
job = myqueue(queuehigh, _ = myqueue(queuehigh,
handle_request_post, handle_request_post,
url, url,
uploadoptions uploadoptions)
)
except BadZipFile: # pragma: no cover except BadZipFile: # pragma: no cover
pass pass
@@ -2227,10 +2221,10 @@ def new_workout_from_file(r, f2,
extension = extension2 extension = extension2
f4 = filename+'a'+extension f4 = filename+'a'+extension
copyfile(f2, f4) copyfile(f2, f4)
job = myqueue(queuehigh, _ = myqueue(queuehigh,
handle_sendemail_unrecognized, handle_sendemail_unrecognized,
f4, f4,
r.user.email) r.user.email)
return (0, message, f2) return (0, message, f2)
@@ -2377,7 +2371,7 @@ def split_workout(r, parent, splitsecond, splitmode):
messages.append(message) messages.append(message)
ids.append(encoder.encode_hex(id)) ids.append(encoder.encode_hex(id))
if not 'keep original' in splitmode: # pragma: no cover if 'keep original' not in splitmode: # pragma: no cover
if 'keep second' in splitmode or 'keep first' in splitmode: if 'keep second' in splitmode or 'keep first' in splitmode:
parent.delete() parent.delete()
messages.append('Deleted Workout: ' + parent.name) messages.append('Deleted Workout: ' + parent.name)
@@ -2448,7 +2442,6 @@ def new_workout_from_df(r, df,
df.rename(columns=columndict, inplace=True) df.rename(columns=columndict, inplace=True)
#starttimeunix = mktime(startdatetime.utctimetuple())
starttimeunix = arrow.get(startdatetime).timestamp() starttimeunix = arrow.get(startdatetime).timestamp()
df[' ElapsedTime (sec)'] = df['TimeStamp (sec)'] df[' ElapsedTime (sec)'] = df['TimeStamp (sec)']
@@ -2458,15 +2451,13 @@ def new_workout_from_df(r, df,
row.write_csv(csvfilename, gzip=True) row.write_csv(csvfilename, gzip=True)
# res = df.to_csv(csvfilename+'.gz',index_label='index',
# compression='gzip')
id, message = save_workout_database(csvfilename, r, id, message = save_workout_database(csvfilename, r,
workouttype=workouttype, workouttype=workouttype,
boattype=boattype, boattype=boattype,
title=title, title=title,
workoutsource=workoutsource, workoutsource=workoutsource,
notes=notes, notes=notes,
summary=summary,
oarlength=oarlength, oarlength=oarlength,
inboard=inboard, inboard=inboard,
makeprivate=makeprivate, makeprivate=makeprivate,
@@ -2474,8 +2465,8 @@ def new_workout_from_df(r, df,
rpe=rpe, rpe=rpe,
consistencychecks=False) consistencychecks=False)
job = myqueue(queuehigh, handle_calctrimp, id, csvfilename, _ = myqueue(queuehigh, handle_calctrimp, id, csvfilename,
r.ftp, r.sex, r.hrftp, r.max, r.rest) r.ftp, r.sex, r.hrftp, r.max, r.rest)
return (id, message) return (id, message)
@@ -2520,7 +2511,7 @@ def delete_strokedata(id):
def update_strokedata(id, df): def update_strokedata(id, df):
delete_strokedata(id) delete_strokedata(id)
rowdata = dataprep(df, id=id, bands=True, barchart=True, otwpower=True) _ = dataprep(df, id=id, bands=True, barchart=True, otwpower=True)
# Test that all data are of a numerical time # Test that all data are of a numerical time
@@ -2555,7 +2546,7 @@ def getrowdata_db(id=0, doclean=False, convertnewtons=True,
else: else:
row = Workout.objects.get(id=id) row = Workout.objects.get(id=id)
if checkefficiency == True and not data.empty: if checkefficiency is True and not data.empty:
try: try:
if data['efficiency'].mean() == 0 and data['power'].mean() != 0: # pragma: no cover if data['efficiency'].mean() == 0 and data['power'].mean() != 0: # pragma: no cover
data = add_efficiency(id=id) data = add_efficiency(id=id)
@@ -2586,15 +2577,13 @@ def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True, comp
if len(ids) > 1: if len(ids) > 1:
for id, f in zip(ids, csvfilenames): for id, f in zip(ids, csvfilenames):
try: try:
#df = dd.read_parquet(f,columns=columns,engine='pyarrow')
df = pd.read_parquet(f, columns=columns) df = pd.read_parquet(f, columns=columns)
data.append(df) data.append(df)
except (OSError, ArrowInvalid, IndexError): # pragma: no cover except (OSError, ArrowInvalid, IndexError): # pragma: no cover
rowdata, row = getrowdata(id=id) rowdata, row = getrowdata(id=id)
if rowdata and len(rowdata.df): if rowdata and len(rowdata.df):
datadf = dataprep(rowdata.df, id=id, _ = dataprep(rowdata.df, id=id,
bands=True, otwpower=True, barchart=True) bands=True, otwpower=True, barchart=True)
# df = dd.read_parquet(f,columns=columns,engine='pyarrow')
df = pd.read_parquet(f, columns=columns) df = pd.read_parquet(f, columns=columns)
data.append(df) data.append(df)
@@ -2649,7 +2638,6 @@ def getrowdata(id=0):
# get user # get user
r = row.user r = row.user
u = r.user
rr = rrower(hrmax=r.max, hrut2=r.ut2, rr = rrower(hrmax=r.max, hrut2=r.ut2,
hrut1=r.ut1, hrat=r.at, hrut1=r.ut1, hrat=r.at,
@@ -2677,8 +2665,8 @@ def prepmultipledata(ids, verbose=False): # pragma: no cover
if verbose: if verbose:
print(id) print(id)
if rowdata and len(rowdata.df): if rowdata and len(rowdata.df):
data = dataprep(rowdata.df, id=id, bands=True, _ = dataprep(rowdata.df, id=id, bands=True,
barchart=True, otwpower=True) barchart=True, otwpower=True)
return ids return ids
# Read a set of columns for a set of workout ids, returns data as a # Read a set of columns for a set of workout ids, returns data as a
@@ -2707,8 +2695,8 @@ def read_cols_df_sql(ids, columns, convertnewtons=True):
except OSError: except OSError:
rowdata, row = getrowdata(id=ids[0]) rowdata, row = getrowdata(id=ids[0])
if rowdata and len(rowdata.df): if rowdata and len(rowdata.df):
datadf = dataprep( _ = dataprep(rowdata.df,
rowdata.df, id=ids[0], bands=True, otwpower=True, barchart=True) id=ids[0], bands=True, otwpower=True, barchart=True)
df = pd.read_parquet(filename, columns=columns) df = pd.read_parquet(filename, columns=columns)
else: else:
data = [] data = []
@@ -2721,8 +2709,8 @@ def read_cols_df_sql(ids, columns, convertnewtons=True):
except (OSError, IndexError, ArrowInvalid): except (OSError, IndexError, ArrowInvalid):
rowdata, row = getrowdata(id=id) rowdata, row = getrowdata(id=id)
if rowdata and len(rowdata.df): # pragma: no cover if rowdata and len(rowdata.df): # pragma: no cover
datadf = dataprep(rowdata.df, id=id, _ = dataprep(rowdata.df, id=id,
bands=True, otwpower=True, barchart=True) bands=True, otwpower=True, barchart=True)
df = pd.read_parquet(f, columns=columns) df = pd.read_parquet(f, columns=columns)
data.append(df) data.append(df)
@@ -2753,8 +2741,8 @@ def read_cols_df_sql(ids, columns, convertnewtons=True):
def initiate_cp(r): def initiate_cp(r):
success = update_rolling_cp(r, otwtypes, 'water') _ = update_rolling_cp(r, otwtypes, 'water')
success = update_rolling_cp(r, otetypes, 'erg') _ = update_rolling_cp(r, otetypes, 'erg')
# Read stroke data from the DB for a Workout ID. Returns a pandas dataframe # Read stroke data from the DB for a Workout ID. Returns a pandas dataframe
@@ -2784,9 +2772,6 @@ def read_df_sql(id):
def datafusion(id1, id2, columns, offset): def datafusion(id1, id2, columns, offset):
workout1 = Workout.objects.get(id=id1)
workout2 = Workout.objects.get(id=id2)
df1, w1 = getrowdata_db(id=id1) df1, w1 = getrowdata_db(id=id1)
df1 = df1.drop([ # 'cumdist', df1 = df1.drop([ # 'cumdist',
'hr_ut2', 'hr_ut2',
@@ -2821,7 +2806,7 @@ def datafusion(id1, id2, columns, offset):
keep1.pop(c) keep1.pop(c)
for c in df1.columns: for c in df1.columns:
if not c in keep1: if c not in keep1:
df1 = df1.drop(c, 1, errors='ignore') df1 = df1.drop(c, 1, errors='ignore')
df = pd.concat([df1, df2], ignore_index=True) df = pd.concat([df1, df2], ignore_index=True)
@@ -2846,7 +2831,6 @@ def fix_newtons(id=0, limit=3000): # pragma: no cover
# rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False)
rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False) rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False)
try: try:
#avgforce = rowdata['averageforce']
peakforce = rowdata['peakforce'] peakforce = rowdata['peakforce']
if peakforce.mean() > limit: if peakforce.mean() > limit:
w = Workout.objects.get(id=id) w = Workout.objects.get(id=id)
@@ -2860,7 +2844,7 @@ def fix_newtons(id=0, limit=3000): # pragma: no cover
def remove_invalid_columns(df): # pragma: no cover def remove_invalid_columns(df): # pragma: no cover
for c in df.columns: for c in df.columns:
if not c in allowedcolumns: if c not in allowedcolumns:
df.drop(labels=c, axis=1, inplace=True) df.drop(labels=c, axis=1, inplace=True)
return df return df
@@ -2907,7 +2891,6 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True,
if rowdatadf.empty: if rowdatadf.empty:
return 0 return 0
#rowdatadf.set_index([range(len(rowdatadf))], inplace=True)
t = rowdatadf.loc[:, 'TimeStamp (sec)'] t = rowdatadf.loc[:, 'TimeStamp (sec)']
t = pd.Series(t - rowdatadf.loc[:, 'TimeStamp (sec)'].iloc[0]) t = pd.Series(t - rowdatadf.loc[:, 'TimeStamp (sec)'].iloc[0])
@@ -2981,7 +2964,7 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True,
if forceunit == 'lbs': if forceunit == 'lbs':
driveenergy = drivelength * averageforce * lbstoN driveenergy = drivelength * averageforce * lbstoN
else: else:
drivenergy = drivelength * averageforce driveenergy = drivelength * averageforce
if forceunit == 'lbs': if forceunit == 'lbs':
averageforce *= lbstoN averageforce *= lbstoN
@@ -3034,7 +3017,7 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True,
data['hr_bottom'] = 0.0 * data['hr'] data['hr_bottom'] = 0.0 * data['hr']
try: try:
tel = rowdatadf.loc[:, ' ElapsedTime (sec)'] _ = rowdatadf.loc[:, ' ElapsedTime (sec)']
except KeyError: # pragma: no cover except KeyError: # pragma: no cover
rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)']
@@ -3218,7 +3201,7 @@ def workout_trimp(w, reset=False):
w.maxhr = maxhr w.maxhr = maxhr
w.save() w.save()
job = myqueue( _ = myqueue(
queuehigh, queuehigh,
handle_calctrimp, handle_calctrimp,
w.id, w.id,
@@ -3246,7 +3229,7 @@ def workout_rscore(w, reset=False):
r.hrftp = int(hrftp) r.hrftp = int(hrftp)
r.save() r.save()
job = myqueue( _ = myqueue(
queuehigh, queuehigh,
handle_calctrimp, handle_calctrimp,
w.id, w.id,
@@ -3274,7 +3257,7 @@ def workout_normv(w, pp=4.0):
r.hrftp = int(hrftp) r.hrftp = int(hrftp)
r.save() r.save()
job = myqueue( _ = myqueue(
queuehigh, queuehigh,
handle_calctrimp, handle_calctrimp,
w.id, w.id,