sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

fixed error in parquet

Browse Source
This commit is contained in:
Sander Roosendaal
2019-10-31 07:59:07 +01:00
parent ec10a6b96b
commit 102efa2cf2
1 changed files with 63 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
rowers/dataprepnodjango.py
View File

@@ -33,7 +33,7 @@ try:
user = DATABASES['default']['USER']
except KeyError:
user = ''
try:
try:
password = DATABASES['default']['PASSWORD']
except KeyError:
password = ''
@@ -98,7 +98,7 @@ def niceformat(values):
for v in values:
formattedv = strfdelta(v)
out.append(formattedv)
return out
def strfdelta(tdelta):
@@ -114,7 +114,7 @@ def strfdelta(tdelta):
seconds=seconds,
tenths=tenths,
)
return res
def nicepaceformat(values):
@@ -122,7 +122,7 @@ def nicepaceformat(values):
for v in values:
formattedv = strfdelta(v)
out.append(formattedv)
return out
@@ -131,8 +131,8 @@ def timedeltaconv(x):
dt = datetime.timedelta(seconds=x)
else:
dt = datetime.timedelta(seconds=350.)
return dt
def rdata(file,rower=rrower()):
@@ -167,7 +167,7 @@ def create_c2_stroke_data_db(
spm = 60.*nr_strokes/totalseconds
except ZeroDivisionError:
spm = 20*zeros(nr_strokes)
step = totalseconds/float(nr_strokes)
elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1))
@@ -186,7 +186,7 @@ def create_c2_stroke_data_db(
else:
power = 0
df = pd.DataFrame({
'TimeStamp (sec)': unixtime,
' Horizontal (meters)': d,
@@ -252,7 +252,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename,
spm = strokedata.loc[:,'spm']
except KeyError:
spm = 0*dist2
try:
hr = strokedata.loc[:,'hr']
except KeyError:
@@ -268,7 +268,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename,
velo = 1000./pace
# save csv
# Create data frame with all necessary data to write to csv
df = pd.DataFrame({'TimeStamp (sec)':unixtime,
@@ -291,11 +291,11 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename,
'cum_dist': dist2
})
df.sort_values(by='TimeStamp (sec)',ascending=True)
timestr = strftime("%Y%m%d-%H%M%S")
# Create CSV file name and save data to CSV file
@@ -307,7 +307,7 @@ def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename,
data = dataprep(df,id=workoutid,bands=False,debug=debug)
except:
return 0
return data
# Processes painsled CSV file to database
@@ -341,7 +341,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
#row.repair()
pass
if row == 0:
return (0,'Error: CSV data file not found')
@@ -349,7 +349,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
# auto smoothing
pace = row.df[' Stroke500mPace (sec/500m)'].values
velo = 500./pace
f = row.df['TimeStamp (sec)'].diff().mean()
if f !=0:
windowsize = 2*(int(10./(f)))+1
@@ -366,9 +366,9 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
velo3 = pd.Series(velo2)
velo3 = velo3.replace([-np.inf,np.inf],np.nan)
velo3 = velo3.fillna(method='ffill')
pace2 = 500./abs(velo3)
row.df[' Stroke500mPace (sec/500m)'] = pace2
row.df = row.df.fillna(0)
@@ -378,7 +378,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
os.remove(f2)
except:
pass
# recalculate power data
if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
try:
@@ -386,10 +386,10 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
row.write_csv(f2,gzip=True)
except:
pass
averagehr = row.df[' HRCur (bpm)'].mean()
maxhr = row.df[' HRCur (bpm)'].max()
if totaldist == 0:
totaldist = row.df['cum_dist'].max()
if totaltime == 0:
@@ -408,7 +408,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
minutes = 59
if not message:
message = 'Warning: there is something wrong with the workout duration'
seconds = int(totaltime - 3600.*hours - 60.*minutes)
if seconds > 59:
seconds = 59
@@ -420,7 +420,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
tenths = 9
if not message:
message = 'Warning: there is something wrong with the workout duration'
duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths)
if dosummary:
@@ -444,7 +444,7 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
message = "Warning: This workout probably already exists in the database"
privacy = 'private'
w = Workout(user=r,name=title,date=workoutdate,
workouttype=workouttype,
@@ -481,7 +481,7 @@ def handle_nonpainsled(f2,fileformat,summary=''):
# handle TCX
if (fileformat == 'tcx'):
row = TCXParser(f2)
# handle Mystery
if (fileformat == 'mystery'):
row = MysteryParser(f2)
@@ -489,7 +489,7 @@ def handle_nonpainsled(f2,fileformat,summary=''):
# handle RowPerfect
if (fileformat == 'rowperfect3'):
row = RowPerfectParser(f2)
# handle ErgData
if (fileformat == 'ergdata'):
row = ErgDataParser(f2)
@@ -501,7 +501,7 @@ def handle_nonpainsled(f2,fileformat,summary=''):
# handle Mike
if (fileformat == 'bcmike'):
row = BoatCoachAdvancedParser(f2)
# handle BoatCoach OTW
if (fileformat == 'boatcoachotw'):
row = BoatCoachOTWParser(f2)
@@ -518,7 +518,7 @@ def handle_nonpainsled(f2,fileformat,summary=''):
if (fileformat == 'speedcoach'):
row = speedcoachParser(f2)
# handle speed coach GPS 2
# handle speed coach GPS 2
if (fileformat == 'speedcoach2'):
row = SpeedCoach2Parser(f2)
try:
@@ -531,7 +531,7 @@ def handle_nonpainsled(f2,fileformat,summary=''):
# handle ErgStick
if (fileformat == 'ergstick'):
row = ErgStickParser(f2)
# handle FIT
if (fileformat == 'fit'):
row = FITParser(f2)
@@ -544,7 +544,7 @@ def handle_nonpainsled(f2,fileformat,summary=''):
# should delete file
f2 = f2[:-4]+'o.csv'
row.write_csv(f2,gzip=True)
#os.remove(f2)
try:
os.remove(f_to_be_deleted)
@@ -573,7 +573,7 @@ def new_workout_from_file(r,f2,
for fname in z.namelist():
f3 = z.extract(fname,path='media/')
id,message,f2 = new_workout_from_file(r,f3,
workouttype=workouttype,
workouttype=workouttype,
makeprivate=makeprivate,
title = title,
notes='')
@@ -590,7 +590,7 @@ def new_workout_from_file(r,f2,
os.remove(f2)
message = "It looks like this file doesn't contain stroke data."
return (0,message,f2)
# Some people try to upload RowPro summary logs
if fileformat == 'rowprolog':
os.remove(f2)
@@ -603,7 +603,7 @@ def new_workout_from_file(r,f2,
# worth supporting
if fileformat == 'unknown':
message = "We couldn't recognize the file type"
if settings.DEBUG:
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2,
r.user.email)
@@ -611,7 +611,7 @@ def new_workout_from_file(r,f2,
res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2,r.user.email)
return (0,message,f2)
# handle non-Painsled by converting it to painsled compatible CSV
if (fileformat != 'csv'):
try:
@@ -670,7 +670,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False):
success = False
try:
df['power empower old'] = df[' Power (watts)']
df['power empower old'] = df[' Power (watts)']
df[' Power (watts)'] = df[' Power (watts)'] * corr_factor
df['driveenergy empower old'] = df['driveenergy']
df['driveenergy'] = df['driveenergy'] * corr_factor
@@ -687,7 +687,7 @@ def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False):
if debug:
print("not updated ",id)
rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True,
debug=debug)
@@ -708,7 +708,7 @@ def testdata(time,distance,pace,spm):
def getsmallrowdata_db(columns,ids=[],debug=False):
csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids]
csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids]
data = []
columns = [c for c in columns if c != 'None']
@@ -720,14 +720,14 @@ def getsmallrowdata_db(columns,ids=[],debug=False):
except OSError:
pass
df = pd.concat(data,axis=0)
else:
df = pd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow')
return df
def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False,
doclean=False):
# test if nan among values
@@ -736,12 +736,12 @@ def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False,
m[key] = -1
if 'inf' in str(m[key]):
m[key] = -1
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
columns = ', '.join(m.keys())
if use_sqlite:
placeholders = ", ".join(["?"] * len(m))
@@ -795,7 +795,7 @@ def read_cols_df_sql(ids,columns,debug=False):
df = pd.concat(data,axis=0)
return df
def read_df_sql(id,debug=False):
try:
@@ -860,15 +860,15 @@ def delete_agegroup_db(age,sex,weightcategory,debug=False):
print("Database locked")
conn.close()
engine.dispose()
def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower,
debug=False):
delete_agegroup_db(age,sex,weightcategory,debug=debug)
wcdurations = [None if type(y) is float and np.isnan(y) else y for y in wcdurations]
wcpower = [None if type(y) is float and np.isnan(y) else y for y in wcpower]
df = pd.DataFrame(
{
'duration':wcdurations,
@@ -893,7 +893,7 @@ def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower,
conn.close()
engine.dispose()
def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),debug=False):
deletecpdata_sql(rower_id,table=table,debug=debug)
df = pd.DataFrame(
@@ -918,8 +918,8 @@ def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),deb
engine.dispose()
def smalldataprep(therows,xparam,yparam1,yparam2):
@@ -936,7 +936,7 @@ def smalldataprep(therows,xparam,yparam1,yparam2):
try:
rowdata = dataprep(rrdata(f1).df)
rowdata = pd.DataFrame({xparam: rowdata[xparam],
yparam1: rowdata[yparam1],
yparam2: rowdata[yparam2],
@@ -960,8 +960,8 @@ def smalldataprep(therows,xparam,yparam1,yparam2):
pass
return df
def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
empower=True,debug=False,inboard=0.88,forceunit='lbs'):
@@ -972,7 +972,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
if debug:
print("dataprep",id)
# rowdatadf.set_index([range(len(rowdatadf))],inplace=True)
t = rowdatadf.loc[:,'TimeStamp (sec)']
t = pd.Series(t-rowdatadf.loc[:,'TimeStamp (sec)'].iloc[0])
@@ -985,7 +985,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
velo = rowdatadf.loc[:,' AverageBoatSpeed (m/s)']
except KeyError:
velo = 500./p
hr = rowdatadf.loc[:,' HRCur (bpm)']
spm = rowdatadf.loc[:,' Cadence (stokes/min)']
cumdist = rowdatadf.loc[:,'cum_dist']
@@ -997,12 +997,12 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
workoutstate = rowdatadf.loc[:,' WorkoutState']
except KeyError:
workoutstate = 0*hr
peakforce = rowdatadf.loc[:,' PeakDriveForce (lbs)']
forceratio = averageforce/peakforce
forceratio = forceratio.fillna(value=0)
try:
drivetime = rowdatadf.loc[:,' DriveTime (ms)']
recoverytime = rowdatadf.loc[:,' StrokeRecoveryTime (ms)']
@@ -1033,7 +1033,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
except TypeError:
t2 = 0*t
p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x))
try:
@@ -1042,7 +1042,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
drivespeed = 0.0*rowdatadf['TimeStamp (sec)']
except TypeError:
drivespeed = 0.0*rowdatadf['TimeStamp (sec)']
drivespeed = drivespeed.fillna(value=0)
try:
@@ -1100,7 +1100,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
tel = rowdatadf.loc[:,' ElapsedTime (sec)']
except KeyError:
rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)']
if empower:
try:
@@ -1132,7 +1132,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
else:
driveenergy = data['driveenergy']
arclength = (inboard-0.05)*(np.radians(finish)-np.radians(catch))
if arclength.mean()>0:
drivelength = arclength
@@ -1151,7 +1151,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
totalangle = 0*t
effectiveangle = 0*t
if windowsize > 3 and windowsize<len(slip):
try:
wash = savgol_filter(wash,windowsize,3)
@@ -1211,7 +1211,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
data['efficiency'] = efficiency
except ValueError:
pass
if otwpower:
try:
nowindpace = rowdatadf.loc[:,'nowindpace']
@@ -1221,7 +1221,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
equivergpower = rowdatadf.loc[:,'equivergpower']
except KeyError:
equivergpower = 0*p+50.
nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x))
ergvelo = (equivergpower/2.8)**(1./3.)
@@ -1244,7 +1244,7 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
data.dropna(axis=0,inplace=True,how='all')
data.dropna(axis=1,inplace=True,how='any')
# write data if id given
if id != 0:
data['workoutid'] = id