diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 9fd3d196..a2ac7764 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -714,8 +714,24 @@ def testdata(time,distance,pace,spm): def getsmallrowdata_db(columns,ids=[],debug=False): + csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + data = [] + columns = [c for c in columns if c != 'None'] - data = read_cols_df_sql(ids,columns,debug=debug) + if len(ids)>1: + for f in csvfilenames: + try: + df = dd.read_parquet(f,columns=columns,engine='pyarrow') + data.append(df) + except OSError: + pass + + + df = dd.concat(data,axis=0) + else: + df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') + + data = df.compute() return data @@ -1101,18 +1117,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - if barchart: - # time increments for bar chart - time_increments = rowdatadf.loc[:,' ElapsedTime (sec)'].diff() - try: - time_increments.iloc[0] = time_increments.iloc[1] - except (KeyError, IndexError): - time_increments.iloc[1] = 1. - - time_increments = 0.5*time_increments+0.5*np.abs(time_increments) - x_right = (t2+time_increments.apply(lambda x:timedeltaconv(x))) - - data['x_right'] = x_right if empower: try: @@ -1260,15 +1264,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, # write data if id given if id != 0: data['workoutid'] = id - - if debug: - engine = create_engine(database_url_debug, echo=False) - else: - engine = create_engine(database_url, echo=False) + data.to_parquet(filename,engine='pyarrow',compression='gzip') - with engine.connect() as conn, conn.begin(): - data.to_sql('strokedata',engine,if_exists='append',index=False) - - conn.close() - engine.dispose() return data