updates in nodjango

rowers/dataprepnodjango.py

@@ -714,8 +714,24 @@ def testdata(time,distance,pace,spm):
 
 
 def getsmallrowdata_db(columns,ids=[],debug=False):
+    csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids]
+    data = []
+    columns = [c for c in columns if c != 'None']
 
-    data = read_cols_df_sql(ids,columns,debug=debug)
+    if len(ids)>1:
+        for f in csvfilenames:
+            try:
+                df = dd.read_parquet(f,columns=columns,engine='pyarrow')
+                data.append(df)
+            except OSError:
+                pass
+
+    
+        df = dd.concat(data,axis=0)
+    else:
+        df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow')
+
+    data = df.compute()
 
     return data
         
@@ -1101,18 +1117,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
     except KeyError:
         rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)']
         
-    if barchart:
-        # time increments for bar chart
-        time_increments = rowdatadf.loc[:,' ElapsedTime (sec)'].diff()
-        try:
-            time_increments.iloc[0] = time_increments.iloc[1]
-        except (KeyError, IndexError):
-            time_increments.iloc[1] = 1.
-            
-        time_increments = 0.5*time_increments+0.5*np.abs(time_increments)
-        x_right =  (t2+time_increments.apply(lambda x:timedeltaconv(x)))
-
-        data['x_right'] = x_right
 
     if empower:
         try:
@@ -1260,15 +1264,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True,
     # write data if id given
     if id != 0:
         data['workoutid'] = id
-        
-        if debug:
-            engine = create_engine(database_url_debug, echo=False)
-        else:
-            engine = create_engine(database_url, echo=False)
+        data.to_parquet(filename,engine='pyarrow',compression='gzip')        
 
-        with engine.connect() as conn, conn.begin():
-            data.to_sql('strokedata',engine,if_exists='append',index=False)
-                    
-        conn.close()
-        engine.dispose()    
     return data