alerts, analysis, api tested

rowers/alerts.py

@@ -1,6 +1,6 @@
 from rowers.models import Alert, Condition, User, Rower, Workout
 from rowers.teams import coach_getcoachees
-from rowers.dataprep import getsmallrowdata_db, getrowdata_db
+from rowers.dataprep import getrowdata_db, read_data, remove_nulls_pl
 import datetime
 import numpy as np
 import math
@@ -101,8 +101,9 @@ def alert_get_stats(alert, nperiod=0):  # pragma: no cover
     ids = [w.id for w in workouts]
 
     try:
-        df = getsmallrowdata_db(columns, ids=ids, doclean=True,
+        df = read_data(columns, ids=ids, doclean=True,
-                                workstrokesonly=workstrokesonly)
+                       workstrokesonly=workstrokesonly)
+        df = remove_nulls_pl(df)
     except:
         return {
             'workouts': workouts.count(),

rowers/dataroutines.py

@@ -1417,10 +1417,10 @@ def update_strokedata(id, df, debug=False):
 
 
 def testdata(time, distance, pace, spm):  # pragma: no cover
-    t1 = np.issubdtype(time, np.number)
+    t1 = time.dtype in pl.NUMERIC_DTYPES
-    t2 = np.issubdtype(distance, np.number)
+    t2 = distance.dtype in pl.NUMERIC_DTYPES
-    t3 = np.issubdtype(pace, np.number)
+    t3 = pace.dtype in pl.NUMERIC_DTYPES
-    t4 = np.issubdtype(spm, np.number)
+    t4 = spm.dtype in pl.NUMERIC_DTYPES
 
     return t1 and t2 and t3 and t4
 
@@ -1630,7 +1630,7 @@ def read_data(columns, ids=[], doclean=True, workstrokesonly=True, debug=False,
 
     return datadf
 
-def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True, compute=True,
+def getsmallrowdata_pd(columns, ids=[], doclean=True, workstrokesonly=True, compute=True,
                        debug=False, for_chart=False):
     #    prepmultipledata(ids)
 

rowers/tasks.py

@@ -130,7 +130,8 @@ from rowers import mytypes
 
 
 from rowers.dataroutines import (
-    getsmallrowdata_db, updatecpdata_sql, update_c2id_sql,
+    getsmallrowdata_pd, updatecpdata_sql, update_c2id_sql,
+    read_data,
     #update_workout_field_sql,
     update_agegroup_db, update_strokedata,
     add_c2_stroke_data_db, totaltime_sec_to_string,

rowers/tests/test_analysis.py

@@ -43,10 +43,10 @@ class ListWorkoutTest(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     @patch('rowers.dataprep.myqueue')
     def test_list_workouts(self, mocked_sqlalchemy,
-                           mocked_getsmallrowdata_db,
+                           mocked_read_data,
                            mocked_myqueue):
 
         login = self.c.login(username=self.u.username, password=self.password)
@@ -170,7 +170,7 @@ class ForcecurveTest(TestCase):
 
            
     @patch('rowers.dataprep.read_data',side_effect = mocked_read_data)
-    def test_forcecurve_plot(self, mocked_getsmallrowdata_db):
+    def test_forcecurve_plot(self, mocked_read_data):
         login = self.c.login(username=self.u.username, password = self.password)
         self.assertTrue(login)
 
@@ -296,9 +296,9 @@ class WorkoutCompareTestNew(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_workouts_compare(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
 
         login = self.c.login(username=self.u.username, password=self.password)
         self.assertTrue(login)
@@ -309,9 +309,9 @@ class WorkoutCompareTestNew(TestCase):
         self.assertEqual(response.status_code,200)
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_compare_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -398,9 +398,9 @@ class WorkoutBoxPlotTestNew(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_workouts_boxplot(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
 
         login = self.c.login(username=self.u.username, password=self.password)
         self.assertTrue(login)
@@ -411,9 +411,9 @@ class WorkoutBoxPlotTestNew(TestCase):
         self.assertEqual(response.status_code,200)
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_boxplot_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -499,9 +499,9 @@ class WorkoutHistoTestNew(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_workouts_histo(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
 
         login = self.c.login(username=self.u.username, password=self.password)
         self.assertTrue(login)
@@ -512,9 +512,9 @@ class WorkoutHistoTestNew(TestCase):
         self.assertEqual(response.status_code,200)
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_histo_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -618,9 +618,9 @@ class History(TestCase):
 
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_history_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -713,9 +713,9 @@ class GoldMedalScores(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_workouts_goldmedalscores(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
 
         ws = Workout.objects.filter(rankingpiece=True)
         self.assertEqual(ws.count(),2)
@@ -729,9 +729,9 @@ class GoldMedalScores(TestCase):
         self.assertEqual(response.status_code,200)
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_goldmedalscores_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -795,9 +795,9 @@ class WorkoutFlexallTestNew(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_workouts_flexall(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
 
         login = self.c.login(username=self.u.username, password=self.password)
         self.assertTrue(login)
@@ -808,9 +808,9 @@ class WorkoutFlexallTestNew(TestCase):
         self.assertEqual(response.status_code,200)
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_flexall_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -903,9 +903,9 @@ class WorkoutStatsTestNew(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_workouts_stats(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
 
         login = self.c.login(username=self.u.username, password=self.password)
         self.assertTrue(login)
@@ -920,7 +920,7 @@ class WorkoutStatsTestNew(TestCase):
     @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql)
     def test_analysis_data(self,
                            mocked_sqlalchemy,
-                           mocked_getsmallrowdata_db,
+                           mocked_read_data,
                            mocked_read_cols_df_sql,
     ):
 
@@ -979,11 +979,11 @@ class WorkoutStatsTestNew(TestCase):
         script, div = comparisondata(workouts,options)
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql)
     def test_analysis_data2(self,
                            mocked_sqlalchemy,
-                           mocked_getsmallrowdata_db,
+                           mocked_read_data,
                            mocked_read_cols_df_sql,
     ):
 
@@ -1042,11 +1042,11 @@ class WorkoutStatsTestNew(TestCase):
 
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql)
     def test_analysis_data2(self,
                            mocked_sqlalchemy,
-                           mocked_getsmallrowdata_db,
+                           mocked_read_data,
                            mocked_read_cols_df_sql,
     ):
 
@@ -1120,9 +1120,9 @@ class WorkoutStatsTestNew(TestCase):
 
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db)
+    @patch('rowers.dataprep.read_data', side_effect=mocked_read_data)
     def test_workouts_stats_submit(self, mocked_sqlalchemy,
-                                     mocked_getsmallrowdata_db):
+                                     mocked_read_data):
 
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
@@ -1241,9 +1241,9 @@ class MarkerPerformanceTest(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_create_marker_workouts(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
 
@@ -1326,9 +1326,9 @@ class MarkerPerformanceTest(TestCase):
 
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.dataprep.getsmallrowdata_db')
+    @patch('rowers.dataprep.read_data')
     def test_performancemanager_view(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
 
@@ -1413,9 +1413,9 @@ class AlertTest(TestCase):
                 pass
 
     @patch('rowers.dataprep.create_engine')
-    @patch('rowers.alerts.getsmallrowdata_db')
+    @patch('rowers.alerts.read_data')
     def test_alerts(self, mocked_sqlalchemy,
-                              mocked_getsmallrowdata_db):
+                              mocked_read_data):
         login = self.c.login(username=self.u.username,password=self.password)
         self.assertTrue(login)
 

rowers/tests/test_api.py

@@ -67,10 +67,11 @@ class OwnApi(TestCase):
         self.assertEqual(response.status_code,200)
 
         # response must be json
+
         strokedata = json.loads(response.content)
-        df = pd.DataFrame(strokedata)
+        df = pl.from_dict(strokedata)
         
-        self.assertFalse(df.empty)
+        self.assertFalse(df.is_empty())
 
         form_data = {
             "distance": [23, 46, 48],
@@ -124,8 +125,8 @@ class OwnApi(TestCase):
 
         # response must be json
         strokedata = json.loads(response.content)
-        df = pd.DataFrame(strokedata)
+        df = pl.from_dicts(strokedata['data'])
-        self.assertFalse(df.empty)
+        self.assertFalse(df.is_empty())
 
 
         form_data = {

rowers/views/analysisviews.py

@@ -612,7 +612,7 @@ def histodata(workouts, options):
     if savedata: # pragma: no cover
         workstrokesonly = not includereststrokes
         ids = [int(w.id) for w in workouts]
-        df = dataprep.getsmallrowdata_db([plotfield], ids=ids,
+        df = dataprep.read_data([plotfield], ids=ids,
                                          workstrokesonly=workstrokesonly,
                                          doclean=True,
                                          )
@@ -885,7 +885,7 @@ def comparisondata(workouts, options):
                    'time', 'pace', 'workoutstate',
                    'workoutid']
 
-        df = dataprep.getsmallrowdata_db(columns, ids=ids,
+        df = dataprep.read_data(columns, ids=ids,
                                          workstrokesonly=workstrokesonly,
                                          doclean=True,
                                          )

rowers/views/apiviews.py

@@ -68,102 +68,102 @@ def strokedataform(request, id=0):
 def api_get_dataframe(startdatetime, df):
     try:
         time = df['time']/1.e3
-    except KeyError:  # pragma: no cover
+    except (KeyError, ColumnNotFoundError):  # pragma: no cover
         try:
             time = df['t']/10.
-        except KeyError:
+        except (KeyError, ColumnNotFoundError):
-            return 400, "Missing time", pd.DataFrame()
+            return 400, "Missing time", pl.DataFrame()
 
     try:
         spm = df['spm']
-    except KeyError:  # pragma: no cover
+    except (KeyError, ColumnNotFoundError):  # pragma: no cover
-        return 400, "Missing spm", pd.DataFrame()
+        return 400, "Missing spm", pl.DataFrame()
     try:
         distance = df['distance']
-    except KeyError:  # pragma: no cover
+    except (KeyError, ColumnNotFoundError):  # pragma: no cover
         try:
             distance = df['d']/10.
-        except KeyError:
+        except (KeyError, ColumnNotFoundError):
-            return 400, "Missing distance", pd.DataFrame()
+            return 400, "Missing distance", pl.DataFrame()
 
     try:
         pace = df['pace']/1.e3
-    except KeyError:  # pragma: no cover
+    except (KeyError, ColumnNotFoundError):  # pragma: no cover
         try:
             pace = df['p']/10.
-        except KeyError:
+        except (KeyError, ColumnNotFoundError):
-            return 400, "Missing pace", pd.DataFrame()
+            return 400, "Missing pace", pl.DataFrame()
 
 
     try:
         power = df['power']
-    except KeyError:  # pragma: no cover
+    except (KeyError, ColumnNotFoundError):  # pragma: no cover
         power = 0*time
     try:
         drivelength = df['drivelength']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         drivelength = 0*time
     try:
         dragfactor = df['dragfactor']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         dragfactor = 0*time
     try:
         drivetime = df['drivetime']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         drivetime = 0*time
     try:
         strokerecoverytime = df['strokerecoverytime']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         strokerecoverytime = 0*time
     try:
         averagedriveforce = df['averagedriveforce']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         averagedriveforce = 0*time
     try:
         peakdriveforce = df['peakdriveforce']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         peakdriveforce = 0*time
     try:
         wash = df['wash']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         wash = 0*time
     try:
         catch = df['catch']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         catch = 0*time
     try:
         finish = df['finish']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         finish = 0*time
     try:
         peakforceangle = df['peakforceangle']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         peakforceangle = 0*time
     try:
         driveenergy = df['driveenergy']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         driveenergy = 60.*power/spm
     try:
         slip = df['slip']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         slip = 0*time
     try:
         lapidx = df['lapidx']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         lapidx = 0*time
     try:
         hr = df['hr']
-    except KeyError:  # pragma: no cover
+    except (KeyError, ColumnNotFoundError):  # pragma: no cover
         hr = 0*df['time']
         
     try:
         latitude = df['latitude']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         latitude = 0*df['time']
 
     try:
         longitude = df['longitude']
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         longitude = 0*df['time']
 
     starttime = totimestamp(startdatetime)+time[0]
@@ -171,7 +171,7 @@ def api_get_dataframe(startdatetime, df):
 
     dologging('apilog.log',"(strokedatajson_v2/3 POST - data parsed)")
 
-    data = pd.DataFrame({'TimeStamp (sec)': unixtime,
+    data = pl.DataFrame({'TimeStamp (sec)': unixtime,
                          ' Horizontal (meters)': distance,
                          ' Cadence (stokes/min)': spm,
                          ' HRCur (bpm)': hr,
@@ -522,7 +522,7 @@ def strokedatajson_v3(request):
     title = request.data.get('name','')
     try:
         elapsedTime = request.data['elapsedTime']
-    except KeyError: # pragma: no cover
+    except (KeyError, ColumnNotFoundError): # pragma: no cover
         try:
             duration = request.data['duration']
             try:
@@ -534,7 +534,7 @@ def strokedatajson_v3(request):
             return HttpResponse("Missing Elapsed Time", status=400)
     try:
         totalDistance = request.data['distance']
-    except KeyError: # pragma: no cover
+    except (KeyError, ColumnNotFoundError): # pragma: no cover
         return HttpResponse("Missing Total Distance", status=400)
     timeZone = request.data.get('timezone','UTC')
     workouttype = request.data.get('workouttype','rower')
@@ -552,23 +552,21 @@ def strokedatajson_v3(request):
     dologging('apilog.log',totalDistance)
     dologging('apilog.log',elapsedTime)
 
-    df = pd.DataFrame()
+    df = pl.DataFrame()
     try:
         strokes = request.data['strokes']
-    except KeyError: # pragma: no cover
+    except (KeyError, ColumnNotFoundError): # pragma: no cover
         return HttpResponse("No Stroke Data in JSON", status=400)
 
     try:
-        df = pd.DataFrame(strokes['data'])
+        df = pl.DataFrame(strokes['data'])
-    except KeyError: # pragma: no cover
+    except (KeyError, ColumnNotFoundError): # pragma: no cover
         try:
-            df = pd.DataFrame(request.data['strokedata'])
+            df = pl.DataFrame(request.data['strokedata'])
         except:
             return HttpResponse("No JSON Object could be decoded", status=400)
 
-    df.index = df.index.astype(int)
+    df = df.sort("time")
-    df.sort_index(inplace=True)
-
     status, comment, data = api_get_dataframe(startdatetime, df)
 
     if status != 200: # pragma: no cover
@@ -576,7 +574,8 @@ def strokedatajson_v3(request):
 
 
     csvfilename = 'media/{code}.csv.gz'.format(code=uuid4().hex[:16])
-    _ = data.to_csv(csvfilename, index_label='index', compression='gzip')
+    with gzip.open(csvfilename, 'w') as f:
+        _ = data.write_csv(f)
 
     duration = datetime.time(0,0,1)
     w = Workout(
@@ -667,10 +666,11 @@ def strokedatajson_v2(request, id):
 
     if request.method == 'GET':
         columns = ['spm', 'time', 'hr', 'pace', 'power', 'distance']
-        datadf = dataprep.getsmallrowdata_db(columns, ids=[id])
+        datadf = dataprep.read_data(columns, ids=[id])
+        datadf = dataprep.remove_nulls_pl(datadf)
         dologging('apilog.log',request.user.username+"(strokedatajson_v2 GET)")
 
-        data = datadf.to_json(orient='records')
+        data = datadf.write_json(row_oriented=True)
         data2 = json.loads(data)
         data2 = {"data": data2}
 
@@ -681,28 +681,27 @@ def strokedatajson_v2(request, id):
         try:
             for d in request.data['data']:
                 dologging('apilog.log',json.dumps(d))
-        except KeyError: # pragma: no cover
+        except (KeyError, ColumnNotFoundError): # pragma: no cover
             try:
                 for d in request.data['strokedata']:
                     dologging('apilog.log',json.dumps(d))
-            except KeyError:
+            except (KeyError, ColumnNotFoundError):
                 dologging('apilog.log','No data in request.data')
         checkdata, r = dataprep.getrowdata_db(id=row.id)
         if not checkdata.empty:  # pragma: no cover
             return HttpResponse("Duplicate Error", status=409)
 
-        df = pd.DataFrame()
+        df = pl.DataFrame()
 
         try:
-            df = pd.DataFrame(request.data['data'])
+            df = pl.DataFrame(request.data['data'])
-        except KeyError:  # pragma: no cover
+        except (KeyError, ColumnNotFoundError):  # pragma: no cover
             try:
-                df = pd.DataFrame(request.data['strokedata'])
+                df = pl.DataFrame(request.data['strokedata'])
             except:
                 return HttpResponse("No JSON object could be decoded", status=400)
 
-        df.index = df.index.astype(int)
+        df = df.sort("time")
-        df.sort_index(inplace=True)
 
         status, comment, data = api_get_dataframe(row.startdatetime, df)
         if status != 200: # pragma: no cover
@@ -724,7 +723,8 @@ def strokedatajson_v2(request, id):
             row.duplicate = True
             row.save()
 
-        _ = data.to_csv(csvfilename+'.gz', index_label='index', compression='gzip')
+        with gzip.open(csvfilename+'.gz', 'w') as f:
+            _ = data.write_csv(f)
         row.csvfilename = csvfilename
         row.save()
 
@@ -796,7 +796,11 @@ def strokedatajson(request, id=0):
     if request.method == 'GET':
         # currently only returns a subset.
         columns = ['spm', 'time', 'hr', 'pace', 'power', 'distance']
-        datadf = dataprep.getsmallrowdata_db(columns, ids=[id])
+        
+        datadf = dataprep.read_data(columns, ids=[id])
+        datadf = dataprep.remove_nulls_pl(datadf)
+        datadf = datadf.to_pandas()
+
         dologging("apilog.log",request.user.username+"(strokedatajson GET) ")
         return JSONResponse(datadf)
 
@@ -817,14 +821,13 @@ def strokedatajson(request, id=0):
                 return HttpResponse("No JSON object could be decoded", status=400)
 
         try:
-            df = pd.DataFrame(strokedata)
+            df = pl.DataFrame(strokedata)
         except ValueError:  # pragma: no cover
             return HttpResponse("Arrays must all be same length", status=400)
-        df.index = df.index.astype(int)
+        df = df.sort("time")
-        df.sort_index(inplace=True)
         try:
             time = df['time']/1.e3
-        except KeyError:  # pragma: no cover
+        except (KeyError, ColumnNotFoundError):  # pragma: no cover
             return HttpResponse("There must be time values", status=400)
         aantal = len(time)
         pace = df['pace']/1.e3
@@ -863,7 +866,7 @@ def strokedatajson(request, id=0):
 
         dologging("apilog.log",request.user.username+"(POST)")
 
-        data = pd.DataFrame({'TimeStamp (sec)': unixtime,
+        data = pl.DataFrame({'TimeStamp (sec)': unixtime,
                              ' Horizontal (meters)': distance,
                              ' Cadence (stokes/min)': spm,
                              ' HRCur (bpm)': hr,
@@ -890,8 +893,8 @@ def strokedatajson(request, id=0):
         timestr = row.startdatetime.strftime("%Y%m%d-%H%M%S")
         csvfilename = 'media/Import_'+timestr+'.csv'
 
-        res = data.to_csv(csvfilename+'.gz', index_label='index',
+        with gzip.open(csvfilename+'.gz','w') as f:
-                          compression='gzip')
+            res = data.write_csv(f)
         row.csvfilename = csvfilename
         row.save()
 

rowers/views/statements.py

@@ -16,7 +16,7 @@ from rowers.utils import (
 from rowers.celery import result as celery_result
 from rowers.interactiveplots import *
 from scipy.interpolate import griddata
-from rowers.dataprep import getsmallrowdata_db, read_data
+from rowers.dataprep import getsmallrowdata_pd, read_data
 from rowers.dataprep import timedeltaconv
 from scipy.special import lambertw
 from io import BytesIO
@@ -35,6 +35,7 @@ import threading
 import redis
 import colorsys
 import re
+import gzip
 import zipfile
 import bleach
 import arrow
@@ -1345,9 +1346,9 @@ def trydf(df, aantal, column):  # pragma: no cover
         s = df[column]
         if len(s) != aantal:
             return np.zeros(aantal)
-        if not np.issubdtype(s, np.number):
+        if not s.dtype in pl.NUMERIC_DTYPES:
             return np.zeros(aantal)
-    except KeyError:
+    except (KeyError, ColumnNotFoundError):
         s = np.zeros(aantal)
 
     return s