From 409f4aff6333c001efeca3fb13c28ab8fbb3ffe0 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 21 Apr 2024 10:56:37 +0200 Subject: [PATCH] alerts, analysis, api tested --- rowers/alerts.py | 7 +- rowers/dataroutines.py | 10 +-- rowers/tasks.py | 3 +- rowers/tests/test_analysis.py | 80 +++++++++++----------- rowers/tests/test_api.py | 9 +-- rowers/views/analysisviews.py | 4 +- rowers/views/apiviews.py | 121 +++++++++++++++++----------------- rowers/views/statements.py | 7 +- 8 files changed, 124 insertions(+), 117 deletions(-) diff --git a/rowers/alerts.py b/rowers/alerts.py index ef74ddf0..bf3587bd 100644 --- a/rowers/alerts.py +++ b/rowers/alerts.py @@ -1,6 +1,6 @@ from rowers.models import Alert, Condition, User, Rower, Workout from rowers.teams import coach_getcoachees -from rowers.dataprep import getsmallrowdata_db, getrowdata_db +from rowers.dataprep import getrowdata_db, read_data, remove_nulls_pl import datetime import numpy as np import math @@ -101,8 +101,9 @@ def alert_get_stats(alert, nperiod=0): # pragma: no cover ids = [w.id for w in workouts] try: - df = getsmallrowdata_db(columns, ids=ids, doclean=True, - workstrokesonly=workstrokesonly) + df = read_data(columns, ids=ids, doclean=True, + workstrokesonly=workstrokesonly) + df = remove_nulls_pl(df) except: return { 'workouts': workouts.count(), diff --git a/rowers/dataroutines.py b/rowers/dataroutines.py index 90f50aeb..8fe36699 100644 --- a/rowers/dataroutines.py +++ b/rowers/dataroutines.py @@ -1417,10 +1417,10 @@ def update_strokedata(id, df, debug=False): def testdata(time, distance, pace, spm): # pragma: no cover - t1 = np.issubdtype(time, np.number) - t2 = np.issubdtype(distance, np.number) - t3 = np.issubdtype(pace, np.number) - t4 = np.issubdtype(spm, np.number) + t1 = time.dtype in pl.NUMERIC_DTYPES + t2 = distance.dtype in pl.NUMERIC_DTYPES + t3 = pace.dtype in pl.NUMERIC_DTYPES + t4 = spm.dtype in pl.NUMERIC_DTYPES return t1 and t2 and t3 and t4 @@ -1630,7 +1630,7 @@ def read_data(columns, ids=[], doclean=True, workstrokesonly=True, debug=False, return datadf -def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True, compute=True, +def getsmallrowdata_pd(columns, ids=[], doclean=True, workstrokesonly=True, compute=True, debug=False, for_chart=False): # prepmultipledata(ids) diff --git a/rowers/tasks.py b/rowers/tasks.py index f2a7355f..104045e5 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -130,7 +130,8 @@ from rowers import mytypes from rowers.dataroutines import ( - getsmallrowdata_db, updatecpdata_sql, update_c2id_sql, + getsmallrowdata_pd, updatecpdata_sql, update_c2id_sql, + read_data, #update_workout_field_sql, update_agegroup_db, update_strokedata, add_c2_stroke_data_db, totaltime_sec_to_string, diff --git a/rowers/tests/test_analysis.py b/rowers/tests/test_analysis.py index 69d98787..4bd98e36 100644 --- a/rowers/tests/test_analysis.py +++ b/rowers/tests/test_analysis.py @@ -43,10 +43,10 @@ class ListWorkoutTest(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') @patch('rowers.dataprep.myqueue') def test_list_workouts(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db, + mocked_read_data, mocked_myqueue): login = self.c.login(username=self.u.username, password=self.password) @@ -170,7 +170,7 @@ class ForcecurveTest(TestCase): @patch('rowers.dataprep.read_data',side_effect = mocked_read_data) - def test_forcecurve_plot(self, mocked_getsmallrowdata_db): + def test_forcecurve_plot(self, mocked_read_data): login = self.c.login(username=self.u.username, password = self.password) self.assertTrue(login) @@ -296,9 +296,9 @@ class WorkoutCompareTestNew(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_workouts_compare(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username, password=self.password) self.assertTrue(login) @@ -309,9 +309,9 @@ class WorkoutCompareTestNew(TestCase): self.assertEqual(response.status_code,200) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_compare_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -398,9 +398,9 @@ class WorkoutBoxPlotTestNew(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_workouts_boxplot(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username, password=self.password) self.assertTrue(login) @@ -411,9 +411,9 @@ class WorkoutBoxPlotTestNew(TestCase): self.assertEqual(response.status_code,200) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_boxplot_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -499,9 +499,9 @@ class WorkoutHistoTestNew(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_workouts_histo(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username, password=self.password) self.assertTrue(login) @@ -512,9 +512,9 @@ class WorkoutHistoTestNew(TestCase): self.assertEqual(response.status_code,200) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_histo_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -618,9 +618,9 @@ class History(TestCase): @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_history_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -713,9 +713,9 @@ class GoldMedalScores(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_workouts_goldmedalscores(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): ws = Workout.objects.filter(rankingpiece=True) self.assertEqual(ws.count(),2) @@ -729,9 +729,9 @@ class GoldMedalScores(TestCase): self.assertEqual(response.status_code,200) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_goldmedalscores_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -795,9 +795,9 @@ class WorkoutFlexallTestNew(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_workouts_flexall(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username, password=self.password) self.assertTrue(login) @@ -808,9 +808,9 @@ class WorkoutFlexallTestNew(TestCase): self.assertEqual(response.status_code,200) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_flexall_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -903,9 +903,9 @@ class WorkoutStatsTestNew(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_workouts_stats(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username, password=self.password) self.assertTrue(login) @@ -920,7 +920,7 @@ class WorkoutStatsTestNew(TestCase): @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql) def test_analysis_data(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db, + mocked_read_data, mocked_read_cols_df_sql, ): @@ -979,11 +979,11 @@ class WorkoutStatsTestNew(TestCase): script, div = comparisondata(workouts,options) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql) def test_analysis_data2(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db, + mocked_read_data, mocked_read_cols_df_sql, ): @@ -1042,11 +1042,11 @@ class WorkoutStatsTestNew(TestCase): @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql) def test_analysis_data2(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db, + mocked_read_data, mocked_read_cols_df_sql, ): @@ -1120,9 +1120,9 @@ class WorkoutStatsTestNew(TestCase): @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) def test_workouts_stats_submit(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -1241,9 +1241,9 @@ class MarkerPerformanceTest(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_create_marker_workouts(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -1326,9 +1326,9 @@ class MarkerPerformanceTest(TestCase): @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db') + @patch('rowers.dataprep.read_data') def test_performancemanager_view(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) @@ -1413,9 +1413,9 @@ class AlertTest(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.alerts.getsmallrowdata_db') + @patch('rowers.alerts.read_data') def test_alerts(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) diff --git a/rowers/tests/test_api.py b/rowers/tests/test_api.py index 69279f53..e17c3532 100644 --- a/rowers/tests/test_api.py +++ b/rowers/tests/test_api.py @@ -67,10 +67,11 @@ class OwnApi(TestCase): self.assertEqual(response.status_code,200) # response must be json + strokedata = json.loads(response.content) - df = pd.DataFrame(strokedata) + df = pl.from_dict(strokedata) - self.assertFalse(df.empty) + self.assertFalse(df.is_empty()) form_data = { "distance": [23, 46, 48], @@ -124,8 +125,8 @@ class OwnApi(TestCase): # response must be json strokedata = json.loads(response.content) - df = pd.DataFrame(strokedata) - self.assertFalse(df.empty) + df = pl.from_dicts(strokedata['data']) + self.assertFalse(df.is_empty()) form_data = { diff --git a/rowers/views/analysisviews.py b/rowers/views/analysisviews.py index 3d0323f6..2b27ed6e 100644 --- a/rowers/views/analysisviews.py +++ b/rowers/views/analysisviews.py @@ -612,7 +612,7 @@ def histodata(workouts, options): if savedata: # pragma: no cover workstrokesonly = not includereststrokes ids = [int(w.id) for w in workouts] - df = dataprep.getsmallrowdata_db([plotfield], ids=ids, + df = dataprep.read_data([plotfield], ids=ids, workstrokesonly=workstrokesonly, doclean=True, ) @@ -885,7 +885,7 @@ def comparisondata(workouts, options): 'time', 'pace', 'workoutstate', 'workoutid'] - df = dataprep.getsmallrowdata_db(columns, ids=ids, + df = dataprep.read_data(columns, ids=ids, workstrokesonly=workstrokesonly, doclean=True, ) diff --git a/rowers/views/apiviews.py b/rowers/views/apiviews.py index 1f2f87bb..37bd0733 100644 --- a/rowers/views/apiviews.py +++ b/rowers/views/apiviews.py @@ -68,102 +68,102 @@ def strokedataform(request, id=0): def api_get_dataframe(startdatetime, df): try: time = df['time']/1.e3 - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover try: time = df['t']/10. - except KeyError: - return 400, "Missing time", pd.DataFrame() + except (KeyError, ColumnNotFoundError): + return 400, "Missing time", pl.DataFrame() try: spm = df['spm'] - except KeyError: # pragma: no cover - return 400, "Missing spm", pd.DataFrame() + except (KeyError, ColumnNotFoundError): # pragma: no cover + return 400, "Missing spm", pl.DataFrame() try: distance = df['distance'] - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover try: distance = df['d']/10. - except KeyError: - return 400, "Missing distance", pd.DataFrame() + except (KeyError, ColumnNotFoundError): + return 400, "Missing distance", pl.DataFrame() try: pace = df['pace']/1.e3 - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover try: pace = df['p']/10. - except KeyError: - return 400, "Missing pace", pd.DataFrame() + except (KeyError, ColumnNotFoundError): + return 400, "Missing pace", pl.DataFrame() try: power = df['power'] - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover power = 0*time try: drivelength = df['drivelength'] - except KeyError: + except (KeyError, ColumnNotFoundError): drivelength = 0*time try: dragfactor = df['dragfactor'] - except KeyError: + except (KeyError, ColumnNotFoundError): dragfactor = 0*time try: drivetime = df['drivetime'] - except KeyError: + except (KeyError, ColumnNotFoundError): drivetime = 0*time try: strokerecoverytime = df['strokerecoverytime'] - except KeyError: + except (KeyError, ColumnNotFoundError): strokerecoverytime = 0*time try: averagedriveforce = df['averagedriveforce'] - except KeyError: + except (KeyError, ColumnNotFoundError): averagedriveforce = 0*time try: peakdriveforce = df['peakdriveforce'] - except KeyError: + except (KeyError, ColumnNotFoundError): peakdriveforce = 0*time try: wash = df['wash'] - except KeyError: + except (KeyError, ColumnNotFoundError): wash = 0*time try: catch = df['catch'] - except KeyError: + except (KeyError, ColumnNotFoundError): catch = 0*time try: finish = df['finish'] - except KeyError: + except (KeyError, ColumnNotFoundError): finish = 0*time try: peakforceangle = df['peakforceangle'] - except KeyError: + except (KeyError, ColumnNotFoundError): peakforceangle = 0*time try: driveenergy = df['driveenergy'] - except KeyError: + except (KeyError, ColumnNotFoundError): driveenergy = 60.*power/spm try: slip = df['slip'] - except KeyError: + except (KeyError, ColumnNotFoundError): slip = 0*time try: lapidx = df['lapidx'] - except KeyError: + except (KeyError, ColumnNotFoundError): lapidx = 0*time try: hr = df['hr'] - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover hr = 0*df['time'] try: latitude = df['latitude'] - except KeyError: + except (KeyError, ColumnNotFoundError): latitude = 0*df['time'] try: longitude = df['longitude'] - except KeyError: + except (KeyError, ColumnNotFoundError): longitude = 0*df['time'] starttime = totimestamp(startdatetime)+time[0] @@ -171,7 +171,7 @@ def api_get_dataframe(startdatetime, df): dologging('apilog.log',"(strokedatajson_v2/3 POST - data parsed)") - data = pd.DataFrame({'TimeStamp (sec)': unixtime, + data = pl.DataFrame({'TimeStamp (sec)': unixtime, ' Horizontal (meters)': distance, ' Cadence (stokes/min)': spm, ' HRCur (bpm)': hr, @@ -522,7 +522,7 @@ def strokedatajson_v3(request): title = request.data.get('name','') try: elapsedTime = request.data['elapsedTime'] - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover try: duration = request.data['duration'] try: @@ -534,7 +534,7 @@ def strokedatajson_v3(request): return HttpResponse("Missing Elapsed Time", status=400) try: totalDistance = request.data['distance'] - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover return HttpResponse("Missing Total Distance", status=400) timeZone = request.data.get('timezone','UTC') workouttype = request.data.get('workouttype','rower') @@ -552,23 +552,21 @@ def strokedatajson_v3(request): dologging('apilog.log',totalDistance) dologging('apilog.log',elapsedTime) - df = pd.DataFrame() + df = pl.DataFrame() try: strokes = request.data['strokes'] - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover return HttpResponse("No Stroke Data in JSON", status=400) try: - df = pd.DataFrame(strokes['data']) - except KeyError: # pragma: no cover + df = pl.DataFrame(strokes['data']) + except (KeyError, ColumnNotFoundError): # pragma: no cover try: - df = pd.DataFrame(request.data['strokedata']) + df = pl.DataFrame(request.data['strokedata']) except: return HttpResponse("No JSON Object could be decoded", status=400) - df.index = df.index.astype(int) - df.sort_index(inplace=True) - + df = df.sort("time") status, comment, data = api_get_dataframe(startdatetime, df) if status != 200: # pragma: no cover @@ -576,7 +574,8 @@ def strokedatajson_v3(request): csvfilename = 'media/{code}.csv.gz'.format(code=uuid4().hex[:16]) - _ = data.to_csv(csvfilename, index_label='index', compression='gzip') + with gzip.open(csvfilename, 'w') as f: + _ = data.write_csv(f) duration = datetime.time(0,0,1) w = Workout( @@ -667,10 +666,11 @@ def strokedatajson_v2(request, id): if request.method == 'GET': columns = ['spm', 'time', 'hr', 'pace', 'power', 'distance'] - datadf = dataprep.getsmallrowdata_db(columns, ids=[id]) + datadf = dataprep.read_data(columns, ids=[id]) + datadf = dataprep.remove_nulls_pl(datadf) dologging('apilog.log',request.user.username+"(strokedatajson_v2 GET)") - data = datadf.to_json(orient='records') + data = datadf.write_json(row_oriented=True) data2 = json.loads(data) data2 = {"data": data2} @@ -681,28 +681,27 @@ def strokedatajson_v2(request, id): try: for d in request.data['data']: dologging('apilog.log',json.dumps(d)) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover try: for d in request.data['strokedata']: dologging('apilog.log',json.dumps(d)) - except KeyError: + except (KeyError, ColumnNotFoundError): dologging('apilog.log','No data in request.data') checkdata, r = dataprep.getrowdata_db(id=row.id) if not checkdata.empty: # pragma: no cover return HttpResponse("Duplicate Error", status=409) - df = pd.DataFrame() + df = pl.DataFrame() try: - df = pd.DataFrame(request.data['data']) - except KeyError: # pragma: no cover + df = pl.DataFrame(request.data['data']) + except (KeyError, ColumnNotFoundError): # pragma: no cover try: - df = pd.DataFrame(request.data['strokedata']) + df = pl.DataFrame(request.data['strokedata']) except: return HttpResponse("No JSON object could be decoded", status=400) - df.index = df.index.astype(int) - df.sort_index(inplace=True) + df = df.sort("time") status, comment, data = api_get_dataframe(row.startdatetime, df) if status != 200: # pragma: no cover @@ -724,7 +723,8 @@ def strokedatajson_v2(request, id): row.duplicate = True row.save() - _ = data.to_csv(csvfilename+'.gz', index_label='index', compression='gzip') + with gzip.open(csvfilename+'.gz', 'w') as f: + _ = data.write_csv(f) row.csvfilename = csvfilename row.save() @@ -796,7 +796,11 @@ def strokedatajson(request, id=0): if request.method == 'GET': # currently only returns a subset. columns = ['spm', 'time', 'hr', 'pace', 'power', 'distance'] - datadf = dataprep.getsmallrowdata_db(columns, ids=[id]) + + datadf = dataprep.read_data(columns, ids=[id]) + datadf = dataprep.remove_nulls_pl(datadf) + datadf = datadf.to_pandas() + dologging("apilog.log",request.user.username+"(strokedatajson GET) ") return JSONResponse(datadf) @@ -817,14 +821,13 @@ def strokedatajson(request, id=0): return HttpResponse("No JSON object could be decoded", status=400) try: - df = pd.DataFrame(strokedata) + df = pl.DataFrame(strokedata) except ValueError: # pragma: no cover return HttpResponse("Arrays must all be same length", status=400) - df.index = df.index.astype(int) - df.sort_index(inplace=True) + df = df.sort("time") try: time = df['time']/1.e3 - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover return HttpResponse("There must be time values", status=400) aantal = len(time) pace = df['pace']/1.e3 @@ -863,7 +866,7 @@ def strokedatajson(request, id=0): dologging("apilog.log",request.user.username+"(POST)") - data = pd.DataFrame({'TimeStamp (sec)': unixtime, + data = pl.DataFrame({'TimeStamp (sec)': unixtime, ' Horizontal (meters)': distance, ' Cadence (stokes/min)': spm, ' HRCur (bpm)': hr, @@ -890,8 +893,8 @@ def strokedatajson(request, id=0): timestr = row.startdatetime.strftime("%Y%m%d-%H%M%S") csvfilename = 'media/Import_'+timestr+'.csv' - res = data.to_csv(csvfilename+'.gz', index_label='index', - compression='gzip') + with gzip.open(csvfilename+'.gz','w') as f: + res = data.write_csv(f) row.csvfilename = csvfilename row.save() diff --git a/rowers/views/statements.py b/rowers/views/statements.py index 83e22708..ac88517b 100644 --- a/rowers/views/statements.py +++ b/rowers/views/statements.py @@ -16,7 +16,7 @@ from rowers.utils import ( from rowers.celery import result as celery_result from rowers.interactiveplots import * from scipy.interpolate import griddata -from rowers.dataprep import getsmallrowdata_db, read_data +from rowers.dataprep import getsmallrowdata_pd, read_data from rowers.dataprep import timedeltaconv from scipy.special import lambertw from io import BytesIO @@ -35,6 +35,7 @@ import threading import redis import colorsys import re +import gzip import zipfile import bleach import arrow @@ -1345,9 +1346,9 @@ def trydf(df, aantal, column): # pragma: no cover s = df[column] if len(s) != aantal: return np.zeros(aantal) - if not np.issubdtype(s, np.number): + if not s.dtype in pl.NUMERIC_DTYPES: return np.zeros(aantal) - except KeyError: + except (KeyError, ColumnNotFoundError): s = np.zeros(aantal) return s