From 30bf61cbe16b8aa43b00e99e13a9bb6341693230 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sat, 20 Apr 2024 17:14:22 +0200 Subject: [PATCH] testing --- rowers/dataprep.py | 17 ++--- rowers/dataroutines.py | 96 +++----------------------- rowers/interactiveplots.py | 70 ++++++++----------- rowers/tests/mocks.py | 4 +- rowers/tests/test_analysis.py | 12 ++-- rowers/tests/test_emails.py | 4 +- rowers/tests/testdata/testdata.tcx.gz | Bin 3999 -> 4000 bytes rowers/utils.py | 1 + rowers/views/analysisviews.py | 27 ++++---- rowers/views/statements.py | 2 +- 10 files changed, 76 insertions(+), 157 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 5983bb91..275f8684 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -29,6 +29,7 @@ import itertools import numpy as np import pandas as pd import polars as pl +from polars.exceptions import ColumnNotFoundError from zipfile import BadZipFile import zipfile import os @@ -422,7 +423,8 @@ def calculate_goldmedalstandard(rower, workout, recurrance=True): try: df = pl.read_parquet(cpfile) except: - df = getsmallrowdata_pl(['power'], ids=[workout.id]) + df = read_data(['power'], ids=[workout.id]) + df = remove_nulls_pl(df) background = True if settings.TESTING: background = False @@ -525,8 +527,9 @@ def setcp(workout, background=False, recurrance=True): except Exception as e: pass - strokesdf = getsmallrowdata_pl( + strokesdf = read_data( ['power', 'workoutid', 'time'], ids=[workout.id]) + strokesdf = remove_nulls_pl(strokesdf) if strokesdf.is_empty(): return pl.DataFrame({'delta': [], 'cp': []}), pl.Series(dtype=pl.Float64), pl.Series(dtype=pl.Float64) @@ -617,14 +620,10 @@ def update_wps(r, types, mode='water', asynchron=True): mode ) - df = getsmallrowdata_db(['time', 'driveenergy'], ids=ids) + df = read_data(['time', 'driveenergy'], ids=ids) try: - mask = df['driveenergy'] > 100 - except (KeyError, TypeError): - return False - try: - wps_median = int(df.loc[mask, 'driveenergy'].median()) + wps_median = int(df.filter(pl.col("driveenergy")>100)["driveenergy"].median()) if mode == 'water': r.median_wps = wps_median else: # pragma: no cover @@ -635,6 +634,8 @@ def update_wps(r, types, mode='water', asynchron=True): pass except OverflowError: pass + except ColumnNotFoundError: + pass return True diff --git a/rowers/dataroutines.py b/rowers/dataroutines.py index ce986ac3..90f50aeb 100644 --- a/rowers/dataroutines.py +++ b/rowers/dataroutines.py @@ -1488,90 +1488,6 @@ def getrowdata_pl(id=0, doclean=False, convertnewtons=True, return data, row -# Fetch a subset of the data from the DB - -def getsmallrowdata_pl(columns, ids=[], doclean=True, workstrokesonly=True, compute=True, - debug=False, for_chart=False): - if ids: - csvfilenames = [ - 'media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] - else: - return pl.DataFrame() - - data = [] - columns = [c for c in columns if c != 'None'] + ['distance', 'spm', 'workoutid'] - columns = list(set(columns)) - - df = pl.DataFrame() - if len(ids) > 1: - for id, f in zip(ids, csvfilenames): - try: - df = pl.read_parquet(f, columns=columns) - data.append(df) - except (IsADirectoryError, FileNotFoundError, OSError, ArrowInvalid, IndexError): # pragma: no cover - rowdata, row = getrowdata(id=id) - try: - shutil.rmtree(f) - except: - pass - if rowdata and len(rowdata.df): - _ = dataplep(rowdata.df, id=id, - bands=True, otwpower=True, barchart=True, - polars=True) - try: - df = pl.read_parquet(f, columns=columns) - data.append(df) - except (OSError, ArrowInvalid, IndexError): - pass - try: - df = pl.concat(data, rechunk=True) - except ValueError: # pragma: no cover - return pl.DataFrame() - except SchemaError: - df = pl.concat(data, rechunk=True, how='vertical_relaxed') - - - else: - try: - df = pl.read_parquet(csvfilenames[0], columns=columns) - rowdata, row = getrowdata(id=ids[0]) - except (OSError, IndexError, ArrowInvalid): - rowdata, row = getrowdata(id=ids[0]) - if rowdata and len(rowdata.df): # pragma: no cover - data = dataplep( - rowdata.df, id=ids[0], bands=True, otwpower=True, barchart=True) - try: - df = pl.read_parquet(csvfilenames[0], columns=columns) - except: - df = pl.DataFrame - else: - df = pl.DataFrame() - except: - rowdata, row = getrowdata(id=ids[0]) - if rowdata and len(rowdata.df): # pragma: no cover - data = dataplep( - rowdata.df, id=ids[0], bands=True, otwpower=True, barchart=True) - try: - df = pl.read_parquet(csvfilenames[0], columns=columns) - except: - df = pl.DataFrame() - else: - df = pl.DataFrame() - - if compute and len(df): - data = df.clone() - if doclean: - data = clean_df_stats_pl(data, ignorehr=True, - workstrokesonly=workstrokesonly, - for_chart=for_chart) - - - data = remove_nulls_pl(data) - - if not df.is_empty(): - df = df.fill_nan(None).drop_nulls() - - return df def read_data(columns, ids=[], doclean=True, workstrokesonly=True, debug=False, for_chart=False, compute=True): @@ -1603,6 +1519,8 @@ def read_data(columns, ids=[], doclean=True, workstrokesonly=True, debug=False, data.append(df) data = pl.collect_all(data) + if len(data)==0: + return pl.DataFrame() try: datadf = pl.concat(data).select(columns) @@ -1635,7 +1553,15 @@ def read_data(columns, ids=[], doclean=True, workstrokesonly=True, debug=False, for df in data ] - datadf = pl.concat(data) + try: + datadf = pl.concat(data) + except SchemaError: + data = [ + df.with_columns(cs.integer().cast(pl.Float64)) for df in data + ] + datadf = pl.concat(data) + + exprs = [] diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 0098c007..06fe2637 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -1612,26 +1612,26 @@ def interactive_cum_flex_chart2(theworkouts, promember=0, try: datadf = datadf.with_columns(pl.col(xparam).alias("x1")) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover try: datadf = datadf.with_columns(pl.col("distance").alias("x1")) - except KeyError: + except (KeyError, ColumnNotFoundError): try: datadf = datadf.with_columns(pl.col('time').alias("x1")) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover return ['', '

No non-zero data in selection

', '', ''] try: datadf = datadf.with_columns(pl.col(yparam1).alias("y1")) - except KeyError: + except (KeyError, ColumnNotFoundError): try: datadf = datadf.with_columns(pl.col('pace').alias("y1")) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover return ['', '

No non-zero data in selection

', '', ''] if yparam2 != 'None': try: datadf = datadf.with_columns(pl.col(yparam2).alias("y2")) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover datadf = datadf.with_columns(pl.col("y1").alias("y2")) else: # pragma: no cover datadf = datadf.with_columns(pl.col("y1").alias("y2")) @@ -2099,14 +2099,11 @@ def interactive_multiple_compare_chart(ids, xparam, yparam, plottype='line', promember=0, workstrokesonly=True, labeldict=None, startenddict={}): - message = '' - errormessage = '' - - columns = [name for name, d in metrics.rowingmetrics] - columns_basic = [name for name, d in metrics.rowingmetrics if d['group'] == 'basic'] + columns = [xparam,yparam] + columns_basic = [xparam,yparam] add_columns = [ 'ftime', 'distance', 'fpace', - 'power', 'hr', 'spm', + 'spm', 'time', 'pace', 'workoutstate', 'workoutid' ] @@ -2122,47 +2119,37 @@ def interactive_multiple_compare_chart(ids, xparam, yparam, plottype='line', datadf = pd.DataFrame() if promember: - datadf = dataprep.getsmallrowdata_db(columns, ids=ids, doclean=doclean, - compute=compute, - workstrokesonly=workstrokesonly, for_chart=True) + datadf = dataprep.read_data(columns, ids=ids, doclean=doclean, + compute=compute, + workstrokesonly=workstrokesonly, for_chart=True) else: - datadf = dataprep.getsmallrowdata_db(columns_basic, ids=ids, doclean=doclean, - compute=compute, - workstrokesonly=workstrokesonly, for_chart=True) + datadf = dataprep.read_data(columns_basic, ids=ids, doclean=doclean, + compute=compute, + workstrokesonly=workstrokesonly, for_chart=True) - + + datadf = dataprep.remove_nulls_pl(datadf) # check if dataframe not empty - if datadf.empty: # pragma: no cover + if datadf.is_empty(): # pragma: no cover return ['

No non-zero data in selection

', ''] - datadf['workoutid'] = datadf['workoutid'].astype(int) - datadf.dropna(axis=1, how='all', inplace=True) - datadf.dropna(axis=0, how='all', inplace=True) + datadf = datadf.with_columns(pl.col("workoutid").cast(pl.UInt32).keep_name()) nrworkouts = len(ids) try: - tseconds = datadf.loc[:, 'time'] - except KeyError: # pragma: no cover + tseconds = datadf['time'] + except (KeyError, ColumnNotFoundError): # pragma: no cover try: - tseconds = datadf.loc[:, xparam] + tseconds = datadf[xparam] except: return ['

A chart data error occurred

', ''] - # check if dataframe not empty - if datadf.empty: # pragma: no cover - return ['

No non-zero data in selection

', ''] - - if (xparam == 'time'): - datadf[xparam] = datadf[xparam] - datadf[xparam].iloc[0] + datadf = datadf.with_columns((pl.col(xparam)-datadf[0,xparam]).alias(xparam)) - datadf = datadf.fillna(0) - datadf.replace([np.inf, -np.inf], np.nan, inplace=True) - datadf = datadf.fillna(0) - - data_dict = datadf.to_dict("records") + data_dict = datadf.to_dicts() metrics_list = [{'name': name, 'rowingmetrics':d } for name, d in metrics.rowingmetrics] @@ -2178,8 +2165,8 @@ def interactive_multiple_compare_chart(ids, xparam, yparam, plottype='line', 'workouts': workoutsdict, } - script, div = get_chart("/compare", chart_data) - return script, div, message, errormessage + script, div = get_chart("/compare", chart_data, debug=False) + return script, div def get_zones_report_pl(rower, startdate, enddate, trainingzones='hr', date_agg='week', yaxis='time'): @@ -2202,7 +2189,10 @@ def get_zones_report_pl(rower, startdate, enddate, trainingzones='hr', date_agg= df = dataprep.read_data(columns, ids=ids, workstrokesonly=False, doclean=False) df = dataprep.remove_nulls_pl(df) - df = df.with_columns((pl.col("time").diff().clip(0, 20*1.e3)).alias("deltat")).lazy() + try: + df = df.with_columns((pl.col("time").diff().clip(0, 20*1.e3)).alias("deltat")).lazy() + except ColumnNotFoundError: + pass hrzones = rower.hrzones powerzones = rower.powerzones diff --git a/rowers/tests/mocks.py b/rowers/tests/mocks.py index cfae2db4..617c7c38 100644 --- a/rowers/tests/mocks.py +++ b/rowers/tests/mocks.py @@ -301,7 +301,7 @@ def mocked_getrowdata_db(*args, **kwargs): return df,row def mocked_getrowdata_uh(*args, **kwargs): # pragma: no cover - df = pd.read_csv('rowers/tests/testdata/uhfull.csv') + df = pl.read_csv('rowers/tests/testdata/uhfull.csv') id = kwargs['id'] @@ -315,7 +315,7 @@ def mocked_getsmallrowdata_uh(*args, **kwargs): # pragma: no cover return df def mocked_getsmallrowdata_forfusion(*args, **kwargs): - df = pd.read_csv('rowers/tests/testdata/getrowdata_mock.csv') + df = pl.read_csv('rowers/tests/testdata/getrowdata_mock.csv') return df diff --git a/rowers/tests/test_analysis.py b/rowers/tests/test_analysis.py index f7101bb7..69d98787 100644 --- a/rowers/tests/test_analysis.py +++ b/rowers/tests/test_analysis.py @@ -169,7 +169,7 @@ class ForcecurveTest(TestCase): pass - @patch('rowers.dataprep.getsmallrowdata_db',side_effect = mocked_getempowerdata_db) + @patch('rowers.dataprep.read_data',side_effect = mocked_read_data) def test_forcecurve_plot(self, mocked_getsmallrowdata_db): login = self.c.login(username=self.u.username, password = self.password) self.assertTrue(login) @@ -600,9 +600,9 @@ class History(TestCase): pass @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data',side_effect=mocked_read_data) def test_workouts_history(self, mocked_sqlalchemy, - mocked_getsmallrowdata_db): + mocked_read_data): login = self.c.login(username=self.u.username, password=self.password) self.assertTrue(login) @@ -916,7 +916,7 @@ class WorkoutStatsTestNew(TestCase): self.assertEqual(response.status_code,200) @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_db) + @patch('rowers.dataprep.read_data', side_effect=mocked_read_data) @patch('rowers.dataprep.read_cols_df_sql', side_effect=mocked_read_cols_df_sql) def test_analysis_data(self, mocked_sqlalchemy, @@ -1257,8 +1257,8 @@ class MarkerPerformanceTest(TestCase): self.assertRedirects(response, expected_url=expected_url, status_code=302,target_status_code=200) - @patch('rowers.dataprep.getsmallrowdata_db', side_effect=mocked_getsmallrowdata_uh) - def test_trainingzones_view(self,mocked_getsmallrowdata_db): + @patch('rowers.dataprep.read_data', side_effect=mocked_getsmallrowdata_uh) + def test_trainingzones_view(self,mocked_getsmallrowdata_uh): login = self.c.login(username=self.u.username,password=self.password) self.assertTrue(login) diff --git a/rowers/tests/test_emails.py b/rowers/tests/test_emails.py index 6523754b..a606263d 100644 --- a/rowers/tests/test_emails.py +++ b/rowers/tests/test_emails.py @@ -55,8 +55,8 @@ class EmailUpload(TestCase): @patch('rowers.dataprep.create_engine') - @patch('rowers.dataprep.getsmallrowdata_db',side_effect=mocked_getsmallrowdata_db) - def test_uploadapi(self,mocked_sqlalchemy,mocked_getsmallrowdata_db): + @patch('rowers.dataprep.read_data',side_effect=mocked_read_data) + def test_uploadapi(self,mocked_sqlalchemy,mocked_read_data): form_data = { 'title': 'test', 'workouttype':'rower', diff --git a/rowers/tests/testdata/testdata.tcx.gz b/rowers/tests/testdata/testdata.tcx.gz index 5bf43e1e916e3a856305c8da0fc7f1e54de12fcf..d8a327745c9963644d6235569cd8ced867417261 100644 GIT binary patch delta 3913 zcmV-P54P~1AD|xwABzYG@79q9AQLJz=TfIGj&VdFt3?7f3MiYmRgpO(f1lo+p6tC@ zUTjup>w9<6z}>x{4(}X1zvx!$)%y6sa@{X49(H~AqT4L@($4G6y}Pga{(OJ2c>DIP zIqoh_S8H>0cDhJkZjN3pPrJ?H`okA*=5ILrN`e)VS6zyIZ3zg*`VymEqf zo7HD$c+2MO;&?#-xcKAA72{{yQ~a=6Z~AV1v`j~BR{Oj2K0cA~7Xi?d2mvvFJ3qVV z@7?`p|7v-0clTxb`q9fn0(3us{6O#=dEeqbITIamguemo0~{>UP9OJw+@0?|>${8o z`Ra6e@9tk6dwHRB=<~C_JIRMVTOOUQUv3VC{lVg^t)C8fxZ1rzPnPM&Zqj|`d--DH zr@cSoJ46fzi_1@+cDSEJtvgdD-=wUzZn8 zR_oXO^56ZHACXS;dw24Bd5AV$WcTN%D=e+ou+GngdpVinYp4Ne1pdw$RraVJ+1_XH8a1qF9XdQ(B};pW__E_t_75qD>ZQ8mh^ zA%Y8XkF(+KY(?B-l6Na!!#$RicZS(;_Z4xsv>oT2g1Zfd3UbF=aL1~+2dId?9gyotqY+EP1{Kd8&&%RWrrYJ|t64 zbfz4ll=R)OosZ13R7BpIn3Q^z%#)ZA$-b`8L`Qk|#8dz^g|4P?r3BB6KJfS#@NM0HR96i?O6VKPU1wR75`M=}f3f zzJgdXN@rNMcDJ-mJ)1v@+SerW zVoiFsKpOIYh?CLnTaZ3-tx8#b-d}vnPDCE7Nh)(0Zwd{)K+lMyV zk3pVc+Y(Eo+gt03>h__<=SLw=#xX=y6FezPeohsYt|IbctT3zQc~`WSc2?x2BJ#l` z?y3xbOk1|1QlQEF800OQq;Wb@wLC4_>sDq|MJ=`>@@%}*MQLq}C3AA-M82LH8_v16 z&Le6@dqJ3t%r|>}n0z!8AO;or*mV2pM{I<0w-i;BRv4OYKYf`OYeLfPRWfhOW?jct zOFIgAZ-jLb8xCiAb#R-a5dRdpmuZM>Z0OG*McI(;?5z zlQT%(<*aK}CqzeBRJWfK`I<)iXaF&2;`5HFXce_@@%b^x=iJ+XLDTovSt^=)J16pg z71{G-4LM}hJTEkv+*@w-^MmBQL5rjs?Hv`h=VwITQAOl)=LBL+e|zV-=sZ6s@)ha! zNn7KHs#VKe;1;Ew0eLY1qUv()BDW~*^hbO4Mnu;M(L*DDQ}Pw*_7)6#QF(Ga+L9XH zt@7}ihBr)}bN5q~_tqy%Ft!ZL$&(j<kH)3JUNhPU3PtVtIgR$A9`Z=WT8x^k>%g(e{0+&+ zipYzBpd9UO(Ul5k<;kgNv`25eh^kvka79-qos;+0Q$6Gfa&IiwcydD7H7bP~`BBKT zLDKcW458?%Wpg55(>zaNf?(8tWMGDfMKd#KM)SO{hFykn_Pd|GBZXX?{glWsvlnDqU+d*ol*DGn#e~J97dJQ z7tY4At?%s^(W*frA_iHtkmtDD{9rviF`$#92*ROMFLH}ee}NQ zXz$uu)MJp}<=*P%`4Ea)b#o$Lktauzg~kLb@-Y<6$#HF-ABDU#!4hktv$+ zt_RTvBS=Up^hq`3i5T^NP3i~H=XR!qtV8c?QQJ=1Tt5nWYpkHkhl}8)N&OAcQ$^@~ zZjt3tg+92+NPT7K+0}zS*=aTqSQYx{i>i5Z8t}0q^ql5;0@YlP5GI4ZwgJB;sdt6} zNj29amMyoL6MC))easqu)Jc8WRSO&XanR>ZsE$>cS(s+5rs2Z@Y$AK=Z0F%-5BU|hNYmczMiFQI~8hb-YDqV z2w>Dny$h2W@UbOm4Wf6(qHZliWFLz*PNQbmkAglLifDBjekdEh;pWX71HGl(KTGG$ z;~1NoH~j&B---cI7G7C|$;9hxOUAA!Xls+kh7_X;JyDzMr;lF1MCHQ8Xqk#e?wSvO z9Q16WL(-wQJelRqnj?1@{kCPISE+ecY;);M-;uj*nTTFD*E^X^f>uk*96%qoEfYbv z#AX6+3*dw0e=kih=y65O6mp5 zE?3VpQuCl9^mf}?L8azN;WkQhs;{>dq4$QYGH5FdMZI~p4ftcACoo8itU_;LGHV+) zZ{9HaZILyp;`PZQXhqgDkb0N9YD67+M@5ah)>UI1^dU>V*A4idCnNQ3z#l}TOzj$HP}Gg(#8)`6Q$szy`u2GJ)w%{tU&={R(4 zz@K_qaTt5T!Vu-YAo`;r^JN3e=kXt5yV%nxdA8;}aRhpVNfIjAyiGUb&c73Fl zmN^6TYyv2~c^=9xXAG^E&M4?zMjusny+KYLZ4T%1`Im% z94CXmtzSQko=p(aMefpzQc}auohTZA`Vr7OC%}MdZ`F~%xl_$PU<}t z9q@fC!59O*kZ~4+DoZDXqLsF{Lf>lchS5u|z6q*~+z^Xa+TLowZ@qMe(TBXItuAsG zq3B{~v(oTu$(;r8R5oY}ZLS{$z0cKuH%j#E zC$qX`o9hSB=asA(N#)Ip))%e5q1HEV6!g4Z#9U?EMTu>7%ZYpQGJ2N_uaxLrEGqS_ zQR+D8V=lbX+4V6%Yo0c7Z(c&bUCCM*xr-57XY3iEwss&I8p!LYTDSAH^`VU(_ zZa7#x?#?eiJU#1|o5Sx;`X7IPIa!`A*Zto2+fCo^b^RW^^M4XAkHQNdJNWLn|M6h4 zJK)alOdo$^{(Sjqwfp|y!A0VDcJX1~r(Jq`F3#R=y7kNMAcv4s|NQ)9b=39gr(eDpm*@WKWa+z~U$0JHUjE#F-_!e;4!!bl zK5Ti?{U=>AI}5IU{(867I&JpyX}|4;%SZdg`466_n?C$)d9v&_%fumn`tazk+}FY5 zZhidd;o&b&Gn?(7yY0WGxA^Jn(-+HhX8=C{?EXJkT=`=6x=D delta 3912 zcmV-O54Z53AD-%@nz}>wc5APhjxad~v)%y71a@{X49(8^9N4Ht-rJdKC`*&aU{rUc4@%HUo zbKG5=uGZ%0>~xX7+#J1Ho_3qX^@lIs(B0iB_PclAaFz#0{p!uCfB*Ble!0#!c;y7| zHmlFh@RrTl#qojwaPj-o6(`TPr?|ITZ~AV1v`j~BR{Oj2J~@%_7XjFl2mvvFKR>(Z z@8A7r|7v-0clTxb`ti#{0(AF4?h(8|-nY0<&O}EX;co!@00)b-)5ravbmx1|`|hHD zu{vGezx!9mUS232`r@qbPV!;Tmq%yomzzUjf3Wy!>!$-At#)tF(`EXxn{=P~UcT7) zY44Bt4iUq_;_}m{9Udf6>yDRy|5&EG|F!FvKb@a$U(NpgdHN-nn;)-#T6X>Bm*vIN z)%tb6{C9ulN2JsI)}6dw9->Vb+5P$H3X5wlapmi$-MiK4>(l?!4T!tu2Clipm9HPK zx36md{&4$NEECzwEmt4p%Vm1NhlKet?*6_)@nUs+oNjH|zdCz)__SMpzwS;F#m}~0 z-Mc&b?`LD zI=ouy3?@mtSOs-tgVVr&o*#5Y+{snMJwZfpLBXAp-c*o#xH)&KOWv(i#N8QURE_d! zh~R?U<7~J)TM_q|I+_5U|0V?7yiF*<% zjW+L%n+W&1@1Y{@&Nv{@=yg$w>htrFcZ7-*JBfx_=cYv{OP+5*p6Vh`)lBiU56P4h zohgSXC4Dz+=Ognh6_K|lCZ%2_^CTt|Brh`}A1flC%sD|0s_={}YR9|zkhiWL@(9LR zB$do3tRY|6cNb?wo+={m3_6_?2c0iD&s%7t{V3#vi6mAN zo`tNQ1lqEl(2pvjARvlR*fT+^&Vk~LT&xyPZ6_HPRIuojr zuOOC;(pj3(Xpa?A^Bfou3qdkjZ2VOJJlPyc$ z&WLe;$(FD7Ns2~?~E4=nkX$37PaT+M7|<>K3iyPRr5SEnm7^i*yj1`$#c%V zWkAw+a?*=ZGV9v9=C>b(ycl2zDw!858nJ2Q$06@cND_}4@{XI#-;g}lLmt6MfS_uU z;kl^H`=;BELY@s<6xBQ*xTuEbo6L_w-WiN2Dw&Uewy5{E$^1Cvx4E~f7HGC?Mdhu? zWAot+kAnX`GH!El-Q~x|JDKQH!mJJR9$HQCb^g$()=yk*}x5hI8(% z^N503DFT2)$Qj*zNXPW8bA!1_`G8(T1D+!e0~h_IrlbT(Dc1^mWt-y&WU_~ zMfN;dLk?Lr&kIc^_m*4z{2+O6&?2cudq+j>`5BRSR1x{yIe}Qy-`;sHI?vCEd_}r_ z($+YlYSl6qxJ7AaKwb=hsJfiH$Sq1c{n4Ji5z%!*^w7xPlzc_Hy#>QwRGu7;SH1L-2GJLz4gfwj4cCm^5n&TcW~~oRK|WQGDJ8W06slljGCd zVkbhrHXmN7hrD2}rB$tI;iYK({G4RICJj&6cp=pI@Vs*+{r0n(=PR=3ll*cdRgI1J zu2odtkbFf^T8RdjKsDM2HyPc&d2a{E%RZp-N*|t&MOPu8mCQ>;@|H}b>eqj71l*NpaoLQ#8uPNRLOhdh$87NaKXI&f?=AqUx3sT+!7@=j6ThR1bNA+#8EEo}5s2jY^?LeiZU- zkaRsTLnyjx*__DNG|!WmAQ&}&8JHnr(aa2*(LC=fA|H$-nb%ZWM4%?~Q;*fjKBKWr zs?3at+{jNoVw1>QL+PK2JQZEXCfA>~%#0Dp`<#cT>W3G(=sGrHXVm?)Ci2k)hfyW- zg|o42>w7x}c}vELuDU7~&c>S4y0jHVX_Nd4D|LJ4iduDZB43dw#|A@xkwBAgAH6R+ z+Pk(E^%&%LxwpD`K7^uH-JHl*EqR4fvKOgC1MI z>p}Fv2ojPCeNqj1B1V0GllnpQxt%E?>(Dz})V7m0*N=kU8Y`&s;UaixQh!7AR1tcg zTV#1up$~2{QePQ*cJ-i7cA5YZUg zQqA>Rk0x5BONYP}HSVZ+^Tt4*l=X-lt{(+Gp@~GGO3-4O&YEV; zhd+$on!w8H8)i3|rfDrpXBfRV-U_QUe70rRxuI5bHwOBgVJWDquV*RSPKBD9Hwt<- z0vL5t@4{pTd~69?gXo>Hs9Vbr*~g-d)2P|?qo9w5B3hk>AIgSrxOwx&KyNAc&(eAG zIL4;tO@F|Dw_-q)g;y3~GV%J_lCdia+S;VCA;qXdPt@l6>7y4gQMqt2TBf3ryXM0m z2R)nUkaXxRPiA?u=ExmJzipZ5Rcf9U+gv)+cjRtcCZgBP^-d;}pw-ec2hfLY%S6yE zd4twVGw!Bt*N5CP(TbqTk!wS%zPTy-P!al&J80>Dx@y>1HhjZvz#juWqG1-Nl6rx% z%hmIY)I6vNz1_A}P^o!RxQ)`B>g#Pq=)EDU4B84qQE#4Y1O6E32@Db=tI%7R%-V*{ zn>UPpTVzeDczv=6T9Ne(q~7JO8c~PdQBmWrb=4RLeaKSpbpyWV$w+-0@CVT|njox; z*GsT}MK$~uv>gY%H4YI~T{R@y=FOYFBbUALOjZ@Nb>JqGs?pTELG;N^vkrAxIu2bM z@TZ=em(a5j%1#;$dYVoNMr)%qjNTawInh#so@STG;*UtdG7$U8j=b>ffPQCCd7?FD(a0S%HE$I3!LVS^NxitDsTxhq8wEWj(>vXXA^{Uk-PMwl+^HZCyIuDegyQ+c(zgH(g_Yrx^!lRUMfN#^P0AxlX{Ou z2Yla3FvdVHWSqsI%F+p;Xr=9~(6^eqVf2!#Z-Oc#H^icqwznGaTQ8kq^dYZltBc%4 zD7x6$tTg;uvUCKtD_H}2l_NKzOva@Ht#95S`t3^Ax^?a%l?~cLo9jnG?{oEkjS@Zk z$*gYK=K4YOc_nK`QhD>D^+l_1sP)Yo1wC&UF;^LPQDR%&a^l{+jNawKDKYM-p$1>d+z&(K7{|AdJU+kXz_Awpa*`i*p({p&3P%bYX WrKj;ny7b|l{{ijikJ7{gfB^uPN%If@ diff --git a/rowers/utils.py b/rowers/utils.py index 95e5fb78..7f96a94a 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -7,6 +7,7 @@ import math import numpy as np import pandas as pd import polars as pl +from polars.exceptions import ColumnNotFoundError import colorsys from django.conf import settings import collections diff --git a/rowers/views/analysisviews.py b/rowers/views/analysisviews.py index 86ed09d3..3d0323f6 100644 --- a/rowers/views/analysisviews.py +++ b/rowers/views/analysisviews.py @@ -571,7 +571,7 @@ def flexalldata(workouts, options): workstrokesonly = not includereststrokes columns = [xparam, yparam1, yparam2, 'spm', 'driveenergy', 'distance'] ids = [int(w.id) for w in workouts] - df = dataprep.getsmallrowdata_pl(columns, ids=ids, + df = dataprep.read_data(columns, ids=ids, workstrokesonly=workstrokesonly, doclean=True, ) @@ -928,9 +928,9 @@ def boxplotdata(workouts, options): ids = [w.id for w in workouts] # prepare data frame - datadf = getsmallrowdata_pl(fieldlist, ids) + datadf = dataprep.read_data(fieldlist, ids) - datadf = dataprep.clean_df_stats_pl(datadf, workstrokesonly=workstrokesonly) + datadf = dataprep.remove_nulls_pl(datadf) try: datadf = datadf.filter( @@ -2361,17 +2361,16 @@ def history_view_data(request, userid=0): ids = [w.id for w in g_workouts] - # columns = ['hr', 'power', 'time'] - columns = [name for name, d in metrics.rowingmetrics]+['workoutstate', 'workoutid'] + columns = ['hr', 'power', 'time', 'workoutstate', 'workoutid'] + + df = dataprep.read_data(columns, ids=ids) + df = dataprep.remove_nulls_pl(df) - df = getsmallrowdata_pl(columns, ids=ids) try: df = df.with_columns(pl.col('time').diff().clip(lower_bound=0).alias("deltat")) except KeyError: # pragma: no cover pass - df = dataprep.clean_df_stats_pl(df, workstrokesonly=True, - ignoreadvanced=True, ignorehr=False) totalmeters, totalhours, totalminutes, totalseconds = get_totals( g_workouts) @@ -2400,7 +2399,8 @@ def history_view_data(request, userid=0): whours=whours, wminutes=wminutes, wseconds=wseconds, ) - ddf = getsmallrowdata_pl(columns, ids=[w.id for w in a_workouts]) + ddf = dataprep.read_data(columns, ids=[w.id for w in a_workouts]) + ddf = dataprep.remove_nulls_pl(ddf) try: ddf = ddf.with_columns(pl.col("time").diff().clip(lower_bound=0).alias("deltat")) except KeyError: # pragma: no cover @@ -2421,7 +2421,7 @@ def history_view_data(request, userid=0): ddict['powermean'] = int(wavg(ddf, 'power', 'deltat')) try: ddict['powermax'] = int(ddf['power'].max()) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover ddict['powermax'] = 0 ddict['nrworkouts'] = a_workouts.count() listofdicts.append(ddict) @@ -2436,13 +2436,13 @@ def history_view_data(request, userid=0): try: totalsdict['powermean'] = int(wavg(df, 'power', 'deltat')) totalsdict['powermax'] = int(df['power'].max()) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover totalsdict['powermean'] = 0 totalsdict['powermax'] = 0 try: totalsdict['hrmean'] = int(wavg(df, 'hr', 'deltat')) totalsdict['hrmax'] = int(df['hr'].max()) - except KeyError: # pragma: no cover + except (KeyError, ColumnNotFoundError): # pragma: no cover totalsdict['hrmean'] = 0 totalsdict['hrmax'] = 0 @@ -2461,7 +2461,8 @@ def history_view_data(request, userid=0): a_workouts = g_workouts.filter(workouttype=typeselect) meters, hours, minutes, seconds = get_totals(a_workouts) totalseconds = 3600 * hours + 60 * minutes + seconds - ddf = getsmallrowdata_pl(columns, ids=[w.id for w in a_workouts]) + ddf = dataprep.read_data(columns, ids=[w.id for w in a_workouts]) + ddf = dataprep.remove_nulls_pl(ddf) if ddf.is_empty(): totalscript = "" totaldiv = "No data" diff --git a/rowers/views/statements.py b/rowers/views/statements.py index 991b440f..83e22708 100644 --- a/rowers/views/statements.py +++ b/rowers/views/statements.py @@ -16,7 +16,7 @@ from rowers.utils import ( from rowers.celery import result as celery_result from rowers.interactiveplots import * from scipy.interpolate import griddata -from rowers.dataprep import getsmallrowdata_db, getsmallrowdata_pl +from rowers.dataprep import getsmallrowdata_db, read_data from rowers.dataprep import timedeltaconv from scipy.special import lambertw from io import BytesIO