From 964416362d22de1db73a20aef63b9c6b35b7d429 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 08:37:09 +0200 Subject: [PATCH 01/14] bug fix in stroke force chart --- rowers/interactiveplots.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index c2e921db..5e37dc78 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -704,12 +704,20 @@ def interactive_forcecurve(theworkouts,workstrokesonly=True,plottype='scatter'): ) if plottype == 'scatter': - sourcepoints = ColumnDataSource( - data = dict( - peakforceangle = rowdata['peakforceangle'], - peakforce = rowdata['peakforce'] + try: + sourcepoints = ColumnDataSource( + data = dict( + peakforceangle = rowdata['peakforceangle'], + peakforce = rowdata['peakforce'] + ) ) - ) + except KeyError: + sourcepoints = ColumnDataSource( + data = dict( + peakforceangle = [], + peakforce = [] + ) + ) else: sourcepoints = ColumnDataSource( data = dict( From 6de6a0dae1f9f062007be3b806cc7d36204527ee Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 14:33:36 +0200 Subject: [PATCH 02/14] ordering in mysql --- rowers/dataprep.py | 6 +++--- rowers/interactiveplots.py | 42 +++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 1aade283..9d43a330 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1911,12 +1911,12 @@ def read_cols_df_sql(ids, columns, convertnewtons=True): # columns=cls, # )) elif len(ids) == 1: - query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid={id}'.format( + query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid={id} ORDER BY time ASC'.format( id=ids[0], columns=cls, )) else: - query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid IN {ids}'.format( + query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid IN {ids} ORDER BY time ASC'.format( columns=cls, ids=tuple(ids), )) @@ -1953,7 +1953,7 @@ def read_cols_df_sql(ids, columns, convertnewtons=True): def read_df_sql(id): engine = create_engine(database_url, echo=False) - df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id}'.format( + df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id} ORDER BY time ASC'.format( id=id)), engine) engine.dispose() diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 5e37dc78..21d48b40 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -2699,11 +2699,11 @@ def interactive_chart(id=0,promember=0,intervaldata = {}): row = Workout.objects.get(id=id) if datadf.empty: return "","No Valid Data Available" - else: - try: - datadf.sort_values(by='time',ascending=True,inplace=True) - except KeyError: - return "","No valid data available" + #else: + # try: + # datadf.sort_values(by='time',ascending=True,inplace=True) + # except KeyError: + # return "","No valid data available" try: spm = datadf['spm'] @@ -2715,10 +2715,6 @@ def interactive_chart(id=0,promember=0,intervaldata = {}): except KeyError: datadf['pace'] = 0 - #datadf,row = dataprep.getrowdata_db(id=id) - #if datadf.empty: - #return "","No Valid Data Available" - source = ColumnDataSource( datadf ) @@ -3526,11 +3522,11 @@ def interactive_flex_chart2(id=0,promember=0, row = Workout.objects.get(id=id) if rowdata.empty: return "","No valid data",'','',workstrokesonly - else: - try: - rowdata.sort_values(by='time',ascending=True,inplace=True) - except KeyError: - pass + #else: + # try: + # rowdata.sort_values(by='time',ascending=True,inplace=True) + # except KeyError: + # pass workoutstateswork = [1,4,5,8,9,6,7] workoutstatesrest = [3] @@ -4094,11 +4090,11 @@ def thumbnails_set(r,id,favorites): 'notes':"" }] - else: - try: - rowdata.sort_values(by='time',ascending=True,inplace=True) - except KeyError: - pass + # else: + # try: + # rowdata.sort_values(by='time',ascending=True,inplace=True) + # except KeyError: + # pass l = len(rowdata) maxlength = 50 @@ -4673,13 +4669,13 @@ def interactive_comparison_chart(id1=0,id2=0,xparam='distance',yparam='spm', if rowdata1.empty: return "","No Valid Data Available" - else: - rowdata1.sort_values(by='time',ascending=True,inplace=True) +# else: +# rowdata1.sort_values(by='time',ascending=True,inplace=True) if rowdata2.empty: return "","No Valid Data Available" - else: - rowdata2.sort_values(by='time',ascending=True,inplace=True) +# else: +# rowdata2.sort_values(by='time',ascending=True,inplace=True) try: x1 = rowdata1.loc[:,xparam] From 7f2c68a90383e774a2f8a80325b23913b7340fd0 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 21:34:03 +0200 Subject: [PATCH 03/14] bla --- requirements.txt | 18 ++++++++++-- rowers/dataprep.py | 72 ++++++++++++++++++++++------------------------ 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/requirements.txt b/requirements.txt index 76e37d6a..86e57035 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ certifi==2019.3.9 cffi==1.12.2 chardet==3.0.4 Click==7.0 +cloudpickle==1.2.2 colorama==0.4.1 colorclass==2.2.0 cookies==2.2.1 @@ -27,7 +28,7 @@ coreschema==0.0.4 coverage==4.5.3 cryptography==2.6.1 cycler==0.10.0 -dask==1.1.4 +dask==2.6.0 decorator==4.4.0 defusedxml==0.5.0 Django==2.1.7 @@ -39,7 +40,7 @@ django-cookie-law==2.0.1 django-cors-headers==2.5.2 django-countries==5.3.3 django-datetime-widget==0.9.3 -django-debug-toolbar==1.11 +django-debug-toolbar==2.0 django-extensions==2.1.6 django-htmlmin==0.11.0 django-leaflet==0.24.0 @@ -64,8 +65,10 @@ entrypoints==0.3 execnet==1.5.0 factory-boy==2.11.1 Faker==1.0.4 +fastparquet==0.3.2 fitparse==1.1.0 Flask==1.0.2 +fsspec==0.5.2 future==0.17.1 geocoder==1.38.1 geos==0.2.1 @@ -74,6 +77,7 @@ html5lib==1.0.1 htmlmin==0.1.12 HTMLParser==0.0.2 httplib2==0.12.1 +hvplot==0.4.0 icalendar==4.0.3 idna==2.8 image==1.5.27 @@ -99,10 +103,12 @@ jupyterlab-server==0.3.0 keyring==18.0.0 kiwisolver==1.0.1 kombu==4.5.0 +llvmlite==0.30.0 lxml==4.3.2 Markdown==3.0.1 MarkupSafe==1.1.1 matplotlib==3.0.3 +minify==0.1.4 MiniMockTest==0.5 mistune==0.8.4 mock==2.0.0 @@ -111,9 +117,11 @@ mpld3==0.3 mysqlclient==1.4.2.post1 nbconvert==5.4.1 nbformat==4.4.0 +newrelic==5.2.1.129 nose==1.3.7 nose-parameterized==0.6.0 notebook==5.7.6 +numba==0.46.0 numpy==1.16.2 oauth2==1.9.0.post1 oauthlib==3.0.1 @@ -135,6 +143,7 @@ prompt-toolkit==2.0.9 psycopg2==2.8.1 ptyprocess==0.6.0 py==1.8.0 +pyarrow==0.15.0 pycparser==2.19 Pygments==2.3.1 pyparsing==2.3.1 @@ -160,7 +169,7 @@ ratelim==0.1.6 redis==3.2.1 requests==2.21.0 requests-oauthlib==1.2.0 -rowingdata==2.5.4 +rowingdata==2.5.5 rowingphysics==0.5.0 rq==0.13.0 scipy==1.2.1 @@ -179,7 +188,9 @@ terminado==0.8.1 terminaltables==3.1.0 testpath==0.4.2 text-unidecode==1.2 +thrift==0.11.0 timezonefinder==4.0.1 +toolz==0.10.0 tornado==6.0.1 tqdm==4.31.1 traitlets==4.3.2 @@ -196,3 +207,4 @@ xlrd==1.2.0 xmltodict==0.12.0 yamjam==0.1.7 yamllint==1.15.0 +yuicompressor==2.4.8 diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 9d43a330..01a649a7 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -4,7 +4,6 @@ from __future__ import print_function from __future__ import unicode_literals - # All the data preparation, data cleaning and data mangling should # be defined here from __future__ import unicode_literals, absolute_import @@ -26,6 +25,8 @@ from rowers.tasks import handle_sendemail_unrecognized from rowers.tasks import handle_zip_file from pandas import DataFrame, Series +import dask.dataframe as dd +from dask.delayed import delayed from django.utils import timezone from django.utils.timezone import get_current_timezone @@ -349,7 +350,7 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, # clean data remove zeros and negative values # bring metrics which have negative values to positive domain - if datadf.empty: + if len(datadf)==0: return datadf try: datadf['catch'] = -datadf['catch'] @@ -1771,8 +1772,31 @@ def getrowdata_db(id=0, doclean=False, convertnewtons=True, # Fetch a subset of the data from the DB +def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True): + prepmultipledata(ids) -def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True): + csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + data = [] + columns = [c for c in columns if c != 'None'] + + for f in csvfilenames: + df = dd.read_parquet(f,columns=columns,engine='pyarrow') + data.append(df) + + df = dd.concat(data,axis=0) + + data = df.compute() + data = data.loc[:,~data.columns.duplicated()] + extracols = [] + if doclean: + data = clean_df_stats(data, ignorehr=True, + workstrokesonly=workstrokesonly) + data.dropna(axis=1,how='all',inplace=True) + data.dropna(axis=0,how='any',inplace=True) + + return data + +def getsmallrowdata_db_old(columns, ids=[], doclean=True, workstrokesonly=True): prepmultipledata(ids) data,extracols = read_cols_df_sql(ids, columns) if extracols and len(ids)==1: @@ -1850,31 +1874,20 @@ def getrowdata(id=0): # safety net for programming errors elsewhere in the app # Also used heavily when I moved from CSV file only to CSV+Stroke data +import glob def prepmultipledata(ids, verbose=False): - query = sa.text('SELECT DISTINCT workoutid FROM strokedata') - engine = create_engine(database_url, echo=False) + filenames = glob.glob('media/*.parquet') + ids = [id for id in ids if 'media/strokedata_{id}.parquet'.format(id=id) not in filenames] - with engine.connect() as conn, conn.begin(): - res = conn.execute(query) - res = list(itertools.chain.from_iterable(res.fetchall())) - conn.close() - engine.dispose() - - try: - ids2 = [int(id) for id in ids] - except ValueError: - ids2 = ids - - res = list(set(ids2) - set(res)) - for id in res: + for id in ids: rowdata, row = getrowdata(id=id) if verbose: print(id) if rowdata and len(rowdata.df): data = dataprep(rowdata.df, id=id, bands=True, barchart=True, otwpower=True) - return res + return ids # Read a set of columns for a set of workout ids, returns data as a # pandas dataframe @@ -2292,19 +2305,6 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - if barchart: - # time increments for bar chart - time_increments = rowdatadf.loc[:, ' ElapsedTime (sec)'].diff() - try: - time_increments.iloc[0] = time_increments.iloc[1] - except (KeyError, IndexError): - time_increments.iloc[0] = 1. - - time_increments = 0.5 * time_increments + 0.5 * np.abs(time_increments) - x_right = (t2 + time_increments.apply(lambda x: timedeltaconv(x))) - - data['x_right'] = x_right - if empower: try: wash = rowdatadf.loc[:, 'wash'] @@ -2441,12 +2441,10 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, # write data if id given if id != 0: data['workoutid'] = id + filename = 'media/strokedata_{id}.parquet'.format(id=id) +# df = dd.from_pandas(data,npartitions=1) + data.to_parquet(filename,engine='pyarrow') - engine = create_engine(database_url, echo=False) - with engine.connect() as conn, conn.begin(): - data.to_sql('strokedata', engine, if_exists='append', index=False) - conn.close() - engine.dispose() return data From 622ae44ea64f9ed580c4499ca7e101f99b4c674e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 21:40:35 +0200 Subject: [PATCH 04/14] using dask --- rowers/dataprep.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 01a649a7..bdc2ef4d 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1779,11 +1779,15 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True): data = [] columns = [c for c in columns if c != 'None'] - for f in csvfilenames: - df = dd.read_parquet(f,columns=columns,engine='pyarrow') - data.append(df) + if len(ids)>1: + for f in csvfilenames: + df = dd.read_parquet(f,columns=columns,engine='pyarrow') + data.append(df) - df = dd.concat(data,axis=0) + + df = dd.concat(data,axis=0) + else: + df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') data = df.compute() data = data.loc[:,~data.columns.duplicated()] @@ -2443,7 +2447,7 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, data['workoutid'] = id filename = 'media/strokedata_{id}.parquet'.format(id=id) # df = dd.from_pandas(data,npartitions=1) - data.to_parquet(filename,engine='pyarrow') + data.to_parquet(filename,engine='pyarrow',compression='gzip') return data From 08c135c6cd3ade4013df9502ff6633ad0758bd6f Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 21:58:09 +0200 Subject: [PATCH 05/14] testing --- 0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx | 2523 ++++++++++++++++++++++ rowers/dataprep.py | 7 +- 2 files changed, 2528 insertions(+), 2 deletions(-) create mode 100644 0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx diff --git a/0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx b/0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx new file mode 100644 index 00000000..6999ba1f --- /dev/null +++ b/0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx @@ -0,0 +1,2523 @@ + + + + + 2016-05-20T13:41:26.962390+00:00 + + 537 + 2000 + 118 + + 148 + + + 156 + + Active + 21 + Manual + + + + 5.4 + + 127 + + 0 + + + 19 + + + + + + 13.1 + + 127 + + 19 + + + 26 + + + + + + 21.0 + + 128 + + 20 + + + 45 + + + + + + 30.3 + + 129 + + 20 + + + 64 + + + + + + 39.0 + + 130 + + 20 + + + 74 + + + + + + 48.2 + + 131 + + 21 + + + 80 + + + + + + 57.6 + + 131 + + 20 + + + 83 + + + + + + 66.4 + + 132 + + 20 + + + 87 + + + + + + 75.5 + + 132 + + 21 + + + 86 + + + + + + 85.1 + + 132 + + 20 + + + 88 + + + + + + 95.0 + + 132 + + 21 + + + 100 + + + + + + 105.0 + + 133 + + 22 + + + 127 + + + + + + 115.3 + + 134 + + 21 + + + 135 + + + + + + 125.8 + + 135 + + 21 + + + 139 + + + + + + 136.6 + + 136 + + 21 + + + 146 + + + + + + 147.2 + + 137 + + 22 + + + 150 + + + + + + 157.6 + + 139 + + 22 + + + 152 + + + + + + 167.8 + + 140 + + 21 + + + 146 + + + + + + 178.5 + + 140 + + 22 + + + 150 + + + + + + 188.5 + + 141 + + 21 + + + 155 + + + + + + 199.3 + + 141 + + 21 + + + 148 + + + + + + 209.4 + + 142 + + 22 + + + 151 + + + + + + 219.4 + + 142 + + 22 + + + 151 + + + + + + 230.2 + + 143 + + 22 + + + 148 + + + + + + 240.2 + + 144 + + 22 + + + 147 + + + + + + 250.1 + + 145 + + 23 + + + 150 + + + + + + 259.6 + + 145 + + 23 + + + 152 + + + + + + 270.3 + + 145 + + 23 + + + 152 + + + + + + 280.6 + + 145 + + 22 + + + 149 + + + + + + 290.7 + + 144 + + 22 + + + 150 + + + + + + 300.8 + + 145 + + 23 + + + 149 + + + + + + 311.1 + + 145 + + 22 + + + 152 + + + + + + 321.2 + + 145 + + 22 + + + 157 + + + + + + 331.9 + + 145 + + 21 + + + 150 + + + + + + 342.0 + + 146 + + 22 + + + 151 + + + + + + 352.4 + + 146 + + 22 + + + 151 + + + + + + 363.0 + + 146 + + 22 + + + 153 + + + + + + 373.4 + + 147 + + 22 + + + 152 + + + + + + 383.9 + + 147 + + 22 + + + 153 + + + + + + 394.6 + + 147 + + 22 + + + 152 + + + + + + 405.0 + + 147 + + 21 + + + 149 + + + + + + 415.3 + + 148 + + 22 + + + 152 + + + + + + 426.0 + + 148 + + 22 + + + 151 + + + + + + 436.5 + + 148 + + 21 + + + 149 + + + + + + 446.9 + + 148 + + 22 + + + 149 + + + + + + 456.9 + + 149 + + 22 + + + 156 + + + + + + 467.6 + + 149 + + 22 + + + 155 + + + + + + 478.5 + + 150 + + 22 + + + 156 + + + + + + 489.0 + + 150 + + 21 + + + 154 + + + + + + 499.1 + + 150 + + 21 + + + 148 + + + + + + 510.0 + + 150 + + 22 + + + 151 + + + + + + 519.9 + + 149 + + 22 + + + 153 + + + + + + 530.6 + + 149 + + 22 + + + 151 + + + + + + 540.8 + + 149 + + 22 + + + 148 + + + + + + 550.8 + + 148 + + 22 + + + 149 + + + + + + 560.8 + + 148 + + 22 + + + 144 + + + + + + 571.0 + + 147 + + 22 + + + 149 + + + + + + 580.7 + + 147 + + 22 + + + 150 + + + + + + 591.2 + + 147 + + 22 + + + 151 + + + + + + 601.4 + + 147 + + 22 + + + 150 + + + + + + 611.4 + + 147 + + 23 + + + 153 + + + + + + 621.8 + + 147 + + 23 + + + 151 + + + + + + 632.1 + + 147 + + 22 + + + 155 + + + + + + 642.3 + + 147 + + 22 + + + 154 + + + + + + 652.0 + + 148 + + 23 + + + 157 + + + + + + 662.7 + + 148 + + 23 + + + 162 + + + + + + 673.1 + + 148 + + 23 + + + 163 + + + + + + 683.6 + + 149 + + 22 + + + 163 + + + + + + 693.8 + + 149 + + 22 + + + 162 + + + + + + 703.8 + + 149 + + 22 + + + 164 + + + + + + 714.7 + + 150 + + 23 + + + 162 + + + + + + 724.9 + + 150 + + 22 + + + 162 + + + + + + 735.2 + + 151 + + 23 + + + 159 + + + + + + 745.4 + + 151 + + 22 + + + 158 + + + + + + 756.0 + + 151 + + 23 + + + 164 + + + + + + 766.3 + + 150 + + 22 + + + 163 + + + + + + 776.5 + + 150 + + 22 + + + 161 + + + + + + 786.9 + + 150 + + 23 + + + 163 + + + + + + 797.2 + + 150 + + 22 + + + 165 + + + + + + 807.8 + + 150 + + 23 + + + 166 + + + + + + 818.2 + + 150 + + 23 + + + 166 + + + + + + 828.4 + + 150 + + 22 + + + 168 + + + + + + 839.2 + + 150 + + 23 + + + 169 + + + + + + 849.6 + + 151 + + 23 + + + 166 + + + + + + 860.1 + + 151 + + 22 + + + 172 + + + + + + 870.3 + + 152 + + 22 + + + 172 + + + + + + 881.1 + + 152 + + 22 + + + 169 + + + + + + 891.7 + + 152 + + 23 + + + 167 + + + + + + 902.1 + + 152 + + 22 + + + 164 + + + + + + 913.1 + + 152 + + 22 + + + 161 + + + + + + 923.9 + + 153 + + 22 + + + 158 + + + + + + 934.6 + + 154 + + 21 + + + 158 + + + + + + 945.4 + + 154 + + 21 + + + 154 + + + + + + 956.0 + + 155 + + 21 + + + 155 + + + + + + 966.7 + + 155 + + 21 + + + 152 + + + + + + 977.4 + + 156 + + 21 + + + 150 + + + + + + 988.1 + + 156 + + 21 + + + 157 + + + + + + 998.8 + + 156 + + 21 + + + 155 + + + + + + 1009.6 + + 156 + + 21 + + + 151 + + + + + + 1020.6 + + 156 + + 21 + + + 147 + + + + + + 1031.5 + + 156 + + 20 + + + 145 + + + + + + 1042.5 + + 156 + + 21 + + + 144 + + + + + + 1053.3 + + 155 + + 20 + + + 145 + + + + + + 1064.1 + + 155 + + 21 + + + 147 + + + + + + 1075.3 + + 155 + + 20 + + + 142 + + + + + + 1086.1 + + 155 + + 20 + + + 136 + + + + + + 1097.5 + + 155 + + 21 + + + 141 + + + + + + 1108.5 + + 155 + + 20 + + + 146 + + + + + + 1119.2 + + 155 + + 20 + + + 143 + + + + + + 1130.6 + + 155 + + 20 + + + 143 + + + + + + 1141.3 + + 155 + + 20 + + + 143 + + + + + + 1152.4 + + 155 + + 21 + + + 142 + + + + + + 1163.3 + + 155 + + 20 + + + 138 + + + + + + 1173.8 + + 154 + + 20 + + + 141 + + + + + + 1184.8 + + 154 + + 21 + + + 146 + + + + + + 1195.8 + + 153 + + 21 + + + 146 + + + + + + 1206.6 + + 152 + + 21 + + + 141 + + + + + + 1217.3 + + 153 + + 21 + + + 141 + + + + + + 1227.8 + + 152 + + 21 + + + 140 + + + + + + 1238.7 + + 152 + + 21 + + + 143 + + + + + + 1249.5 + + 151 + + 21 + + + 149 + + + + + + 1260.1 + + 151 + + 20 + + + 141 + + + + + + 1270.9 + + 151 + + 21 + + + 141 + + + + + + 1281.8 + + 150 + + 21 + + + 145 + + + + + + 1292.7 + + 151 + + 20 + + + 142 + + + + + + 1303.4 + + 151 + + 20 + + + 141 + + + + + + 1314.3 + + 151 + + 21 + + + 141 + + + + + + 1325.2 + + 151 + + 21 + + + 146 + + + + + + 1336.1 + + 152 + + 20 + + + 143 + + + + + + 1346.9 + + 152 + + 21 + + + 144 + + + + + + 1357.4 + + 152 + + 20 + + + 141 + + + + + + 1368.1 + + 152 + + 21 + + + 138 + + + + + + 1379.0 + + 152 + + 20 + + + 142 + + + + + + 1389.5 + + 153 + + 21 + + + 145 + + + + + + 1399.9 + + 152 + + 21 + + + 138 + + + + + + 1410.7 + + 152 + + 20 + + + 139 + + + + + + 1422.0 + + 152 + + 20 + + + 139 + + + + + + 1432.8 + + 151 + + 20 + + + 141 + + + + + + 1443.6 + + 151 + + 21 + + + 146 + + + + + + 1454.4 + + 152 + + 20 + + + 143 + + + + + + 1465.1 + + 151 + + 21 + + + 143 + + + + + + 1475.9 + + 152 + + 21 + + + 145 + + + + + + 1486.6 + + 152 + + 21 + + + 148 + + + + + + 1497.4 + + 153 + + 21 + + + 143 + + + + + + 1508.2 + + 153 + + 20 + + + 140 + + + + + + 1519.2 + + 154 + + 20 + + + 144 + + + + + + 1530.0 + + 154 + + 21 + + + 143 + + + + + + 1540.9 + + 153 + + 20 + + + 141 + + + + + + 1551.3 + + 153 + + 21 + + + 143 + + + + + + 1562.6 + + 153 + + 21 + + + 146 + + + + + + 1573.3 + + 153 + + 20 + + + 141 + + + + + + 1584.2 + + 152 + + 20 + + + 139 + + + + + + 1594.6 + + 152 + + 21 + + + 145 + + + + + + 1606.0 + + 152 + + 21 + + + 143 + + + + + + 1616.2 + + 152 + + 20 + + + 138 + + + + + + 1627.4 + + 152 + + 21 + + + 140 + + + + + + 1638.0 + + 152 + + 21 + + + 144 + + + + + + 1649.2 + + 151 + + 20 + + + 143 + + + + + + 1660.2 + + 152 + + 20 + + + 143 + + + + + + 1670.8 + + 151 + + 20 + + + 142 + + + + + + 1681.4 + + 151 + + 21 + + + 140 + + + + + + 1692.1 + + 151 + + 21 + + + 140 + + + + + + 1702.5 + + 150 + + 21 + + + 141 + + + + + + 1713.7 + + 150 + + 21 + + + 144 + + + + + + 1724.4 + + 150 + + 21 + + + 146 + + + + + + 1735.1 + + 150 + + 20 + + + 141 + + + + + + 1745.6 + + 150 + + 21 + + + 140 + + + + + + 1756.3 + + 150 + + 21 + + + 141 + + + + + + 1766.2 + + 151 + + 20 + + + 142 + + + + + + 1777.1 + + 150 + + 22 + + + 138 + + + + + + 1787.5 + + 150 + + 21 + + + 138 + + + + + + 1797.7 + + 150 + + 22 + + + 140 + + + + + + 1808.4 + + 150 + + 21 + + + 140 + + + + + + 1818.4 + + 149 + + 21 + + + 138 + + + + + + 1828.9 + + 149 + + 22 + + + 146 + + + + + + 1839.9 + + 149 + + 21 + + + 142 + + + + + + 1850.5 + + 148 + + 21 + + + 142 + + + + + + 1861.2 + + 148 + + 21 + + + 145 + + + + + + 1871.9 + + 147 + + 21 + + + 143 + + + + + + 1882.6 + + 147 + + 20 + + + 139 + + + + + + 1893.3 + + 148 + + 20 + + + 140 + + + + + + 1904.3 + + 149 + + 21 + + + 144 + + + + + + 1915.4 + + 149 + + 20 + + + 148 + + + + + + 1926.2 + + 150 + + 20 + + + 139 + + + + + + 1937.3 + + 151 + + 20 + + + 140 + + + + + + 1947.8 + + 152 + + 20 + + + 144 + + + + + + 1959.1 + + 152 + + 20 + + + 142 + + + + + + 1969.8 + + 153 + + 20 + + + 140 + + + + + + 1980.6 + + 153 + + 21 + + + 143 + + + + + + 1991.4 + + 153 + + 21 + + + 143 + + + + + + 2000.0 + + 154 + + 21 + + + 147 + + + + + + <Element 'Notes' at 0x7fe4d25cdef0> + + + + rowsandall.com/rowingdata + + + rowingdata + + + 0 + 75 + + Release + + EN + 000-00000-00 + + diff --git a/rowers/dataprep.py b/rowers/dataprep.py index bdc2ef4d..25f02385 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1781,8 +1781,11 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True): if len(ids)>1: for f in csvfilenames: - df = dd.read_parquet(f,columns=columns,engine='pyarrow') - data.append(df) + try: + df = dd.read_parquet(f,columns=columns,engine='pyarrow') + data.append(df) + except OSError: + pass df = dd.concat(data,axis=0) From 450a32221f1456a42015e464c281b36f1b737ef9 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 21:58:39 +0200 Subject: [PATCH 06/14] removed temp file --- 0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx | 2523 ---------------------- 1 file changed, 2523 deletions(-) delete mode 100644 0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx diff --git a/0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx b/0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx deleted file mode 100644 index 6999ba1f..00000000 --- a/0f302a72-04b8-49a4-a435-5d2f7b48703b.tcx +++ /dev/null @@ -1,2523 +0,0 @@ - - - - - 2016-05-20T13:41:26.962390+00:00 - - 537 - 2000 - 118 - - 148 - - - 156 - - Active - 21 - Manual - - - - 5.4 - - 127 - - 0 - - - 19 - - - - - - 13.1 - - 127 - - 19 - - - 26 - - - - - - 21.0 - - 128 - - 20 - - - 45 - - - - - - 30.3 - - 129 - - 20 - - - 64 - - - - - - 39.0 - - 130 - - 20 - - - 74 - - - - - - 48.2 - - 131 - - 21 - - - 80 - - - - - - 57.6 - - 131 - - 20 - - - 83 - - - - - - 66.4 - - 132 - - 20 - - - 87 - - - - - - 75.5 - - 132 - - 21 - - - 86 - - - - - - 85.1 - - 132 - - 20 - - - 88 - - - - - - 95.0 - - 132 - - 21 - - - 100 - - - - - - 105.0 - - 133 - - 22 - - - 127 - - - - - - 115.3 - - 134 - - 21 - - - 135 - - - - - - 125.8 - - 135 - - 21 - - - 139 - - - - - - 136.6 - - 136 - - 21 - - - 146 - - - - - - 147.2 - - 137 - - 22 - - - 150 - - - - - - 157.6 - - 139 - - 22 - - - 152 - - - - - - 167.8 - - 140 - - 21 - - - 146 - - - - - - 178.5 - - 140 - - 22 - - - 150 - - - - - - 188.5 - - 141 - - 21 - - - 155 - - - - - - 199.3 - - 141 - - 21 - - - 148 - - - - - - 209.4 - - 142 - - 22 - - - 151 - - - - - - 219.4 - - 142 - - 22 - - - 151 - - - - - - 230.2 - - 143 - - 22 - - - 148 - - - - - - 240.2 - - 144 - - 22 - - - 147 - - - - - - 250.1 - - 145 - - 23 - - - 150 - - - - - - 259.6 - - 145 - - 23 - - - 152 - - - - - - 270.3 - - 145 - - 23 - - - 152 - - - - - - 280.6 - - 145 - - 22 - - - 149 - - - - - - 290.7 - - 144 - - 22 - - - 150 - - - - - - 300.8 - - 145 - - 23 - - - 149 - - - - - - 311.1 - - 145 - - 22 - - - 152 - - - - - - 321.2 - - 145 - - 22 - - - 157 - - - - - - 331.9 - - 145 - - 21 - - - 150 - - - - - - 342.0 - - 146 - - 22 - - - 151 - - - - - - 352.4 - - 146 - - 22 - - - 151 - - - - - - 363.0 - - 146 - - 22 - - - 153 - - - - - - 373.4 - - 147 - - 22 - - - 152 - - - - - - 383.9 - - 147 - - 22 - - - 153 - - - - - - 394.6 - - 147 - - 22 - - - 152 - - - - - - 405.0 - - 147 - - 21 - - - 149 - - - - - - 415.3 - - 148 - - 22 - - - 152 - - - - - - 426.0 - - 148 - - 22 - - - 151 - - - - - - 436.5 - - 148 - - 21 - - - 149 - - - - - - 446.9 - - 148 - - 22 - - - 149 - - - - - - 456.9 - - 149 - - 22 - - - 156 - - - - - - 467.6 - - 149 - - 22 - - - 155 - - - - - - 478.5 - - 150 - - 22 - - - 156 - - - - - - 489.0 - - 150 - - 21 - - - 154 - - - - - - 499.1 - - 150 - - 21 - - - 148 - - - - - - 510.0 - - 150 - - 22 - - - 151 - - - - - - 519.9 - - 149 - - 22 - - - 153 - - - - - - 530.6 - - 149 - - 22 - - - 151 - - - - - - 540.8 - - 149 - - 22 - - - 148 - - - - - - 550.8 - - 148 - - 22 - - - 149 - - - - - - 560.8 - - 148 - - 22 - - - 144 - - - - - - 571.0 - - 147 - - 22 - - - 149 - - - - - - 580.7 - - 147 - - 22 - - - 150 - - - - - - 591.2 - - 147 - - 22 - - - 151 - - - - - - 601.4 - - 147 - - 22 - - - 150 - - - - - - 611.4 - - 147 - - 23 - - - 153 - - - - - - 621.8 - - 147 - - 23 - - - 151 - - - - - - 632.1 - - 147 - - 22 - - - 155 - - - - - - 642.3 - - 147 - - 22 - - - 154 - - - - - - 652.0 - - 148 - - 23 - - - 157 - - - - - - 662.7 - - 148 - - 23 - - - 162 - - - - - - 673.1 - - 148 - - 23 - - - 163 - - - - - - 683.6 - - 149 - - 22 - - - 163 - - - - - - 693.8 - - 149 - - 22 - - - 162 - - - - - - 703.8 - - 149 - - 22 - - - 164 - - - - - - 714.7 - - 150 - - 23 - - - 162 - - - - - - 724.9 - - 150 - - 22 - - - 162 - - - - - - 735.2 - - 151 - - 23 - - - 159 - - - - - - 745.4 - - 151 - - 22 - - - 158 - - - - - - 756.0 - - 151 - - 23 - - - 164 - - - - - - 766.3 - - 150 - - 22 - - - 163 - - - - - - 776.5 - - 150 - - 22 - - - 161 - - - - - - 786.9 - - 150 - - 23 - - - 163 - - - - - - 797.2 - - 150 - - 22 - - - 165 - - - - - - 807.8 - - 150 - - 23 - - - 166 - - - - - - 818.2 - - 150 - - 23 - - - 166 - - - - - - 828.4 - - 150 - - 22 - - - 168 - - - - - - 839.2 - - 150 - - 23 - - - 169 - - - - - - 849.6 - - 151 - - 23 - - - 166 - - - - - - 860.1 - - 151 - - 22 - - - 172 - - - - - - 870.3 - - 152 - - 22 - - - 172 - - - - - - 881.1 - - 152 - - 22 - - - 169 - - - - - - 891.7 - - 152 - - 23 - - - 167 - - - - - - 902.1 - - 152 - - 22 - - - 164 - - - - - - 913.1 - - 152 - - 22 - - - 161 - - - - - - 923.9 - - 153 - - 22 - - - 158 - - - - - - 934.6 - - 154 - - 21 - - - 158 - - - - - - 945.4 - - 154 - - 21 - - - 154 - - - - - - 956.0 - - 155 - - 21 - - - 155 - - - - - - 966.7 - - 155 - - 21 - - - 152 - - - - - - 977.4 - - 156 - - 21 - - - 150 - - - - - - 988.1 - - 156 - - 21 - - - 157 - - - - - - 998.8 - - 156 - - 21 - - - 155 - - - - - - 1009.6 - - 156 - - 21 - - - 151 - - - - - - 1020.6 - - 156 - - 21 - - - 147 - - - - - - 1031.5 - - 156 - - 20 - - - 145 - - - - - - 1042.5 - - 156 - - 21 - - - 144 - - - - - - 1053.3 - - 155 - - 20 - - - 145 - - - - - - 1064.1 - - 155 - - 21 - - - 147 - - - - - - 1075.3 - - 155 - - 20 - - - 142 - - - - - - 1086.1 - - 155 - - 20 - - - 136 - - - - - - 1097.5 - - 155 - - 21 - - - 141 - - - - - - 1108.5 - - 155 - - 20 - - - 146 - - - - - - 1119.2 - - 155 - - 20 - - - 143 - - - - - - 1130.6 - - 155 - - 20 - - - 143 - - - - - - 1141.3 - - 155 - - 20 - - - 143 - - - - - - 1152.4 - - 155 - - 21 - - - 142 - - - - - - 1163.3 - - 155 - - 20 - - - 138 - - - - - - 1173.8 - - 154 - - 20 - - - 141 - - - - - - 1184.8 - - 154 - - 21 - - - 146 - - - - - - 1195.8 - - 153 - - 21 - - - 146 - - - - - - 1206.6 - - 152 - - 21 - - - 141 - - - - - - 1217.3 - - 153 - - 21 - - - 141 - - - - - - 1227.8 - - 152 - - 21 - - - 140 - - - - - - 1238.7 - - 152 - - 21 - - - 143 - - - - - - 1249.5 - - 151 - - 21 - - - 149 - - - - - - 1260.1 - - 151 - - 20 - - - 141 - - - - - - 1270.9 - - 151 - - 21 - - - 141 - - - - - - 1281.8 - - 150 - - 21 - - - 145 - - - - - - 1292.7 - - 151 - - 20 - - - 142 - - - - - - 1303.4 - - 151 - - 20 - - - 141 - - - - - - 1314.3 - - 151 - - 21 - - - 141 - - - - - - 1325.2 - - 151 - - 21 - - - 146 - - - - - - 1336.1 - - 152 - - 20 - - - 143 - - - - - - 1346.9 - - 152 - - 21 - - - 144 - - - - - - 1357.4 - - 152 - - 20 - - - 141 - - - - - - 1368.1 - - 152 - - 21 - - - 138 - - - - - - 1379.0 - - 152 - - 20 - - - 142 - - - - - - 1389.5 - - 153 - - 21 - - - 145 - - - - - - 1399.9 - - 152 - - 21 - - - 138 - - - - - - 1410.7 - - 152 - - 20 - - - 139 - - - - - - 1422.0 - - 152 - - 20 - - - 139 - - - - - - 1432.8 - - 151 - - 20 - - - 141 - - - - - - 1443.6 - - 151 - - 21 - - - 146 - - - - - - 1454.4 - - 152 - - 20 - - - 143 - - - - - - 1465.1 - - 151 - - 21 - - - 143 - - - - - - 1475.9 - - 152 - - 21 - - - 145 - - - - - - 1486.6 - - 152 - - 21 - - - 148 - - - - - - 1497.4 - - 153 - - 21 - - - 143 - - - - - - 1508.2 - - 153 - - 20 - - - 140 - - - - - - 1519.2 - - 154 - - 20 - - - 144 - - - - - - 1530.0 - - 154 - - 21 - - - 143 - - - - - - 1540.9 - - 153 - - 20 - - - 141 - - - - - - 1551.3 - - 153 - - 21 - - - 143 - - - - - - 1562.6 - - 153 - - 21 - - - 146 - - - - - - 1573.3 - - 153 - - 20 - - - 141 - - - - - - 1584.2 - - 152 - - 20 - - - 139 - - - - - - 1594.6 - - 152 - - 21 - - - 145 - - - - - - 1606.0 - - 152 - - 21 - - - 143 - - - - - - 1616.2 - - 152 - - 20 - - - 138 - - - - - - 1627.4 - - 152 - - 21 - - - 140 - - - - - - 1638.0 - - 152 - - 21 - - - 144 - - - - - - 1649.2 - - 151 - - 20 - - - 143 - - - - - - 1660.2 - - 152 - - 20 - - - 143 - - - - - - 1670.8 - - 151 - - 20 - - - 142 - - - - - - 1681.4 - - 151 - - 21 - - - 140 - - - - - - 1692.1 - - 151 - - 21 - - - 140 - - - - - - 1702.5 - - 150 - - 21 - - - 141 - - - - - - 1713.7 - - 150 - - 21 - - - 144 - - - - - - 1724.4 - - 150 - - 21 - - - 146 - - - - - - 1735.1 - - 150 - - 20 - - - 141 - - - - - - 1745.6 - - 150 - - 21 - - - 140 - - - - - - 1756.3 - - 150 - - 21 - - - 141 - - - - - - 1766.2 - - 151 - - 20 - - - 142 - - - - - - 1777.1 - - 150 - - 22 - - - 138 - - - - - - 1787.5 - - 150 - - 21 - - - 138 - - - - - - 1797.7 - - 150 - - 22 - - - 140 - - - - - - 1808.4 - - 150 - - 21 - - - 140 - - - - - - 1818.4 - - 149 - - 21 - - - 138 - - - - - - 1828.9 - - 149 - - 22 - - - 146 - - - - - - 1839.9 - - 149 - - 21 - - - 142 - - - - - - 1850.5 - - 148 - - 21 - - - 142 - - - - - - 1861.2 - - 148 - - 21 - - - 145 - - - - - - 1871.9 - - 147 - - 21 - - - 143 - - - - - - 1882.6 - - 147 - - 20 - - - 139 - - - - - - 1893.3 - - 148 - - 20 - - - 140 - - - - - - 1904.3 - - 149 - - 21 - - - 144 - - - - - - 1915.4 - - 149 - - 20 - - - 148 - - - - - - 1926.2 - - 150 - - 20 - - - 139 - - - - - - 1937.3 - - 151 - - 20 - - - 140 - - - - - - 1947.8 - - 152 - - 20 - - - 144 - - - - - - 1959.1 - - 152 - - 20 - - - 142 - - - - - - 1969.8 - - 153 - - 20 - - - 140 - - - - - - 1980.6 - - 153 - - 21 - - - 143 - - - - - - 1991.4 - - 153 - - 21 - - - 143 - - - - - - 2000.0 - - 154 - - 21 - - - 147 - - - - - - <Element 'Notes' at 0x7fe4d25cdef0> - - - - rowsandall.com/rowingdata - - - rowingdata - - - 0 - 75 - - Release - - EN - 000-00000-00 - - From 8e2f7b0cce62b13774f84a0c3a21de4c30542693 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 22 Oct 2019 22:02:45 +0200 Subject: [PATCH 07/14] updates in nodjango --- rowers/dataprepnodjango.py | 41 +++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 9fd3d196..a2ac7764 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -714,8 +714,24 @@ def testdata(time,distance,pace,spm): def getsmallrowdata_db(columns,ids=[],debug=False): + csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + data = [] + columns = [c for c in columns if c != 'None'] - data = read_cols_df_sql(ids,columns,debug=debug) + if len(ids)>1: + for f in csvfilenames: + try: + df = dd.read_parquet(f,columns=columns,engine='pyarrow') + data.append(df) + except OSError: + pass + + + df = dd.concat(data,axis=0) + else: + df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') + + data = df.compute() return data @@ -1101,18 +1117,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, except KeyError: rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - if barchart: - # time increments for bar chart - time_increments = rowdatadf.loc[:,' ElapsedTime (sec)'].diff() - try: - time_increments.iloc[0] = time_increments.iloc[1] - except (KeyError, IndexError): - time_increments.iloc[1] = 1. - - time_increments = 0.5*time_increments+0.5*np.abs(time_increments) - x_right = (t2+time_increments.apply(lambda x:timedeltaconv(x))) - - data['x_right'] = x_right if empower: try: @@ -1260,15 +1264,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, # write data if id given if id != 0: data['workoutid'] = id - - if debug: - engine = create_engine(database_url_debug, echo=False) - else: - engine = create_engine(database_url, echo=False) + data.to_parquet(filename,engine='pyarrow',compression='gzip') - with engine.connect() as conn, conn.begin(): - data.to_sql('strokedata',engine,if_exists='append',index=False) - - conn.close() - engine.dispose() return data From 4ea24fa5aace43a2c444dc3d3633921dde529538 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 23 Oct 2019 06:48:58 +0200 Subject: [PATCH 08/14] small improvements dataprep --- rowers/dataprep.py | 62 ++++++++++++++++++++------------------ rowers/dataprepnodjango.py | 2 ++ 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 25f02385..b5f74f84 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -379,6 +379,7 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, pass datadf.replace(to_replace=0, value=np.nan, inplace=True) + # datadf = datadf.map_partitions(lambda df:df.replace(to_replace=0,value=np.nan)) # bring spm back to real values try: @@ -406,55 +407,55 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, if not ignorehr: try: mask = datadf['hr'] < 30 - datadf.loc[mask, 'hr'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['spm'] < 0 - datadf.loc[mask,'spm'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['efficiency'] > 200. - datadf.loc[mask, 'efficiency'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['spm'] < 10 - datadf.loc[mask, 'spm'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['pace'] / 1000. > 300. - datadf.loc[mask, 'pace'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['efficiency'] < 0. - datadf.loc[mask, 'efficiency'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['pace'] / 1000. < 60. - datadf.loc[mask, 'pace'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['spm'] > 60 - datadf.loc[mask, 'spm'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: - mask = datadf['wash'] < 1 + mask = datadf['wash'] > 1 datadf.loc[mask, 'wash'] = np.nan except KeyError: pass @@ -462,67 +463,67 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, if not ignoreadvanced: try: mask = datadf['rhythm'] < 5 - datadf.loc[mask, 'rhythm'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['rhythm'] > 70 - datadf.loc[mask, 'rhythm'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['power'] < 20 - datadf.loc[mask, 'power'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['drivelength'] < 0.5 - datadf.loc[mask, 'drivelength'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['forceratio'] < 0.2 - datadf.loc[mask, 'forceratio'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['forceratio'] > 1.0 - datadf.loc[mask, 'forceratio'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['drivespeed'] < 0.5 - datadf.loc[mask, 'drivespeed'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['drivespeed'] > 4 - datadf.loc[mask, 'drivespeed'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['driveenergy'] > 2000 - datadf.loc[mask, 'driveenergy'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['driveenergy'] < 100 - datadf.loc[mask, 'driveenergy'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass try: mask = datadf['catch'] > -30. - datadf.loc[mask, 'catch'] = np.nan + datadf.mask(mask,inplace=True) except KeyError: pass @@ -1772,7 +1773,7 @@ def getrowdata_db(id=0, doclean=False, convertnewtons=True, # Fetch a subset of the data from the DB -def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True): +def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True,compute=True): prepmultipledata(ids) csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] @@ -1792,16 +1793,19 @@ def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True): else: df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') - data = df.compute() - data = data.loc[:,~data.columns.duplicated()] - extracols = [] - if doclean: - data = clean_df_stats(data, ignorehr=True, - workstrokesonly=workstrokesonly) + df = df.loc[:,~df.columns.duplicated()] + + + if compute: + data = df.compute() + if doclean: + data = clean_df_stats(data, ignorehr=True, + workstrokesonly=workstrokesonly) data.dropna(axis=1,how='all',inplace=True) data.dropna(axis=0,how='any',inplace=True) - - return data + return data + + return df def getsmallrowdata_db_old(columns, ids=[], doclean=True, workstrokesonly=True): prepmultipledata(ids) diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index a2ac7764..3347ac49 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -16,6 +16,8 @@ from pandas import DataFrame,Series import pandas as pd import numpy as np import itertools +import dask.dataframe as dd +from dask.delayed import delayed from sqlalchemy import create_engine import sqlalchemy as sa From fa373d781ff231ceef6d2d3ce811b4367a60b462 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 23 Oct 2019 18:15:07 +0200 Subject: [PATCH 09/14] ds --- rowers/dataprep.py | 190 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 153 insertions(+), 37 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index b5f74f84..f96b6c3f 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -27,6 +27,8 @@ from rowers.tasks import handle_zip_file from pandas import DataFrame, Series import dask.dataframe as dd from dask.delayed import delayed +import pyarrow.parquet as pq +import pyarrow as pa from django.utils import timezone from django.utils.timezone import get_current_timezone @@ -114,6 +116,36 @@ columndict = { 'cumdist': 'cum_dist', } +# dtypes +dtypes = {'workoutid':int, + 'hr':int, + 'pace':float, + 'velo':float, + 'spm':float, + 'driveenergy':float, + 'power':float, + 'averageforce':float, + 'peakforce':float, + 'drivelength':float, + 'distance':float, + 'cumdist':float, + 'drivespeed':float, + 'catch':float, + 'slip':float, + 'finish':float, + 'wash':float, + 'peakforceangle':float, + 'totalangle':float, + 'effectiveangle':float, + 'rhythm':float, + 'efficiency':float, + 'distanceperstroke':float, + 'ftime':str, + 'fpace':str, + 'fergpace':str, + 'fnowindpace':str, +} + from scipy.signal import savgol_filter import datetime @@ -349,22 +381,23 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, ignoreadvanced=False): # clean data remove zeros and negative values + # bring metrics which have negative values to positive domain if len(datadf)==0: return datadf try: datadf['catch'] = -datadf['catch'] - except KeyError: + except (KeyError,TypeError): pass try: datadf['peakforceangle'] = datadf['peakforceangle'] + 1000 - except KeyError: + except (KeyError,TypeError): pass try: datadf['hr'] = datadf['hr'] + 10 - except KeyError: + except (KeyError,TypeError): pass # protect 0 spm values from being nulled @@ -390,17 +423,17 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, # return from positive domain to negative try: datadf['catch'] = -datadf['catch'] - except KeyError: + except (KeyError,TypeError): pass try: datadf['peakforceangle'] = datadf['peakforceangle'] - 1000 - except KeyError: + except (KeyError,TypeError): pass try: datadf['hr'] = datadf['hr'] - 10 - except KeyError: + except (KeyError,TypeError): pass # clean data for useful ranges per column @@ -408,123 +441,123 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, try: mask = datadf['hr'] < 30 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['spm'] < 0 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['efficiency'] > 200. datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['spm'] < 10 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['pace'] / 1000. > 300. datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['efficiency'] < 0. datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['pace'] / 1000. < 60. datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['spm'] > 60 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['wash'] > 1 datadf.loc[mask, 'wash'] = np.nan - except KeyError: + except (KeyError,TypeError): pass if not ignoreadvanced: try: mask = datadf['rhythm'] < 5 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['rhythm'] > 70 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['power'] < 20 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['drivelength'] < 0.5 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['forceratio'] < 0.2 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['forceratio'] > 1.0 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['drivespeed'] < 0.5 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['drivespeed'] > 4 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['driveenergy'] > 2000 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['driveenergy'] < 100 datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass try: mask = datadf['catch'] > -30. datadf.mask(mask,inplace=True) - except KeyError: + except (KeyError,TypeError): pass workoutstateswork = [1, 4, 5, 8, 9, 6, 7] @@ -1774,27 +1807,97 @@ def getrowdata_db(id=0, doclean=False, convertnewtons=True, # Fetch a subset of the data from the DB def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True,compute=True): - prepmultipledata(ids) + # prepmultipledata(ids) - csvfilenames = ['media/strokedata_{id}.parquet'.format(id=id) for id in ids] + csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] data = [] columns = [c for c in columns if c != 'None'] + columns = list(set(columns)) if len(ids)>1: - for f in csvfilenames: + for id,f in zip(ids,csvfilenames): try: - df = dd.read_parquet(f,columns=columns,engine='pyarrow') + #df = dd.read_parquet(f,columns=columns,engine='pyarrow') + df = pd.read_parquet(f,columns=columns) data.append(df) except OSError: - pass + rowdata, row = getrowdata(id=id) + if rowdata and len(rowdata.df): + datadf = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) + # df = dd.read_parquet(f,columns=columns,engine='pyarrow') + df = pd.read_parquet(f,columns=columns) + data.append(df) + + df = pd.concat(data,axis=0) + # df = dd.concat(data,axis=0) + + else: + try: + df = pd.read_parquet(csvfilenames[0],columns=columns) + except OSError: + rowdata,row = getrowdata(id=ids[0]) + if rowdata and len(rowdata.df): + data = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) + df = pd.read_parquet(csvfilenames[0],columns=columns) + # df = dd.read_parquet(csvfilenames[0], + # column=columns,engine='pyarrow', + # ) + + # df = df.loc[:,~df.columns.duplicated()] + + if compute: + data = df.copy() + if doclean: + data = clean_df_stats(data, ignorehr=True, + workstrokesonly=workstrokesonly) + data.dropna(axis=1,how='all',inplace=True) + data.dropna(axis=0,how='any',inplace=True) + return data + + return df + +def getsmallrowdata_db_dask(columns, ids=[], doclean=True,workstrokesonly=True,compute=True): + # prepmultipledata(ids) + + csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] + data = [] + columns = [c for c in columns if c != 'None'] + columns = list(set(columns)) + + if len(ids)>1: + for id,f in zip(ids,csvfilenames): + try: + #df = dd.read_parquet(f,columns=columns,engine='pyarrow') + df = dd.read_parquet(f,columns=columns) + data.append(df) + except OSError: + rowdata, row = getrowdata(id=id) + if rowdata and len(rowdata.df): + datadf = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) + # df = dd.read_parquet(f,columns=columns,engine='pyarrow') + df = dd.read_parquet(f,columns=columns) + data.append(df) + df = dd.concat(data,axis=0) + # df = dd.concat(data,axis=0) + else: - df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') + try: + df = dd.read_parquet(csvfilenames[0],columns=columns) + except OSError: + rowdata,row = getrowdata(id=ids[0]) + if rowdata and len(rowdata.df): + data = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) + df = dd.read_parquet(csvfilenames[0],columns=columns) + # df = dd.read_parquet(csvfilenames[0], + # column=columns,engine='pyarrow', + # ) - df = df.loc[:,~df.columns.duplicated()] + # df = df.loc[:,~df.columns.duplicated()] + if compute: data = df.compute() @@ -1889,7 +1992,7 @@ import glob def prepmultipledata(ids, verbose=False): filenames = glob.glob('media/*.parquet') - ids = [id for id in ids if 'media/strokedata_{id}.parquet'.format(id=id) not in filenames] + ids = [id for id in ids if 'media/strokedata_{id}.parquet.gz'.format(id=id) not in filenames] for id in ids: rowdata, row = getrowdata(id=id) @@ -2452,9 +2555,22 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, # write data if id given if id != 0: data['workoutid'] = id - filename = 'media/strokedata_{id}.parquet'.format(id=id) -# df = dd.from_pandas(data,npartitions=1) - data.to_parquet(filename,engine='pyarrow',compression='gzip') + data.fillna(0,inplace=True) + data = data.astype( + dtype=dtypes, + ) + + + filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = dd.from_pandas(data,npartitions=1) + #df = df.loc[:,~df.columns.duplicated()] + # data.to_csv(filename,compression='gzip') + + df.to_parquet(filename,engine='fastparquet',compression='GZIP') + + # data.to_parquet(filename,engine='fastparquet',compression='gzip') + # table = pa.Table.from_pandas(data) + #pq.write_table(table,filename) return data From c0e8e448e3f7aed696c08cab9eb9833320620d6a Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 23 Oct 2019 20:04:28 +0200 Subject: [PATCH 10/14] updated dataprepnodjango --- rowers/dataprep.py | 191 +++++++++++++++++++------------------ rowers/dataprepnodjango.py | 119 +++++++++++++---------- rowers/models.py | 8 ++ 3 files changed, 172 insertions(+), 146 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index f96b6c3f..158c74a3 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -15,6 +15,7 @@ from rowingdata import rowingdata as rrdata from rowingdata import rower as rrower +import shutil from shutil import copyfile from rowingdata import ( @@ -1651,75 +1652,7 @@ def new_workout_from_df(r, df, return (id, message) -# Compare the data from the CSV file and the database -# Currently only calculates number of strokes. To be expanded with -# more elaborate testing if needed -def compare_data(id): - row = Workout.objects.get(id=id) - f1 = row.csvfilename - try: - rowdata = rdata(f1) - l1 = len(rowdata.df) - except AttributeError: - rowdata = 0 - l1 = 0 - engine = create_engine(database_url, echo=False) - query = sa.text('SELECT COUNT(*) FROM strokedata WHERE workoutid={id};'.format( - id=id, - )) - with engine.connect() as conn, conn.begin(): - try: - res = conn.execute(query) - l2 = res.fetchall()[0][0] - except: - print("Database Locked") - conn.close() - engine.dispose() - lfile = l1 - ldb = l2 - return l1 == l2 and l1 != 0, ldb, lfile - -# Repair data for workouts where the CSV file is lost (or the DB entries -# don't exist) - - -def repair_data(verbose=False): - ws = Workout.objects.all() - for w in ws: - if verbose: - sys.stdout.write(".") - test, ldb, lfile = compare_data(w.id) - if not test: - if verbose: - print(w.id, lfile, ldb) - try: - rowdata = rdata(w.csvfilename) - if rowdata and len(rowdata.df): - update_strokedata(w.id, rowdata.df) - - except (IOError, AttributeError): - pass - - if lfile == 0: - # if not ldb - delete workout - - try: - data = read_df_sql(w.id) - try: - datalength = len(data) - except AttributeError: - datalength = 0 - - if datalength != 0: - data.rename(columns=columndict, inplace=True) - res = data.to_csv(w.csvfilename + '.gz', - index_label='index', - compression='gzip') - else: - w.delete() - except: - pass # A wrapper around the rowingdata class, with some error catching @@ -1745,17 +1678,11 @@ def rdata(file, rower=rrower()): def delete_strokedata(id): - engine = create_engine(database_url, echo=False) - query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( - id=id, - )) - with engine.connect() as conn, conn.begin(): - try: - result = conn.execute(query) - except: - print("Database Locked") - conn.close() - engine.dispose() + dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id) + try: + shutil.rmtree(dirname) + except FileNotFoundError: + pass # Replace stroke data in DB with data from CSV file @@ -1782,7 +1709,6 @@ def testdata(time, distance, pace, spm): def getrowdata_db(id=0, doclean=False, convertnewtons=True, checkefficiency=True): data = read_df_sql(id) - data['x_right'] = data['x_right'] / 1.0e6 data['deltat'] = data['time'].diff() if data.empty: @@ -2010,6 +1936,66 @@ def prepmultipledata(ids, verbose=False): def read_cols_df_sql(ids, columns, convertnewtons=True): # drop columns that are not in offical list # axx = [ax[0] for ax in axes] + + extracols = [] + + columns = list(columns) + ['distance', 'spm', 'workoutid'] + columns = [x for x in columns if x != 'None'] + columns = list(set(columns)) + ids = [int(id) for id in ids] + + + if len(ids) == 0: + return pd.DataFrame(),extracols + elif len(ids) == 1: + try: + filename = 'media/strokedata_{id}.parquet.gz'.format(id=ids[0]) + df = pd.read_parquet(filename,columns=columns) + except OSError: + rowdata,row = getrowdata(id=ids[0]) + if rowdata and len(rowdata.df): + datadf = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) + df = pd.read_parquet(filename,columns=columns) + else: + data = [] + filenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] + for id,f in zip(ids,filenames): + try: + df = pd.read_parquet(f,columns=columns) + data.append(df) + except OSError: + rowdata,row = getrowdata(id=id) + if rowdata and len(rowdata.df): + datadf = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) + df = pd.read_parquet(f,columns=columns) + data.append(df) + + df = pd.concat(data,axis=0) + + + df = df.fillna(value=0) + + if 'peakforce' in columns: + funits = ((w.id, w.forceunit) + for w in Workout.objects.filter(id__in=ids)) + for id, u in funits: + if u == 'lbs': + mask = df['workoutid'] == id + df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN + if 'averageforce' in columns: + funits = ((w.id, w.forceunit) + for w in Workout.objects.filter(id__in=ids)) + for id, u in funits: + if u == 'lbs': + mask = df['workoutid'] == id + df.loc[mask, 'averageforce'] = df.loc[mask, + 'averageforce'] * lbstoN + + return df,extracols + +def read_cols_df_sql_old(ids, columns, convertnewtons=True): + # drop columns that are not in offical list + # axx = [ax[0] for ax in axes] prepmultipledata(ids) axx = [f.name for f in StrokeData._meta.get_fields()] @@ -2076,8 +2062,34 @@ def read_cols_df_sql(ids, columns, convertnewtons=True): # Read stroke data from the DB for a Workout ID. Returns a pandas dataframe - def read_df_sql(id): + try: + f = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = pd.read_parquet(f) + except OSError: + rowdata,row = getrowdata(id=ids[0]) + if rowdata and len(rowdata.df): + data = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) + df = pd.read_parquet(f) + + df = df.fillna(value=0) + + funit = Workout.objects.get(id=id).forceunit + + if funit == 'lbs': + try: + df['peakforce'] = df['peakforce'] * lbstoN + except KeyError: + pass + + try: + df['averageforce'] = df['averageforce'] * lbstoN + except KeyError: + pass + + return df + +def read_df_sql_old(id): engine = create_engine(database_url, echo=False) df = pd.read_sql_query(sa.text('SELECT * FROM strokedata WHERE workoutid={id} ORDER BY time ASC'.format( @@ -2269,14 +2281,13 @@ def add_efficiency(id=0): rowdata = rowdata.fillna(method='ffill') delete_strokedata(id) + if id != 0: rowdata['workoutid'] = id - engine = create_engine(database_url, echo=False) - with engine.connect() as conn, conn.begin(): - rowdata.to_sql('strokedata', engine, - if_exists='append', index=False) - conn.close() - engine.dispose() + filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = dd.from_pandas(rowdata,npartitions=1) + df.to_parquet(filename,engine='fastparquet',compression='GZIP') + return rowdata # This is the main routine. @@ -2563,14 +2574,8 @@ def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) df = dd.from_pandas(data,npartitions=1) - #df = df.loc[:,~df.columns.duplicated()] - # data.to_csv(filename,compression='gzip') - df.to_parquet(filename,engine='fastparquet',compression='GZIP') - # data.to_parquet(filename,engine='fastparquet',compression='gzip') - # table = pa.Table.from_pandas(data) - #pq.write_table(table,filename) return data diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 3347ac49..11a3a45d 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -28,6 +28,35 @@ from rowsandall_app.settings_dev import use_sqlite from rowers.utils import lbstoN +# dtypes +dtypes = {'workoutid':int, + 'hr':int, + 'pace':float, + 'velo':float, + 'spm':float, + 'driveenergy':float, + 'power':float, + 'averageforce':float, + 'peakforce':float, + 'drivelength':float, + 'distance':float, + 'cumdist':float, + 'drivespeed':float, + 'catch':float, + 'slip':float, + 'finish':float, + 'wash':float, + 'peakforceangle':float, + 'totalangle':float, + 'effectiveangle':float, + 'rhythm':float, + 'efficiency':float, + 'distanceperstroke':float, + 'ftime':str, + 'fpace':str, + 'fergpace':str, + 'fnowindpace':str, +} try: user = DATABASES['default']['USER'] @@ -637,20 +666,11 @@ def new_workout_from_file(r,f2, return (id,message,f2) def delete_strokedata(id,debug=False): - if debug: - engine = create_engine(database_url_debug, echo=False) - else: - engine = create_engine(database_url, echo=False) - query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( - id=id, - )) - with engine.connect() as conn, conn.begin(): - try: - result = conn.execute(query) - except: - print("Database Locked") - conn.close() - engine.dispose() + dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id) + try: + shutil.rmtree(dirname) + except FileNotFoundError: + pass def update_strokedata(id,df,debug=False): delete_strokedata(id,debug=debug) @@ -723,19 +743,18 @@ def getsmallrowdata_db(columns,ids=[],debug=False): if len(ids)>1: for f in csvfilenames: try: - df = dd.read_parquet(f,columns=columns,engine='pyarrow') + df = pd.read_parquet(f,columns=columns,engine='pyarrow') data.append(df) except OSError: pass - df = dd.concat(data,axis=0) + df = pd.concat(data,axis=0) else: - df = dd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') + df = pd.read_parquet(csvfilenames[0],columns=columns,engine='pyarrow') - data = df.compute() - return data + return df def fitnessmetric_to_sql(m,table='powertimefitnessmetric',debug=False, doclean=False): @@ -779,51 +798,42 @@ def read_cols_df_sql(ids,columns,debug=False): columns = list(columns)+['distance','spm'] columns = [x for x in columns if x != 'None'] columns = list(set(columns)) - cls = '' + ids = [int(id) for id in ids] - if debug: - engine = create_engine(database_url_debug, echo=False) - else: - engine = create_engine(database_url, echo=False) - for column in columns: - cls += column+', ' - cls = cls[:-2] if len(ids) == 0: - query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid=0'.format( - columns = cls, - )) + return pd.DataFrame() elif len(ids) == 1: - query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid={id}'.format( - id = ids[0], - columns = cls, - )) + try: + filename = 'media/strokedata_{id}.parquet.gz'.format(id=ids[0]) + df = pd.read_parquet(filename,columns=columns) + except OSError: + pass else: - query = sa.text('SELECT {columns} FROM strokedata WHERE workoutid IN {ids}'.format( - columns = cls, - ids = tuple(ids), - )) - - df = pd.read_sql_query(query,engine) - engine.dispose() + data = [] + filenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] + for id,f in zip(ids,filenames): + try: + df = pd.read_parquet(f,columns=columns) + data.append(df) + except OSError: + pass + + df = pd.concat(data,axis=0) + return df def read_df_sql(id,debug=False): - if debug: - engine = create_engine(database_url_debug, echo=False) - print("read_df",id) - print(database_url_debug) - else: - engine = create_engine(database_url, echo=False) + try: + f = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = pd.read_parquet(f) + except OSError: + pass - df = pd.read_sql_query(sa.text( - 'SELECT * FROM strokedata WHERE workoutid={id}'.format( - id=id - )), engine) + df = df.fillna(value=0) - engine.dispose() return df def getcpdata_sql(rower_id,table='cpdata',debug=False): @@ -1266,6 +1276,9 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, # write data if id given if id != 0: data['workoutid'] = id - data.to_parquet(filename,engine='pyarrow',compression='gzip') + data = data.astype(dtype=dtypes) + filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = dd.from_pandas(data,npartitions=1) + df.to_parquet(filename,engine='fastparquet',compression='GZIP') return data diff --git a/rowers/models.py b/rowers/models.py index 02fb531f..dbe4062a 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -28,6 +28,8 @@ from django_countries.fields import CountryField from scipy.interpolate import splprep, splev, CubicSpline import numpy as np +import shutil + from django.conf import settings from sqlalchemy import create_engine import sqlalchemy as sa @@ -2805,6 +2807,12 @@ def auto_delete_file_on_delete(sender, instance, **kwargs): if instance.csvfilename+'.gz': if os.path.isfile(instance.csvfilename+'.gz'): os.remove(instance.csvfilename+'.gz') + # remove parquet file + try: + dirname = 'media/strokedata_{id}.parquet.gz'.format(id=instance.id) + shutil.rmtree(dirname) + except FileNotFoundError: + pass @receiver(models.signals.post_delete,sender=Workout) def update_duplicates_on_delete(sender, instance, **kwargs): From 7a26669cd703168a622010c30e84567a2d250056 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 23 Oct 2019 20:17:49 +0200 Subject: [PATCH 11/14] removed StrokeData table and all references to it --- rowers/dataprep.py | 68 ++------------------------------------ rowers/dataprepnodjango.py | 30 +---------------- rowers/metrics.py | 10 ++++-- rowers/models.py | 6 ++-- rowers/serializers.py | 2 +- rowers/urls.py | 2 +- rowers/views/statements.py | 2 +- 7 files changed, 17 insertions(+), 103 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 158c74a3..1fb142d0 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -7,7 +7,7 @@ from __future__ import unicode_literals # All the data preparation, data cleaning and data mangling should # be defined here from __future__ import unicode_literals, absolute_import -from rowers.models import Workout, StrokeData,Team +from rowers.models import Workout, Team import pytz @@ -55,7 +55,7 @@ from rowingdata import ( from rowingdata.csvparsers import HumonParser -from rowers.metrics import axes,calc_trimp,rowingmetrics +from rowers.metrics import axes,calc_trimp,rowingmetrics,dtypes from rowers.models import strokedatafields #allowedcolumns = [item[0] for item in rowingmetrics] @@ -117,35 +117,6 @@ columndict = { 'cumdist': 'cum_dist', } -# dtypes -dtypes = {'workoutid':int, - 'hr':int, - 'pace':float, - 'velo':float, - 'spm':float, - 'driveenergy':float, - 'power':float, - 'averageforce':float, - 'peakforce':float, - 'drivelength':float, - 'distance':float, - 'cumdist':float, - 'drivespeed':float, - 'catch':float, - 'slip':float, - 'finish':float, - 'wash':float, - 'peakforceangle':float, - 'totalangle':float, - 'effectiveangle':float, - 'rhythm':float, - 'efficiency':float, - 'distanceperstroke':float, - 'ftime':str, - 'fpace':str, - 'fergpace':str, - 'fnowindpace':str, -} from scipy.signal import savgol_filter @@ -605,41 +576,6 @@ def getstatsfields(): return fieldlist, fielddict -def getstatsfields_old(): - # Get field names and remove those that are not useful in stats - fields = StrokeData._meta.get_fields() - - fielddict = {field.name: field.verbose_name for field in fields} - - # fielddict.pop('workoutid') - fielddict.pop('ergpace') - fielddict.pop('hr_an') - fielddict.pop('hr_tr') - fielddict.pop('hr_at') - fielddict.pop('hr_ut2') - fielddict.pop('hr_ut1') - fielddict.pop('time') - fielddict.pop('distance') - fielddict.pop('nowindpace') - fielddict.pop('fnowindpace') - fielddict.pop('fergpace') - fielddict.pop('equivergpower') -# fielddict.pop('workoutstate') - fielddict.pop('fpace') - fielddict.pop('pace') - fielddict.pop('id') - fielddict.pop('ftime') - fielddict.pop('x_right') - fielddict.pop('hr_max') - fielddict.pop('hr_bottom') - fielddict.pop('cumdist') - - try: - fieldlist = [field for field, value in fielddict.iteritems()] - except AttributeError: - fieldlist = [field for field, value in fielddict.items()] - - return fieldlist, fielddict # A string representation for time deltas diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 11a3a45d..c7e2871c 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -28,35 +28,6 @@ from rowsandall_app.settings_dev import use_sqlite from rowers.utils import lbstoN -# dtypes -dtypes = {'workoutid':int, - 'hr':int, - 'pace':float, - 'velo':float, - 'spm':float, - 'driveenergy':float, - 'power':float, - 'averageforce':float, - 'peakforce':float, - 'drivelength':float, - 'distance':float, - 'cumdist':float, - 'drivespeed':float, - 'catch':float, - 'slip':float, - 'finish':float, - 'wash':float, - 'peakforceangle':float, - 'totalangle':float, - 'effectiveangle':float, - 'rhythm':float, - 'efficiency':float, - 'distanceperstroke':float, - 'ftime':str, - 'fpace':str, - 'fergpace':str, - 'fnowindpace':str, -} try: user = DATABASES['default']['USER'] @@ -176,6 +147,7 @@ def rdata(file,rower=rrower()): return res from rowers.utils import totaltime_sec_to_string +from rowers.metrics import dtypes # Creates C2 stroke data diff --git a/rowers/metrics.py b/rowers/metrics.py index dd1852b2..65886fca 100644 --- a/rowers/metrics.py +++ b/rowers/metrics.py @@ -290,8 +290,14 @@ rowingmetrics = ( ) - - +dtypes = {} + +for name,d in rowingmetrics: + if d['numtype'] == 'float': + dtypes[name] = float + elif d['numtype'] == 'int': + dtypes[name] = int + axesnew = [ (name,d['verbose_name'],d['ax_min'],d['ax_max'],d['type']) for name,d in rowingmetrics ] diff --git a/rowers/models.py b/rowers/models.py index dbe4062a..1d720e5f 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -3073,9 +3073,9 @@ attrs.update(strokedatafields) # when the StrokeData are expanded. # No Django Instances of this model are managed. Strokedata table is # accesssed directly with SQL commands -StrokeData = type(str('StrokeData'), (models.Model,), - attrs - ) +#StrokeData = type(str('StrokeData'), (models.Model,), +# attrs +# ) # Storing data for the OTW CP chart class cpdata(models.Model): diff --git a/rowers/serializers.py b/rowers/serializers.py index 7a99f2a1..d5ea3e84 100644 --- a/rowers/serializers.py +++ b/rowers/serializers.py @@ -7,7 +7,7 @@ from __future__ import unicode_literals # Also optionally define POST, PATCH methods (create, update) from rest_framework import serializers -from rowers.models import Workout,Rower,StrokeData,FavoriteChart +from rowers.models import Workout,Rower,FavoriteChart import datetime diff --git a/rowers/urls.py b/rowers/urls.py index b6bcc91b..cabc1753 100644 --- a/rowers/urls.py +++ b/rowers/urls.py @@ -7,7 +7,7 @@ from django.conf.urls import url, include from django.urls import path, re_path from django.contrib.auth.models import User from django.contrib.auth.decorators import login_required, permission_required -from rowers.models import Workout,Rower,StrokeData,FavoriteChart +from rowers.models import Workout,Rower,FavoriteChart from rest_framework import routers, serializers, viewsets,permissions from rest_framework.urlpatterns import format_suffix_patterns diff --git a/rowers/views/statements.py b/rowers/views/statements.py index 85df6cb6..15fd34ef 100644 --- a/rowers/views/statements.py +++ b/rowers/views/statements.py @@ -99,7 +99,7 @@ from rowers.models import ( ) from rowers.models import ( RowerPowerForm,RowerForm,GraphImage,AdvancedWorkoutForm, - RowerPowerZonesForm,AccountRowerForm,UserForm,StrokeData, + RowerPowerZonesForm,AccountRowerForm,UserForm, Team,TeamForm,TeamInviteForm,TeamInvite,TeamRequest, WorkoutComment,WorkoutCommentForm,RowerExportForm, CalcAgePerformance, From 83987013202495ca81ab4b6bcb1021a993e5d444 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 23 Oct 2019 20:34:16 +0200 Subject: [PATCH 12/14] moving age group records to c2stuff --- rowers/c2stuff.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++ rowers/metrics.py | 63 +--------------------------------------------- 2 files changed, 65 insertions(+), 62 deletions(-) diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 4d034e6a..5c71d85a 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -29,6 +29,70 @@ queue = django_rq.get_queue('default') queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('low') from rowers.utils import myqueue +from rowers.models import C2WorldClassAgePerformance + +def getagegrouprecord(age,sex='male',weightcategory='hwt', + distance=2000,duration=None,indf=pd.DataFrame()): + + if not indf.empty: + if not duration: + df = indf[indf['distance'] == distance] + else: + duration = 60*int(duration) + df = indf[indf['duration'] == duration] + else: + if not duration: + df = pd.DataFrame( + list( + C2WorldClassAgePerformance.objects.filter( + distance=distance, + sex=sex, + weightcategory=weightcategory + ).values() + ) + ) + else: + duration=60*int(duration) + df = pd.DataFrame( + list( + C2WorldClassAgePerformance.objects.filter( + duration=duration, + sex=sex, + weightcategory=weightcategory + ).values() + ) + ) + + if not df.empty: + ages = df['age'] + powers = df['power'] + + #poly_coefficients = np.polyfit(ages,powers,6) + fitfunc = lambda pars, x: np.abs(pars[0])*(1-x/max(120,pars[1]))-np.abs(pars[2])*np.exp(-x/np.abs(pars[3]))+np.abs(pars[4])*(np.sin(np.pi*x/max(50,pars[5]))) + errfunc = lambda pars, x,y: fitfunc(pars,x)-y + + p0 = [700,120,700,10,100,100] + + try: + p1, success = optimize.leastsq(errfunc,p0[:], + args = (ages,powers)) + except: + p1 = p0 + success = 0 + + if success: + power = fitfunc(p1, float(age)) + + #power = np.polyval(poly_coefficients,age) + + power = 0.5*(np.abs(power)+power) + else: + power = 0 + else: + power = 0 + + return power + oauth_data = { 'client_id': C2_CLIENT_ID, diff --git a/rowers/metrics.py b/rowers/metrics.py index 65886fca..1a488afa 100644 --- a/rowers/metrics.py +++ b/rowers/metrics.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals from __future__ import absolute_import from rowers.utils import lbstoN import numpy as np -from rowers.models import C2WorldClassAgePerformance + import pandas as pd from scipy import optimize from django.utils import timezone @@ -397,64 +397,3 @@ def calc_trimp(df,sex,hrmax,hrmin,hrftp): return trimp,hrtss -def getagegrouprecord(age,sex='male',weightcategory='hwt', - distance=2000,duration=None,indf=pd.DataFrame()): - - if not indf.empty: - if not duration: - df = indf[indf['distance'] == distance] - else: - duration = 60*int(duration) - df = indf[indf['duration'] == duration] - else: - if not duration: - df = pd.DataFrame( - list( - C2WorldClassAgePerformance.objects.filter( - distance=distance, - sex=sex, - weightcategory=weightcategory - ).values() - ) - ) - else: - duration=60*int(duration) - df = pd.DataFrame( - list( - C2WorldClassAgePerformance.objects.filter( - duration=duration, - sex=sex, - weightcategory=weightcategory - ).values() - ) - ) - - if not df.empty: - ages = df['age'] - powers = df['power'] - - #poly_coefficients = np.polyfit(ages,powers,6) - fitfunc = lambda pars, x: np.abs(pars[0])*(1-x/max(120,pars[1]))-np.abs(pars[2])*np.exp(-x/np.abs(pars[3]))+np.abs(pars[4])*(np.sin(np.pi*x/max(50,pars[5]))) - errfunc = lambda pars, x,y: fitfunc(pars,x)-y - - p0 = [700,120,700,10,100,100] - - try: - p1, success = optimize.leastsq(errfunc,p0[:], - args = (ages,powers)) - except: - p1 = p0 - success = 0 - - if success: - power = fitfunc(p1, float(age)) - - #power = np.polyval(poly_coefficients,age) - - power = 0.5*(np.abs(power)+power) - else: - power = 0 - else: - power = 0 - - return power From 2099dde846152df47a9560648d10ead7510e291e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Wed, 23 Oct 2019 21:25:47 +0200 Subject: [PATCH 13/14] bug fixes --- rowers/interactiveplots.py | 27 ++++++++++----------------- rowers/views/statements.py | 20 +++++++++----------- rowers/views/workoutviews.py | 1 - 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 21d48b40..95c5b5ed 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -77,6 +77,7 @@ import rowers.stravastuff as stravastuff from rowers.dataprep import rdata import rowers.dataprep as dataprep import rowers.metrics as metrics +import rowers.c2stuff as c2stuff from rowers.metrics import axes,axlabels,yaxminima,yaxmaxima @@ -1815,7 +1816,7 @@ def interactive_agegroupcpchart(age,normalized=False): fhpower = [] for distance in distances: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='female', distance=distance, @@ -1829,7 +1830,7 @@ def interactive_agegroupcpchart(age,normalized=False): except ZeroDivisionError: pass for duration in durations: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='female', duration=duration, @@ -1847,7 +1848,7 @@ def interactive_agegroupcpchart(age,normalized=False): flpower = [] for distance in distances: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='female', distance=distance, @@ -1861,7 +1862,7 @@ def interactive_agegroupcpchart(age,normalized=False): except ZeroDivisionError: pass for duration in durations: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='female', duration=duration, @@ -1879,7 +1880,7 @@ def interactive_agegroupcpchart(age,normalized=False): mlpower = [] for distance in distances: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='male', distance=distance, @@ -1893,7 +1894,7 @@ def interactive_agegroupcpchart(age,normalized=False): except ZeroDivisionError: pass for duration in durations: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='male', duration=duration, @@ -1912,7 +1913,7 @@ def interactive_agegroupcpchart(age,normalized=False): mhpower = [] for distance in distances: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='male', distance=distance, @@ -1926,7 +1927,7 @@ def interactive_agegroupcpchart(age,normalized=False): except ZeroDivisionError: pass for duration in durations: - worldclasspower = metrics.getagegrouprecord( + worldclasspower = c2stuff.getagegrouprecord( age, sex='male', duration=duration, @@ -4060,15 +4061,7 @@ def thumbnails_set(r,id,favorites): columns += [f.yparam2 for f in favorites] columns += ['time'] - try: - rowdata = dataprep.getsmallrowdata_db(columns,ids=[id],doclean=True) - except: - return [ - {'script':"", - 'div':"", - 'notes':"" - }] - + rowdata = dataprep.getsmallrowdata_db(columns,ids=[id],doclean=True) rowdata.dropna(axis=1,how='all',inplace=True) diff --git a/rowers/views/statements.py b/rowers/views/statements.py index 15fd34ef..069bd438 100644 --- a/rowers/views/statements.py +++ b/rowers/views/statements.py @@ -256,19 +256,17 @@ def getfavorites(r,row): if 'speedcoach2' in row.workoutsource: workoutsource = 'speedcoach2' - try: - favorites = FavoriteChart.objects.filter(user=r, - workouttype__in=matchworkouttypes).order_by("id") - favorites2 = FavoriteChart.objects.filter(user=r, - workouttype__in=[workoutsource]).order_by("id") + favorites = FavoriteChart.objects.filter(user=r, + workouttype__in=matchworkouttypes).order_by("id") + favorites2 = FavoriteChart.objects.filter(user=r, + workouttype__in=[workoutsource]).order_by("id") - favorites = favorites | favorites2 - - maxfav = len(favorites)-1 - except: - favorites = None - maxfav = 0 + + favorites = favorites | favorites2 + + maxfav = len(favorites)-1 + return favorites,maxfav diff --git a/rowers/views/workoutviews.py b/rowers/views/workoutviews.py index 0293bfea..f8c0efbf 100644 --- a/rowers/views/workoutviews.py +++ b/rowers/views/workoutviews.py @@ -2767,7 +2767,6 @@ def workout_workflow_view(request,id): aantalcomments = len(comments) favorites,maxfav = getfavorites(r,row) - charts = get_call() From ff5a8a7d3ea71fcfef4669a0d5c6485f52bc99ae Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 24 Oct 2019 08:16:28 +0200 Subject: [PATCH 14/14] restored strokedata temporarily --- rowers/models.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/rowers/models.py b/rowers/models.py index 1d720e5f..defc99f8 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -2850,20 +2850,20 @@ def update_duplicates_on_delete(sender, instance, **kwargs): # Delete stroke data from the database when a workout is deleted -@receiver(models.signals.post_delete,sender=Workout) -def auto_delete_strokedata_on_delete(sender, instance, **kwargs): - if instance.id: - query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( - id=instance.id, - )) - engine = create_engine(database_url, echo=False) - with engine.connect() as conn, conn.begin(): - try: - result = conn.execute(query) - except: - print("Database Locked") - conn.close() - engine.dispose() +#@receiver(models.signals.post_delete,sender=Workout) +#def auto_delete_strokedata_on_delete(sender, instance, **kwargs): +# if instance.id: +# query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( +# id=instance.id, +# )) +# engine = create_engine(database_url, echo=False) +# with engine.connect() as conn, conn.begin(): +# try: +# result = conn.execute(query) +# except: +# print("Database Locked") +# conn.close() +# engine.dispose() # Virtual Race results (for keeping results when workouts are deleted) @python_2_unicode_compatible @@ -3073,9 +3073,9 @@ attrs.update(strokedatafields) # when the StrokeData are expanded. # No Django Instances of this model are managed. Strokedata table is # accesssed directly with SQL commands -#StrokeData = type(str('StrokeData'), (models.Model,), -# attrs -# ) +StrokeData = type(str('StrokeData'), (models.Model,), + attrs + ) # Storing data for the OTW CP chart class cpdata(models.Model):