From 287d2da73f517c3fb5b495317a3d983c0a2ed6dc Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 1 Oct 2020 08:39:10 +0200 Subject: [PATCH 1/7] dataprep is now saving cpdata as a parquet file --- rowers/dataprep.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 3874fa3f..9868964a 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1448,6 +1448,8 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', logarr = datautils.getlogarr(maxt) dfgrouped = df.groupby(['workoutid']) delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) + filename = 'media/cpdata_{id}.parquet.gz'.format(id=w.id) + df.to_parquet(filename,engine='fastparquet',compression='GZIP') res, btvalues, res2 = utils.isbreakthrough( delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) From 52725cb593e0e2acc85d06c27bb5f25bf15e4a52 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 1 Oct 2020 08:51:37 +0200 Subject: [PATCH 2/7] adding automatic cp calculation for ote --- rowers/dataprep.py | 14 +++++++++----- rowers/datautils.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 9868964a..469413b9 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -96,7 +96,7 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') from rowsandall_app.settings import SITE_URL -from rowers.mytypes import otwtypes,otetypes +from rowers.mytypes import otwtypes,otetypes,rowtypes from rowers import mytypes from rowers.database import * @@ -1434,7 +1434,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', isbreakthrough = False ishard = False - if workouttype == 'water': + if workouttype in rowtypes: df = getsmallrowdata_db(['power', 'workoutid', 'time'], ids=[w.id]) try: powermean = df['power'].mean() @@ -1451,14 +1451,18 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', filename = 'media/cpdata_{id}.parquet.gz'.format(id=w.id) df.to_parquet(filename,engine='fastparquet',compression='GZIP') - res, btvalues, res2 = utils.isbreakthrough( - delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) + if workouttype in otwtypes: + res, btvalues, res2 = utils.isbreakthrough( + delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) + elif workouttype in otetypes: + res, btvalues, res2 = utils.isbreakthrough( + delta, cpvalues, r.ep0, r.ep1, r.ep2, r.ep3, r.ecpratio) else: res = 0 res2 = 0 if res: isbreakthrough = True - res = datautils.updatecp(delta, cpvalues, r) + res = datautils.updatecp(delta, cpvalues, r,workouttype=workouttype) if res2 and not isbreakthrough: ishard = True diff --git a/rowers/datautils.py b/rowers/datautils.py index 10f5cd13..7cc434d7 100644 --- a/rowers/datautils.py +++ b/rowers/datautils.py @@ -8,12 +8,24 @@ import numpy as np from scipy.interpolate import griddata from scipy import optimize +from rowers.mytypes import otwtypes,otetypes,rowtypes + #p0 = [500,350,10,8000] p0 = [190,200,33,16000] -def updatecp(delta,cpvalues,r): - cp2 = r.p0/(1+delta/r.p2) - cp2 += r.p1/(1+delta/r.p3) +def updatecp(delta,cpvalues,r,workouttype='water'): + if workouttype in otwtypes: + p0 = r.p0 + p1 = r.p1 + p2 = r.p2 + p3 = r.p3 + else: + p0 = r.ep0 + p1 = r.ep1 + p2 = r.ep2 + p3 = r.ep3 + cp2 = p0/(1+delta/p2) + cp2 += p1/(1+delta/p3) delta = delta.append(delta) cp = cpvalues.append(cp2) @@ -31,11 +43,18 @@ def updatecp(delta,cpvalues,r): res = cpfit(powerdf) p1 = res[0] - r.p0 = p1[0] - r.p1 = p1[1] - r.p2 = p1[2] - r.p3 = p1[3] - r.cpratio = res[3] + if workouttype in otwtypes: + r.p0 = p1[0] + r.p1 = p1[1] + r.p2 = p1[2] + r.p3 = p1[3] + r.cpratio = res[3] + else: + r.ep0 = p1[0] + r.ep1 = p1[1] + r.ep2 = p1[2] + r.ep3 = p1[3] + r.ecpratio = res[3] r.save() From 88107530a5515ffb6ed31bb515c81441a41e724a Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 11 Oct 2020 13:47:44 +0200 Subject: [PATCH 3/7] start of UI for Analysis -> CP chart --- rowers/forms.py | 1 + rowers/templates/user_analysis_select.html | 71 +++++++++++++++------- rowers/views/analysisviews.py | 7 ++- 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/rowers/forms.py b/rowers/forms.py index c4519a1d..3bfbe8ce 100644 --- a/rowers/forms.py +++ b/rowers/forms.py @@ -1036,6 +1036,7 @@ analysischoices = ( ('flexall','Cumulative Flex Chart'), ('stats','Statistics'), ('compare','Compare'), + ('cp','CP chart'), ) diff --git a/rowers/templates/user_analysis_select.html b/rowers/templates/user_analysis_select.html index 3b71d66b..799df584 100644 --- a/rowers/templates/user_analysis_select.html +++ b/rowers/templates/user_analysis_select.html @@ -16,14 +16,14 @@
@@ -285,9 +312,9 @@
- + {% if workouts %} - + Toggle All
{{ form.as_table }} diff --git a/rowers/views/analysisviews.py b/rowers/views/analysisviews.py index 69a17863..fed7795e 100644 --- a/rowers/views/analysisviews.py +++ b/rowers/views/analysisviews.py @@ -500,6 +500,9 @@ def histodata(workouts, options): return(script,div) +def cpdata(workouts, options): + return ('','Not Yet Implemented') + def statsdata(workouts, options): includereststrokes = options['includereststrokes'] spmmin = options['spmmin'] @@ -725,6 +728,8 @@ def analysis_view_data(request,userid=0): script,div = statsdata(workouts,options) elif function == 'compare': script,div = comparisondata(workouts,options) + elif function == 'cp': + script, div = cpdata(workouts, options) else: script = '' div = 'Unknown analysis functions' @@ -4849,7 +4854,7 @@ def history_view_data(request,userid=0): typeselect = 'All' yaxis = request.GET.get('yaxis','duration') - + if yaxis.lower() not in ['duration','rscore','trimp']: yaxis = 'duration' From e0f11a3c1f65171478989720f7bae49a48bedcc9 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 11 Oct 2020 17:27:39 +0200 Subject: [PATCH 4/7] sort of gets CP data --- rowers/dataprep.py | 69 +++++++++++++++++++++- rowers/models.py | 7 +++ rowers/templates/user_analysis_select.html | 6 +- rowers/views/analysisviews.py | 53 ++++++++++++++++- 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 7218d06a..f6d40497 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1016,6 +1016,68 @@ def fetchcperg(rower,theworkouts): return cpdf +def fetchcp_new(rower,workouts): + + data = [] + for workout in workouts: + cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) + try: + df = pd.read_parquet(cpfile) + data.append(df) + except OSError: + # CP data file doesn't exist yet. has to be created + strokesdf = getsmallrowdata_db(['power','workoutid','time'],ids = [workout.id]) + if not strokesdf.empty: + totaltime = strokesdf['time'].max() + try: + powermean = strokesdf['power'].mean() + except KeyError: + powermean = 0 + + + if powermean != 0: + thesecs = totaltime + maxt = 1.05 * thesecs + if maxt > 0: + logarr = datautils.getlogarr(maxt) + dfgrouped = strokesdf.groupby(['workoutid']) + delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) + filename = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) + df = pd.DataFrame({ + 'delta':delta, + 'cp':cpvalues, + 'id':workout.id, + }) + df.to_parquet(filename,engine='fastparquet',compression='GZIP') + data.append(df) + + + if len(data)>1: + df = pd.concat(data,axis=0) + + df = df.groupby(['delta']).max() + + + df = df.sort_values(['delta']).reset_index() + dindex = df['id'].shift(1)-df['id'] + dpowerplus = df['cp'].shift(1)-df['cp'] + dpowermin = df['cp'].shift(-1)-df['cp'] + badrows = [] + badid = 0 + for index,row in df.iterrows(): + if dindex[index] != 0 and dpowermin[index] > 0: + badrows.append(index) + badid = row['id'] + elif row['id'] == badid: + badrows.append(index) + else: + badid = 0 + + + df = df.drop(index = badrows) + + + return df['delta'],df['cp'],0 def fetchcp(rower,theworkouts,table='cpdata'): # get all power data from database (plus workoutid) @@ -1449,7 +1511,12 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', dfgrouped = df.groupby(['workoutid']) delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) filename = 'media/cpdata_{id}.parquet.gz'.format(id=w.id) - df.to_parquet(filename,engine='fastparquet',compression='GZIP') + cpdf = pd.DataFrame({ + 'delta':delta, + 'cp':cpvalues, + 'id':w.id, + }) + cpdf.to_parquet(filename,engine='fastparquet',compression='GZIP') if workouttype in otwtypes: res, btvalues, res2 = utils.isbreakthrough( diff --git a/rowers/models.py b/rowers/models.py index c302b3ad..2ec434b7 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -2985,6 +2985,13 @@ def auto_delete_file_on_delete(sender, instance, **kwargs): except FileNotFoundError: pass + # remove parquet file + try: + dirname = 'media/cpdata_{id}.parquet.gz'.format(id=instance.id) + shutil.rmtree(dirname) + except FileNotFoundError: + pass + @receiver(models.signals.post_delete,sender=Workout) def update_duplicates_on_delete(sender, instance, **kwargs): if instance.id: diff --git a/rowers/templates/user_analysis_select.html b/rowers/templates/user_analysis_select.html index 799df584..bfd6f13c 100644 --- a/rowers/templates/user_analysis_select.html +++ b/rowers/templates/user_analysis_select.html @@ -104,7 +104,11 @@ yaxis1.hide(); yaxis2.hide(); plottype.hide(); - reststrokes.show(); + reststrokes.hide(); + workmin.hide(); + workmax.hide(); + spmmin.hide(); + spmmax.hide(); if (functionfield.val() == 'boxplot') { plotfield.show(); diff --git a/rowers/views/analysisviews.py b/rowers/views/analysisviews.py index fed7795e..43b2563b 100644 --- a/rowers/views/analysisviews.py +++ b/rowers/views/analysisviews.py @@ -501,7 +501,58 @@ def histodata(workouts, options): return(script,div) def cpdata(workouts, options): - return ('','Not Yet Implemented') + userid = options['userid'] + + + u = User.objects.get(id=userid) + r = u.rower + + + ids = [w.id for w in workouts] + delta, cpvalue, avgpower = dataprep.fetchcp_new(r,workouts) + powerdf = pd.DataFrame({ + 'Delta':delta, + 'CP':cpvalue, + }) + + + if powerdf.empty: + return('','

No valid data found

') + + powerdf = powerdf[powerdf['CP']>0] + powerdf.dropna(axis=0,inplace=True) + powerdf.sort_values(['Delta','CP'],ascending=[1,0],inplace=True) + powerdf.drop_duplicates(subset='Delta',keep='first',inplace=True) + + rowername = r.user.first_name+" "+r.user.last_name + + if len(powerdf) !=0 : + res = interactive_otwcpchart(powerdf,promember=True,rowername=rowername) + script = res[0] + div = res[1] + p1 = res[2] + ratio = res[3] + r.p0 = p1[0] + r.p1 = p1[1] + r.p2 = p1[2] + r.p3 = p1[3] + r.cpratio = ratio + r.save() + paulslope = 1 + paulintercept = 1 + message = res[4] + else: + script = '' + div = '

No ranking pieces found.

' + paulslope = 1 + paulintercept = 1 + p1 = [1,1,1,1] + message = "" + + scripta = script.split('\n')[2:-1] + script = ''.join(scripta) + + return (script,div) def statsdata(workouts, options): includereststrokes = options['includereststrokes'] From a42c275a096db395286428559b96fbefc049bb2d Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 11 Oct 2020 20:50:21 +0200 Subject: [PATCH 5/7] fix --- rowers/views/analysisviews.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rowers/views/analysisviews.py b/rowers/views/analysisviews.py index 8d75146e..43b2563b 100644 --- a/rowers/views/analysisviews.py +++ b/rowers/views/analysisviews.py @@ -780,11 +780,7 @@ def analysis_view_data(request,userid=0): elif function == 'compare': script,div = comparisondata(workouts,options) elif function == 'cp': -<<<<<<< HEAD script, div = cpdata(workouts, options) -======= - script, div = cpdata(workouts,options) ->>>>>>> develop else: script = '' div = 'Unknown analysis functions' From 60a68f5611f9d89fa7778b17ba9b62b3f2c7175a Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 12 Oct 2020 08:50:32 +0200 Subject: [PATCH 6/7] same basis of logarithmic delta axis --- rowers/dataprep.py | 32 +++++++++++++++++--------------- rowers/datautils.py | 5 ++++- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index f6d40497..d3d040c0 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1034,10 +1034,10 @@ def fetchcp_new(rower,workouts): except KeyError: powermean = 0 - if powermean != 0: thesecs = totaltime maxt = 1.05 * thesecs + if maxt > 0: logarr = datautils.getlogarr(maxt) dfgrouped = strokesdf.groupby(['workoutid']) @@ -1052,6 +1052,8 @@ def fetchcp_new(rower,workouts): data.append(df) + if len(data) == 0: + return pd.Series(),pd.Series(),0 if len(data)>1: df = pd.concat(data,axis=0) @@ -1059,22 +1061,22 @@ def fetchcp_new(rower,workouts): df = df.sort_values(['delta']).reset_index() - dindex = df['id'].shift(1)-df['id'] - dpowerplus = df['cp'].shift(1)-df['cp'] - dpowermin = df['cp'].shift(-1)-df['cp'] - badrows = [] - badid = 0 - for index,row in df.iterrows(): - if dindex[index] != 0 and dpowermin[index] > 0: - badrows.append(index) - badid = row['id'] - elif row['id'] == badid: - badrows.append(index) - else: - badid = 0 + #dindex = df['id'].shift(1)-df['id'] + #dpowerplus = df['cp'].shift(1)-df['cp'] + #dpowermin = df['cp'].shift(-1)-df['cp'] + #badrows = [] + #badid = 0 + #for index,row in df.iterrows(): + # if dindex[index] != 0 and dpowermin[index] > 0: + # badrows.append(index) + # badid = row['id'] + # elif row['id'] == badid: + # badrows.append(index) + # else: + # badid = 0 - df = df.drop(index = badrows) + #df = df.drop(index = badrows) return df['delta'],df['cp'],0 diff --git a/rowers/datautils.py b/rowers/datautils.py index 7cc434d7..554ea19e 100644 --- a/rowers/datautils.py +++ b/rowers/datautils.py @@ -108,7 +108,10 @@ def cpfit(powerdf): def getlogarr(maxt): maxlog10 = np.log10(maxt-5) - logarr = np.arange(50)*maxlog10/50. + #print(maxlog10,round(maxlog10)) + aantal = 10*round(maxlog10) + logarr = np.arange(aantal+1)/10. + res = [] for la in logarr: try: From cdc6b57c97eee9e3c6cfc1c9ae3793c9429bb25a Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 12 Oct 2020 18:42:43 +0200 Subject: [PATCH 7/7] bug fixes --- rowers/dataprep.py | 17 ----------------- rowers/models.py | 2 +- rowers/templates/user_analysis_select.html | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index d3d040c0..a68f05c6 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1061,23 +1061,6 @@ def fetchcp_new(rower,workouts): df = df.sort_values(['delta']).reset_index() - #dindex = df['id'].shift(1)-df['id'] - #dpowerplus = df['cp'].shift(1)-df['cp'] - #dpowermin = df['cp'].shift(-1)-df['cp'] - #badrows = [] - #badid = 0 - #for index,row in df.iterrows(): - # if dindex[index] != 0 and dpowermin[index] > 0: - # badrows.append(index) - # badid = row['id'] - # elif row['id'] == badid: - # badrows.append(index) - # else: - # badid = 0 - - - #df = df.drop(index = badrows) - return df['delta'],df['cp'],0 diff --git a/rowers/models.py b/rowers/models.py index 2ec434b7..9e5bc625 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -2988,7 +2988,7 @@ def auto_delete_file_on_delete(sender, instance, **kwargs): # remove parquet file try: dirname = 'media/cpdata_{id}.parquet.gz'.format(id=instance.id) - shutil.rmtree(dirname) + os.remove(dirname) except FileNotFoundError: pass diff --git a/rowers/templates/user_analysis_select.html b/rowers/templates/user_analysis_select.html index bfd6f13c..37974888 100644 --- a/rowers/templates/user_analysis_select.html +++ b/rowers/templates/user_analysis_select.html @@ -2,7 +2,7 @@ {% load staticfiles %} {% load rowerfilters %} -{% block title %}Workouts{% endblock %} +{% block title %}Analysis{% endblock %} {% block main %}