sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

Offline CP calculations for OTW

Browse Source 
A new table in the database with precalculated CP values. The data
are updated through RQ/Celery asynchronous functions
This commit is contained in:
Sander Roosendaal
2017-10-25 15:17:14 +02:00
parent 470e809ebf
commit 74acd35e7a
6 changed files with 194 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
rowers/dataprep.py
View File

@@ -17,6 +17,7 @@ from django.utils import timezone
from time import strftime, strptime, mktime, time, daylight
import arrow
from django.utils.timezone import get_current_timezone
thetimezone = get_current_timezone()
from rowingdata import (
TCXParser, RowProParser, ErgDataParser,
@@ -40,7 +41,7 @@ import itertools
import math
from tasks import (
handle_sendemail_unrecognized, handle_sendemail_breakthrough,
handle_sendemail_hard
handle_sendemail_hard, handle_updatecp
)
from django.conf import settings
@@ -425,6 +426,112 @@ def paceformatsecs(values):
return out
def getcpdata_sql(rower_id):
engine = create_engine(database_url, echo=False)
query = sa.text('SELECT delta,cp from cpdata WHERE user={rower_id};'.format(
rower_id=rower_id
))
connection = engine.raw_connection()
df = pd.read_sql_query(query, engine)
return df
def deletecpdata_sql(rower_id):
engine = create_engine(database_url, echo=False)
query = sa.text('DELETE from cpdata WHERE user={rower_id};'.format(
rower_id=rower_id
))
with engine.connect() as conn, conn.begin():
try:
result = conn.execute(query)
except:
print "Database locked"
conn.close()
engine.dispose()
def updatecpdata_sql(rower_id,delta,cp):
deletecpdata_sql(rower_id)
df = pd.DataFrame(
{
'delta':delta,
'cp':cp,
'user':rower_id
}
)
engine = create_engine(database_url, echo=False)
with engine.connect() as conn, conn.begin():
df.to_sql('cpdata', engine, if_exists='append', index=False)
conn.close()
engine.dispose()
def runcpupdate(rower):
startdate = timezone.now()-datetime.timedelta(days=365)
enddate = timezone.now()+datetime.timedelta(days=5)
theworkouts = Workout.objects.filter(user=rower,rankingpiece=True,
workouttype='water',
startdatetime__gte=startdate,
startdatetime__lte=enddate)
theids = [w.id for w in theworkouts]
if settings.DEBUG:
res = handle_updatecp.delay(rower.id,theids,debug=True)
else:
res = queue.enqueue(handle_updatecp,rower.id,theids)
def fetchcp(rower,theworkouts):
# get all power data from database (plus workoutid)
theids = [int(w.id) for w in theworkouts]
columns = ['power','workoutid','time']
df = getsmallrowdata_db(columns,ids=theids)
dfgrouped = df.groupby(['workoutid'])
avgpower2 = dict(dfgrouped.mean()['power'].astype(int))
cpdf = getcpdata_sql(rower.id)
if not cpdf.empty:
return cpdf['delta'],cpdf['cp'],avgpower2
else:
if settings.DEBUG:
res = handle_updatecp.delay(rower.id,theids,debug=True)
else:
res = queue.enqueue(handle_updatecp,rower.id,theids)
return [],[],avgpower2
# below is redundant
thesecs = []
for w in theworkouts:
timesecs = 3600*w.duration.hour
timesecs += 60*w.duration.minute
timesecs += w.duration.second
timesecs += 1.e-5*w.duration.microsecond
thesecs.append(timesecs)
if len(thesecs) != 0:
maxt = 1.05*pd.Series(thesecs).max()
else:
maxt = 1000.
logarr = datautils.getlogarr(maxt)
delta,cpvalue,avgpower = datautils.getcp(dfgrouped,logarr)
updatecpdata_sql(rower.id,delta,cpvalue)
return delta,cpvalue,avgpower2
# Processes painsled CSV file to database

42
rowers/dataprepnodjango.py
View File

@@ -489,9 +489,9 @@ def testdata(time,distance,pace,spm):
return t1 and t2 and t3 and t4
def getsmallrowdata_db(columns,ids=[]):
def getsmallrowdata_db(columns,ids=[],debug=False):
data = read_cols_df_sql(ids,columns)
data = read_cols_df_sql(ids,columns,debug=debug)
return data
@@ -548,6 +548,44 @@ def read_df_sql(id,debug=False):
engine.dispose()
return df
def deletecpdata_sql(rower_id,debug=False):
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
query = sa.text('DELETE from cpdata WHERE user={rower_id};'.format(
rower_id=rower_id
))
with engine.connect() as conn, conn.begin():
try:
result = conn.execute(query)
except:
print "Database locked"
conn.close()
engine.dispose()
def updatecpdata_sql(rower_id,delta,cp,debug=False):
deletecpdata_sql(rower_id,debug=debug)
df = pd.DataFrame(
{
'delta':delta,
'cp':cp,
'user':rower_id
}
)
if debug:
engine = create_engine(database_url_debug, echo=False)
else:
engine = create_engine(database_url, echo=False)
with engine.connect() as conn, conn.begin():
df.to_sql('cpdata', engine, if_exists='append', index=False)
conn.close()
engine.dispose()

2
rowers/datautils.py
View File

@@ -107,6 +107,7 @@ def getsinglecp(df):
return delta,cpvalue,avgpower
def getcp(dfgrouped,logarr):
delta = []
cpvalue = []
@@ -206,3 +207,4 @@ def getmaxwattinterval(tt,ww,i):
deltat = 0
return deltat,wmax

13
rowers/models.py
View File

@@ -641,7 +641,18 @@ attrs.update(strokedatafields)
StrokeData = type(str('StrokeData'), (models.Model,),
attrs
)
# Storing data for the OTW CP chart
class cpdata(models.Model):
delta = models.IntegerField(default=0)
cp = models.FloatField(default=0)
user = models.IntegerField(default=0)
class Meta:
db_table = 'cpdata'
index_together = ['user']
app_label = 'rowers'
# A wrapper around the png files
class GraphImage(models.Model):
filename = models.CharField(default='',max_length=150,blank=True,null=True)

21
rowers/tasks.py
View File

@@ -26,7 +26,7 @@ from utils import deserialize_list
from rowers.dataprepnodjango import (
update_strokedata, new_workout_from_file,
getsmallrowdata_db,
getsmallrowdata_db, updatecpdata_sql
)
from django.core.mail import send_mail, EmailMessage
@@ -402,7 +402,26 @@ def handle_otwsetpower(f1, boattype, weightvalue,
# This function generates all the static (PNG image) plots
@app.task
def handle_updatecp(rower_id,workoutids,debug=False):
columns = ['power','workoutid','time']
df = getsmallrowdata_db(columns,ids=workoutids,debug=debug)
dfgrouped = df.groupby(['workoutid'])
if not df.empty:
maxt = 1.05*df['time'].max()/1000.
else:
maxt = 1000.
logarr = datautils.getlogarr(maxt)
delta,cpvalue,avgpower = datautils.getcp(dfgrouped,logarr)
updatecpdata_sql(rower_id,delta,cpvalue,debug=debug)
return 1
@app.task
def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename):

42
rowers/views.py
View File

@@ -98,7 +98,8 @@ from rowers.rows import handle_uploaded_file
from rowers.tasks import handle_makeplot,handle_otwsetpower,handle_sendemailtcx,handle_sendemailcsv
from rowers.tasks import (
handle_sendemail_unrecognized,handle_sendemailnewcomment,
handle_sendemailnewresponse, handle_updatedps
handle_sendemailnewresponse, handle_updatedps,
handle_updatecp
)
from scipy.signal import savgol_filter
@@ -2977,6 +2978,10 @@ def workout_update_cp_view(request,id=0):
row.rankingpiece = True
row.save()
r = getrower(request.user)
dataprep.runcpupdate(r)
url = reverse(otwrankings_view)
return HttpResponseRedirect(url)
@@ -3089,39 +3094,12 @@ def otwrankings_view(request,theuser=0,
startdatetime__lte=enddate)
# get all power data from database (plus workoutid)
theids = [int(w.id) for w in theworkouts]
columns = ['power','workoutid','time']
df = dataprep.getsmallrowdata_db(columns,ids=theids)
thesecs = []
for w in theworkouts:
timesecs = 3600*w.duration.hour
timesecs += 60*w.duration.minute
timesecs += w.duration.second
timesecs += 1.e-5*w.duration.microsecond
thesecs.append(timesecs)
if len(thesecs) != 0:
maxt = 1.05*pd.Series(thesecs).max()
else:
maxt = 1000.
logarr = datautils.getlogarr(maxt)
dfgrouped = df.groupby(['workoutid'])
delta,cpvalue,avgpower = datautils.getcp(dfgrouped,logarr)
delta,cpvalue,avgpower = dataprep.fetchcp(r,theworkouts)
powerdf = pd.DataFrame({
'Delta':delta,
'CP':cpvalue,
})
powerdf = powerdf[powerdf['CP']>0]
powerdf.dropna(axis=0,inplace=True)
@@ -3203,7 +3181,7 @@ def otwrankings_view(request,theuser=0,
del form.fields["pieceunit"]
messages.error(request,message)
return render(request, 'otwrankings.html',
{'rankingworkouts':theworkouts,
@@ -6655,6 +6633,10 @@ def workout_edit_view(request,id=0,message="",successmessage=""):
r.write_csv(row.csvfilename,gzip=True)
dataprep.update_strokedata(id,r.df)
successmessage = "Changes saved"
if rankingpiece:
dataprep.runcpupdate(row.user)
messages.info(request,successmessage)
url = reverse(workout_edit_view,
kwargs = {