diff --git a/rowers/dataprep.py b/rowers/dataprep.py index d116dbc0..727eaae3 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -36,13 +36,15 @@ import pandas as pd import numpy as np import itertools import math -from tasks import handle_sendemail_unrecognized +from tasks import handle_sendemail_unrecognized,handle_sendemail_breakthrough from django.conf import settings from sqlalchemy import create_engine import sqlalchemy as sa import sys +import utils +import datautils from utils import lbstoN from scipy.interpolate import griddata @@ -139,71 +141,6 @@ def filter_df(datadf,fieldname,value,largerthan=True): return datadf -def getcp(dfgrouped,logarr): - delta = [] - cpvalue = [] - avgpower = {} - #avgpower[0] = 0 - - for id,group in dfgrouped: - tt = group['time'].copy() - ww = group['power'].copy() - - tmax = tt.max() - newlen = int(tmax/2000.) - print newlen,len(ww) - newt = np.arange(newlen)*tmax/float(newlen) - neww = griddata(tt.values, - ww.values, - newt,method='linear', - rescale=True) - - #tt = pd.Series(newt) - #ww = pd.Series(neww) - - try: - avgpower[id] = int(ww.mean()) - except ValueError: - avgpower[id] = '---' - if not np.isnan(ww.mean()): - length = len(ww) - dt = [] - cpw = [] - for i in range(length-2): - w_roll = ww.rolling(i+2).mean().dropna() - if len(w_roll): - # now goes with # data points - should be fixed seconds - indexmax = w_roll.idxmax(axis=1) - try: - t_0 = tt.ix[indexmax] - t_1 = tt.ix[indexmax-i] - deltat = 1.0e-3*(t_0-t_1) - wmax = w_roll.ix[indexmax] - if not np.isnan(deltat) and not np.isnan(wmax): - dt.append(deltat) - cpw.append(wmax) - except KeyError: - pass - - - dt = pd.Series(dt) - cpw = pd.Series(cpw) - - - cpvalues = griddata(dt.values, - cpw.values, - logarr,method='linear', - rescale=True) - - for cpv in cpvalues: - cpvalue.append(cpv) - for d in logarr: - delta.append(d) - - delta = pd.Series(delta,name='Delta') - cpvalue = pd.Series(cpvalue,name='CP') - return delta,cpvalue,avgpower - def df_resample(datadf): # time stamps must be in seconds @@ -488,6 +425,13 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', powerperc=powerperc,powerzones=r.powerzones) row = rdata(f2,rower=rr) + isbreakthrough = False + if workouttype == 'water': + delta,cpvalues,avgpower = datautils.getsinglecp(row.df) + if utils.isbreakthrough(delta,cpvalues,r.p0,r.p1,r.p2,r.p3): + isbreakthrough = True + res = datautils.updatecp(delta,cpvalues,r) + dtavg = row.df['TimeStamp (sec)'].diff().mean() if dtavg < 1: @@ -643,6 +587,25 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', w.save() + # submit email task to send email about breakthrough workout + if isbreakthrough: + a_messages.info(r.user,'It looks like you have a new breakthrough workout') + if settings.DEBUG and r.getemailnotifications: + res = handle_sendemail_breakthrough.delay(w.id,r.user.email, + r.user.first_name, + r.user.last_name) + elif r.getemailnotifications: + try: + res = queuehigh.enqueue( + handle_sendemail_breakthrough(w.id, + r.user.email, + r.user.first_name, + r.user.last_name)) + except AttributeError: + pass + else: + pass + if privacy == 'visible': ts = Team.objects.filter(rower=r) for t in ts: diff --git a/rowers/datautils.py b/rowers/datautils.py new file mode 100644 index 00000000..7c1d9136 --- /dev/null +++ b/rowers/datautils.py @@ -0,0 +1,185 @@ +import pandas as pd +import numpy as np +from scipy.interpolate import griddata +from scipy import optimize + +def updatecp(delta,cpvalues,r): + cp2 = r.p0/(1+delta/r.p2) + cp2 += r.p1/(1+delta/r.p3) + + delta = delta.append(delta) + cp = cpvalues.append(cp2) + + powerdf = pd.DataFrame({ + 'Delta':delta, + 'CP':cp, + }) + + powerdf.dropna(axis=0,inplace=True) + powerdf.sort_values(['Delta','CP'],ascending=[1,0],inplace=True) + powerdf.drop_duplicates(subset='Delta',keep='first',inplace=True) + + + res = cpfit(powerdf) + p1 = res[0] + + r.p0 = p1[0] + r.p1 = p1[1] + r.p2 = p1[2] + r.p3 = p1[3] + + r.save() + + return 1 + +def cpfit(powerdf): + # Fit the data to thee parameter CP model + fitfunc = lambda pars,x: abs(pars[0])/(1+(x/abs(pars[2]))) + abs(pars[1])/(1+(x/abs(pars[3]))) + errfunc = lambda pars,x,y: fitfunc(pars,x)-y + + p0 = [500,350,10,8000] + + p1 = p0 + + thesecs = powerdf['Delta'] + theavpower = powerdf['CP'] + + if len(thesecs)>=4: + p1, success = optimize.leastsq(errfunc, p0[:], args = (thesecs,theavpower)) + else: + factor = fitfunc(p0,thesecs.mean())/theavpower.mean() + p1 = [p0[0]/factor,p0[1]/factor,p0[2],p0[3]] + + + p1 = [abs(p) for p in p1] + fitt = pd.Series(10**(4*np.arange(100)/100.)) + + fitpower = fitfunc(p1,fitt) + + fitpoints = fitfunc(p1,thesecs) + fitpoints0 = fitpoints.copy() + dd = fitpoints-theavpower + + ddmin = dd.min() + + frac = abs(ddmin)/fitpoints.mean() + + while frac>0.0001: + fitpoints = fitpoints*(fitpoints.mean()-ddmin)/(fitpoints.mean()) + dd = fitpoints-theavpower + ddmin = dd.min() + frac = abs(ddmin)/fitpoints.mean() + + ratio = fitpoints.mean()/fitpoints0.mean() + + return p1,fitt,fitpower,ratio + +def getlogarr(maxt): + maxlog10 = np.log10(maxt-5) + logarr = np.log10(5.)+np.arange(50)*maxlog10/50. + logarr = [int(10.**(la)) for la in logarr] + logarr = pd.Series(logarr) + logarr.drop_duplicates(keep='first',inplace=True) + + logarr = logarr.values + return logarr + +def getsinglecp(df): + thesecs = df['TimeStamp (sec)'].max()-df['TimeStamp (sec)'].min() + if thesecs != 0: + maxt = 2*thesecs + else: + maxt = 1000. + + logarr = getlogarr(maxt) + + + dfnew = pd.DataFrame({ + 'time':1000*(df['TimeStamp (sec)']-df.ix[0,'TimeStamp (sec)']), + 'power':df[' Power (watts)'] + }) + + dfnew['workoutid'] = 0 + + dfgrouped = dfnew.groupby(['workoutid']) + delta,cpvalue,avgpower = getcp(dfgrouped,logarr) + + return delta,cpvalue,avgpower + +def getcp(dfgrouped,logarr): + delta = [] + cpvalue = [] + avgpower = {} + #avgpower[0] = 0 + + for id,group in dfgrouped: + tt = group['time'].copy() + ww = group['power'].copy() + + tmax = tt.max() + if tmax > 500000: + newlen = int(tmax/2000.) + else: + newlen = len(tt) + if newlen < len(tt): + newt = np.arange(newlen)*tmax/float(newlen) + ww = griddata(tt.values, + ww.values, + newt,method='linear', + rescale=True) + + tt = pd.Series(newt) + ww = pd.Series(ww) + + + try: + avgpower[id] = int(ww.mean()) + except ValueError: + avgpower[id] = '---' + if not np.isnan(ww.mean()): + length = len(ww) + dt = [] + cpw = [] + for i in xrange(length-2): + deltat,wmax = getmaxwattinterval(tt,ww,i) + if not np.isnan(deltat) and not np.isnan(wmax): + dt.append(deltat) + cpw.append(wmax) + + + + dt = pd.Series(dt) + cpw = pd.Series(cpw) + if len(dt): + + cpvalues = griddata(dt.values, + cpw.values, + logarr,method='linear', + rescale=True) + + for cpv in cpvalues: + cpvalue.append(cpv) + for d in logarr: + delta.append(d) + + delta = pd.Series(delta,name='Delta') + cpvalue = pd.Series(cpvalue,name='CP') + return delta,cpvalue,avgpower + +def getmaxwattinterval(tt,ww,i): + w_roll = ww.rolling(i+2).mean().dropna() + if len(w_roll): + # now goes with # data points - should be fixed seconds + indexmax = w_roll.idxmax(axis=1) + try: + t_0 = tt.ix[indexmax] + t_1 = tt.ix[indexmax-i] + deltat = 1.0e-3*(t_0-t_1) + wmax = w_roll.ix[indexmax] + except KeyError: + pass + else: + wmax = 0 + deltat = 0 + + return deltat,wmax diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index feaf14c5..7200497e 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -55,6 +55,7 @@ import rowers.dataprep as dataprep from rowers.metrics import axes,axlabels,yaxminima,yaxmaxima from utils import lbstoN +import datautils watermarkurl = "/static/img/logo7.png" watermarksource = ColumnDataSource(dict( @@ -618,28 +619,11 @@ def interactive_otwcpchart(powerdf,promember=0): # there is no Paul's law for OTW - # Fit the data to thee parameter CP model - fitfunc = lambda pars,x: pars[0]/(1+(x/pars[2])) + pars[1]/(1+(x/pars[3])) - errfunc = lambda pars,x,y: fitfunc(pars,x)-y - - p0 = [500,350,10,8000] - - p1 = p0 - thesecs = powerdf['Delta'] theavpower = powerdf['CP'] - - if len(thesecs)>=4: - p1, success = optimize.leastsq(errfunc, p0[:], args = (thesecs,theavpower)) - else: - factor = fitfunc(p0,thesecs.mean())/theavpower.mean() - p1 = [p0[0]/factor,p0[1]/factor,p0[2],p0[3]] - - - fitt = pd.Series(10**(4*np.arange(100)/100.)) - - fitpower = fitfunc(p1,fitt) - + + p1,fitt,fitpower,ratio = datautils.cpfit(powerdf) + message = "" #if len(fitpower[fitpower<0]) > 0: # message = "CP model fit didn't give correct results" @@ -651,6 +635,7 @@ def interactive_otwcpchart(powerdf,promember=0): sourcecomplex = ColumnDataSource( data = dict( CP = fitpower, + CPmax = ratio*fitpower, duration = fitt, ftime = ftime ) @@ -689,6 +674,7 @@ def interactive_otwcpchart(powerdf,promember=0): hover.tooltips = OrderedDict([ ('Duration ','@ftime'), ('Power (W)','@CP{int}'), + ('Power (W) upper','@CPmax{int}'), ]) hover.mode = 'mouse' @@ -696,6 +682,9 @@ def interactive_otwcpchart(powerdf,promember=0): plot.line('duration','CP',source=sourcecomplex,legend="CP Model", color='green') + plot.line('duration','CPmax',source=sourcecomplex,legend="CP Model", + color='red') + script, div = components(plot) return [script,div,p1,message] diff --git a/rowers/models.py b/rowers/models.py index bdde5013..6922265f 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -232,6 +232,9 @@ class Rower(models.Model): ('hidden','Hidden'), ) + getemailnotifications = models.BooleanField(default=False, + verbose_name='Receive email notifications') + rowerplan = models.CharField(default='basic',max_length=30, choices=plans) @@ -746,7 +749,7 @@ class RowerPowerZonesForm(ModelForm): class AccountRowerForm(ModelForm): class Meta: model = Rower - fields = ['weightcategory'] + fields = ['weightcategory','getemailnotifications'] class UserForm(ModelForm): class Meta: diff --git a/rowers/tasks.py b/rowers/tasks.py index 34e33726..4d488f7c 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -29,8 +29,8 @@ from rowers.dataprepnodjango import new_workout_from_file from django.core.mail import send_mail, BadHeaderError,EmailMessage - - +import datautils +import utils # testing task @app.task @@ -47,6 +47,43 @@ def handle_new_workout_from_file(r,f2, return new_workout_from_file(r,f2,workouttype, title,makeprivate,notes) +# send email when a breakthrough workout is uploaded +@app.task +def handle_sendemail_breakthrough(workoutid,useremail,userfirstname,userlastname): + + # send email with attachment + subject = "A breakthrough workout on rowsandall.com" + message = "Dear "+userfirstname+",\n" + message += "Congratulations! Your recent workout has been analyzed" + message += " by Rowsandall.com and it appears your fitness," + message += " as measured by Critical Power, has improved!" + message += " Critical Power (CP) is the power that you can " + message += "sustain for a given duration. For more, see this " + message += " article in the analytics blog:\n\n" + message += " [link to article to be written]\n\n" + message += "Link to the workout http://rowsandall.com/rowers/workout/" + message += str(workoutid) + message +="/edit\n\n" + message +="To add the workout to your Ranking workouts and see the updated CP plot, click the following link:\n" + message += "http://rowsandall.com/rowers/workout/" + message += str(workoutid) + message += "/updatecp\n\n" + + message += "To opt out of these email notifications, deselect the checkbox on your Profile page under Account Information.\n\n" + + message += "Best Regards, the Rowsandall Team" + + email = EmailMessage(subject, message, + 'Rowsandall ', + [useremail]) + + + res = email.send() + + # remove tcx file + return 1 + + # send email to me when an unrecognized file is uploaded @app.task def handle_sendemail_unrecognized(unrecognizedfile,useremail): @@ -141,7 +178,7 @@ def handle_sendemailcsv(first_name,last_name,email,csvfile): # Calculate wind and stream corrections for OTW rowing @app.task def handle_otwsetpower(f1,boattype,weightvalue, - first_name,last_name,email,workoutid, + first_name,last_name,email,workoutid,ps=[1,1,1,1], debug=False): try: rowdata = rdata(f1) @@ -183,6 +220,12 @@ def handle_otwsetpower(f1,boattype,weightvalue, rowdata.write_csv(f1,gzip=True) update_strokedata(workoutid,rowdata.df,debug=debug) + delta,cpvalues,avgpower = datautils.getsinglecp(rowdata.df) + if utils.isbreakthrough(delta,cpvalues,ps[0],ps[1],ps[2],ps[3]): + handle_sendemail_breakthrough(workoutid,email, + first_name, + last_name) + # send email fullemail = first_name + " " + last_name + " " + "<" + email + ">" subject = "Your Rowsandall OTW calculations are ready" diff --git a/rowers/tests.py b/rowers/tests.py index 8ddbfaed..415154ef 100644 --- a/rowers/tests.py +++ b/rowers/tests.py @@ -68,6 +68,9 @@ class C2Objects(DjangoTestCase): u = User.objects.create_user('john', 'sander@ds.ds', 'koeinsloot') + u.first_name = 'John' + u.last_name = 'Sander' + u.save() r = Rower.objects.create(user=u) res = add_workout_from_strokedata(u,1,data,strokedata,source='c2') @@ -88,6 +91,9 @@ class C2Objects(DjangoTestCase): u = User.objects.create_user('john', 'sander@ds.ds', 'koeinsloot') + u.first_name = 'John' + u.last_name = 'Sander' + u.save() r = Rower.objects.create(user=u) res = add_workout_from_strokedata(u,1,data,strokedata,source='c2') @@ -162,6 +168,9 @@ class StravaObjects(DjangoTestCase): u = User.objects.create_user('john', 'sander@ds.ds', 'koeinsloot') + u.first_name = 'John' + u.last_name = 'Sander' + u.save() r = Rower.objects.create(user=u) res = add_workout_from_strokedata(u,1,workoutsummary,strokedata, @@ -235,6 +244,9 @@ class StravaObjects(DjangoTestCase): u = User.objects.create_user('john', 'sander@ds.ds', 'koeinsloot') + u.first_name = 'John' + u.last_name = 'Sander' + u.save() r = Rower.objects.create(user=u) res = add_workout_from_strokedata(u,1,workoutsummary,strokedata, diff --git a/rowers/urls.py b/rowers/urls.py index e850a4c6..844f9ae0 100644 --- a/rowers/urls.py +++ b/rowers/urls.py @@ -183,6 +183,7 @@ urlpatterns = [ url(r'^workout/compare/(?P\d+)/(?P\d+-\d+-\d+)/(?P\w+.*)$',views.workout_comparison_list), url(r'^workout/(?P\d+)/edit$',views.workout_edit_view), url(r'^workout/(?P\d+)/setprivate$',views.workout_setprivate_view), + url(r'^workout/(?P\d+)/updatecp$',views.workout_update_cp_view), url(r'^workout/(?P\d+)/makepublic$',views.workout_makepublic_view), url(r'^workout/(?P\d+)/geeky$',views.workout_geeky_view), url(r'^workout/(?P\d+)/advanced$',views.workout_advanced_view), diff --git a/rowers/utils.py b/rowers/utils.py index 17dfffe6..22c4680f 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -74,3 +74,13 @@ def geo_distance(lat1,lon1,lat2,lon2): bearing = math.degrees(tc1) return [distance,bearing] + + +def isbreakthrough(delta,cpvalues,p0,p1,p2,p3): + + pwr = p0/(1+delta/p2) + pwr += p1/(1+delta/p3) + + res = np.sum(cpvalues>pwr) + + return res>1 diff --git a/rowers/views.py b/rowers/views.py index c1a8a455..68781044 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -274,6 +274,8 @@ from utils import ( str2bool ) +import datautils + from rowers.models import checkworkoutuser # Check if a user is a Coach member @@ -343,6 +345,10 @@ def rower_register_view(request): response = dataprep.new_workout_from_file(therower,f2, title='New User Sample Data', notes='This is an example workout to get you started') + newworkoutid = response[0] + w = Workout.objects.get(id=newworkoutid) + w.startdatetime = timezone.now() + w.save() # Create and send email fullemail = first_name + " " + last_name + " " + "<" + email + ">" @@ -2774,6 +2780,27 @@ def rankings_view(request,theuser=0, 'teams':get_my_teams(request.user), }) +@user_passes_test(ispromember,login_url="/",redirect_field_name=None) +def workout_update_cp_view(request,id=0): + try: + row = Workout.objects.get(id=id) + except Workout.DoesNotExist: + raise Http404("Workout doesn't exist") + + if (checkworkoutuser(request.user,row)==False): + message = "You are not allowed to edit this workout" + messages.error(request,message) + url = reverse(workouts_view) + + return HttpResponseRedirect(url) + + row.rankingpiece = True + row.save() + + url = reverse(otwrankings_view) + + return HttpResponseRedirect(url) + # Show ranking distances including predicted paces @user_passes_test(ispromember,login_url="/",redirect_field_name=None) def otwrankings_view(request,theuser=0, @@ -2904,18 +2931,10 @@ def otwrankings_view(request,theuser=0, maxt = 1000. - maxlog10 = np.log10(maxt) - logarr = np.arange(50)*maxlog10/50. - logarr = [int(10.**(la)) for la in logarr] - logarr = pd.Series(logarr) - logarr.drop_duplicates(keep='first',inplace=True) - - logarr = logarr.values - - + logarr = datautils.getlogarr(maxt) dfgrouped = df.groupby(['workoutid']) - delta,cpvalue,avgpower = dataprep.getcp(dfgrouped,logarr) + delta,cpvalue,avgpower = datautils.getcp(dfgrouped,logarr) powerdf = pd.DataFrame({ @@ -7364,7 +7383,10 @@ def workout_summary_edit_view(request,id,message="",successmessage="" if form.is_valid(): cd = form.cleaned_data s = cd["intervalstring"] - rowdata.updateinterval_string(s) + try: + rowdata.updateinterval_string(s) + except IndexError: + messages.error(request,'Interval string parse error') intervalstats = rowdata.allstats() itime,idist,itype = rowdata.intervalstats_values() nrintervals = len(idist) @@ -7409,7 +7431,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" iunits = [] itypes = [] iresults = [] - for i in range(nrintervals): + for i in xrange(nrintervals): try: t = datetime.datetime.strptime(request.POST['intervalt_%s' % i],"%H:%M:%S.%f") except ValueError: @@ -7462,7 +7484,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" iunits = [] itypes = [] iresults = [] - for i in range(nrintervals): + for i in xrange(nrintervals): t = cd['intervalt_%s' % i] timesecs = t.total_seconds() itime += [timesecs] @@ -7496,7 +7518,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" form = SummaryStringForm() initial = {} - for i in range(nrintervals): + for i in xrange(nrintervals): initial['intervald_%s' % i] = idist[i] initial['intervalt_%s' % i] = get_time(itime[i]) initial['type_%s' % i] = itype[i] @@ -7775,6 +7797,7 @@ def rower_edit_view(request,message=""): last_name = ucd['last_name'] email = ucd['email'] weightcategory = cd['weightcategory'] + getemailnotifications = cd['getemailnotifications'] u = request.user if len(first_name): u.first_name = first_name @@ -7784,6 +7807,7 @@ def rower_edit_view(request,message=""): u.save() r = getrower(u) r.weightcategory = weightcategory + r.getemailnotifications = getemailnotifications r.save() form = RowerForm(instance=r) powerform = RowerPowerForm(instance=r) @@ -8072,8 +8096,10 @@ def team_view(request,id=0): if inviteid: teams.send_invite_email(inviteid) successmessage = text + messages.info(request,successmessage) else: message = text + messages.error(request,message) elif request.user == t.manager: inviteform = TeamInviteForm() diff --git a/rowsanda_107501 b/rowsanda_107501 index 93b5d653..c733aeb4 100644 Binary files a/rowsanda_107501 and b/rowsanda_107501 differ diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index 168952ac..2831f3ee 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -54,7 +54,6 @@ INSTALLED_APPS = [ 'django_rq', 'django_rq_dashboard', 'translation_manager', -# 'debug_toolbar', 'django_mailbox', 'rest_framework', 'rest_framework_swagger', @@ -90,7 +89,6 @@ MIDDLEWARE_CLASSES = [ 'django.contrib.messages.middleware.MessageMiddleware', 'async_messages.middleware.AsyncMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', -# 'debug_toolbar.middleware.DebugToolbarMiddleware', ] ROOT_URLCONF = 'rowsandall_app.urls' diff --git a/rowsandall_app/settings_dev.py b/rowsandall_app/settings_dev.py index 82d0dffe..69cbd074 100644 --- a/rowsandall_app/settings_dev.py +++ b/rowsandall_app/settings_dev.py @@ -50,6 +50,10 @@ TEMPLATES[0]['OPTIONS']['debug'] = DEBUG ALLOWED_HOSTS = ['localhost'] +INSTALLED_APPS += ['debug_toolbar',] + +MIDDLEWARE_CLASSES += ['debug_toolbar.middleware.DebugToolbarMiddleware',] + CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',