using async job to calculate predicted age records and store in db

2018-01-06 13:33:59 +01:00
parent 89b0a4f61f
commit bdf4546a29
7 changed files with 4062 additions and 3879 deletions
--- a/rowers/dataprepnodjango.py
+++ b/rowers/dataprepnodjango.py
@@ -582,6 +582,52 @@ def deletecpdata_sql(rower_id,table='cpdata',debug=False):
    conn.close()
    engine.dispose()

+def delete_agegroup_db(age,sex,weightcategory,debug=False):
+    if debug:
+        engine = create_engine(database_url_debug, echo=False)
+    else:
+        engine = create_engine(database_url, echo=False)
+
+    query = sa.text('DELETE from {table} WHERE age={age} and weightcategory = {weightcategory} and sex={sex};'.format(
+        sex=sex,
+        age=age,
+        weightcategory=weightcategory,
+        table='calcagegrouprecords'
+        ))
+    with engine.connect() as conn, conn.begin():
+        try:
+            result = conn.execute(query)
+        except:
+            print "Database locked"
+    conn.close()
+    engine.dispose()
+    
+def update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower,
+                       debug=False):
+                    
+    delete_agegroup_db(age,sex,weightcategory,debug=debug)
+
+    df = pd.DataFrame(
+        {
+            'duration':wcdurations,
+            'power':wcpower,
+         }
+    )
+
+    df['sex'] = sex
+    df['age'] = age
+    df['weightcategory'] = weightcategory
+
+    if debug:
+        engine = create_engine(database_url_debug, echo=False)
+    else:
+        engine = create_engine(database_url, echo=False)
+
+    table = 'calcagegrouprecords'
+    with engine.connect() as conn, conn.begin():
+        df.to_sql(table, engine, if_exists='append', index=False)
+    conn.close()
+    engine.dispose()

    
 def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=pd.Series([]),debug=False):
--- a/rowers/metrics.py
+++ b/rowers/metrics.py
@@ -3,6 +3,7 @@ import numpy as np
 from models import C2WorldClassAgePerformance
 import pandas as pd
 from scipy import optimize
+from django.utils import timezone

 rowingmetrics = (
    ('time',{
@@ -321,28 +322,36 @@ def calc_trimp(df,sex,hrmax,hrmin):
    return trimp

 def getagegrouprecord(age,sex='male',weightcategory='hwt',
-                      distance=2000,duration=None):
-    if not duration:
-        df = pd.DataFrame(
-            list(
-                C2WorldClassAgePerformance.objects.filter(
-                    distance=distance,
-                    sex=sex,
-                    weightcategory=weightcategory
-                ).values()
-            )
-        )
+                      distance=2000,duration=None,indf=pd.DataFrame()):
+
+    if not indf.empty:
+        if not duration:
+            df = indf[indf['distance'] == distance]
+        else:
+            duration = 60*int(duration)
+            df = indf[indf['duration'] == duration]
    else:
-        duration=60*int(duration)
-        df = pd.DataFrame(
-            list(
-                C2WorldClassAgePerformance.objects.filter(
-                    duration=duration,
-                    sex=sex,
-                    weightcategory=weightcategory
-                ).values()
+        if not duration:
+            df = pd.DataFrame(
+                list(
+                    C2WorldClassAgePerformance.objects.filter(
+                        distance=distance,
+                        sex=sex,
+                        weightcategory=weightcategory
+                    ).values()
+                )
+            )
+        else:
+            duration=60*int(duration)
+            df = pd.DataFrame(
+                list(
+                    C2WorldClassAgePerformance.objects.filter(
+                        duration=duration,
+                        sex=sex,
+                        weightcategory=weightcategory
+                    ).values()
+                )
            )
-        )

    if not df.empty:
        ages = df['age']
--- a/rowers/models.py
+++ b/rowers/models.py
@@ -215,6 +215,33 @@ def update_records(url=c2url):
            print record
            

+class CalcAgePerformance(models.Model):
+    weightcategories = (
+	('hwt','heavy-weight'),
+	('lwt','light-weight'),
+	)
+
+    sexcategories = (
+        ('male','male'),
+        ('female','female'),
+        )
+
+    weightcategory = models.CharField(default="hwt",
+                                      max_length=30,
+                                      choices=weightcategories)
+
+    sex = models.CharField(default="female",
+                           max_length=30,
+                           choices=sexcategories)
+
+    age = models.IntegerField(default=19,verbose_name="Age")
+
+    duration = models.FloatField(default=1,blank=True)
+    power = models.IntegerField(default=200)
+
+    class Meta:
+        db_table = 'calcagegrouprecords'
+        
 class C2WorldClassAgePerformance(models.Model):
    weightcategories = (
 	('hwt','heavy-weight'),
--- a/rowers/tasks.py
+++ b/rowers/tasks.py
@@ -6,6 +6,8 @@ import gzip
 import shutil
 import numpy as np

+from scipy import optimize
+
 import rowingdata

 from rowingdata import rowingdata as rdata
@@ -28,7 +30,8 @@ from utils import deserialize_list

 from rowers.dataprepnodjango import (
    update_strokedata, new_workout_from_file,
-    getsmallrowdata_db, updatecpdata_sql
+    getsmallrowdata_db, updatecpdata_sql,
+    update_agegroup_db,
 )

 from django.core.mail import send_mail, EmailMessage
@@ -46,6 +49,96 @@ import longtask
 def add(x, y):
    return x + y

+def getagegrouprecord(age,sex='male',weightcategory='hwt',
+                      distance=2000,duration=None,indf=pd.DataFrame()):
+
+    if not duration:
+        df = indf[indf['distance'] == distance]
+    else:
+        duration = 60*int(duration)
+        df = indf[indf['duration'] == duration]
+
+    if not df.empty:
+        ages = df['age']
+        powers = df['power']
+
+        #poly_coefficients = np.polyfit(ages,powers,6)
+        fitfunc = lambda pars, x: np.abs(pars[0])*(1-x/max(120,pars[1]))-np.abs(pars[2])*np.exp(-x/np.abs(pars[3]))+np.abs(pars[4])*(np.sin(np.pi*x/max(50,pars[5])))
+        errfunc = lambda pars, x,y: fitfunc(pars,x)-y
+
+        p0 = [700,120,700,10,100,100]
+
+        p1, success = optimize.leastsq(errfunc,p0[:],
+                                       args = (ages,powers))
+
+        if success:
+            power = fitfunc(p1, float(age))
+
+            #power = np.polyval(poly_coefficients,age)
+
+            power = 0.5*(np.abs(power)+power)
+        else:
+            power = 0
+    else:
+        power = 0
+
+    return power
+
+
+@app.task(bind=True)
+def handle_getagegrouprecords(self,
+                              df,
+                              distances,durations,
+                              age,sex,weightcategory,
+                              **kwargs):
+    wcdurations = []
+    wcpower = []
+
+    if 'debug' in kwargs:
+        debug = kwargs['debug']
+    else:
+        debug = False
+
+
+    df = pd.read_json(df)
+    
+    for distance in distances:
+        worldclasspower = getagegrouprecord(
+            age,
+            sex=sex,
+            distance=distance,
+            weightcategory=weightcategory,indf=df,
+        )
+        velo = (worldclasspower/2.8)**(1./3.)
+        try:
+            duration = distance/velo
+            wcdurations.append(duration)
+            wcpower.append(worldclasspower)
+        except ZeroDivisionError:
+            pass
+
+
+
+    for duration in durations:
+        worldclasspower = getagegrouprecord(
+            age,
+            sex=sex,
+            duration=duration,
+            weightcategory=weightcategory,indf=df
+        )
+        try:
+            velo = (worldclasspower/2.8)**(1./3.)
+            distance = int(60*duration*velo)
+            wcdurations.append(60.*duration)
+            wcpower.append(worldclasspower)
+        except ValueError:
+            pass
+
+    update_agegroup_db(age,sex,weightcategory,wcdurations,wcpower,
+                       debug=debug)
+
+    return 1
+

@app.task(bind=True)
 def long_test_task(self,aantal,debug=False,job=None,session_key=None):
--- a/rowers/templates/rankings.html
+++ b/rowers/templates/rankings.html
@@ -9,21 +9,6 @@
        src='https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js'>
 </script>

-<script>
-  $(function($) {
-  console.log('loading script');
-  console.log('{{ recalc }}');
-  if ('{{ recalc }}' == 'True') {
-  theurl = '/rowers/ajax_agegroup/{{ age }}/{{ weightcategory }}/{{ sex }}/{{ user.id }}'
-    console.log(theurl);
-    $.getJSON(window.location.protocol + '//'+window.location.host + theurl, function(json) {
-  location.reload();
-  });
-  } else {
-  console.log('No need to reload');
-  }
-  });
-</script>
 {% endblock %}

 {% block content %}
--- a/rowers/views.py
+++ b/rowers/views.py
@@ -56,6 +56,7 @@ from rowers.models import (
    RowerPowerZonesForm,AccountRowerForm,UserForm,StrokeData,
    Team,TeamForm,TeamInviteForm,TeamInvite,TeamRequest,
    WorkoutComment,WorkoutCommentForm,RowerExportForm,
+    CalcAgePerformance
    )
 from rowers.models import FavoriteForm,BaseFavoriteFormSet,SiteAnnouncement
 from rowers.metrics import rowingmetrics,defaultfavoritecharts
@@ -108,7 +109,7 @@ from rowers.tasks import (
    handle_sendemail_unrecognized,handle_sendemailnewcomment,
    handle_sendemailnewresponse, handle_updatedps,
    handle_updatecp,long_test_task,long_test_task2,
-    handle_zip_file
+    handle_zip_file,handle_getagegrouprecords
    )

 from scipy.signal import savgol_filter
@@ -284,6 +285,7 @@ verbose_job_status = {
    'updatecp': 'Critical Power Calculation for Ergometer Workouts',
    'updatecpwater': 'Critical Power Calculation for OTW Workouts',
    'otwsetpower': 'Rowing Physics OTW Power Calculation',
+    'agegrouprecords': 'Calculate age group records',
    'make_plot': 'Create static chart',
    'long_test_task': 'Long Test Task',
    'long_test_task2': 'Long Test Task 2',
@@ -3526,51 +3528,28 @@ def ajax_agegrouprecords(request,
    wcpower = []
    durations = [1,4,30,60]
    distances = [100,500,1000,2000,5000,6000,10000,21097,42195]
-    for distance in distances:
-        worldclasspower = metrics.getagegrouprecord(
-            age,
-            sex=sex,
-            distance=distance,
-            weightcategory=weightcategory
-        )
-        velo = (worldclasspower/2.8)**(1./3.)
-        try:
-            duration = distance/velo
-            wcdurations.append(duration)
-            wcpower.append(worldclasspower)
-        except ZeroDivisionError:
-            pass
-        for duration in durations:
-            worldclasspower = metrics.getagegrouprecord(
-                age,
+
+    df = pd.DataFrame(
+        list(
+            C2WorldClassAgePerformance.objects.filter(
                sex=sex,
-                duration=duration,
                weightcategory=weightcategory
-            )
-            try:
-                velo = (worldclasspower/2.8)**(1./3.)
-                distance = int(60*duration*velo)
-                wcdurations.append(60.*duration)
-                wcpower.append(worldclasspower)
-            except ValueError:
-                pass
+            ).values()
+        )
+    )

-    options = {}
-    options['wcpower'] = wcpower
-    options['wcdurations'] = wcdurations
+    jsondf = df.to_json()
    
-    thenowtime = timezone.now().isoformat()
-    options['lastupdated'] = thenowtime
-    options['userid'] = userid
+    job = myqueue(queue,
+                  handle_getagegrouprecords,
+                  jsondf,distances,durations,age,sex,weightcategory,
+    )

-    request.session['options'] = options
    
            
    return JSONResponse(
        {
-        'wcdurations': wcdurations,
-            'wcpower': wcpower,
-            'lastupdated': thenowtime,
+            'job':job.id
            }
        )

@@ -3636,6 +3615,7 @@ def rankings_view2(request,theuser=0,
        recalc = False
        if str(userid) != str(theuser) or deltatime_seconds > 3600:
            recalc = True
+            options['lastupdated'] = arrow.utcnow().isoformat()
        else:
            recalc = False

@@ -3647,6 +3627,23 @@ def rankings_view2(request,theuser=0,
            worldclasspower = None
            age = 0

+        agerecords = CalcAgePerformance.objects.filter(
+            age = age,
+            sex = r.sex,
+            weightcategory = r.weightcategory)
+
+        print len(agerecords),'aap'
+        if len(agerecords) == 0:
+            recalc = True
+            wcpower = []
+            wcduration = []
+        else:
+            wcdurations = []
+            wcpower = []
+            for record in agerecords:
+                wcdurations.append(record.duration)
+                wcpower.append(record.power)
+
        options['wcpower'] = wcpower
        options['wcdurations'] = wcdurations
        if theuser:
@@ -3914,6 +3911,32 @@ def rankings_view2(request,theuser=0,
 	     'power':int(pwr)}
 	cpredictions.append(a)

+    if recalc:
+        wcdurations = []
+        wcpower = []
+        durations = [1,4,30,60]
+        distances = [100,500,1000,2000,5000,6000,10000,21097,42195]
+
+        df = pd.DataFrame(
+            list(
+                C2WorldClassAgePerformance.objects.filter(
+                    sex=r.sex,
+                    weightcategory=r.weightcategory
+                ).values()
+            )
+        )
+
+        jsondf = df.to_json()
+    
+        job = myqueue(queue,
+                      handle_getagegrouprecords,
+                      jsondf,distances,durations,age,r.sex,r.weightcategory)
+        try:
+            request.session['async_tasks'] += [(job.id,'agegrouprecords')]
+        except KeyError:
+            request.session['async_tasks'] = [(job.id,'agegrouprecords')]
+
+        

    messages.error(request,message)
    return render(request, 'rankings.html',