Merge branch 'feature/nostrokedataimport' into develop

rowers/c2stuff.py

@@ -7,8 +7,10 @@
 import oauth2 as oauth
 import cgi
 import requests
+import arrow
 import requests.auth
 import json
+import iso8601
 from django.utils import timezone
 from datetime import datetime
 from datetime import timedelta
@@ -21,8 +23,8 @@ from django.conf import settings
 from django.contrib.auth import authenticate, login, logout
 from django.contrib.auth.models import User
 from django.contrib.auth.decorators import login_required
-
+import dataprep
-
+import pytz
 from rowingdata import rowingdata
 import pandas as pd
 import numpy as np
@@ -32,8 +34,16 @@ import sys
 import urllib
 from requests import Request, Session
 
+from utils import myqueue
+
 from rowsandall_app.settings import C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET
 
+from rowers.tasks import handle_c2_import_stroke_data
+import django_rq
+queue = django_rq.get_queue('default')
+queuelow = django_rq.get_queue('low')
+queuehigh = django_rq.get_queue('low')
+
 # Custom error class - to raise a NoTokenError
 class C2NoTokenError(Exception):
     def __init__(self,value):
@@ -82,7 +92,87 @@ def c2_open(user):
 
     return thetoken
 
+def add_stroke_data(user,c2id,workoutid,startdatetime,csvfilename):
+    r = Rower.objects.get(user=user)
+    if (r.c2token == '') or (r.c2token is None):
+	return custom_exception_handler(401,s)
+	s = "Token doesn't exist. Need to authorize"
+    elif (timezone.now()>r.tokenexpirydate):
+	s = "Token expired. Needs to refresh."
+	return custom_exception_handler(401,s)
+    else:
+        starttimeunix = arrow.get(startdatetime).timestamp
 
+        job = myqueue(queue,
+                      handle_c2_import_stroke_data,
+                      r.c2token,
+                      c2id,
+                      workoutid,
+                      starttimeunix,
+                      csvfilename)
+
+    return 1
+    
+
+# get workout metrics, then relay stroke data to an asynchronous task
+def create_async_workout(alldata,user,c2id):
+    data = alldata[c2id]
+    splitdata = None
+
+    distance = data['distance']
+    c2id = data['id']
+    workouttype = data['type']
+    verified = data['verified']
+    startdatetime = iso8601.parse_date(data['date'])
+    weightclass = data['weight_class']
+    weightcategory = 'hwt'
+    if weightclass == "L":
+        weightcategory = 'lwt'
+
+    # Create CSV file name and save data to CSV file
+    csvfilename ='media/Import_'+str(c2id)+'.csv.gz'
+
+    totaltime = data['time']/10.
+    duration = dataprep.totaltime_sec_to_string(totaltime)
+
+    try:
+        timezone_str = tz(data['timezone'])
+    except:
+        timezone_str = 'UTC'
+
+    workoutdate = startdatetime.astimezone(
+        pytz.timezone(timezone_str)
+    ).strftime('%Y-%m-%d')
+    starttime = startdatetime.astimezone(
+        pytz.timezone(timezone_str)
+    ).strftime('%H:%M:%S')
+
+    r = Rower.objects.get(user=user)
+
+
+    w = Workout(
+        user=r,
+        workouttype = workouttype,
+        name = 'Imported workout',
+        date = workoutdate,
+        starttime = starttime,
+        startdatetime = startdatetime,
+        timezone = timezone_str,
+        duration = duration,
+        distance=distance,
+        weightcategory = weightcategory,
+        uploadedtoc2 = c2id,
+        csvfilename = csvfilename,
+        notes = 'imported from Concept2 log'
+    )
+
+    w.save()
+
+    # Check if workout has stroke data, and get the stroke data
+
+    result = add_stroke_data(user,c2id,w.id,startdatetime,csvfilename)
+
+    return w.id
 
 # convert datetime object to seconds
 def makeseconds(t):

rowers/dataprep.py

@@ -723,9 +723,35 @@ def create_row_df(r,distance,duration,startdatetime,
     return (id, message)
 
 
+def totaltime_sec_to_string(totaltime):
+    hours = int(totaltime / 3600.)
+    if hours > 23:
+        message = 'Warning: The workout duration was longer than 23 hours. '
+        hours = 23
+
+    minutes = int((totaltime - 3600. * hours) / 60.)
+    if minutes > 59:
+        minutes = 59
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    seconds = int(totaltime - 3600. * hours - 60. * minutes)
+    if seconds > 59:
+        seconds = 59
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    tenths = int(10 * (totaltime - 3600. * hours - 60. * minutes - seconds))
+    if tenths > 9:
+        tenths = 9
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    duration = "%s:%s:%s.%s" % (hours, minutes, seconds, tenths)
+
+    return duration
+    
 # Processes painsled CSV file to database
-
-
 def save_workout_database(f2, r, dosmooth=True, workouttype='rower',
                           dosummary=True, title='Workout',
                           workoutsource='unknown',
@@ -838,30 +864,6 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower',
     if np.isnan(totaltime):
         totaltime = 0
 
-    hours = int(totaltime / 3600.)
-    if hours > 23:
-        message = 'Warning: The workout duration was longer than 23 hours. '
-        hours = 23
-
-    minutes = int((totaltime - 3600. * hours) / 60.)
-    if minutes > 59:
-        minutes = 59
-        if not message:
-            message = 'Warning: there is something wrong with the workout duration'
-
-    seconds = int(totaltime - 3600. * hours - 60. * minutes)
-    if seconds > 59:
-        seconds = 59
-        if not message:
-            message = 'Warning: there is something wrong with the workout duration'
-
-    tenths = int(10 * (totaltime - 3600. * hours - 60. * minutes - seconds))
-    if tenths > 9:
-        tenths = 9
-        if not message:
-            message = 'Warning: there is something wrong with the workout duration'
-
-    duration = "%s:%s:%s.%s" % (hours, minutes, seconds, tenths)
 
     if dosummary:
         summary = row.allstats()
@@ -897,6 +899,8 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower',
     except KeyError:
         timezone_str = r.defaulttimezone
 
+    duration = totaltime_sec_to_string(totaltime)
+
     workoutdate = workoutstartdatetime.astimezone(
         pytz.timezone(timezone_str)
     ).strftime('%Y-%m-%d')

rowers/dataprepnodjango.py

@@ -1,10 +1,10 @@
 # This is Data prep used for testing purposes (no Django environment)
 # Uses the debug SQLite database for stroke data
 from rowingdata import rowingdata as rrdata
-
+from rowingdata import make_cumvalues
 from rowingdata import rower as rrower
 from rowingdata import main as rmain
-
+from time import strftime
 from pandas import DataFrame,Series
 
 import pandas as pd
@@ -131,6 +131,185 @@ def rdata(file,rower=rrower()):
 
     return res
 
+
+def totaltime_sec_to_string(totaltime):
+    hours = int(totaltime / 3600.)
+    if hours > 23:
+        message = 'Warning: The workout duration was longer than 23 hours. '
+        hours = 23
+
+    minutes = int((totaltime - 3600. * hours) / 60.)
+    if minutes > 59:
+        minutes = 59
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    seconds = int(totaltime - 3600. * hours - 60. * minutes)
+    if seconds > 59:
+        seconds = 59
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    tenths = int(10 * (totaltime - 3600. * hours - 60. * minutes - seconds))
+    if tenths > 9:
+        tenths = 9
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    duration = "%s:%s:%s.%s" % (hours, minutes, seconds, tenths)
+
+    return duration
+
+# Creates C2 stroke data
+def create_c2_stroke_data_db(
+        distance,duration,workouttype,
+        workoutid,starttimeunix,csvfilename,debug=False):
+
+    nr_strokes = int(distance/10.)
+
+
+    totalseconds = duration.hour*3600.
+    totalseconds += duration.minute*60.
+    totalseconds += duration.second
+    totalseconds += duration.microsecond/1.e6
+
+    
+    spm = 60.*nr_strokes/totalseconds
+    
+    step = totalseconds/float(nr_strokes)
+
+    elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1))
+
+    dstep = distance/float(nr_strokes)
+
+    d = np.arange(nr_strokes)*distance/(float(nr_strokes-1))
+
+    unixtime = starttimeunix + elapsed
+
+    pace = 500.*totalseconds/distance
+
+    if workouttype in ['rower','slides','dynamic']:
+        velo = distance/totalseconds
+        power = 2.8*velo**3
+    else:
+        power = 0
+
+        
+    df = pd.DataFrame({
+        'TimeStamp (sec)': unixtime,
+	' Horizontal (meters)': d,
+        ' Cadence (stokes/min)': spm,
+        ' Stroke500mPace (sec/500m)':pace,
+        ' ElapsedTime (sec)':elapsed,
+        ' Power (watts)':power,
+	' HRCur (bpm)':np.zeros(nr_strokes),
+	' longitude':np.zeros(nr_strokes),
+	' latitude':np.zeros(nr_strokes),
+	' DragFactor':np.zeros(nr_strokes),
+	' DriveLength (meters)':np.zeros(nr_strokes),
+	' StrokeDistance (meters)':np.zeros(nr_strokes),
+	' DriveTime (ms)':np.zeros(nr_strokes),
+	' StrokeRecoveryTime (ms)':np.zeros(nr_strokes),
+	' AverageDriveForce (lbs)':np.zeros(nr_strokes),
+	' PeakDriveForce (lbs)':np.zeros(nr_strokes),
+	' lapIdx':np.zeros(nr_strokes),
+        'cum_dist': d
+    })
+
+    timestr = strftime("%Y%m%d-%H%M%S")
+
+    df[' ElapsedTime (sec)'] = df['TimeStamp (sec)']
+
+    res =  df.to_csv(csvfilename,index_label='index',
+                     compression='gzip')
+
+    data = dataprep(df,id=workoutid,bands=False,debug=debug)
+
+    return data
+
+# Saves C2 stroke data to CSV and database
+def add_c2_stroke_data_db(strokedata,workoutid,starttimeunix,csvfilename,
+                          debug=False):
+
+    res = make_cumvalues(0.1*strokedata['t'])
+    cum_time = res[0]
+    lapidx = res[1]
+
+    unixtime = cum_time+starttimeunix
+    # unixtime[0] = starttimeunix
+    seconds = 0.1*strokedata.ix[:,'t']
+
+    nr_rows = len(unixtime)
+
+    try:
+	latcoord = strokedata.ix[:,'lat']
+	loncoord = strokedata.ix[:,'lon']
+    except:
+	latcoord = np.zeros(nr_rows)
+	loncoord = np.zeros(nr_rows)
+
+
+    try:
+	strokelength = strokedata.ix[:,'strokelength']
+    except:
+	strokelength = np.zeros(nr_rows)
+
+    dist2 = 0.1*strokedata.ix[:,'d']
+
+    try:
+        spm = strokedata.ix[:,'spm']
+    except KeyError:
+        spm = 0*dist2
+        
+    try:
+        hr = strokedata.ix[:,'hr']
+    except KeyError:
+        hr = 0*spm
+    pace = strokedata.ix[:,'p']/10.
+    pace = np.clip(pace,0,1e4)
+    pace = pace.replace(0,300)
+
+    velo = 500./pace
+
+    power = 2.8*velo**3
+    
+    # save csv
+    # Create data frame with all necessary data to write to csv
+    df = pd.DataFrame({'TimeStamp (sec)':unixtime,
+		       ' Horizontal (meters)': dist2,
+		       ' Cadence (stokes/min)':spm,
+		       ' HRCur (bpm)':hr,
+		       ' longitude':loncoord,
+		       ' latitude':latcoord,
+		       ' Stroke500mPace (sec/500m)':pace,
+		       ' Power (watts)':power,
+		       ' DragFactor':np.zeros(nr_rows),
+		       ' DriveLength (meters)':np.zeros(nr_rows),
+		       ' StrokeDistance (meters)':strokelength,
+		       ' DriveTime (ms)':np.zeros(nr_rows),
+		       ' StrokeRecoveryTime (ms)':np.zeros(nr_rows),
+		       ' AverageDriveForce (lbs)':np.zeros(nr_rows),
+		       ' PeakDriveForce (lbs)':np.zeros(nr_rows),
+		       ' lapIdx':lapidx,
+		       ' ElapsedTime (sec)':seconds,
+                       'cum_dist': dist2
+		       })
+
+    
+    df.sort_values(by='TimeStamp (sec)',ascending=True)
+	
+    timestr = strftime("%Y%m%d-%H%M%S")
+    
+
+    # Create CSV file name and save data to CSV file
+
+    res =  df.to_csv(csvfilename,index_label='index',
+                     compression='gzip')
+
+    data = dataprep(df,id=workoutid,bands=False,debug=debug)
+
+    return data
+
 # Processes painsled CSV file to database
 def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
                           dosummary=True,title='Workout',

rowers/tasks.py

@@ -13,6 +13,9 @@ import rowingdata
 from rowingdata import rowingdata as rdata
 
 from celery import app
+import datetime
+import pytz
+import iso8601
 
 from matplotlib.backends.backend_agg import FigureCanvas
 #from matplotlib.backends.backend_cairo import FigureCanvasCairo as FigureCanvas
@@ -32,7 +35,9 @@ from utils import deserialize_list
 from rowers.dataprepnodjango import (
     update_strokedata, new_workout_from_file,
     getsmallrowdata_db, updatecpdata_sql,
-    update_agegroup_db,fitnessmetric_to_sql
+    update_agegroup_db,fitnessmetric_to_sql,
+    add_c2_stroke_data_db,totaltime_sec_to_string,
+    create_c2_stroke_data_db
 )
 
 from django.core.mail import send_mail, EmailMessage
@@ -40,8 +45,9 @@ from django.db.utils import OperationalError
 
 import datautils
 import utils
-
+import requests
 import longtask
+import arrow
 
 # testing task
 
@@ -50,6 +56,70 @@ import longtask
 def add(x, y):
     return x + y
 
+
+@app.task
+def handle_c2_import_stroke_data(c2token,
+                                 c2id,workoutid,
+                                 starttimeunix,
+                                 csvfilename,debug=True):
+    authorizationstring = str('Bearer ' + c2token)
+    headers = {'Authorization': authorizationstring,
+	       'user-agent': 'sanderroosendaal',
+	       'Content-Type': 'application/json'}
+    url = "https://log.concept2.com/api/users/me/results/"+str(c2id)+"/strokes"
+    s = requests.get(url,headers=headers)
+    if s.status_code == 200:
+        strokedata = pd.DataFrame.from_dict(s.json()['data'])
+        result = add_c2_stroke_data_db(
+            strokedata,workoutid,starttimeunix,
+            csvfilename,debug=debug,
+        )
+
+        return 1
+    else:
+	url = "https://log.concept2.com/api/users/me/results/"+str(c2id)
+	s = requests.get(url,headers=headers)
+
+        if s.status_code == 200:
+            workoutdata = s.json()['data']
+            distance = workoutdata['distance']
+            c2id = workoutdata['id']
+            workouttype = workoutdata['type']
+            verified = workoutdata['verified']
+            startdatetime = iso8601.parse_date(workoutdata['date'])
+            weightclass = workoutdata['weight_class']
+            weightcategory = 'hwt'
+            if weightclass == "L":
+                weightcategory = 'lwt'
+            totaltime = workoutdata['time']/10.
+            duration = totaltime_sec_to_string(totaltime)
+            duration = datetime.datetime.strptime(duration,'%H:%M:%S.%f').time()
+
+            try:
+                timezone_str = tz(workoutdata['timezone'])
+            except:
+                timezone_str = 'UTC'
+
+            workoutdate = startdatetime.astimezone(
+                pytz.timezone(timezone_str)
+            ).strftime('%Y-%m-%d')
+            starttime = startdatetime.astimezone(
+                pytz.timezone(timezone_str)
+            ).strftime('%H:%M:%S')
+
+            result = create_c2_stroke_data_db(
+                distance,duration,workouttype,
+                workoutid,starttimeunix,
+                csvfilename,debug=debug,
+            )
+            
+            
+            return 1
+        
+        return 0
+
+    return 0
+
 def getagegrouprecord(age,sex='male',weightcategory='hwt',
                       distance=2000,duration=None,indf=pd.DataFrame()):
 

rowers/templates/c2_list_import2.html

@@ -46,11 +46,7 @@
     {% for workout in workouts %}
     <tr>
       <td>
-        {% if workout|lookup:'source' != 'Web' %}
         <a href="/rowers/workout/c2import/{{ workout|lookup:'id' }}/">Import</a></td>
-      {% else %}
-      &nbsp;
-      {% endif %}
       <td>{{ workout|lookup:'starttime' }}</td>
       <td>{{ workout|lookup:'duration' }}</td>
       <td>{{ workout|lookup:'distance' }}</td>
@@ -58,11 +54,8 @@
       <td>{{ workout|lookup:'source' }}</td>
       <td>{{ workout|lookup:'comment' }}</td>
       <td>
-        {% if workout|lookup:'source' != 'Web' %}
         {{ workout|lookup:'new' }}
-        {% else %}
+      </td>
-        &nbsp;
-      {% endif %}</td>
     </tr>
     
     {% endfor %}

rowers/views.py

@@ -3190,7 +3190,6 @@ def addmanual_view(request):
             )
 
 
-            print duration,'aap'
             id,message = dataprep.create_row_df(r,
                                                 distance,
                                                 duration,startdatetime,
@@ -9109,40 +9108,19 @@ def workout_getc2workout_all(request,page=1,message=""):
         messages.error(request,message)
     else:
         r = getrower(request.user)
-        c2ids = [item['id'] for item in res.json()['data'] if item['source'] != 'Web']
+        c2ids = [item['id'] for item in res.json()['data']]
+        alldata = {}
+        for item in res.json()['data']:
+            alldata[item['id']] = item
+        
         knownc2ids = uniqify([
             w.uploadedtoc2 for w in Workout.objects.filter(user=r)
         ])
         newids = [c2id for c2id in c2ids if not c2id in knownc2ids]
+        
         for c2id in newids:
-            res = c2stuff.get_c2_workout(request.user,c2id)
+            workoutid = c2stuff.create_async_workout(alldata,
-            if (res.status_code == 200):
+                                                     request.user,c2id)
-	        data = res.json()['data']
-                splitdata = None
-                if 'workout' in data:
-                    if 'splits' in data['workout']:
-                        splitdata = data['workout']['splits']
-                    if 'intervals' in data['workout']:
-                        splitdata = data['workout']['intervals']
-
-                # Check if workout has stroke data, and get the stroke data
-                if data['stroke_data']:
-	            res2 = c2stuff.get_c2_workout_strokes(request.user,c2id)
-                    # We have stroke data
-	            if res2.status_code == 200:
-	                strokedata = pd.DataFrame.from_dict(res2.json()['data'])
-                        # create the workout
-                        try:
-	                    id,message = add_workout_from_strokedata(
-                                request.user,c2id,data,strokedata,
-			        source='c2')
-	                    w = Workout.objects.get(id=id)
-	                    w.uploadedtoc2=c2id
-                            w.save()
-                            if message:
-                                messages.error(request,message)
-                        except KeyError:
-                            pass
 
     url = reverse(workouts_view)
     return HttpResponseRedirect(url)
@@ -9393,9 +9371,51 @@ def workout_getc2workout_view(request,c2id):
         if data['stroke_data']:
 	    res2 = c2stuff.get_c2_workout_strokes(request.user,c2id)
         else:
-            message = "This workout does not have any stroke data associated with it"
+            distance = data['distance']
-            messages.error(request,message)
+            c2id = data['id']
-            url = reverse(workout_c2import_view)
+            workouttype = data['type']
+            verified = data['verified']
+            startdatetime = iso8601.parse_date(data['date'])
+            weightclass = data['weight_class']
+            weightcategory = 'hwt'
+            if weightclass == "L":
+                weightcategory = 'lwt'
+            totaltime = data['time']/10.
+            duration = dataprep.totaltime_sec_to_string(totaltime)
+            duration = datetime.datetime.strptime(duration,'%H:%M:%S.%f').time()
+
+            try:
+                timezone_str = tz(data['timezone'])
+            except:
+                timezone_str = 'UTC'
+
+            workoutdate = startdatetime.astimezone(
+                pytz.timezone(timezone_str)
+            ).strftime('%Y-%m-%d')
+            starttime = startdatetime.astimezone(
+                pytz.timezone(timezone_str)
+            ).strftime('%H:%M:%S')
+
+            r = getrower(request.user)
+            
+            id, message = dataprep.create_row_df(r,
+                                                 distance,
+                                                 duration,
+                                                 startdatetime,
+                                                 title = 'Imported from C2',
+                                                 workouttype=workouttype)
+
+            w = Workout.objects.get(id=id)
+            w.uploadedtoc2 = c2id
+            w.save()
+
+            message = "This workout does not have any stroke data associated with it. We created synthetic stroke data."
+            messages.info(request,message)
+            url = reverse(r.defaultlandingpage,
+                          kwargs = {
+                              'id':int(id),
+                          })
+
 	    return HttpResponseRedirect(url)
 
         # We have stroke data