Merge branch 'feature/autostrava' into develop

rowers/management/commands/processemail.py

@@ -22,6 +22,7 @@ import rowers.uploads as uploads
 from rowers.mailprocessing import make_new_workout_from_email, send_confirm
 import rowers.polarstuff as polarstuff
 import rowers.c2stuff as c2stuff
+import rowers.stravastuff as stravastuff
 
 workoutmailbox = Mailbox.objects.get(name='workouts')
 failedmailbox = Mailbox.objects.get(name='Failed')
@@ -157,6 +158,11 @@ class Command(BaseCommand):
         rowers = Rower.objects.filter(c2_auto_import=True)
         for r in rowers:
             c2stuff.get_c2_workouts(r)
+
+        # Strava
+        rowers = Rower.objects.filter(strava_auto_import=True)
+        for r in rowers:
+            stravastuff.get_strava_workouts(r)
         
         messages = Message.objects.filter(mailbox_id = workoutmailbox.id)
         message_ids = [m.id for m in messages]

rowers/models.py

@@ -674,6 +674,7 @@ class Rower(models.Model):
                                       verbose_name="Export Workouts to Strava as")
 
     strava_auto_export = models.BooleanField(default=False)
+    strava_auto_import = models.BooleanField(default=False)
     runkeepertoken = models.CharField(default='',max_length=200,
                                       blank=True,null=True)
     runkeeper_auto_export = models.BooleanField(default=False)
@@ -2026,6 +2027,7 @@ class RowerImportExportForm(ModelForm):
             'runkeeper_auto_export',
             'sporttracks_auto_export',
             'strava_auto_export',
+            'strava_auto_import',
             'trainingpeaks_auto_export',
          ]
 

rowers/stravastuff.py

@@ -16,6 +16,8 @@ from math import sin,cos,atan2,sqrt
 import os,sys
 import gzip
 
+from pytz import timezone as tz,utc
+
 # Django
 from django.shortcuts import render_to_response
 from django.http import HttpResponseRedirect, HttpResponse,JsonResponse
@@ -24,6 +26,12 @@ from django.contrib.auth import authenticate, login, logout
 from django.contrib.auth.models import User
 from django.contrib.auth.decorators import login_required
 
+import django_rq
+queue = django_rq.get_queue('default')
+queuelow = django_rq.get_queue('low')
+queuehigh = django_rq.get_queue('low')
+
+
 # Project
 # from .models import Profile
 from rowingdata import rowingdata
@@ -32,9 +40,17 @@ from rowers.models import Rower,Workout
 from rowers.models import checkworkoutuser
 import dataprep
 from dataprep import columndict
-
+from utils import uniqify,isprorower,myqueue
+from uuid import uuid4
 import stravalib
 from stravalib.exc import ActivityUploadFailed,TimeoutExceeded
+import iso8601
+from iso8601 import ParseError
+
+import pytz
+import arrow
+
+from rowers.tasks import handle_strava_import_stroke_data
 
 from rowsandall_app.settings import C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET, STRAVA_CLIENT_ID, STRAVA_REDIRECT_URI, STRAVA_CLIENT_SECRET
 
@@ -43,28 +59,8 @@ try:
 except ImportError:
     JSONDecodeError = ValueError
 
-# Exponentially weighted moving average
-# Used for data smoothing of the jagged data obtained by Strava
-# See bitbucket issue 72
-def ewmovingaverage(interval,window_size):
-    # Experimental code using Exponential Weighted moving average
 
-    try:
+from utils import geo_distance,ewmovingaverage
-        intervaldf = pd.DataFrame({'v':interval})
-        idf_ewma1 = intervaldf.ewm(span=window_size)
-        idf_ewma2 = intervaldf[::-1].ewm(span=window_size)
-        
-        i_ewma1 = idf_ewma1.mean().ix[:,'v']
-        i_ewma2 = idf_ewma2.mean().ix[:,'v']
-
-        interval2 = np.vstack((i_ewma1,i_ewma2[::-1]))
-        interval2 = np.mean( interval2, axis=0) # average
-    except ValueError:
-        interval2 = interval
-
-    return interval2
-
-from utils import geo_distance
 
 
 # Custom exception handler, returns a 401 HTTP message
@@ -120,7 +116,7 @@ def get_token(code):
 def make_authorization_url(request):
     # Generate a random string for the state parameter
     # Save it for use later to prevent xsrf attacks
-    from uuid import uuid4
+
     state = str(uuid4())
 
     params = {"client_id": STRAVA_CLIENT_ID,
@@ -149,6 +145,137 @@ def get_strava_workout_list(user):
 
     return s
 
+def add_stroke_data(user,stravaid,workoutid,startdatetime,csvfilename):
+    r = Rower.objects.get(user=user)
+
+    starttimeunix = arrow.get(startdatetime).timestamp
+
+    
+    job = myqueue(queue,
+                  handle_strava_import_stroke_data,
+                  r.stravatoken,
+                  stravaid,
+                  workoutid,
+                  starttimeunix,
+                  csvfilename)
+
+# gets all new Strava workouts for a rower
+def get_strava_workouts(rower):
+
+    if not isprorower(rower):
+        return 0
+    
+    res = get_strava_workout_list(rower.user)
+
+    if (res.status_code != 200):
+        return 0
+    else:
+        stravaids = [int(item['id']) for item in res.json()]
+
+        alldata = {}
+        for item in res.json():
+            alldata[item['id']] = item
+        
+        knownstravaids = uniqify([
+            w.uploadedtostrava for w in Workout.objects.filter(user=rower)
+            ])
+        newids = [stravaid for stravaid in stravaids if not stravaid in knownstravaids]
+
+        for stravaid in newids:
+            workoutid = create_async_workout(alldata,rower.user,stravaid)
+
+    return 1
+
+def create_async_workout(alldata,user,stravaid):
+    data = alldata[stravaid]
+    r = Rower.objects.get(user=user)
+    distance = data['distance']
+    stravaid = data['id']
+    try:
+        workouttype = data['type']
+    except:
+        workouttype = 'rower'
+
+    if workouttype not in [x[0] for x in Workout.workouttypes]:
+        workouttype = 'other'
+
+    try:
+        comments = data['comments']
+    except:
+        comments = ' '
+
+    try:
+        thetimezone = tz(data['timezone'])
+    except:
+        thetimezone = 'UTC'
+
+    try:
+	rowdatetime = iso8601.parse_date(data['date_utc'])
+    except KeyError:
+	rowdatetime = iso8601.parse_date(data['start_date'])
+    except ParseError:
+	rowdatetime = iso8601.parse_date(data['date'])
+
+    try:
+        c2intervaltype = data['workout_type']
+	
+    except KeyError:
+	c2intervaltype = ''
+	
+    try:
+	title = data['name']
+    except KeyError:
+	title = ""
+	try:
+	    t = data['comments'].split('\n', 1)[0]
+	    title += t[:20]
+	except:
+	    title = 'Imported'
+
+    workoutdate = rowdatetime.astimezone(
+        pytz.timezone(thetimezone)
+    ).strftime('%Y-%m-%d')
+            
+    starttime = rowdatetime.astimezone(
+        pytz.timezone(thetimezone)
+    ).strftime('%H:%m:%S')
+            
+    totaltime = data['elapsed_time']
+    duration = dataprep.totaltime_sec_to_string(totaltime)
+
+    weightcategory = 'hwt'
+            
+    # Create CSV file name and save data to CSV file
+    csvfilename ='media/{code}_{importid}.csv'.format(
+        importid=stravaid,
+        code = uuid4().hex[:16]
+        )
+    
+    w = Workout(
+        user=r,
+        workouttype = workouttype,
+        name = title,
+        date = workoutdate,
+        starttime = starttime,
+        startdatetime = rowdatetime,
+        timezone = thetimezone,
+        duration = duration,
+        distance=distance,
+        weightcategory = weightcategory,
+        uploadedtostrava = stravaid,
+        csvfilename = csvfilename,
+        notes = ''
+    )
+
+    w.save()
+
+    # Check if workout has stroke data, and get the stroke data
+
+    result = add_stroke_data(user,stravaid,w.id,rowdatetime,csvfilename)
+
+    return w.id
+
+
 # Get a Strava workout summary data and stroke data by ID
 def get_strava_workout(user,stravaid):
     r = Rower.objects.get(user=user)

rowers/tasks.py

@@ -8,6 +8,7 @@ import numpy as np
 import re
 
 from scipy import optimize
+from scipy.signal import savgol_filter
 
 import rowingdata
 
@@ -20,6 +21,7 @@ import datetime
 import pytz
 import iso8601
 
+
 from matplotlib.backends.backend_agg import FigureCanvas
 #from matplotlib.backends.backend_cairo import FigureCanvasCairo as FigureCanvas
 import matplotlib.pyplot as plt
@@ -37,7 +39,7 @@ from django_rq import job
 from django.utils import timezone
 from django.utils.html import strip_tags
 
-from utils import deserialize_list
+from utils import deserialize_list,ewmovingaverage
 
 from rowers.dataprepnodjango import (
     update_strokedata, new_workout_from_file,
@@ -45,7 +47,7 @@ from rowers.dataprepnodjango import (
     update_agegroup_db,fitnessmetric_to_sql,
     add_c2_stroke_data_db,totaltime_sec_to_string,
     create_c2_stroke_data_db,update_empower,
-    database_url_debug,database_url,
+    database_url_debug,database_url,dataprep
 )
 
 
@@ -77,6 +79,142 @@ def add(x, y):
     return x + y
 
 
+@app.task
+def handle_strava_import_stroke_data(stravatoken,
+                                     stravaid,workoutid,
+                                     starttimeunix,
+                                     csvfilename,debug=True,**kwargs):
+    # ready to fetch. Hurray
+    fetchresolution = 'high'
+    series_type = 'time'
+    authorizationstring = str('Bearer ' + stravatoken)
+    headers = {'Authorization': authorizationstring,
+	       'user-agent': 'sanderroosendaal',
+	       'Content-Type': 'application/json',
+	       'resolution': 'medium',}
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)
+    workoutsummary = requests.get(url,headers=headers).json()
+
+    workoutsummary['timezone'] = "Etc/UTC"
+    startdatetime = workoutsummary['start_date']
+
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/cadence?resolution="+fetchresolution+"&series_type="+series_type
+    spmjson = requests.get(url,headers=headers)
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/heartrate?resolution="+fetchresolution+"&series_type="+series_type
+    hrjson = requests.get(url,headers=headers)
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/time?resolution="+fetchresolution+"&series_type="+series_type
+    timejson = requests.get(url,headers=headers)
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/velocity_smooth?resolution="+fetchresolution+"&series_type="+series_type
+    velojson = requests.get(url,headers=headers)
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/distance?resolution="+fetchresolution+"&series_type="+series_type
+    distancejson = requests.get(url,headers=headers)
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/latlng?resolution="+fetchresolution+"&series_type="+series_type
+    latlongjson = requests.get(url,headers=headers)
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/watts?resolution="+fetchresolution+"&series_type="+series_type
+    wattsjson = requests.get(url,headers=headers)
+
+    try:
+	t = np.array(timejson.json()[0]['data'])
+	nr_rows = len(t)
+	d =  np.array(distancejson.json()[1]['data'])
+        if nr_rows == 0:
+            return 0
+    except IndexError:
+        d = 0*t
+        # return (0,"Error: No Distance information in the Strava data")
+    except KeyError:
+        return 0
+
+    try:
+	spm = np.array(spmjson.json()[1]['data'])
+    except:
+	spm = np.zeros(nr_rows)
+
+    try:
+        watts = np.array(wattsjson.json()[1]['data'])
+    except:
+        watts = np.zeros(nr_rows)
+
+    try:
+	hr = np.array(hrjson.json()[1]['data'])
+    except IndexError:
+	hr = np.zeros(nr_rows)
+    except KeyError:
+	hr = np.zeros(nr_rows)
+
+    try:
+	velo = np.array(velojson.json()[1]['data'])
+    except IndexError:
+	velo = np.zeros(nr_rows)
+    except KeyError:
+        velo = np.zeros(nr_rows)
+
+    f = np.diff(t).mean()
+    if f != 0:
+        windowsize = 2*(int(10./(f)))+1
+    else:
+        windowsize = 1
+
+    if windowsize > 3 and windowsize < len(velo):
+        velo2 = savgol_filter(velo,windowsize,3)
+    else:
+        velo2 = velo
+        
+    coords = np.array(latlongjson.json()[0]['data'])
+    try:
+	lat = coords[:,0]
+	lon = coords[:,1]
+    except IndexError:
+	lat = np.zeros(len(t))
+	lon = np.zeros(len(t))
+    except KeyError:
+	lat = np.zeros(len(t))
+	lon = np.zeros(len(t))
+
+    strokelength = velo*60./(spm)
+    strokelength[np.isinf(strokelength)] = 0.0
+
+    pace = 500./(1.0*velo2)
+    pace[np.isinf(pace)] = 0.0
+
+    unixtime = starttimeunix+t
+
+    strokedistance = 60.*velo2/spm
+    
+    nr_strokes = len(t)
+
+    df = pd.DataFrame({'TimeStamp (sec)':unixtime,
+                       ' ElapsedTime (sec)':t,
+		       ' Horizontal (meters)':d,
+		       ' Stroke500mPace (sec/500m)':pace,
+		       ' Cadence (stokes/min)':spm,
+		       ' HRCur (bpm)':hr,
+		       ' latitude':lat,
+		       ' longitude':lon,
+		       ' StrokeDistance (meters)':strokelength,
+                       'cum_dist':d,
+	               ' DragFactor':np.zeros(nr_strokes),
+	               ' DriveLength (meters)':np.zeros(nr_strokes),
+	               ' StrokeDistance (meters)':strokedistance,
+	               ' DriveTime (ms)':np.zeros(nr_strokes),
+	               ' StrokeRecoveryTime (ms)':np.zeros(nr_strokes),
+	               ' AverageDriveForce (lbs)':np.zeros(nr_strokes),
+	               ' PeakDriveForce (lbs)':np.zeros(nr_strokes),
+	               ' lapIdx':np.zeros(nr_strokes),
+                       ' Power (watts)':watts,
+    })
+
+
+    df.sort_values(by='TimeStamp (sec)',ascending=True)
+
+    res = df.to_csv(csvfilename+'.gz',index_label='index',compression='gzip')
+
+    data = dataprep(df,id=workoutid,bands=False,debug=debug)
+	# startdatetime = datetime.datetime.strptime(startdatetime,"%Y-%m-%d-%H:%M:%S")
+
+    return 1
+    
+
 @app.task
 def handle_c2_import_stroke_data(c2token,
                                  c2id,workoutid,

rowers/utils.py

@@ -378,3 +378,23 @@ def isprorower(r):
 
     return result
                                                                                      
+# Exponentially weighted moving average
+# Used for data smoothing of the jagged data obtained by Strava
+# See bitbucket issue 72
+def ewmovingaverage(interval,window_size):
+    # Experimental code using Exponential Weighted moving average
+
+    try:
+        intervaldf = pd.DataFrame({'v':interval})
+        idf_ewma1 = intervaldf.ewm(span=window_size)
+        idf_ewma2 = intervaldf[::-1].ewm(span=window_size)
+        
+        i_ewma1 = idf_ewma1.mean().ix[:,'v']
+        i_ewma2 = idf_ewma2.mean().ix[:,'v']
+
+        interval2 = np.vstack((i_ewma1,i_ewma2[::-1]))
+        interval2 = np.mean( interval2, axis=0) # average
+    except ValueError:
+        interval2 = interval
+
+    return interval2