diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index e8f7f8d8..5183af9c 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -22,6 +22,7 @@ import rowers.uploads as uploads from rowers.mailprocessing import make_new_workout_from_email, send_confirm import rowers.polarstuff as polarstuff import rowers.c2stuff as c2stuff +import rowers.stravastuff as stravastuff workoutmailbox = Mailbox.objects.get(name='workouts') failedmailbox = Mailbox.objects.get(name='Failed') @@ -157,6 +158,11 @@ class Command(BaseCommand): rowers = Rower.objects.filter(c2_auto_import=True) for r in rowers: c2stuff.get_c2_workouts(r) + + # Strava + rowers = Rower.objects.filter(strava_auto_import=True) + for r in rowers: + stravastuff.get_strava_workouts(r) messages = Message.objects.filter(mailbox_id = workoutmailbox.id) message_ids = [m.id for m in messages] diff --git a/rowers/models.py b/rowers/models.py index b237088a..b6146910 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -674,6 +674,7 @@ class Rower(models.Model): verbose_name="Export Workouts to Strava as") strava_auto_export = models.BooleanField(default=False) + strava_auto_import = models.BooleanField(default=False) runkeepertoken = models.CharField(default='',max_length=200, blank=True,null=True) runkeeper_auto_export = models.BooleanField(default=False) @@ -2026,6 +2027,7 @@ class RowerImportExportForm(ModelForm): 'runkeeper_auto_export', 'sporttracks_auto_export', 'strava_auto_export', + 'strava_auto_import', 'trainingpeaks_auto_export', ] diff --git a/rowers/stravastuff.py b/rowers/stravastuff.py index c502b2f0..b1ee7d3c 100644 --- a/rowers/stravastuff.py +++ b/rowers/stravastuff.py @@ -16,6 +16,8 @@ from math import sin,cos,atan2,sqrt import os,sys import gzip +from pytz import timezone as tz,utc + # Django from django.shortcuts import render_to_response from django.http import HttpResponseRedirect, HttpResponse,JsonResponse @@ -24,6 +26,12 @@ from django.contrib.auth import authenticate, login, logout from django.contrib.auth.models import User from django.contrib.auth.decorators import login_required +import django_rq +queue = django_rq.get_queue('default') +queuelow = django_rq.get_queue('low') +queuehigh = django_rq.get_queue('low') + + # Project # from .models import Profile from rowingdata import rowingdata @@ -32,9 +40,17 @@ from rowers.models import Rower,Workout from rowers.models import checkworkoutuser import dataprep from dataprep import columndict - +from utils import uniqify,isprorower,myqueue +from uuid import uuid4 import stravalib from stravalib.exc import ActivityUploadFailed,TimeoutExceeded +import iso8601 +from iso8601 import ParseError + +import pytz +import arrow + +from rowers.tasks import handle_strava_import_stroke_data from rowsandall_app.settings import C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET, STRAVA_CLIENT_ID, STRAVA_REDIRECT_URI, STRAVA_CLIENT_SECRET @@ -43,28 +59,8 @@ try: except ImportError: JSONDecodeError = ValueError -# Exponentially weighted moving average -# Used for data smoothing of the jagged data obtained by Strava -# See bitbucket issue 72 -def ewmovingaverage(interval,window_size): - # Experimental code using Exponential Weighted moving average - try: - intervaldf = pd.DataFrame({'v':interval}) - idf_ewma1 = intervaldf.ewm(span=window_size) - idf_ewma2 = intervaldf[::-1].ewm(span=window_size) - - i_ewma1 = idf_ewma1.mean().ix[:,'v'] - i_ewma2 = idf_ewma2.mean().ix[:,'v'] - - interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) - interval2 = np.mean( interval2, axis=0) # average - except ValueError: - interval2 = interval - - return interval2 - -from utils import geo_distance +from utils import geo_distance,ewmovingaverage # Custom exception handler, returns a 401 HTTP message @@ -120,7 +116,7 @@ def get_token(code): def make_authorization_url(request): # Generate a random string for the state parameter # Save it for use later to prevent xsrf attacks - from uuid import uuid4 + state = str(uuid4()) params = {"client_id": STRAVA_CLIENT_ID, @@ -149,6 +145,137 @@ def get_strava_workout_list(user): return s +def add_stroke_data(user,stravaid,workoutid,startdatetime,csvfilename): + r = Rower.objects.get(user=user) + + starttimeunix = arrow.get(startdatetime).timestamp + + + job = myqueue(queue, + handle_strava_import_stroke_data, + r.stravatoken, + stravaid, + workoutid, + starttimeunix, + csvfilename) + +# gets all new Strava workouts for a rower +def get_strava_workouts(rower): + + if not isprorower(rower): + return 0 + + res = get_strava_workout_list(rower.user) + + if (res.status_code != 200): + return 0 + else: + stravaids = [int(item['id']) for item in res.json()] + + alldata = {} + for item in res.json(): + alldata[item['id']] = item + + knownstravaids = uniqify([ + w.uploadedtostrava for w in Workout.objects.filter(user=rower) + ]) + newids = [stravaid for stravaid in stravaids if not stravaid in knownstravaids] + + for stravaid in newids: + workoutid = create_async_workout(alldata,rower.user,stravaid) + + return 1 + +def create_async_workout(alldata,user,stravaid): + data = alldata[stravaid] + r = Rower.objects.get(user=user) + distance = data['distance'] + stravaid = data['id'] + try: + workouttype = data['type'] + except: + workouttype = 'rower' + + if workouttype not in [x[0] for x in Workout.workouttypes]: + workouttype = 'other' + + try: + comments = data['comments'] + except: + comments = ' ' + + try: + thetimezone = tz(data['timezone']) + except: + thetimezone = 'UTC' + + try: + rowdatetime = iso8601.parse_date(data['date_utc']) + except KeyError: + rowdatetime = iso8601.parse_date(data['start_date']) + except ParseError: + rowdatetime = iso8601.parse_date(data['date']) + + try: + c2intervaltype = data['workout_type'] + + except KeyError: + c2intervaltype = '' + + try: + title = data['name'] + except KeyError: + title = "" + try: + t = data['comments'].split('\n', 1)[0] + title += t[:20] + except: + title = 'Imported' + + workoutdate = rowdatetime.astimezone( + pytz.timezone(thetimezone) + ).strftime('%Y-%m-%d') + + starttime = rowdatetime.astimezone( + pytz.timezone(thetimezone) + ).strftime('%H:%m:%S') + + totaltime = data['elapsed_time'] + duration = dataprep.totaltime_sec_to_string(totaltime) + + weightcategory = 'hwt' + + # Create CSV file name and save data to CSV file + csvfilename ='media/{code}_{importid}.csv'.format( + importid=stravaid, + code = uuid4().hex[:16] + ) + + w = Workout( + user=r, + workouttype = workouttype, + name = title, + date = workoutdate, + starttime = starttime, + startdatetime = rowdatetime, + timezone = thetimezone, + duration = duration, + distance=distance, + weightcategory = weightcategory, + uploadedtostrava = stravaid, + csvfilename = csvfilename, + notes = '' + ) + + w.save() + + # Check if workout has stroke data, and get the stroke data + + result = add_stroke_data(user,stravaid,w.id,rowdatetime,csvfilename) + + return w.id + + # Get a Strava workout summary data and stroke data by ID def get_strava_workout(user,stravaid): r = Rower.objects.get(user=user) diff --git a/rowers/tasks.py b/rowers/tasks.py index b6fe15e2..1cad32e8 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -8,6 +8,7 @@ import numpy as np import re from scipy import optimize +from scipy.signal import savgol_filter import rowingdata @@ -20,6 +21,7 @@ import datetime import pytz import iso8601 + from matplotlib.backends.backend_agg import FigureCanvas #from matplotlib.backends.backend_cairo import FigureCanvasCairo as FigureCanvas import matplotlib.pyplot as plt @@ -37,7 +39,7 @@ from django_rq import job from django.utils import timezone from django.utils.html import strip_tags -from utils import deserialize_list +from utils import deserialize_list,ewmovingaverage from rowers.dataprepnodjango import ( update_strokedata, new_workout_from_file, @@ -45,7 +47,7 @@ from rowers.dataprepnodjango import ( update_agegroup_db,fitnessmetric_to_sql, add_c2_stroke_data_db,totaltime_sec_to_string, create_c2_stroke_data_db,update_empower, - database_url_debug,database_url, + database_url_debug,database_url,dataprep ) @@ -77,6 +79,142 @@ def add(x, y): return x + y +@app.task +def handle_strava_import_stroke_data(stravatoken, + stravaid,workoutid, + starttimeunix, + csvfilename,debug=True,**kwargs): + # ready to fetch. Hurray + fetchresolution = 'high' + series_type = 'time' + authorizationstring = str('Bearer ' + stravatoken) + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json', + 'resolution': 'medium',} + url = "https://www.strava.com/api/v3/activities/"+str(stravaid) + workoutsummary = requests.get(url,headers=headers).json() + + workoutsummary['timezone'] = "Etc/UTC" + startdatetime = workoutsummary['start_date'] + + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/cadence?resolution="+fetchresolution+"&series_type="+series_type + spmjson = requests.get(url,headers=headers) + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/heartrate?resolution="+fetchresolution+"&series_type="+series_type + hrjson = requests.get(url,headers=headers) + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/time?resolution="+fetchresolution+"&series_type="+series_type + timejson = requests.get(url,headers=headers) + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/velocity_smooth?resolution="+fetchresolution+"&series_type="+series_type + velojson = requests.get(url,headers=headers) + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/distance?resolution="+fetchresolution+"&series_type="+series_type + distancejson = requests.get(url,headers=headers) + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/latlng?resolution="+fetchresolution+"&series_type="+series_type + latlongjson = requests.get(url,headers=headers) + url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/watts?resolution="+fetchresolution+"&series_type="+series_type + wattsjson = requests.get(url,headers=headers) + + try: + t = np.array(timejson.json()[0]['data']) + nr_rows = len(t) + d = np.array(distancejson.json()[1]['data']) + if nr_rows == 0: + return 0 + except IndexError: + d = 0*t + # return (0,"Error: No Distance information in the Strava data") + except KeyError: + return 0 + + try: + spm = np.array(spmjson.json()[1]['data']) + except: + spm = np.zeros(nr_rows) + + try: + watts = np.array(wattsjson.json()[1]['data']) + except: + watts = np.zeros(nr_rows) + + try: + hr = np.array(hrjson.json()[1]['data']) + except IndexError: + hr = np.zeros(nr_rows) + except KeyError: + hr = np.zeros(nr_rows) + + try: + velo = np.array(velojson.json()[1]['data']) + except IndexError: + velo = np.zeros(nr_rows) + except KeyError: + velo = np.zeros(nr_rows) + + f = np.diff(t).mean() + if f != 0: + windowsize = 2*(int(10./(f)))+1 + else: + windowsize = 1 + + if windowsize > 3 and windowsize < len(velo): + velo2 = savgol_filter(velo,windowsize,3) + else: + velo2 = velo + + coords = np.array(latlongjson.json()[0]['data']) + try: + lat = coords[:,0] + lon = coords[:,1] + except IndexError: + lat = np.zeros(len(t)) + lon = np.zeros(len(t)) + except KeyError: + lat = np.zeros(len(t)) + lon = np.zeros(len(t)) + + strokelength = velo*60./(spm) + strokelength[np.isinf(strokelength)] = 0.0 + + pace = 500./(1.0*velo2) + pace[np.isinf(pace)] = 0.0 + + unixtime = starttimeunix+t + + strokedistance = 60.*velo2/spm + + nr_strokes = len(t) + + df = pd.DataFrame({'TimeStamp (sec)':unixtime, + ' ElapsedTime (sec)':t, + ' Horizontal (meters)':d, + ' Stroke500mPace (sec/500m)':pace, + ' Cadence (stokes/min)':spm, + ' HRCur (bpm)':hr, + ' latitude':lat, + ' longitude':lon, + ' StrokeDistance (meters)':strokelength, + 'cum_dist':d, + ' DragFactor':np.zeros(nr_strokes), + ' DriveLength (meters)':np.zeros(nr_strokes), + ' StrokeDistance (meters)':strokedistance, + ' DriveTime (ms)':np.zeros(nr_strokes), + ' StrokeRecoveryTime (ms)':np.zeros(nr_strokes), + ' AverageDriveForce (lbs)':np.zeros(nr_strokes), + ' PeakDriveForce (lbs)':np.zeros(nr_strokes), + ' lapIdx':np.zeros(nr_strokes), + ' Power (watts)':watts, + }) + + + df.sort_values(by='TimeStamp (sec)',ascending=True) + + res = df.to_csv(csvfilename+'.gz',index_label='index',compression='gzip') + + data = dataprep(df,id=workoutid,bands=False,debug=debug) + # startdatetime = datetime.datetime.strptime(startdatetime,"%Y-%m-%d-%H:%M:%S") + + return 1 + + @app.task def handle_c2_import_stroke_data(c2token, c2id,workoutid, diff --git a/rowers/utils.py b/rowers/utils.py index 635d255a..b51948ef 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -378,3 +378,23 @@ def isprorower(r): return result +# Exponentially weighted moving average +# Used for data smoothing of the jagged data obtained by Strava +# See bitbucket issue 72 +def ewmovingaverage(interval,window_size): + # Experimental code using Exponential Weighted moving average + + try: + intervaldf = pd.DataFrame({'v':interval}) + idf_ewma1 = intervaldf.ewm(span=window_size) + idf_ewma2 = intervaldf[::-1].ewm(span=window_size) + + i_ewma1 = idf_ewma1.mean().ix[:,'v'] + i_ewma2 = idf_ewma2.mean().ix[:,'v'] + + interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) + interval2 = np.mean( interval2, axis=0) # average + except ValueError: + interval2 = interval + + return interval2