From 3564a89e02757dd517091838b26e2e16665a67e9 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 12 Jul 2020 12:54:50 +0200 Subject: [PATCH 1/2] First attempt at processing strava webhook create --- rowers/dataprepnodjango.py | 2 +- rowers/models.py | 4 +- rowers/stravastuff.py | 13 +- rowers/tasks.py | 233 ++++++++++++++++++++++++++++++++++++ rowers/utils.py | 5 +- rowers/views/importviews.py | 34 ++++++ 6 files changed, 284 insertions(+), 7 deletions(-) diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 8df22bd4..ec4972ee 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -754,7 +754,7 @@ def update_workout_field_sql(workoutid,fieldname,value,debug=False): table = 'rowers_workout' - query = "UPDATE %s SET %s = %s WHERE `id` = %s;" % (table,fieldname,value,workoutid) + query = "UPDATE %s SET %s = '%s' WHERE `id` = %s;" % (table,fieldname,value,workoutid) with engine.connect() as conn, conn.begin(): diff --git a/rowers/models.py b/rowers/models.py index b676f4a7..a205bac4 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -2762,7 +2762,7 @@ class Workout(models.Model): plannedsession = models.ForeignKey(PlannedSession, blank=True,null=True, verbose_name='Session',on_delete=models.SET_NULL) name = models.CharField(max_length=150,blank=True,null=True) - date = models.DateField() + date = models.DateField(default=timezone.now) workouttype = models.CharField(choices=workouttypes,max_length=50, verbose_name='Exercise/Boat Class') workoutsource = models.CharField(max_length=100, @@ -2779,7 +2779,7 @@ class Workout(models.Model): choices=timezones, max_length=100) distance = models.IntegerField(default=0,blank=True) - duration = models.TimeField(default=1,blank=True) + duration = models.TimeField(default=datetime.time(0,0),blank=True) dragfactor = models.IntegerField(default=0,blank=True) trimp = models.IntegerField(default=-1,blank=True) rscore = models.IntegerField(default=-1,blank=True) diff --git a/rowers/stravastuff.py b/rowers/stravastuff.py index e46649d5..2a2c1c9c 100644 --- a/rowers/stravastuff.py +++ b/rowers/stravastuff.py @@ -29,7 +29,7 @@ from rowers.utils import myqueue import rowers.mytypes as mytypes import gzip -from rowers.tasks import handle_strava_sync +from rowers.tasks import handle_strava_sync,fetch_strava_workout from rowsandall_app.settings import ( C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET, @@ -325,7 +325,16 @@ def create_async_workout(alldata,user,stravaid,debug=False): from rowers.utils import get_strava_stream - +def async_get_workout(user,stravaid,workout): + job = myqueue(queue, + fetch_strava_workout, + user.rower.stravatoken, + oauth_data, + workout.id, + stravaid, + workout.csvfilename, + ) + return job # Get a Strava workout summary data and stroke data by ID def get_workout(user,stravaid): diff --git a/rowers/tasks.py b/rowers/tasks.py index 8d12d008..ff3ec212 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -18,6 +18,7 @@ from scipy import optimize from scipy.signal import savgol_filter import rowingdata +from rowingdata import make_cumvalues from uuid import uuid4 from rowingdata import rowingdata as rdata from datetime import timedelta @@ -30,6 +31,7 @@ from celery import shared_task import datetime import pytz import iso8601 +from iso8601 import ParseError from json.decoder import JSONDecodeError @@ -61,6 +63,7 @@ from django.utils.html import strip_tags from rowers.utils import deserialize_list,ewmovingaverage,wavg from rowers.emails import htmlstrip +from rowers import mytypes #from HTMLParser import HTMLParser from html.parser import HTMLParser @@ -2708,3 +2711,233 @@ def handle_sendemail_invite_accept(email, name, teamname, managername, # Another simple task for debugging purposes def add2(x, y,debug=False,**kwargs): return x + y + +@app.task +def fetch_strava_workout(stravatoken,oauth_data,workoutid,stravaid,csvfilename,debug=False,**kwargs): + fetchresolution = 'high' + authorizationstring = str('Bearer '+stravatoken) + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json', + 'resolution': 'medium',} + url = "https://www.strava.com/api/v3/activities/"+str(stravaid) + workoutsummary = requests.get(url,headers=headers).json() + try: + startdatetime = workoutsummary['start_date'] + except KeyError: + startdatetime = timezone.now() + + spm = get_strava_stream(None,'cadence',stravaid,authorizationstring=authorizationstring) + hr = get_strava_stream(None,'heartrate',stravaid,authorizationstring=authorizationstring) + t = get_strava_stream(None,'time',stravaid,authorizationstring=authorizationstring) + velo = get_strava_stream(None,'velocity_smooth',stravaid,authorizationstring=authorizationstring) + d = get_strava_stream(None,'distance',stravaid,authorizationstring=authorizationstring) + coords = get_strava_stream(None,'latlng',stravaid,authorizationstring=authorizationstring) + power = get_strava_stream(None,'watts',stravaid,authorizationstring=authorizationstring) + + if t is not None: + nr_rows = len(t) + else: + duration = int(workoutsummary['elapsed_time']) + t = pd.Series(range(duration+1)) + + nr_rows = len(t) + + + if nr_rows == 0: + return 0 + + if d is None: + d = 0*t + + if spm is None: + spm = np.zeros(nr_rows) + + if power is None: + power = np.zeros(nr_rows) + + if hr is None: + hr = np.zeros(nr_rows) + + if velo is None: + velo = np.zeros(nr_rows) + + dt = np.diff(t).mean() + wsize = round(5./dt) + + velo2 = ewmovingaverage(velo,wsize) + + if coords is not None: + try: + lat = coords[:,0] + lon = coords[:,1] + except IndexError: + lat = np.zeros(len(t)) + lon = np.zeros(len(t)) + else: + lat = np.zeros(len(t)) + lon = np.zeros(len(t)) + + + + + strokelength = velo*60./(spm) + strokelength[np.isinf(strokelength)] = 0.0 + + + pace = 500./(1.0*velo2) + pace[np.isinf(pace)] = 0.0 + + strokedata = pd.DataFrame({'t':10*t, + 'd':10*d, + 'p':10*pace, + 'spm':spm, + 'hr':hr, + 'lat':lat, + 'lon':lon, + 'power':power, + 'strokelength':strokelength, + }) + + try: + workouttype = mytypes.stravamappinginv[workoutsummary['type']] + except KeyError: + workouttype = 'other' + + if workouttype.lower() == 'rowing': + workouttype = 'rower' + + if 'summary_polyline' in workoutsummary['map'] and workouttype=='rower': + workouttype = 'water' + + try: + comments = workoutsummary['comments'] + except: + comments = ' ' + + try: + thetimezone = tz(workoutsummary['timezone']) + except: + thetimezone = 'UTC' + + try: + rowdatetime = iso8601.parse_date(workoutsummary['date_utc']) + except KeyError: + rowdatetime = iso8601.parse_date(workoutsummary['start_date']) + except ParseError: + rowdatetime = iso8601.parse_date(workoutsummary['date']) + + + try: + intervaltype = workoutsummary['workout_type'] + + except KeyError: + intervaltype = '' + + try: + title = workoutsummary['name'] + except KeyError: + title = "" + try: + t = data['comments'].split('\n', 1)[0] + title += t[:20] + except: + title = 'Imported' + + starttimeunix = arrow.get(rowdatetime).timestamp + + res = make_cumvalues(0.1*strokedata['t']) + cum_time = res[0] + lapidx = res[1] + + unixtime = cum_time+starttimeunix + seconds = 0.1*strokedata.loc[:,'t'] + + nr_rows = len(unixtime) + + try: + latcoord = strokedata.loc[:,'lat'] + loncoord = strokedata.loc[:,'lon'] + if latcoord.std() == 0 and loncoord.std() == 0 and workouttype == 'water': + workouttype = 'rower' + except: + latcoord = np.zeros(nr_rows) + loncoord = np.zeros(nr_rows) + if workouttype == 'water': + workouttype = 'rower' + + + + try: + strokelength = strokedata.loc[:,'strokelength'] + except: + strokelength = np.zeros(nr_rows) + + dist2 = 0.1*strokedata.loc[:,'d'] + + try: + spm = strokedata.loc[:,'spm'] + except KeyError: + spm = 0*dist2 + + try: + hr = strokedata.loc[:,'hr'] + except KeyError: + hr = 0*spm + pace = strokedata.loc[:,'p']/10. + pace = np.clip(pace,0,1e4) + pace = pace.replace(0,300) + + velo = 500./pace + + try: + power = strokedata.loc[:,'power'] + except KeyError: + power = 2.8*velo**3 + + #if power.std() == 0 and power.mean() == 0: + # power = 2.8*velo**3 + + # save csv + # Create data frame with all necessary data to write to csv + df = pd.DataFrame({'TimeStamp (sec)':unixtime, + ' Horizontal (meters)': dist2, + ' Cadence (stokes/min)':spm, + ' HRCur (bpm)':hr, + ' longitude':loncoord, + ' latitude':latcoord, + ' Stroke500mPace (sec/500m)':pace, + ' Power (watts)':power, + ' DragFactor':np.zeros(nr_rows), + ' DriveLength (meters)':np.zeros(nr_rows), + ' StrokeDistance (meters)':strokelength, + ' DriveTime (ms)':np.zeros(nr_rows), + ' StrokeRecoveryTime (ms)':np.zeros(nr_rows), + ' AverageDriveForce (lbs)':np.zeros(nr_rows), + ' PeakDriveForce (lbs)':np.zeros(nr_rows), + ' lapIdx':lapidx, + ' ElapsedTime (sec)':seconds, + 'cum_dist':dist2, + }) + + + + df.sort_values(by='TimeStamp (sec)',ascending=True) + + row = rowingdata.rowingdata(df=df) + row.write_csv(csvfilename,gzip=True) + summary = row.allstats() + maxdist = df['cum_dist'].max() + duration = row.duration + + + update_strokedata(workoutid,row.df,debug=debug) + res = update_workout_field_sql(workoutid,'workouttype',workouttype,debug=debug) + res = update_workout_field_sql(workoutid,'name',title,debug=debug) + res = update_workout_field_sql(workoutid,'notes',comments,debug=debug) + res = update_workout_field_sql(workoutid,'summary',summary,debug=debug) + res = update_workout_field_sql(workoutid,'distance',maxdist,debug=debug) + res = update_workout_field_sql(workoutid,'duration',duration,debug=debug) + + + return 1 diff --git a/rowers/utils.py b/rowers/utils.py index f019fbe9..fdef303a 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -467,8 +467,9 @@ def custom_exception_handler(exc,message): return res -def get_strava_stream(r,metric,stravaid,series_type='time',fetchresolution='high'): - authorizationstring = str('Bearer ' + r.stravatoken) +def get_strava_stream(r,metric,stravaid,series_type='time',fetchresolution='high',authorizationstring=''): + if r is not None: + authorizationstring = str('Bearer ' + r.stravatoken) headers = {'Authorization': authorizationstring, 'user-agent': 'sanderroosendaal', 'Content-Type': 'application/json', diff --git a/rowers/views/importviews.py b/rowers/views/importviews.py index a0149a41..f0ecc68d 100644 --- a/rowers/views/importviews.py +++ b/rowers/views/importviews.py @@ -1009,6 +1009,7 @@ def workout_stravaimport_view(request,message="",userid=0): return HttpResponse(res) # for Strava webhook request validation +@csrf_exempt def strava_webhook_view(request): if request.method == 'GET': challenge = request.GET.get('hub.challenge') @@ -1019,6 +1020,39 @@ def strava_webhook_view(request): return JSONResponse(data) # POST - does nothing so far + data = json.loads(request.body) + try: + aspect_type = data['aspect_type'] + object_type = data['object_type'] + strava_owner = data['owner_id'] + starttimeunix = data['event_time'] + except KeyError: + return HttpResponse(status=200) + + if aspect_type == 'create': + if object_type == 'activity': + try: + stravaid = data['object_id'] + except KeyError: + return HttpResponse(status=200) + + try: + r = Rower.objects.get(strava_owner_id=strava_owner) + except Rower.DoesNotExist: + return HttpResponse(status=200) + + w = Workout( + user = r, + csvfilename = 'media/{code}_{stravaid}'.format(code=uuid4().hex[:16],stravaid=stravaid), + startdatetime = timezone.now(), + uploadedtostrava=stravaid, + ) + w.save() + # too slow ... + job = stravastuff.async_get_workout(r.user,stravaid,w) + + print(w.id) + return HttpResponse(status=200) # For push notifications from Garmin From b85441f7eb80a37f3adfa4fdb990cf02cd770372 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 12 Jul 2020 13:20:13 +0200 Subject: [PATCH 2/2] hoping strava webhooks will work --- rowers/stravastuff.py | 7 ++++--- rowers/tasks.py | 29 ++++++++++++++++++++--------- rowers/views/importviews.py | 10 +--------- rowers/views/workoutviews.py | 2 +- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/rowers/stravastuff.py b/rowers/stravastuff.py index 2a2c1c9c..7c38e7f2 100644 --- a/rowers/stravastuff.py +++ b/rowers/stravastuff.py @@ -325,14 +325,15 @@ def create_async_workout(alldata,user,stravaid,debug=False): from rowers.utils import get_strava_stream -def async_get_workout(user,stravaid,workout): +def async_get_workout(user,stravaid): + csvfilename = 'media/{code}_{stravaid}.csv'.format(code=uuid4().hex[:16],stravaid=stravaid) job = myqueue(queue, fetch_strava_workout, user.rower.stravatoken, oauth_data, - workout.id, stravaid, - workout.csvfilename, + csvfilename, + user.id, ) return job diff --git a/rowers/tasks.py b/rowers/tasks.py index ff3ec212..72a2bd32 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -2713,7 +2713,7 @@ def add2(x, y,debug=False,**kwargs): return x + y @app.task -def fetch_strava_workout(stravatoken,oauth_data,workoutid,stravaid,csvfilename,debug=False,**kwargs): +def fetch_strava_workout(stravatoken,oauth_data,stravaid,csvfilename,userid,debug=False,**kwargs): fetchresolution = 'high' authorizationstring = str('Bearer '+stravatoken) headers = {'Authorization': authorizationstring, @@ -2925,19 +2925,30 @@ def fetch_strava_workout(stravatoken,oauth_data,workoutid,stravaid,csvfilename,d df.sort_values(by='TimeStamp (sec)',ascending=True) row = rowingdata.rowingdata(df=df) - row.write_csv(csvfilename,gzip=True) + row.write_csv(csvfilename,gzip=False) + summary = row.allstats() maxdist = df['cum_dist'].max() duration = row.duration + uploadoptions = { + 'secret':UPLOAD_SERVICE_SECRET, + 'user':userid, + 'file': csvfilename, + 'title': title, + 'workouttype':workouttype, + 'boattype':'1x', + 'stravaid':stravaid, + } + + print(uploadoptions) + + session = requests.session() + newHeaders = {'Content-type': 'application/json', 'Accept': 'text/plain'} + session.headers.update(newHeaders) + response = session.post(UPLOAD_SERVICE_URL,json=uploadoptions) + - update_strokedata(workoutid,row.df,debug=debug) - res = update_workout_field_sql(workoutid,'workouttype',workouttype,debug=debug) - res = update_workout_field_sql(workoutid,'name',title,debug=debug) - res = update_workout_field_sql(workoutid,'notes',comments,debug=debug) - res = update_workout_field_sql(workoutid,'summary',summary,debug=debug) - res = update_workout_field_sql(workoutid,'distance',maxdist,debug=debug) - res = update_workout_field_sql(workoutid,'duration',duration,debug=debug) return 1 diff --git a/rowers/views/importviews.py b/rowers/views/importviews.py index f0ecc68d..115fcfec 100644 --- a/rowers/views/importviews.py +++ b/rowers/views/importviews.py @@ -1041,17 +1041,9 @@ def strava_webhook_view(request): except Rower.DoesNotExist: return HttpResponse(status=200) - w = Workout( - user = r, - csvfilename = 'media/{code}_{stravaid}'.format(code=uuid4().hex[:16],stravaid=stravaid), - startdatetime = timezone.now(), - uploadedtostrava=stravaid, - ) - w.save() # too slow ... - job = stravastuff.async_get_workout(r.user,stravaid,w) + job = stravastuff.async_get_workout(r.user,stravaid) - print(w.id) return HttpResponse(status=200) diff --git a/rowers/views/workoutviews.py b/rowers/views/workoutviews.py index 83dffac6..44ed35f7 100644 --- a/rowers/views/workoutviews.py +++ b/rowers/views/workoutviews.py @@ -4717,7 +4717,7 @@ def workout_upload_api(request): message = {'status':'false','message':'unable to process file: '+message} else: message = {'status': 'false', 'message': 'unable to process file'} - print(message) + return JSONResponse(status=400,data=message) if id == -1: message = {'status': 'true', 'message':message}