diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index dca87e85..41201a7e 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -34,7 +34,7 @@ import sys import urllib from requests import Request, Session -from utils import myqueue,uniqify,isprorower +from utils import myqueue,uniqify,isprorower, custom_exception_handler, NoTokenError from rowers.types import otwtypes @@ -46,32 +46,7 @@ queue = django_rq.get_queue('default') queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('low') -# Custom error class - to raise a NoTokenError -class C2NoTokenError(Exception): - def __init__(self,value): - self.value=value - def __str__(self): - return repr(self.value) - -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): - - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res # Checks if user has Concept2 tokens, resets tokens if they are # expired. @@ -79,16 +54,16 @@ def c2_open(user): r = Rower.objects.get(user=user) if (r.c2token == '') or (r.c2token is None): s = "Token doesn't exist. Need to authorize" - raise C2NoTokenError("User has no token") + raise NoTokenError("User has no token") else: if (timezone.now()>r.tokenexpirydate): res = rower_c2_token_refresh(user) if res == None: - raise C2NoTokenError("User has no token") + raise NoTokenError("User has no token") if res[0] != None: thetoken = res[0] else: - raise C2NoTokenError("User has no token") + raise NoTokenError("User has no token") else: thetoken = r.c2token @@ -122,7 +97,7 @@ def get_c2_workouts(rower): try: thetoken = c2_open(rower.user) - except C2NoTokenError: + except NoTokenError: return 0 res = get_c2_workout_list(rower.user,page=1) @@ -693,7 +668,7 @@ def workout_c2_upload(user,w): if (checkworkoutuser(user,w)): c2userid = get_userid(r.c2token) if not c2userid: - raise C2NoTokenError + raise NoTokenError data = createc2workoutdata(w) if data == 0: diff --git a/rowers/courses.py b/rowers/courses.py index 4658462e..4d204100 100644 --- a/rowers/courses.py +++ b/rowers/courses.py @@ -46,30 +46,6 @@ from utils import geo_distance from rowers.courseutils import coursetime_paths, coursetime_first,time_in_path - -def get_course_timezone(course): - polygons = GeoPolygon.objects.filter(course = course) - points = GeoPoint.objects.filter(polygon = polygons[0]) - lat = points[0].latitude - lon = points[0].longitude - - tf = TimezoneFinder() - try: - timezone_str = tf.timezone_at(lng=lon,lat=lat) - except ValueError: - timezone_str = 'UTC' - - if timezone_str is None: - timezone_str = tf.closest_timezone_at(lng=lon,lat=lat) - if timezone_str is None: - timezone_str = 'UTC' - - return timezone_str - - - - - def crewnerdcourse(doc): courses = [] for course in doc: diff --git a/rowers/database.py b/rowers/database.py new file mode 100644 index 00000000..cd420ad2 --- /dev/null +++ b/rowers/database.py @@ -0,0 +1,18 @@ +from django.conf import settings + +user = settings.DATABASES['default']['USER'] +password = settings.DATABASES['default']['PASSWORD'] +database_name = settings.DATABASES['default']['NAME'] +host = settings.DATABASES['default']['HOST'] +port = settings.DATABASES['default']['PORT'] + +database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format( + user=user, + password=password, + database_name=database_name, + host=host, + port=port, + ) + +if settings.DEBUG or user=='': + database_url = 'sqlite:///db.sqlite3' diff --git a/rowers/dataprep.py b/rowers/dataprep.py index ddcb0b05..bd238950 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -71,24 +71,8 @@ queuehigh = django_rq.get_queue('default') from rowsandall_app.settings import SITE_URL from rowers.types import otwtypes -user = settings.DATABASES['default']['USER'] -password = settings.DATABASES['default']['PASSWORD'] -database_name = settings.DATABASES['default']['NAME'] -host = settings.DATABASES['default']['HOST'] -port = settings.DATABASES['default']['PORT'] +from rowers.database import * -database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format( - user=user, - password=password, - database_name=database_name, - host=host, - port=port, -) - -# Use SQLite local database when we're in debug mode -if settings.DEBUG or user == '': - # database_url = 'sqlite:///db.sqlite3' - database_url = 'sqlite:///' + database_name # mapping the DB column names to the CSV file column names diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 3828afdb..0dd4d25b 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -133,34 +133,8 @@ def rdata(file,rower=rrower()): return res +from utils import totaltime_sec_to_string -def totaltime_sec_to_string(totaltime): - hours = int(totaltime / 3600.) - if hours > 23: - message = 'Warning: The workout duration was longer than 23 hours. ' - hours = 23 - - minutes = int((totaltime - 3600. * hours) / 60.) - if minutes > 59: - minutes = 59 - if not message: - message = 'Warning: there is something wrong with the workout duration' - - seconds = int(totaltime - 3600. * hours - 60. * minutes) - if seconds > 59: - seconds = 59 - if not message: - message = 'Warning: there is something wrong with the workout duration' - - tenths = int(10 * (totaltime - 3600. * hours - 60. * minutes - seconds)) - if tenths > 9: - tenths = 9 - if not message: - message = 'Warning: there is something wrong with the workout duration' - - duration = "%s:%s:%s.%s" % (hours, minutes, seconds, tenths) - - return duration # Creates C2 stroke data def create_c2_stroke_data_db( diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 057c97d0..8e0ac7b4 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -328,7 +328,8 @@ def interactive_forcecurve(theworkouts,workstrokesonly=False): 'peakforceangle','peakforce','spm','distance', 'workoutstate','driveenergy'] - rowdata = dataprep.getsmallrowdata_db(columns,ids=ids) + rowdata = dataprep.getsmallrowdata_db(columns,ids=ids, + workstrokesonly=workstrokesonly) rowdata.dropna(axis=1,how='all',inplace=True) rowdata.dropna(axis=0,how='any',inplace=True) diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index e8f7f8d8..5183af9c 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -22,6 +22,7 @@ import rowers.uploads as uploads from rowers.mailprocessing import make_new_workout_from_email, send_confirm import rowers.polarstuff as polarstuff import rowers.c2stuff as c2stuff +import rowers.stravastuff as stravastuff workoutmailbox = Mailbox.objects.get(name='workouts') failedmailbox = Mailbox.objects.get(name='Failed') @@ -157,6 +158,11 @@ class Command(BaseCommand): rowers = Rower.objects.filter(c2_auto_import=True) for r in rowers: c2stuff.get_c2_workouts(r) + + # Strava + rowers = Rower.objects.filter(strava_auto_import=True) + for r in rowers: + stravastuff.get_strava_workouts(r) messages = Message.objects.filter(mailbox_id = workoutmailbox.id) message_ids = [m.id for m in messages] diff --git a/rowers/models.py b/rowers/models.py index b237088a..6510f4dd 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -49,22 +49,7 @@ tweetapi = twitter.Api(consumer_key=TWEET_CONSUMER_KEY, access_token_key=TWEET_ACCESS_TOKEN_KEY, access_token_secret=TWEET_ACCESS_TOKEN_SECRET) -user = settings.DATABASES['default']['USER'] -password = settings.DATABASES['default']['PASSWORD'] -database_name = settings.DATABASES['default']['NAME'] -host = settings.DATABASES['default']['HOST'] -port = settings.DATABASES['default']['PORT'] - -database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format( - user=user, - password=password, - database_name=database_name, - host=host, - port=port, - ) - -if settings.DEBUG or user=='': - database_url = 'sqlite:///db.sqlite3' +from rowers.database import * timezones = ( (x,x) for x in pytz.common_timezones @@ -674,6 +659,7 @@ class Rower(models.Model): verbose_name="Export Workouts to Strava as") strava_auto_export = models.BooleanField(default=False) + strava_auto_import = models.BooleanField(default=False) runkeepertoken = models.CharField(default='',max_length=200, blank=True,null=True) runkeeper_auto_export = models.BooleanField(default=False) @@ -2026,6 +2012,7 @@ class RowerImportExportForm(ModelForm): 'runkeeper_auto_export', 'sporttracks_auto_export', 'strava_auto_export', + 'strava_auto_import', 'trainingpeaks_auto_export', ] diff --git a/rowers/plots.py b/rowers/plots.py index 8193a088..b57d263e 100644 --- a/rowers/plots.py +++ b/rowers/plots.py @@ -2,43 +2,8 @@ from matplotlib.ticker import MultipleLocator,FuncFormatter,NullFormatter import matplotlib.pyplot as plt import numpy as np +from rows import format_pace_tick, format_pace, format_time, format_time_tick -# Make pace ticks for pace axis '2:00', '1:50', etc -def format_pace_tick(x,pos=None): - min=int(x/60) - sec=int(x-min*60.) - sec_str=str(sec).zfill(2) - template='%d:%s' - return template % (min,sec_str) - -# Returns a pace string from a pace in seconds/500m, '1:45.4' -def format_pace(x,pos=None): - if isinf(x) or isnan(x): - x=0 - - min=int(x/60) - sec=(x-min*60.) - - str1 = "{min:0>2}:{sec:0>4.1f}".format( - min = min, - sec = sec - ) - - return str1 - -# Returns a time string from a time in seconds -def format_time(x,pos=None): - - - min = int(x/60.) - sec = int(x-min*60) - - str1 = "{min:0>2}:{sec:0>4.1f}".format( - min=min, - sec=sec, - ) - - return str1 # Formatting the distance tick marks def format_dist_tick(x,pos=None): @@ -46,13 +11,6 @@ def format_dist_tick(x,pos=None): template='%6.3f' return template % (km) -# Formatting the time tick marks (h:mm) -def format_time_tick(x,pos=None): - hour=int(x/3600) - min=int((x-hour*3600.)/60) - min_str=str(min).zfill(2) - template='%d:%s' - return template % (hour,min_str) # Utility to select reasonable y axis range # Basically the data range plus some padding, but with ultimate diff --git a/rowers/polarstuff.py b/rowers/polarstuff.py index 9b9c0e9e..a1962bcc 100644 --- a/rowers/polarstuff.py +++ b/rowers/polarstuff.py @@ -50,32 +50,8 @@ from rowsandall_app.settings import ( #baseurl = 'https://polaraccesslink.com/v3-example' baseurl = 'https://polaraccesslink.com/v3' -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res - -# Custom error class - to raise a NoTokenError -class PolarNoTokenError(Exception): - def __init__(self,value): - self.value=value - - def __str__(self): - return repr(self.value) +from utils import NoTokenError, custom_exception_handler # Exchange access code for long-lived access token diff --git a/rowers/runkeeperstuff.py b/rowers/runkeeperstuff.py index 512cd1a9..8f8afb71 100644 --- a/rowers/runkeeperstuff.py +++ b/rowers/runkeeperstuff.py @@ -36,63 +36,15 @@ from rowsandall_app.settings import ( RUNKEEPER_CLIENT_ID, RUNKEEPER_CLIENT_SECRET,RUNKEEPER_REDIRECT_URI, ) -# Custom error class - to raise a NoTokenError -class RunKeeperNoTokenError(Exception): - def __init__(self,value): - self.value=value +from utils import geo_distance,ewmovingaverage,NoTokenError, custom_exception_handler - def __str__(self): - return repr(self.value) - -# Exponentially weighted moving average -# Used for data smoothing of the jagged data obtained by Strava -# See bitbucket issue 72 -def ewmovingaverage(interval,window_size): - # Experimental code using Exponential Weighted moving average - - try: - intervaldf = pd.DataFrame({'v':interval}) - idf_ewma1 = intervaldf.ewm(span=window_size) - idf_ewma2 = intervaldf[::-1].ewm(span=window_size) - - i_ewma1 = idf_ewma1.mean().ix[:,'v'] - i_ewma2 = idf_ewma2.mean().ix[:,'v'] - - interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) - interval2 = np.mean( interval2, axis=0) # average - except ValueError: - interval2 = interval - - return interval2 - -from utils import geo_distance - - -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): - - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res # Checks if user has SportTracks token, renews them if they are expired def runkeeper_open(user): r = Rower.objects.get(user=user) if (r.runkeepertoken == '') or (r.runkeepertoken is None): s = "Token doesn't exist. Need to authorize" - raise RunKeeperNoTokenError("User has no token") + raise NoTokenError("User has no token") else: thetoken = r.runkeepertoken diff --git a/rowers/sporttracksstuff.py b/rowers/sporttracksstuff.py index d8de7b20..8190cbdb 100644 --- a/rowers/sporttracksstuff.py +++ b/rowers/sporttracksstuff.py @@ -34,40 +34,15 @@ from rowers.models import Rower,Workout,checkworkoutuser from rowsandall_app.settings import C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET, STRAVA_CLIENT_ID, STRAVA_REDIRECT_URI, STRAVA_CLIENT_SECRET, SPORTTRACKS_CLIENT_SECRET, SPORTTRACKS_CLIENT_ID, SPORTTRACKS_REDIRECT_URI -# Custom error class - to raise a NoTokenError -class SportTracksNoTokenError(Exception): - def __init__(self,value): - self.value=value +from utils import NoTokenError, custom_exception_handler - def __str__(self): - return repr(self.value) - - -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): - - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res # Checks if user has SportTracks token, renews them if they are expired def sporttracks_open(user): r = Rower.objects.get(user=user) if (r.sporttrackstoken == '') or (r.sporttrackstoken is None): s = "Token doesn't exist. Need to authorize" - raise SportTracksNoTokenError("User has no token") + raise NoTokenError("User has no token") else: if (timezone.now()>r.sporttrackstokenexpirydate): thetoken = rower_sporttracks_token_refresh(user) diff --git a/rowers/stravastuff.py b/rowers/stravastuff.py index c14e6817..b1c2d3ed 100644 --- a/rowers/stravastuff.py +++ b/rowers/stravastuff.py @@ -16,6 +16,8 @@ from math import sin,cos,atan2,sqrt import os,sys import gzip +from pytz import timezone as tz,utc + # Django from django.shortcuts import render_to_response from django.http import HttpResponseRedirect, HttpResponse,JsonResponse @@ -24,6 +26,12 @@ from django.contrib.auth import authenticate, login, logout from django.contrib.auth.models import User from django.contrib.auth.decorators import login_required +import django_rq +queue = django_rq.get_queue('default') +queuelow = django_rq.get_queue('low') +queuehigh = django_rq.get_queue('low') + + # Project # from .models import Profile from rowingdata import rowingdata @@ -32,62 +40,29 @@ from rowers.models import Rower,Workout from rowers.models import checkworkoutuser import dataprep from dataprep import columndict - +from utils import uniqify,isprorower,myqueue,NoTokenError, custom_exception_handler +from uuid import uuid4 import stravalib from stravalib.exc import ActivityUploadFailed,TimeoutExceeded +import iso8601 +from iso8601 import ParseError + +import pytz +import arrow + +from rowers.tasks import handle_strava_import_stroke_data from rowsandall_app.settings import C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET, STRAVA_CLIENT_ID, STRAVA_REDIRECT_URI, STRAVA_CLIENT_SECRET -# Exponentially weighted moving average -# Used for data smoothing of the jagged data obtained by Strava -# See bitbucket issue 72 -def ewmovingaverage(interval,window_size): - # Experimental code using Exponential Weighted moving average - - try: - intervaldf = pd.DataFrame({'v':interval}) - idf_ewma1 = intervaldf.ewm(span=window_size) - idf_ewma2 = intervaldf[::-1].ewm(span=window_size) - - i_ewma1 = idf_ewma1.mean().ix[:,'v'] - i_ewma2 = idf_ewma2.mean().ix[:,'v'] - - interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) - interval2 = np.mean( interval2, axis=0) # average - except ValueError: - interval2 = interval - - return interval2 - -from utils import geo_distance +try: + from json.decoder import JSONDecodeError +except ImportError: + JSONDecodeError = ValueError -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): +from utils import geo_distance,ewmovingaverage - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res - -# Custom error class - to raise a NoTokenError -class StravaNoTokenError(Exception): - def __init__(self,value): - self.value=value - - def __str__(self): - return repr(self.value) # Exchange access code for long-lived access token @@ -106,7 +81,7 @@ def get_token(code): try: token_json = response.json() thetoken = token_json['access_token'] - except KeyError: + except (KeyError,JSONDecodeError): thetoken = 0 return [thetoken] @@ -115,7 +90,7 @@ def get_token(code): def make_authorization_url(request): # Generate a random string for the state parameter # Save it for use later to prevent xsrf attacks - from uuid import uuid4 + state = str(uuid4()) params = {"client_id": STRAVA_CLIENT_ID, @@ -144,6 +119,138 @@ def get_strava_workout_list(user): return s +def add_stroke_data(user,stravaid,workoutid,startdatetime,csvfilename): + r = Rower.objects.get(user=user) + + starttimeunix = arrow.get(startdatetime).timestamp + + + job = myqueue(queue, + handle_strava_import_stroke_data, + r.stravatoken, + stravaid, + workoutid, + starttimeunix, + csvfilename) + +# gets all new Strava workouts for a rower +def get_strava_workouts(rower): + + if not isprorower(rower): + return 0 + + res = get_strava_workout_list(rower.user) + + if (res.status_code != 200): + return 0 + else: + stravaids = [int(item['id']) for item in res.json()] + + alldata = {} + for item in res.json(): + alldata[item['id']] = item + + knownstravaids = uniqify([ + w.uploadedtostrava for w in Workout.objects.filter(user=rower) + ]) + newids = [stravaid for stravaid in stravaids if not stravaid in knownstravaids] + + for stravaid in newids: + workoutid = create_async_workout(alldata,rower.user,stravaid) + + return 1 + +def create_async_workout(alldata,user,stravaid): + data = alldata[stravaid] + r = Rower.objects.get(user=user) + distance = data['distance'] + stravaid = data['id'] + try: + workouttype = data['type'] + except: + workouttype = 'rower' + + if workouttype not in [x[0] for x in Workout.workouttypes]: + workouttype = 'other' + + try: + comments = data['comments'] + except: + comments = ' ' + + try: + thetimezone = tz(data['timezone']) + except: + thetimezone = 'UTC' + + try: + rowdatetime = iso8601.parse_date(data['date_utc']) + except KeyError: + rowdatetime = iso8601.parse_date(data['start_date']) + except ParseError: + rowdatetime = iso8601.parse_date(data['date']) + + try: + c2intervaltype = data['workout_type'] + + except KeyError: + c2intervaltype = '' + + try: + title = data['name'] + except KeyError: + title = "" + try: + t = data['comments'].split('\n', 1)[0] + title += t[:20] + except: + title = 'Imported' + + workoutdate = rowdatetime.astimezone( + pytz.timezone(thetimezone) + ).strftime('%Y-%m-%d') + + starttime = rowdatetime.astimezone( + pytz.timezone(thetimezone) + ).strftime('%H:%m:%S') + + totaltime = data['elapsed_time'] + duration = dataprep.totaltime_sec_to_string(totaltime) + + weightcategory = 'hwt' + + # Create CSV file name and save data to CSV file + csvfilename ='media/{code}_{importid}.csv'.format( + importid=stravaid, + code = uuid4().hex[:16] + ) + + w = Workout( + user=r, + workouttype = workouttype, + name = title, + date = workoutdate, + starttime = starttime, + startdatetime = rowdatetime, + timezone = thetimezone, + duration = duration, + distance=distance, + weightcategory = weightcategory, + uploadedtostrava = stravaid, + csvfilename = csvfilename, + notes = '' + ) + + w.save() + + # Check if workout has stroke data, and get the stroke data + + result = add_stroke_data(user,stravaid,w.id,rowdatetime,csvfilename) + + return w.id + +from utils import get_strava_stream + # Get a Strava workout summary data and stroke data by ID def get_strava_workout(user,stravaid): r = Rower.objects.get(user=user) @@ -165,18 +272,12 @@ def get_strava_workout(user,stravaid): workoutsummary['timezone'] = "Etc/UTC" startdatetime = workoutsummary['start_date'] - url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/cadence?resolution="+fetchresolution+"&series_type="+series_type - spmjson = requests.get(url,headers=headers) - url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/heartrate?resolution="+fetchresolution+"&series_type="+series_type - hrjson = requests.get(url,headers=headers) - url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/time?resolution="+fetchresolution+"&series_type="+series_type - timejson = requests.get(url,headers=headers) - url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/velocity_smooth?resolution="+fetchresolution+"&series_type="+series_type - velojson = requests.get(url,headers=headers) - url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/distance?resolution="+fetchresolution+"&series_type="+series_type - distancejson = requests.get(url,headers=headers) - url = "https://www.strava.com/api/v3/activities/"+str(stravaid)+"/streams/latlng?resolution="+fetchresolution+"&series_type="+series_type - latlongjson = requests.get(url,headers=headers) + spmjson = get_strava_stream(r,'cadence',stravaid) + hrjson = get_strava_stream(r,'heartrate',stravaid) + timejson = get_strava_stream(r,'time',stravaid) + velojson = get_strava_stream(r,'velocity_smooth',stravaid) + distancejson = get_strava_stream(r,'distance',stravaid) + latlongjson = get_strava_stream(r,'latlng',stravaid) try: t = np.array(timejson.json()[0]['data']) @@ -335,7 +436,7 @@ def workout_strava_upload(user,w): res = -1 if (r.stravatoken == '') or (r.stravatoken is None): s = "Token doesn't exist. Need to authorize" - raise StravaNoTokenError("Your hovercraft is full of eels") + raise NoTokenError("Your hovercraft is full of eels") else: if (checkworkoutuser(user,w)): try: diff --git a/rowers/tasks.py b/rowers/tasks.py index b6fe15e2..b565754c 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -8,6 +8,7 @@ import numpy as np import re from scipy import optimize +from scipy.signal import savgol_filter import rowingdata @@ -20,6 +21,7 @@ import datetime import pytz import iso8601 + from matplotlib.backends.backend_agg import FigureCanvas #from matplotlib.backends.backend_cairo import FigureCanvasCairo as FigureCanvas import matplotlib.pyplot as plt @@ -37,7 +39,7 @@ from django_rq import job from django.utils import timezone from django.utils.html import strip_tags -from utils import deserialize_list +from utils import deserialize_list,ewmovingaverage from rowers.dataprepnodjango import ( update_strokedata, new_workout_from_file, @@ -45,7 +47,7 @@ from rowers.dataprepnodjango import ( update_agegroup_db,fitnessmetric_to_sql, add_c2_stroke_data_db,totaltime_sec_to_string, create_c2_stroke_data_db,update_empower, - database_url_debug,database_url, + database_url_debug,database_url,dataprep ) @@ -65,6 +67,8 @@ import requests import longtask import arrow +from utils import get_strava_stream + siteurl = SITE_URL # testing task @@ -77,6 +81,135 @@ def add(x, y): return x + y +@app.task +def handle_strava_import_stroke_data(stravatoken, + stravaid,workoutid, + starttimeunix, + csvfilename,debug=True,**kwargs): + # ready to fetch. Hurray + fetchresolution = 'high' + series_type = 'time' + authorizationstring = str('Bearer ' + stravatoken) + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json', + 'resolution': 'medium',} + url = "https://www.strava.com/api/v3/activities/"+str(stravaid) + workoutsummary = requests.get(url,headers=headers).json() + + workoutsummary['timezone'] = "Etc/UTC" + startdatetime = workoutsummary['start_date'] + + spmjson = get_strava_stream(r,'cadence',stravaid) + hrjson = get_strava_stream(r,'heartrate',stravaid) + timejson = get_strava_stream(r,'time',stravaid) + velojson = get_strava_stream(r,'velocity_smooth',stravaid) + distancejson = get_strava_stream(r,'distance',stravaid) + latlongjson = get_strava_stream(r,'latlng',stravaid) + wattsjson = get_strava_stream(r,'watts',stravaid) + + try: + t = np.array(timejson.json()[0]['data']) + nr_rows = len(t) + d = np.array(distancejson.json()[1]['data']) + if nr_rows == 0: + return 0 + except IndexError: + d = 0*t + # return (0,"Error: No Distance information in the Strava data") + except KeyError: + return 0 + + try: + spm = np.array(spmjson.json()[1]['data']) + except: + spm = np.zeros(nr_rows) + + try: + watts = np.array(wattsjson.json()[1]['data']) + except: + watts = np.zeros(nr_rows) + + try: + hr = np.array(hrjson.json()[1]['data']) + except IndexError: + hr = np.zeros(nr_rows) + except KeyError: + hr = np.zeros(nr_rows) + + try: + velo = np.array(velojson.json()[1]['data']) + except IndexError: + velo = np.zeros(nr_rows) + except KeyError: + velo = np.zeros(nr_rows) + + f = np.diff(t).mean() + if f != 0: + windowsize = 2*(int(10./(f)))+1 + else: + windowsize = 1 + + if windowsize > 3 and windowsize < len(velo): + velo2 = savgol_filter(velo,windowsize,3) + else: + velo2 = velo + + coords = np.array(latlongjson.json()[0]['data']) + try: + lat = coords[:,0] + lon = coords[:,1] + except IndexError: + lat = np.zeros(len(t)) + lon = np.zeros(len(t)) + except KeyError: + lat = np.zeros(len(t)) + lon = np.zeros(len(t)) + + strokelength = velo*60./(spm) + strokelength[np.isinf(strokelength)] = 0.0 + + pace = 500./(1.0*velo2) + pace[np.isinf(pace)] = 0.0 + + unixtime = starttimeunix+t + + strokedistance = 60.*velo2/spm + + nr_strokes = len(t) + + df = pd.DataFrame({'TimeStamp (sec)':unixtime, + ' ElapsedTime (sec)':t, + ' Horizontal (meters)':d, + ' Stroke500mPace (sec/500m)':pace, + ' Cadence (stokes/min)':spm, + ' HRCur (bpm)':hr, + ' latitude':lat, + ' longitude':lon, + ' StrokeDistance (meters)':strokelength, + 'cum_dist':d, + ' DragFactor':np.zeros(nr_strokes), + ' DriveLength (meters)':np.zeros(nr_strokes), + ' StrokeDistance (meters)':strokedistance, + ' DriveTime (ms)':np.zeros(nr_strokes), + ' StrokeRecoveryTime (ms)':np.zeros(nr_strokes), + ' AverageDriveForce (lbs)':np.zeros(nr_strokes), + ' PeakDriveForce (lbs)':np.zeros(nr_strokes), + ' lapIdx':np.zeros(nr_strokes), + ' Power (watts)':watts, + }) + + + df.sort_values(by='TimeStamp (sec)',ascending=True) + + res = df.to_csv(csvfilename+'.gz',index_label='index',compression='gzip') + + data = dataprep(df,id=workoutid,bands=False,debug=debug) + # startdatetime = datetime.datetime.strptime(startdatetime,"%Y-%m-%d-%H:%M:%S") + + return 1 + + @app.task def handle_c2_import_stroke_data(c2token, c2id,workoutid, diff --git a/rowers/tests.py b/rowers/tests.py index 59afd8e3..bbdcc5ef 100644 --- a/rowers/tests.py +++ b/rowers/tests.py @@ -20,7 +20,7 @@ from time import strftime,strptime,mktime,time,daylight import os from rowers.tasks import handle_makeplot from rowers.utils import serialize_list,deserialize_list -from rowers.c2stuff import C2NoTokenError +from rowers.utils import NoTokenError from shutil import copyfile from minimocktest import MockTestCase @@ -424,8 +424,8 @@ class C2Tests(TestCase): def c2_notokentest(self): thetoken = c2_open(self.u) - # should raise C2NoTokenError - self.assertRaises(C2NoTokenError) + # should raise NoTokenError + self.assertRaises(NoTokenError) class DataTest(TestCase): def setUp(self): diff --git a/rowers/tpstuff.py b/rowers/tpstuff.py index 511e14ab..9a783d9a 100644 --- a/rowers/tpstuff.py +++ b/rowers/tpstuff.py @@ -52,63 +52,17 @@ from async_messages import message_user,messages -# Custom error class - to raise a NoTokenError -class TPNoTokenError(Exception): - def __init__(self,value): - self.value=value - def __str__(self): - return repr(self.value) - - # Exponentially weighted moving average -# Used for data smoothing of the jagged data obtained by Strava -# See bitbucket issue 72 -def ewmovingaverage(interval,window_size): - # Experimental code using Exponential Weighted moving average - - try: - intervaldf = pd.DataFrame({'v':interval}) - idf_ewma1 = intervaldf.ewm(span=window_size) - idf_ewma2 = intervaldf[::-1].ewm(span=window_size) - - i_ewma1 = idf_ewma1.mean().ix[:,'v'] - i_ewma2 = idf_ewma2.mean().ix[:,'v'] - - interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) - interval2 = np.mean( interval2, axis=0) # average - except ValueError: - interval2 = interval - - return interval2 - -from utils import geo_distance +from utils import geo_distance, NoTokenError,ewmovingaverage, custom_exception_handler -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): - - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res # Checks if user has UnderArmour token, renews them if they are expired def tp_open(user): r = Rower.objects.get(user=user) if (r.tptoken == '') or (r.tptoken is None): s = "Token doesn't exist. Need to authorize" - raise TPNoTokenError("User has no token") + raise NoTokenError("User has no token") else: if (timezone.now()>r.tptokenexpirydate): res = do_refresh_token(r.tprefreshtoken) @@ -120,7 +74,7 @@ def tp_open(user): r.save() thetoken = r.tptoken else: - raise TPNoTokenError("Refresh token invalid") + raise NoTokenError("Refresh token invalid") else: thetoken = r.tptoken diff --git a/rowers/traverselinktest.py b/rowers/traverselinktest.py index bb9826a2..75a22fd0 100644 --- a/rowers/traverselinktest.py +++ b/rowers/traverselinktest.py @@ -20,7 +20,7 @@ from time import strftime,strptime,mktime,time,daylight import os from rowers.tasks import handle_makeplot from rowers.utils import serialize_list,deserialize_list -from rowers.c2stuff import C2NoTokenError + from shutil import copyfile from minimocktest import MockTestCase diff --git a/rowers/underarmourstuff.py b/rowers/underarmourstuff.py index d47d944a..cb5a479e 100644 --- a/rowers/underarmourstuff.py +++ b/rowers/underarmourstuff.py @@ -39,63 +39,18 @@ from rowsandall_app.settings import ( UNDERARMOUR_REDIRECT_URI,UNDERARMOUR_CLIENT_KEY, ) -# Custom error class - to raise a NoTokenError -class UnderArmourNoTokenError(Exception): - def __init__(self,value): - self.value=value +from utils import NoTokenError,ewmovingaverage - def __str__(self): - return repr(self.value) - - # Exponentially weighted moving average -# Used for data smoothing of the jagged data obtained by Strava -# See bitbucket issue 72 -def ewmovingaverage(interval,window_size): - # Experimental code using Exponential Weighted moving average - - try: - intervaldf = pd.DataFrame({'v':interval}) - idf_ewma1 = intervaldf.ewm(span=window_size) - idf_ewma2 = intervaldf[::-1].ewm(span=window_size) - - i_ewma1 = idf_ewma1.mean().ix[:,'v'] - i_ewma2 = idf_ewma2.mean().ix[:,'v'] - - interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) - interval2 = np.mean( interval2, axis=0) # average - except ValueError: - interval2 = interval - - return interval2 - -from utils import geo_distance +from utils import geo_distance, custom_exception_handler -# Custom exception handler, returns a 401 HTTP message -# with exception details in the json data -def custom_exception_handler(exc,message): - - response = { - "errors": [ - { - "code": str(exc), - "detail": message, - } - ] - } - - res = HttpResponse(message) - res.status_code = 401 - res.json = json.dumps(response) - - return res # Checks if user has UnderArmour token, renews them if they are expired def underarmour_open(user): r = Rower.objects.get(user=user) if (r.underarmourtoken == '') or (r.underarmourtoken is None): s = "Token doesn't exist. Need to authorize" - raise UnderArmourNoTokenError("User has no token") + raise NoTokenError("User has no token") else: if (timezone.now()>r.underarmourtokenexpirydate): res = do_refresh_token( diff --git a/rowers/uploads.py b/rowers/uploads.py index 07b5cdce..47a6f1c1 100644 --- a/rowers/uploads.py +++ b/rowers/uploads.py @@ -36,7 +36,7 @@ except: from rowers.utils import ( geo_distance,serialize_list,deserialize_list,uniqify, - str2bool,range_to_color_hex,absolute,myqueue + str2bool,range_to_color_hex,absolute,myqueue,NoTokenError ) def cleanbody(body): @@ -379,7 +379,7 @@ def do_sync(w,options): if ('upload_to_C2' in options and options['upload_to_C2']) or (w.user.c2_auto_export and isprorower(w.user)): try: message,id = c2stuff.workout_c2_upload(w.user.user,w) - except c2stuff.C2NoTokenError: + except NoTokenError: id = 0 message = "Something went wrong with the Concept2 sync" @@ -388,7 +388,7 @@ def do_sync(w,options): message,id = stravastuff.workout_strava_upload( w.user.user,w ) - except stravastuff.StravaNoTokenError: + except NoTokenError: id = 0 message = "Please connect to Strava first" @@ -398,7 +398,7 @@ def do_sync(w,options): message,id = sporttracksstuff.workout_sporttracks_upload( w.user.user,w ) - except sporttracksstuff.SportTracksNoTokenError: + except NoTokenError: message = "Please connect to SportTracks first" id = 0 @@ -408,7 +408,7 @@ def do_sync(w,options): message,id = runkeeperstuff.workout_runkeeper_upload( w.user.user,w ) - except runkeeperstuff.RunKeeperNoTokenError: + except NoTokenError: message = "Please connect to Runkeeper first" id = 0 @@ -417,7 +417,7 @@ def do_sync(w,options): message,id = underarmourstuff.workout_ua_upload( w.user.user,w ) - except underarmourstuff.UnderArmourNoTokenError: + except NoTokenError: message = "Please connect to MapMyFitness first" id = 0 @@ -427,7 +427,7 @@ def do_sync(w,options): message,id = tpstuff.workout_tp_upload( w.user.user,w ) - except tpstuff.TPNoTokenError: + except NoTokenError: message = "Please connect to TrainingPeaks first" id = 0 diff --git a/rowers/utils.py b/rowers/utils.py index 635d255a..1efd8aed 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -7,6 +7,7 @@ from django.conf import settings import uuid import datetime +import requests lbstoN = 4.44822 @@ -378,3 +379,67 @@ def isprorower(r): return result +# Exponentially weighted moving average +# Used for data smoothing of the jagged data obtained by Strava +# See bitbucket issue 72 +def ewmovingaverage(interval,window_size): + # Experimental code using Exponential Weighted moving average + + try: + intervaldf = pd.DataFrame({'v':interval}) + idf_ewma1 = intervaldf.ewm(span=window_size) + idf_ewma2 = intervaldf[::-1].ewm(span=window_size) + + i_ewma1 = idf_ewma1.mean().ix[:,'v'] + i_ewma2 = idf_ewma2.mean().ix[:,'v'] + + interval2 = np.vstack((i_ewma1,i_ewma2[::-1])) + interval2 = np.mean( interval2, axis=0) # average + except ValueError: + interval2 = interval + + return interval2 + +# Exceptions +# Custom error class - to raise a NoTokenError +class NoTokenError(Exception): + def __init__(self,value): + self.value=value + + def __str__(self): + return repr(self.value) + +# Custom exception handler, returns a 401 HTTP message +# with exception details in the json data +def custom_exception_handler(exc,message): + + response = { + "errors": [ + { + "code": str(exc), + "detail": message, + } + ] + } + + res = HttpResponse(message) + res.status_code = 401 + res.json = json.dumps(response) + + return res + +def get_strava_stream(r,metric,stravaid,series_type='time',fetchresolution='high'): + authorizationstring = str('Bearer ' + r.stravatoken) + headers = {'Authorization': authorizationstring, + 'user-agent': 'sanderroosendaal', + 'Content-Type': 'application/json', + 'resolution': 'medium',} + + url = "https://www.strava.com/api/v3/activities/{stravaid}/streams/{metric}?resolution={fetchresolutions}&series_type={series_type}".format( + stravaid=stravid, + fetchresolution=fetchresolution, + series_type=series_type, + metric=metric + ) + + return requests.get(url,headers=headers) diff --git a/rowers/views.py b/rowers/views.py index c6e5b1e9..60416947 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -90,18 +90,16 @@ import os,sys import datetime import iso8601 import c2stuff -from c2stuff import C2NoTokenError,c2_open -from runkeeperstuff import RunKeeperNoTokenError,runkeeper_open -from sporttracksstuff import SportTracksNoTokenError,sporttracks_open -from tpstuff import TPNoTokenError,tp_open +from c2stuff import c2_open +from runkeeperstuff import runkeeper_open +from sporttracksstuff import sporttracks_open +from tpstuff import tp_open from iso8601 import ParseError import stravastuff import polarstuff -from polarstuff import PolarNoTokenError -from stravastuff import StravaNoTokenError import sporttracksstuff import underarmourstuff -from underarmourstuff import UnderArmourNoTokenError,underarmour_open +from underarmourstuff import underarmour_open import tpstuff import runkeeperstuff import ownapistuff @@ -887,7 +885,7 @@ from utils import ( geo_distance,serialize_list,deserialize_list,uniqify, str2bool,range_to_color_hex,absolute,myqueue,get_call, calculate_age,rankingdistances,rankingdurations, - is_ranking_piece,my_dict_from_instance,wavg + is_ranking_piece,my_dict_from_instance,wavg,NoTokenError ) import datautils @@ -1409,8 +1407,11 @@ def add_workout_from_runkeeperdata(user,importid,data): unixtime = cum_time+starttimeunix - unixtime[0] = starttimeunix - + try: + unixtime[0] = starttimeunix + except IndexError: + return (0,'No data to import') + df['TimeStamp (sec)'] = unixtime @@ -2062,7 +2063,7 @@ def workout_tp_upload_view(request,id=0): res = -1 try: thetoken = tp_open(r.user) - except TPNoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/tpauthorize/") # ready to upload. Hurray @@ -2213,7 +2214,7 @@ def workout_c2_upload_view(request,id=0): try: message,c2id = c2stuff.workout_c2_upload(request.user,w) - except C2NoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/c2authorize/") if message and c2id <=0: @@ -2238,7 +2239,7 @@ def workout_runkeeper_upload_view(request,id=0): try: thetoken = runkeeper_open(r.user) - except RunKeeperNoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/runkeeperauthorize/") # ready to upload. Hurray @@ -2300,7 +2301,7 @@ def workout_underarmour_upload_view(request,id=0): try: thetoken = underarmour_open(r.user) - except UnderArmourNoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/underarmourauthorize/") # ready to upload. Hurray @@ -2364,7 +2365,7 @@ def workout_sporttracks_upload_view(request,id=0): try: thetoken = sporttracks_open(r.user) - except SportTracksNoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/sporttracksauthorize/") @@ -3365,7 +3366,7 @@ def workout_forcecurve_view(request,id=0,workstrokesonly=False): workstrokesonly = True else: workstrokesonly = False - + script,div,js_resources,css_resources = interactive_forcecurve([row], workstrokesonly=workstrokesonly) @@ -8947,7 +8948,7 @@ def workout_export_view(request,id=0, message="", successmessage=""): try: thetoken = c2_open(request.user) - except C2NoTokenError: + except NoTokenError: thetoken = 0 if (checkworkoutuser(request.user,row)) and thetoken: @@ -8957,7 +8958,7 @@ def workout_export_view(request,id=0, message="", successmessage=""): try: rktoken = runkeeper_open(request.user) - except RunKeeperNoTokenError: + except NoTokenError: rktoken = 0 if (checkworkoutuser(request.user,row)) and rktoken: @@ -9937,7 +9938,7 @@ def workout_sporttracksimport_view(request,message=""): def c2listdebug_view(request,page=1,message=""): try: thetoken = c2_open(request.user) - except C2NoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/c2authorize/") r = getrower(request.user) @@ -9980,7 +9981,7 @@ def c2listdebug_view(request,page=1,message=""): def workout_getc2workout_all(request,page=1,message=""): try: thetoken = c2_open(request.user) - except C2NoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/c2authorize/") res = c2stuff.get_c2_workout_list(request.user,page=page) @@ -10013,7 +10014,7 @@ def workout_getc2workout_all(request,page=1,message=""): def workout_c2import_view(request,page=1,message=""): try: thetoken = c2_open(request.user) - except C2NoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/c2authorize/") res = c2stuff.get_c2_workout_list(request.user,page=page) @@ -10239,7 +10240,7 @@ def workout_getstravaworkout_all(request): def workout_getc2workout_view(request,c2id): try: thetoken = c2_open(request.user) - except C2NoTokenError: + except NoTokenError: return HttpResponseRedirect("/rowers/me/c2authorize/") res = c2stuff.get_c2_workout(request.user,c2id) @@ -10640,7 +10641,7 @@ def workout_upload_view(request, if (upload_to_c2) or (w.user.c2_auto_export and isprorower(w.user)): try: message,id = c2stuff.workout_c2_upload(request.user,w) - except C2NoTokenError: + except NoTokenError: id = 0 message = "Something went wrong with the Concept2 sync" if id>1: @@ -10653,7 +10654,7 @@ def workout_upload_view(request, message,id = stravastuff.workout_strava_upload( request.user,w ) - except StravaNoTokenError: + except NoTokenError: id = 0 message = "Please connect to Strava first" if id>1: @@ -10666,7 +10667,7 @@ def workout_upload_view(request, message,id = sporttracksstuff.workout_sporttracks_upload( request.user,w ) - except SportTracksNoTokenError: + except NoTokenError: message = "Please connect to SportTracks first" id = 0 if id>1: @@ -10679,7 +10680,7 @@ def workout_upload_view(request, message,id = runkeeperstuff.workout_runkeeper_upload( request.user,w ) - except RunKeeperNoTokenError: + except NoTokenError: message = "Please connect to Runkeeper first" id = 0 @@ -10694,7 +10695,7 @@ def workout_upload_view(request, message,id = underarmourstuff.workout_ua_upload( request.user,w ) - except UnderArmourNoTokenError: + except NoTokenError: message = "Please connect to MapMyFitness first" id = 0 @@ -10709,7 +10710,7 @@ def workout_upload_view(request, message,id = tpstuff.workout_tp_upload( request.user,w ) - except TPNoTokenError: + except NoTokenError: message = "Please connect to TrainingPeaks first" id = 0 @@ -11138,7 +11139,7 @@ def workout_split_view(request,id=id): r,row,splitsecond,splitmode ) except IndexError: - messages.error("Something went wrong in Split") + messages.error(request,"Something went wrong in Split") for message in mesgs: messages.info(request,message) @@ -11301,7 +11302,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" try: avpace = datetime.timedelta(seconds=int(500./normv)) - except: + except (OverflowError, ZeroDivisionError): avpace = datetime.timedelta(seconds=130) data = {