Private
Public Access
1
0
Files
rowsandall/rowers/stravastuff.py
Sander Roosendaal 1d528e6489 fixing strava imports
2019-09-28 15:22:40 +02:00

824 lines
24 KiB
Python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import unicode_literals, absolute_import
# All the functionality needed to connect to Strava
from scipy import optimize
from scipy.signal import savgol_filter
from django_mailbox.models import Message,Mailbox,MessageAttachment
import django_rq
queue = django_rq.get_queue('default')
queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('low')
from rowers.dataprep import columndict
import stravalib
from stravalib.exc import ActivityUploadFailed,TimeoutExceeded
from iso8601 import ParseError
from rowers.utils import myqueue
import rowers.mytypes as mytypes
import gzip
from rowsandall_app.settings import (
C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET,
STRAVA_CLIENT_ID, STRAVA_REDIRECT_URI, STRAVA_CLIENT_SECRET
)
try:
from json.decoder import JSONDecodeError
except ImportError:
JSONDecodeError = ValueError
from rowers.imports import *
headers = {'Accept': 'application/json',
'Api-Key': STRAVA_CLIENT_ID,
'Content-Type': 'application/json',
'user-agent': 'sanderroosendaal'}
oauth_data = {
'client_id': STRAVA_CLIENT_ID,
'client_secret': STRAVA_CLIENT_SECRET,
'redirect_uri': STRAVA_REDIRECT_URI,
'autorization_uri': "https://www.strava.com/oauth/authorize",
'content_type': 'application/json',
'tokenname': 'stravatoken',
'refreshtokenname': 'stravarefreshtoken',
'expirydatename': 'stravatokenexpirydate',
'bearer_auth': True,
'base_url': "https://www.strava.com/oauth/token",
'grant_type': 'refresh_token',
'headers': headers,
'scope':'activity:write,activity:read_all',
}
# Exchange access code for long-lived access token
def get_token(code):
return imports_get_token(code, oauth_data)
def strava_open(user):
return imports_open(user, oauth_data)
def do_refresh_token(refreshtoken):
return imports_do_refresh_token(refreshtoken, oauth_data)
def rower_strava_token_refresh(user):
r = Rower.objects.get(user=user)
res = do_refresh_token(r.stravarefreshtoken)
access_token = res[0]
expires_in = res[1]
refresh_token = res[2]
expirydatetime = timezone.now()+timedelta(seconds=expires_in)
r.stravatoken = access_token
r.stravatokenexpirydate = expirydatetime
r.stravarefreshtoken = refresh_token
r.save()
return r.stravatoken
# Make authorization URL including random string
def make_authorization_url(request):
return imports_make_authorization_url(oauth_data)
# Get list of workouts available on Strava
def get_strava_workout_list(user,limit_n=0):
r = Rower.objects.get(user=user)
if (r.stravatoken == '') or (r.stravatoken is None):
s = "Token doesn't exist. Need to authorize"
return custom_exception_handler(401,s)
elif (r.stravatokenexpirydate is None or timezone.now()+timedelta(seconds=3599)>r.stravatokenexpirydate):
s = "Token expired. Needs to refresh."
return custom_exception_handler(401,s)
else:
# ready to fetch. Hurray
authorizationstring = str('Bearer ' + r.stravatoken)
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json'}
url = "https://www.strava.com/api/v3/athlete/activities"
if limit_n==0:
params = {}
else:
params = {'per_page':limit_n}
s = requests.get(url,headers=headers,params=params)
return s
# gets all new Strava workouts for a rower
def get_strava_workouts(rower):
if not isprorower(rower):
return 0
try:
thetoken = strava_open(rower.user)
except NoTokenError:
return 0
res = get_strava_workout_list(rower.user,limit_n=10)
if (res.status_code != 200):
return 0
else:
stravaids = [int(item['id']) for item in res.json()]
stravadata = [{
'id':int(item['id']),
'elapsed_time':item['elapsed_time'],
'start_date':item['start_date'],
} for item in res.json()]
alldata = {}
for item in res.json():
alldata[item['id']] = item
wfailed = Workout.objects.filter(user=rower,uploadedtostrava=-1)
for w in wfailed:
for item in stravadata:
elapsed_time = item['elapsed_time']
start_date = item['start_date']
stravaid = item['id']
if arrow.get(start_date) == arrow.get(w.startdatetime):
dd = datetime.min + timedelta(
seconds=int(elapsed_time)
)
if datetime.time(dd) == w.duration:
w.uploadedtostrava = int(stravaid)
w.save()
knownstravaids = [
w.uploadedtostrava for w in Workout.objects.filter(user=rower)
]
tombstones = [
t.uploadedtostrava for t in TombStone.objects.filter(user=rower)
]
knownstravaids = uniqify(knownstravaids+tombstones)
newids = [stravaid for stravaid in stravaids if not stravaid in knownstravaids]
for stravaid in newids:
result = create_async_workout(alldata,rower.user,stravaid)
return 1
def create_async_workout(alldata,user,stravaid,debug=False):
data = alldata[stravaid]
r = Rower.objects.get(user=user)
distance = data['distance']
stravaid = data['id']
try:
workouttype = mytypes.stravamappinginv[data['type']]
except:
workouttype = 'other'
if workouttype not in [x[0] for x in Workout.workouttypes]:
workouttype = 'other'
if workouttype.lower() == 'rowing':
workouttype = 'rower'
if 'summary_polyline' in data['map']:
workouttype = 'water'
try:
comments = data['comments']
except:
comments = ' '
try:
thetimezone = tz(data['timezone'])
except:
thetimezone = 'UTC'
try:
rowdatetime = iso8601.parse_date(data['date_utc'])
except KeyError:
rowdatetime = iso8601.parse_date(data['start_date'])
except ParseError:
rowdatetime = iso8601.parse_date(data['date'])
try:
c2intervaltype = data['workout_type']
except KeyError:
c2intervaltype = ''
try:
title = data['name']
except KeyError:
title = ""
try:
t = data['comments'].split('\n', 1)[0]
title += t[:20]
except:
title = 'Imported'
workoutdate = rowdatetime.astimezone(
pytz.timezone(thetimezone)
).strftime('%Y-%m-%d')
starttime = rowdatetime.astimezone(
pytz.timezone(thetimezone)
).strftime('%H:%m:%S')
totaltime = data['elapsed_time']
duration = dataprep.totaltime_sec_to_string(totaltime)
weightcategory = 'hwt'
# Create CSV file name and save data to CSV file
csvfilename ='media/mailbox_attachments/{code}_{importid}.csv'.format(
importid=stravaid,
code = uuid4().hex[:16]
)
# Check if workout has stroke data, and get the stroke data
starttimeunix = arrow.get(rowdatetime).timestamp
result = handle_strava_import_stroke_data(
title,
user.email,
r.stravatoken,
stravaid,
starttimeunix,
csvfilename,
workouttype = workouttype,
)
return 1
from rowers.utils import get_strava_stream
# Get a Strava workout summary data and stroke data by ID
def get_workout(user,stravaid):
try:
thetoken = strava_open(user)
except NoTokenError:
s = "Token error"
return custom_exception_handler(401,s)
r = Rower.objects.get(user=user)
if (r.stravatoken == '') or (r.stravatoken is None):
s = "Token doesn't exist. Need to authorize"
return custom_exception_handler(401,s)
elif (r.stravatokenexpirydate is not None and timezone.now()>r.stravatokenexpirydate):
s = "Token expired. Needs to refresh."
return custom_exception_handler(401,s)
else:
# ready to fetch. Hurray
fetchresolution = 'high'
series_type = 'time'
authorizationstring = str('Bearer ' + r.stravatoken)
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json',
'resolution': 'medium',}
url = "https://www.strava.com/api/v3/activities/"+str(stravaid)
workoutsummary = requests.get(url,headers=headers).json()
workoutsummary['timezone'] = "Etc/UTC"
try:
startdatetime = workoutsummary['start_date']
except KeyError:
startdatetime = timezone.now()
spm = get_strava_stream(r,'cadence',stravaid)
hr = get_strava_stream(r,'heartrate',stravaid)
t = get_strava_stream(r,'time',stravaid)
velo = get_strava_stream(r,'velocity_smooth',stravaid)
d = get_strava_stream(r,'distance',stravaid)
coords = get_strava_stream(r,'latlng',stravaid)
power = get_strava_stream(r,'power',stravaid)
if t is not None:
nr_rows = len(t)
else:
duration = int(workoutsummary['elapsed_time'])
t = pd.Series(range(duration+1))
nr_rows = len(t)
if nr_rows == 0:
return (0,"Error: Time data had zero length")
if d is None:
d = 0*t
if spm is None:
spm = np.zeros(nr_rows)
if power is None:
power = np.zeros(nr_rows)
if hr is None:
hr = np.zeros(nr_rows)
if velo is None:
velo = np.zeros(nr_rows)
dt = np.diff(t).mean()
wsize = round(5./dt)
velo2 = ewmovingaverage(velo,wsize)
if coords is not None:
try:
lat = coords[:,0]
lon = coords[:,1]
except IndexError:
lat = np.zeros(len(t))
lon = np.zeros(len(t))
else:
lat = np.zeros(len(t))
lon = np.zeros(len(t))
strokelength = velo*60./(spm)
strokelength[np.isinf(strokelength)] = 0.0
pace = 500./(1.0*velo2)
pace[np.isinf(pace)] = 0.0
df = pd.DataFrame({'t':10*t,
'd':10*d,
'p':10*pace,
'spm':spm,
'hr':hr,
'lat':lat,
'lon':lon,
'power':power,
'strokelength':strokelength,
})
# startdatetime = datetime.datetime.strptime(startdatetime,"%Y-%m-%d-%H:%M:%S")
return [workoutsummary,df]
# Generate Workout data for Strava (a TCX file)
def createstravaworkoutdata(w,dozip=True):
filename = w.csvfilename
try:
row = rowingdata(filename)
except IOError:
data = dataprep.read_df_sql(w.id)
try:
datalength = len(data)
except AttributeError:
datalength = 0
if datalength != 0:
data.rename(columns = columndict,inplace=True)
res = data.to_csv(w.csvfilename+'.gz',
index_label='index',
compression='gzip')
try:
row = rowingdata(filename)
except IOError:
return '','Error - could not find rowing data'
else:
return '','Error - could not find rowing data'
tcxfilename = filename[:-4]+'.tcx'
try:
newnotes = w.notes+'\n from '+w.workoutsource+' via rowsandall.com'
except TypeError:
newnotes = 'from '+w.workoutsource+' via rowsandall.com'
row.exporttotcx(tcxfilename,notes=newnotes)
if dozip:
gzfilename = tcxfilename+'.gz'
with open(tcxfilename,'rb') as inF:
s = inF.read()
with gzip.GzipFile(gzfilename,'wb') as outF:
outF.write(s)
try:
os.remove(tcxfilename)
except WindowError:
pass
return gzfilename,""
else:
return tcxfilename,""
# Upload the TCX file to Strava and set the workout activity type
# to rowing on Strava
def handle_stravaexport(f2,workoutname,stravatoken,description='',
activity_type='Rowing'):
# w = Workout.objects.get(id=workoutid)
client = stravalib.Client(access_token=stravatoken)
act = client.upload_activity(f2,'tcx.gz',name=workoutname)
try:
res = act.wait(poll_interval=5.0,timeout=30)
message = 'Workout successfully synchronized to Strava'
except:
res = 0
message = 'Strava upload timed out'
# description doesn't work yet. Have to wait for stravalib to update
if res:
try:
act = client.update_activity(res.id,activity_type=activity_type,description=description,device_name='Rowsandall.com')
except TypeError:
act = client.update_activity(res.id,activity_type=activity_type,description=description)
else:
message = 'Strava activity update timed out.'
return (0,message)
return (res.id,message)
# Create workout data from Strava or Concept2
# data and create the associated Workout object and save it
def add_workout_from_data(user,importid,data,strokedata,
source='strava',splitdata=None,
workoutsource='strava'):
try:
workouttype = mytypes.stravamappinginv[data['type']]
except KeyError:
workouttype = 'other'
if workouttype.lower() == 'rowing':
workouttype = 'rower'
if 'summary_polyline' in data['map'] and workouttype=='rower':
workouttype = 'water'
if workouttype not in [x[0] for x in Workout.workouttypes]:
workouttype = 'other'
try:
comments = data['comments']
except:
comments = ' '
try:
thetimezone = tz(data['timezone'])
except:
thetimezone = 'UTC'
r = Rower.objects.get(user=user)
try:
rowdatetime = iso8601.parse_date(data['date_utc'])
except KeyError:
rowdatetime = iso8601.parse_date(data['start_date'])
except ParseError:
rowdatetime = iso8601.parse_date(data['date'])
try:
intervaltype = data['workout_type']
except KeyError:
intervaltype = ''
try:
title = data['name']
except KeyError:
title = ""
try:
t = data['comments'].split('\n', 1)[0]
title += t[:20]
except:
title = 'Imported'
starttimeunix = arrow.get(rowdatetime).timestamp
res = make_cumvalues(0.1*strokedata['t'])
cum_time = res[0]
lapidx = res[1]
unixtime = cum_time+starttimeunix
seconds = 0.1*strokedata.loc[:,'t']
nr_rows = len(unixtime)
try:
latcoord = strokedata.loc[:,'lat']
loncoord = strokedata.loc[:,'lon']
if latcoord.std() == 0 and loncoord.std() == 0 and workouttype == 'water':
workouttype = 'rower'
except:
latcoord = np.zeros(nr_rows)
loncoord = np.zeros(nr_rows)
if workouttype == 'water':
workouttype = 'rower'
try:
strokelength = strokedata.loc[:,'strokelength']
except:
strokelength = np.zeros(nr_rows)
dist2 = 0.1*strokedata.loc[:,'d']
try:
spm = strokedata.loc[:,'spm']
except KeyError:
spm = 0*dist2
try:
hr = strokedata.loc[:,'hr']
except KeyError:
hr = 0*spm
pace = strokedata.loc[:,'p']/10.
pace = np.clip(pace,0,1e4)
pace = pace.replace(0,300)
velo = 500./pace
try:
power = strokedata.loc[:,'power']
except KeyError:
power = 2.8*velo**3
#if power.std() == 0 and power.mean() == 0:
# power = 2.8*velo**3
# save csv
# Create data frame with all necessary data to write to csv
df = pd.DataFrame({'TimeStamp (sec)':unixtime,
' Horizontal (meters)': dist2,
' Cadence (stokes/min)':spm,
' HRCur (bpm)':hr,
' longitude':loncoord,
' latitude':latcoord,
' Stroke500mPace (sec/500m)':pace,
' Power (watts)':power,
' DragFactor':np.zeros(nr_rows),
' DriveLength (meters)':np.zeros(nr_rows),
' StrokeDistance (meters)':strokelength,
' DriveTime (ms)':np.zeros(nr_rows),
' StrokeRecoveryTime (ms)':np.zeros(nr_rows),
' AverageDriveForce (lbs)':np.zeros(nr_rows),
' PeakDriveForce (lbs)':np.zeros(nr_rows),
' lapIdx':lapidx,
' ElapsedTime (sec)':seconds
})
df.sort_values(by='TimeStamp (sec)',ascending=True)
timestr = strftime("%Y%m%d-%H%M%S")
# Create CSV file name and save data to CSV file
csvfilename ='media/{code}_{importid}.csv'.format(
importid=importid,
code = uuid4().hex[:16]
)
res = df.to_csv(csvfilename+'.gz',index_label='index',
compression='gzip')
id,message = dataprep.save_workout_database(
csvfilename,r,
workouttype=workouttype,
title=title,notes=comments,
workoutsource=workoutsource,
dosummary=True
)
return id,message
def workout_strava_upload(user,w):
try:
thetoken = strava_open(user)
except NoTokenError:
return "Please connect to Strava first",0
message = "Uploading to Strava"
stravaid=-1
r = Rower.objects.get(user=user)
res = -1
if (r.stravatoken == '') or (r.stravatoken is None):
s = "Token doesn't exist. Need to authorize"
raise NoTokenError("Your hovercraft is full of eels")
else:
if (checkworkoutuser(user,w)):
try:
tcxfile,tcxmesg = createstravaworkoutdata(w)
if tcxfile:
with open(tcxfile,'rb') as f:
res,mes = handle_stravaexport(
f,w.name,
r.stravatoken,
description=w.notes+'\n from '+w.workoutsource+' via rowsandall.com',
activity_type=r.stravaexportas)
if res==0:
message = mes
w.uploadedtostrava = -1
stravaid = -1
w.save()
try:
os.remove(tcxfile)
except WindowsError:
pass
return message,stravaid
w.uploadedtostrava = res
w.save()
try:
os.remove(tcxfile)
except WindowsError:
pass
message = mes
stravaid = res
return message,stravaid
else:
message = "Strava TCX data error "+tcxmesg
w.uploadedtostrava = -1
stravaid = -1
w.save()
return message, stravaid
except ActivityUploadFailed as e:
message = "Strava Upload error: %s" % e
w.uploadedtostrava = -1
stravaid = -1
w.save()
os.remove(tcxfile)
return message,stravaid
return message,stravaid
return message,stravaid
def handle_strava_import_stroke_data(title,
useremail,
stravatoken,
stravaid,
starttimeunix,
csvfilename,debug=True,
workouttype = 'rower',
**kwargs):
# ready to fetch. Hurray
fetchresolution = 'high'
series_type = 'time'
authorizationstring = str('Bearer ' + stravatoken)
headers = {'Authorization': authorizationstring,
'user-agent': 'sanderroosendaal',
'Content-Type': 'application/json',
'resolution': 'medium',}
url = "https://www.strava.com/api/v3/activities/"+str(stravaid)
workoutsummary = requests.get(url,headers=headers).json()
workoutsummary['timezone'] = "Etc/UTC"
startdatetime = workoutsummary['start_date']
r = type('Rower', (object,), {"stravatoken": stravatoken})
spm = get_strava_stream(r,'cadence',stravaid)
hr = get_strava_stream(r,'heartrate',stravaid)
t = get_strava_stream(r,'time',stravaid)
velo = get_strava_stream(r,'velocity_smooth',stravaid)
d = get_strava_stream(r,'distance',stravaid)
coords = get_strava_stream(r,'latlng',stravaid)
power = get_strava_stream(r,'power',stravaid)
if t is not None:
nr_rows = len(t)
else:
return 0
if nr_rows == 0:
return 0
if d is None:
d = 0*t
if spm is None:
spm = np.zeros(nr_rows)
if power is None:
power = np.zeros(nr_rows)
if hr is None:
hr = np.zeros(nr_rows)
if velo is None:
velo = np.zeros(nr_rows)
f = np.diff(t).mean()
if f != 0:
windowsize = 2*(int(10./(f)))+1
else:
windowsize = 1
if windowsize > 3 and windowsize < len(velo):
velo2 = savgol_filter(velo,windowsize,3)
else:
velo2 = velo
if coords is not None:
try:
lat = coords[:,0]
lon = coords[:,1]
if lat.std() == 0 and lon.std() == 0 and workouttype == 'water':
workouttype = 'rower'
except IndexError:
lat = np.zeros(len(t))
lon = np.zeros(len(t))
if workouttype == 'water':
workouttype = 'rower'
else:
lat = np.zeros(len(t))
lon = np.zeros(len(t))
if workouttype == 'water':
workouttype = 'rower'
strokelength = velo*60./(spm)
strokelength[np.isinf(strokelength)] = 0.0
pace = 500./(1.0*velo2)
pace[np.isinf(pace)] = 0.0
unixtime = starttimeunix+t
strokedistance = 60.*velo2/spm
nr_strokes = len(t)
df = pd.DataFrame({'TimeStamp (sec)':unixtime,
' ElapsedTime (sec)':t,
' Horizontal (meters)':d,
' Stroke500mPace (sec/500m)':pace,
' Cadence (stokes/min)':spm,
' HRCur (bpm)':hr,
' latitude':lat,
' longitude':lon,
' StrokeDistance (meters)':strokelength,
'cum_dist':d,
' DragFactor':np.zeros(nr_strokes),
' DriveLength (meters)':np.zeros(nr_strokes),
' StrokeDistance (meters)':strokedistance,
' DriveTime (ms)':np.zeros(nr_strokes),
' StrokeRecoveryTime (ms)':np.zeros(nr_strokes),
' AverageDriveForce (lbs)':np.zeros(nr_strokes),
' PeakDriveForce (lbs)':np.zeros(nr_strokes),
' lapIdx':np.zeros(nr_strokes),
' Power (watts)':power,
})
df.sort_values(by='TimeStamp (sec)',ascending=True)
res = df.to_csv(csvfilename,index_label='index')
workoutsbox = Mailbox.objects.filter(name='workouts')[0]
body = """stravaid {stravaid}
workouttype {workouttype}""".format(
stravaid=stravaid,
workouttype=workouttype
)
msg = Message(mailbox=workoutsbox,
from_header=useremail,
subject=title,
body=body)
msg.save()
a = MessageAttachment(message=msg,document=csvfilename[6:])
a.save()
return res