Merge branch 'develop' into feature/revisedimports

rowers/stravastuff.py

@@ -1,3 +1,9 @@
+# All the functionality needed to connect to Strava
+from scipy import optimize
+from scipy.signal import savgol_filter
+
+
+from django_mailbox.models import Message,Mailbox,MessageAttachment
 
 import django_rq
 queue = django_rq.get_queue('default')
@@ -12,7 +18,7 @@ from stravalib.exc import ActivityUploadFailed,TimeoutExceeded
 from iso8601 import ParseError
 
 
-from rowers.tasks import handle_strava_import_stroke_data
+
 
 from rowsandall_app.settings import (
     C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET,
@@ -78,7 +84,7 @@ def make_authorization_url(request):
     return HttpResponseRedirect(url)
 
 # Get list of workouts available on Strava
-def get_strava_workout_list(user):
+def get_strava_workout_list(user,limit_n=0):
     r = Rower.objects.get(user=user)
     if (r.stravatoken == '') or (r.stravatoken is None):
 	s = "Token doesn't exist. Need to authorize"
@@ -89,8 +95,15 @@ def get_strava_workout_list(user):
 	headers = {'Authorization': authorizationstring,
 		   'user-agent': 'sanderroosendaal',
 		   'Content-Type': 'application/json'}
+
 	url = "https://www.strava.com/api/v3/athlete/activities"
-	s = requests.get(url,headers=headers)
+
+        if limit_n==0:
+            params = {}
+        else:
+            params = {'per_page':limit_n}
+            
+	s = requests.get(url,headers=headers,params=params)
 
     return s
 
@@ -101,7 +114,7 @@ def get_strava_workouts(rower):
     if not isprorower(rower):
         return 0
     
-    res = get_strava_workout_list(rower.user)
+    res = get_strava_workout_list(rower.user,limit_n=10)
 
     if (res.status_code != 200):
         return 0
@@ -117,6 +130,7 @@ def get_strava_workouts(rower):
             ])
         newids = [stravaid for stravaid in stravaids if not stravaid in knownstravaids]
 
+
         for stravaid in newids:
             result = create_async_workout(alldata,rower.user,stravaid)
 
@@ -182,40 +196,24 @@ def create_async_workout(alldata,user,stravaid,debug=False):
     weightcategory = 'hwt'
             
     # Create CSV file name and save data to CSV file
-    csvfilename ='media/{code}_{importid}.csv'.format(
+    csvfilename ='media/mailbox_attachments/{code}_{importid}.csv'.format(
         importid=stravaid,
         code = uuid4().hex[:16]
         )
     
-#    w = Workout(
-#        user=r,
-#        workouttype = workouttype,
-#        name = title,
-#        date = workoutdate,
-#        starttime = starttime,
-#        startdatetime = rowdatetime,
-#        timezone = thetimezone,
-#        duration = duration,
-#        distance=distance,
-#        weightcategory = weightcategory,
-#        uploadedtostrava = stravaid,
-#        csvfilename = csvfilename,
-#        notes = ''
-#    )
 
 
     # Check if workout has stroke data, and get the stroke data
 
     starttimeunix = arrow.get(rowdatetime).timestamp
 
-    job = myqueue(queue,
+    result = handle_strava_import_stroke_data(
-                  handle_strava_import_stroke_data,
+        title,
-                  title,
+        user.email,
-                  user.email,
+        r.stravatoken,
-                  r.stravatoken,
+        stravaid,
-                  stravaid,
+        starttimeunix,
-                  starttimeunix,
+        csvfilename,
-                  csvfilename,
     )
     
     return 1
@@ -454,141 +452,145 @@ def workout_strava_upload(user,w):
 	return message,stravaid
     return message,stravaid
 
-# Create workout data from Strava or Concept2
-# data and create the associated Workout object and save it
-def add_workout_from_data(user,importid,data,strokedata,
-                          source='strava',splitdata=None,
-                          workoutsource='strava'):
-    try:
-        workouttype = data['type']
-    except KeyError:
-        workouttype = 'rower'
-        
-    if workouttype not in [x[0] for x in Workout.workouttypes]:
-	workouttype = 'other'
-    try:
-	comments = data['comments']
-    except:
-	comments = ' '
 
-    try:
+def handle_strava_import_stroke_data(title,
-	thetimezone = tz(data['timezone'])
+                                     useremail,
-    except:
+                                     stravatoken,
-	thetimezone = 'UTC'
+                                     stravaid,
+                                     starttimeunix,
+                                     csvfilename,debug=True,**kwargs):
+    # ready to fetch. Hurray
 
-    r = Rower.objects.get(user=user)
+    fetchresolution = 'high'
-    try:
+    series_type = 'time'
-	rowdatetime = iso8601.parse_date(data['date_utc'])
+    authorizationstring = str('Bearer ' + stravatoken)
-    except KeyError:
+    headers = {'Authorization': authorizationstring,
-	rowdatetime = iso8601.parse_date(data['start_date'])
+	       'user-agent': 'sanderroosendaal',
-    except ParseError:
+	       'Content-Type': 'application/json',
-	rowdatetime = iso8601.parse_date(data['date'])
+	       'resolution': 'medium',}
+    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)
+    workoutsummary = requests.get(url,headers=headers).json()
+
+    workoutsummary['timezone'] = "Etc/UTC"
+    startdatetime = workoutsummary['start_date']
+
+    r = type('Rower', (object,), {"stravatoken": stravatoken})
     
-	
+    spmjson = get_strava_stream(r,'cadence',stravaid)
+    hrjson = get_strava_stream(r,'heartrate',stravaid)
+    timejson = get_strava_stream(r,'time',stravaid)
+    velojson = get_strava_stream(r,'velocity_smooth',stravaid)
+    distancejson = get_strava_stream(r,'distance',stravaid)
+    latlongjson = get_strava_stream(r,'latlng',stravaid)
+    wattsjson = get_strava_stream(r,'watts',stravaid)
+
     try:
-        intervaltype = data['workout_type']
+	t = np.array(timejson.json()[0]['data'])
-	
+	nr_rows = len(t)
+	d =  np.array(distancejson.json()[1]['data'])
+        if nr_rows == 0:
+            return 0
+    except IndexError:
+        d = 0*t
+        # return (0,"Error: No Distance information in the Strava data")
     except KeyError:
-	intervaltype = ''
+        return 0
-	
-    try:
-	title = data['name']
-    except KeyError:
-	title = ""
-	try:
-	    t = data['comments'].split('\n', 1)[0]
-	    title += t[:20]
-	except:
-	    title = 'Imported'
-
-    starttimeunix = arrow.get(rowdatetime).timestamp
-
-    res = make_cumvalues(0.1*strokedata['t'])
-    cum_time = res[0]
-    lapidx = res[1]
-
-    unixtime = cum_time+starttimeunix
-    seconds = 0.1*strokedata.ix[:,'t']
-
-    nr_rows = len(unixtime)
 
     try:
-	latcoord = strokedata.ix[:,'lat']
+	spm = np.array(spmjson.json()[1]['data'])
-	loncoord = strokedata.ix[:,'lon']
     except:
-	latcoord = np.zeros(nr_rows)
+	spm = np.zeros(nr_rows)
-	loncoord = np.zeros(nr_rows)
-
 
     try:
-	strokelength = strokedata.ix[:,'strokelength']
+        watts = np.array(wattsjson.json()[1]['data'])
     except:
-	strokelength = np.zeros(nr_rows)
+        watts = np.zeros(nr_rows)
-
-    dist2 = 0.1*strokedata.ix[:,'d']
 
     try:
-        spm = strokedata.ix[:,'spm']
+	hr = np.array(hrjson.json()[1]['data'])
+    except IndexError:
+	hr = np.zeros(nr_rows)
     except KeyError:
-        spm = 0*dist2
+	hr = np.zeros(nr_rows)
+
+    try:
+	velo = np.array(velojson.json()[1]['data'])
+    except IndexError:
+	velo = np.zeros(nr_rows)
+    except KeyError:
+        velo = np.zeros(nr_rows)
+
+    f = np.diff(t).mean()
+    if f != 0:
+        windowsize = 2*(int(10./(f)))+1
+    else:
+        windowsize = 1
+
+    if windowsize > 3 and windowsize < len(velo):
+        velo2 = savgol_filter(velo,windowsize,3)
+    else:
+        velo2 = velo
         
+    coords = np.array(latlongjson.json()[0]['data'])
     try:
-        hr = strokedata.ix[:,'hr']
+	lat = coords[:,0]
+	lon = coords[:,1]
+    except IndexError:
+	lat = np.zeros(len(t))
+	lon = np.zeros(len(t))
     except KeyError:
-        hr = 0*spm
+	lat = np.zeros(len(t))
-    pace = strokedata.ix[:,'p']/10.
+	lon = np.zeros(len(t))
-    pace = np.clip(pace,0,1e4)
-    pace = pace.replace(0,300)
 
-    velo = 500./pace
+    strokelength = velo*60./(spm)
+    strokelength[np.isinf(strokelength)] = 0.0
 
-    power = 2.8*velo**3
+    pace = 500./(1.0*velo2)
+    pace[np.isinf(pace)] = 0.0
+
+    unixtime = starttimeunix+t
+
+    strokedistance = 60.*velo2/spm
     
-    # save csv
+    nr_strokes = len(t)
-    # Create data frame with all necessary data to write to csv
+
     df = pd.DataFrame({'TimeStamp (sec)':unixtime,
-		       ' Horizontal (meters)': dist2,
+                       ' ElapsedTime (sec)':t,
+		       ' Horizontal (meters)':d,
+		       ' Stroke500mPace (sec/500m)':pace,
 		       ' Cadence (stokes/min)':spm,
 		       ' HRCur (bpm)':hr,
-		       ' longitude':loncoord,
+		       ' latitude':lat,
-		       ' latitude':latcoord,
+		       ' longitude':lon,
-		       ' Stroke500mPace (sec/500m)':pace,
-		       ' Power (watts)':power,
-		       ' DragFactor':np.zeros(nr_rows),
-		       ' DriveLength (meters)':np.zeros(nr_rows),
 		       ' StrokeDistance (meters)':strokelength,
-		       ' DriveTime (ms)':np.zeros(nr_rows),
+                       'cum_dist':d,
-		       ' StrokeRecoveryTime (ms)':np.zeros(nr_rows),
+	               ' DragFactor':np.zeros(nr_strokes),
-		       ' AverageDriveForce (lbs)':np.zeros(nr_rows),
+	               ' DriveLength (meters)':np.zeros(nr_strokes),
-		       ' PeakDriveForce (lbs)':np.zeros(nr_rows),
+	               ' StrokeDistance (meters)':strokedistance,
-		       ' lapIdx':lapidx,
+	               ' DriveTime (ms)':np.zeros(nr_strokes),
-		       ' ElapsedTime (sec)':seconds
+	               ' StrokeRecoveryTime (ms)':np.zeros(nr_strokes),
-		       })
+	               ' AverageDriveForce (lbs)':np.zeros(nr_strokes),
+	               ' PeakDriveForce (lbs)':np.zeros(nr_strokes),
+	               ' lapIdx':np.zeros(nr_strokes),
+                       ' Power (watts)':watts,
+    })
+
 
-    
     df.sort_values(by='TimeStamp (sec)',ascending=True)
-	
-    timestr = strftime("%Y%m%d-%H%M%S")
 
+    res = df.to_csv(csvfilename,index_label='index')
 
-    # Create CSV file name and save data to CSV file
+    workoutsbox = Mailbox.objects.filter(name='workouts')[0]
-    csvfilename ='media/{code}_{importid}.csv'.format(
-        importid=importid,
-        code = uuid4().hex[:16]
-        )
 
-    res =  df.to_csv(csvfilename+'.gz',index_label='index',
+    body = 'stravaid {stravaid}'.format(stravaid=stravaid)
-                     compression='gzip')
+    
+    msg = Message(mailbox=workoutsbox,
+                  from_header=useremail,
+                  subject=title,
+                  body=body)
+    msg.save()
+    
+    a = MessageAttachment(message=msg,document=csvfilename[6:])
+    a.save()
 
-
+    return res
-    id,message = dataprep.save_workout_database(
-        csvfilename,r,
-        workouttype=workouttype,
-        title=title,notes=comments,
-        workoutsource=workoutsource,
-        dosummary=True
-    )
-
-            
-
-    return id,message

rowers/tasks.py

@@ -82,148 +82,6 @@ def add(x, y):
     return x + y
 
 
-@app.task
-def handle_strava_import_stroke_data(title,
-                                     useremail,
-                                     stravatoken,
-                                     stravaid,
-                                     starttimeunix,
-                                     csvfilename,debug=True,**kwargs):
-    # ready to fetch. Hurray
-    fetchresolution = 'high'
-    series_type = 'time'
-    authorizationstring = str('Bearer ' + stravatoken)
-    headers = {'Authorization': authorizationstring,
-	       'user-agent': 'sanderroosendaal',
-	       'Content-Type': 'application/json',
-	       'resolution': 'medium',}
-    url = "https://www.strava.com/api/v3/activities/"+str(stravaid)
-    workoutsummary = requests.get(url,headers=headers).json()
-
-    workoutsummary['timezone'] = "Etc/UTC"
-    startdatetime = workoutsummary['start_date']
-
-    r = type('Rower', (object,), {"stravatoken": stravatoken})
-    
-    spmjson = get_strava_stream(r,'cadence',stravaid)
-    hrjson = get_strava_stream(r,'heartrate',stravaid)
-    timejson = get_strava_stream(r,'time',stravaid)
-    velojson = get_strava_stream(r,'velocity_smooth',stravaid)
-    distancejson = get_strava_stream(r,'distance',stravaid)
-    latlongjson = get_strava_stream(r,'latlng',stravaid)
-    wattsjson = get_strava_stream(r,'watts',stravaid)
-
-    try:
-	t = np.array(timejson.json()[0]['data'])
-	nr_rows = len(t)
-	d =  np.array(distancejson.json()[1]['data'])
-        if nr_rows == 0:
-            return 0
-    except IndexError:
-        d = 0*t
-        # return (0,"Error: No Distance information in the Strava data")
-    except KeyError:
-        return 0
-
-    try:
-	spm = np.array(spmjson.json()[1]['data'])
-    except:
-	spm = np.zeros(nr_rows)
-
-    try:
-        watts = np.array(wattsjson.json()[1]['data'])
-    except:
-        watts = np.zeros(nr_rows)
-
-    try:
-	hr = np.array(hrjson.json()[1]['data'])
-    except IndexError:
-	hr = np.zeros(nr_rows)
-    except KeyError:
-	hr = np.zeros(nr_rows)
-
-    try:
-	velo = np.array(velojson.json()[1]['data'])
-    except IndexError:
-	velo = np.zeros(nr_rows)
-    except KeyError:
-        velo = np.zeros(nr_rows)
-
-    f = np.diff(t).mean()
-    if f != 0:
-        windowsize = 2*(int(10./(f)))+1
-    else:
-        windowsize = 1
-
-    if windowsize > 3 and windowsize < len(velo):
-        velo2 = savgol_filter(velo,windowsize,3)
-    else:
-        velo2 = velo
-        
-    coords = np.array(latlongjson.json()[0]['data'])
-    try:
-	lat = coords[:,0]
-	lon = coords[:,1]
-    except IndexError:
-	lat = np.zeros(len(t))
-	lon = np.zeros(len(t))
-    except KeyError:
-	lat = np.zeros(len(t))
-	lon = np.zeros(len(t))
-
-    strokelength = velo*60./(spm)
-    strokelength[np.isinf(strokelength)] = 0.0
-
-    pace = 500./(1.0*velo2)
-    pace[np.isinf(pace)] = 0.0
-
-    unixtime = starttimeunix+t
-
-    strokedistance = 60.*velo2/spm
-    
-    nr_strokes = len(t)
-
-    df = pd.DataFrame({'TimeStamp (sec)':unixtime,
-                       ' ElapsedTime (sec)':t,
-		       ' Horizontal (meters)':d,
-		       ' Stroke500mPace (sec/500m)':pace,
-		       ' Cadence (stokes/min)':spm,
-		       ' HRCur (bpm)':hr,
-		       ' latitude':lat,
-		       ' longitude':lon,
-		       ' StrokeDistance (meters)':strokelength,
-                       'cum_dist':d,
-	               ' DragFactor':np.zeros(nr_strokes),
-	               ' DriveLength (meters)':np.zeros(nr_strokes),
-	               ' StrokeDistance (meters)':strokedistance,
-	               ' DriveTime (ms)':np.zeros(nr_strokes),
-	               ' StrokeRecoveryTime (ms)':np.zeros(nr_strokes),
-	               ' AverageDriveForce (lbs)':np.zeros(nr_strokes),
-	               ' PeakDriveForce (lbs)':np.zeros(nr_strokes),
-	               ' lapIdx':np.zeros(nr_strokes),
-                       ' Power (watts)':watts,
-    })
-
-
-    df.sort_values(by='TimeStamp (sec)',ascending=True)
-
-    res = df.to_csv(csvfilename,index_label='index')
-
-    d = {
-        'stravaid':stravaid
-        }
-    
-    res = send_template_email(useremail,[workoutemailbox],
-                              title,'workoutemail.html',
-                              d,
-                              attach_file=csvfilename)
-
-
-    time.sleep(1)
-    
-    os.remove(csvfilename)
-
-    return res
     
 
 @app.task