diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index dc55a821..0e27bad9 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -1,3 +1,8 @@ +# The interactions with the Concept2 logbook API +# All C2 related functions should be defined here +# (There is still some stuff defined directly in views.py. Need to +# move that here.) + # Python import oauth2 as oauth import cgi @@ -17,8 +22,7 @@ from django.contrib.auth import authenticate, login, logout from django.contrib.auth.models import User from django.contrib.auth.decorators import login_required -# Project -# from .models import Profile + from rowingdata import rowingdata import pandas as pd import numpy as np @@ -27,9 +31,9 @@ import sys import urllib from requests import Request, Session - from rowsandall_app.settings import C2_CLIENT_ID, C2_REDIRECT_URI, C2_CLIENT_SECRET +# Custom error class - to raise a NoTokenError class C2NoTokenError(Exception): def __init__(self,value): self.value=value @@ -37,8 +41,8 @@ class C2NoTokenError(Exception): def __str__(self): return repr(self.value) - - +# Custom exception handler, returns a 401 HTTP message +# with exception details in the json data def custom_exception_handler(exc,message): response = { @@ -56,7 +60,7 @@ def custom_exception_handler(exc,message): return res - +# Check if workout is owned by this user def checkworkoutuser(user,workout): try: r = Rower.objects.get(user=user) @@ -64,11 +68,12 @@ def checkworkoutuser(user,workout): except Rower.DoesNotExist: return(False) - +# convert datetime object to seconds def makeseconds(t): seconds = t.hour*3600.+t.minute*60.+t.second+0.1*int(t.microsecond/1.e5) return seconds +# convert our weight class code to Concept2 weight class code def c2wc(weightclass): if (weightclass=="lwt"): res = "L" @@ -77,7 +82,9 @@ def c2wc(weightclass): return res - +# Concept2 logbook sends over split data for each interval +# We use it here to generate a custom summary +# Some users complained about small differences def summaryfromsplitdata(splitdata,data,filename,sep='|'): totaldist = data['distance'] @@ -177,6 +184,8 @@ def summaryfromsplitdata(splitdata,data,filename,sep='|'): return sums,sa,results +# Not used now. Could be used to add workout split data to Concept2 +# logbook but needs to be reviewed. def createc2workoutdata_as_splits(w): filename = w.csvfilename row = rowingdata(filename) @@ -216,7 +225,6 @@ def createc2workoutdata_as_splits(w): data = { "type": w.workouttype, -# "date": str(w.date)+" "+str(w.starttime), "date": w.startdatetime.isoformat(), "distance": int(w.distance), "time": int(10*makeseconds(durationstr)), @@ -233,59 +241,8 @@ def createc2workoutdata_as_splits(w): return data -def createc2workoutdata_grouped(w): - filename = w.csvfilename - row = rowingdata(filename) - - # resize per minute - df = row.df.groupby(lambda x:x/10).mean() - - averagehr = int(df[' HRCur (bpm)'].mean()) - maxhr = int(df[' HRCur (bpm)'].max()) - - # adding diff, trying to see if this is valid - t = 10*df.ix[:,' ElapsedTime (sec)'].values - t[0] = t[1] - d = df.ix[:,' Horizontal (meters)'].values - d[0] = d[1] - p = 10*df.ix[:,' Stroke500mPace (sec/500m)'].values - t = t.astype(int) - d = d.astype(int) - p = p.astype(int) - spm = df[' Cadence (stokes/min)'].astype(int) - spm[0] = spm[1] - hr = df[' HRCur (bpm)'].astype(int) - stroke_data = [] - for i in range(len(t)): - thisrecord = {"t":t[i],"d":d[i],"p":p[i],"spm":spm[i],"hr":hr[i]} - stroke_data.append(thisrecord) - - - try: - durationstr = datetime.strptime(str(w.duration),"%H:%M:%S.%f") - except ValueError: - durationstr = datetime.strptime(str(w.duration),"%H:%M:%S") - - - data = { - "type": w.workouttype, -# "date": str(w.date)+" "+str(w.starttime), - "date": w.startdatetime.isoformat(), - "distance": int(w.distance), - "time": int(10*makeseconds(durationstr)), - "weight_class": c2wc(w.weightcategory), - "timezone": "Etc/UTC", - "comments": w.notes, - "heart_rate": { - "average": averagehr, - "max": maxhr, - }, - "stroke_data": stroke_data, - } - - - return data - +# Create the Data object for the stroke data to be sent to Concept2 logbook +# API def createc2workoutdata(w): filename = w.csvfilename row = rowingdata(filename) @@ -318,7 +275,6 @@ def createc2workoutdata(w): data = { "type": w.workouttype, -# "date": str(w.date)+" "+str(w.starttime), "date": w.startdatetime.isoformat(), "timezone": "Etc/UTC", "distance": int(w.distance), @@ -335,6 +291,7 @@ def createc2workoutdata(w): return data +# Refresh Concept2 authorization token def do_refresh_token(refreshtoken): scope = "results:write,user:read" client_auth = requests.auth.HTTPBasicAuth(C2_CLIENT_ID, C2_CLIENT_SECRET) @@ -347,10 +304,6 @@ def do_refresh_token(refreshtoken): url = "https://log.concept2.com/oauth/access_token" s = Session() req = Request('POST',url, data=post_data, headers=headers) - # response = requests.post("https://log.concept2.com/oauth/access_token", - # data=post_data, - # data=post_data, - # headers=headers) prepped = req.prepare() prepped.body+="&scope=" @@ -374,13 +327,12 @@ def do_refresh_token(refreshtoken): return [thetoken,expires_in,refresh_token] - +# Exchange authorization code for authorization token def get_token(code): scope = "user:read,results:write" client_auth = requests.auth.HTTPBasicAuth(C2_CLIENT_ID, C2_CLIENT_SECRET) post_data = {"grant_type": "authorization_code", "code": code, -# "scope": scope, "redirect_uri": C2_REDIRECT_URI, "client_secret": C2_CLIENT_SECRET, "client_id":C2_CLIENT_ID, @@ -396,9 +348,6 @@ def get_token(code): response = s.send(prepped) -# response = requests.post("https://log.concept2.com/oauth/access_token", -# data=post_data, -# headers=headers) token_json = response.json() thetoken = token_json['access_token'] expires_in = token_json['expires_in'] @@ -406,6 +355,7 @@ def get_token(code): return [thetoken,expires_in,refresh_token] +# Make URL for authorization and load it def make_authorization_url(request): # Generate a random string for the state parameter # Save it for use later to prevent xsrf attacks @@ -421,6 +371,7 @@ def make_authorization_url(request): return HttpResponseRedirect(url) +# Get workout from C2 ID def get_c2_workout(user,c2id): r = Rower.objects.get(user=user) if (r.c2token == '') or (r.c2token is None): @@ -440,6 +391,7 @@ def get_c2_workout(user,c2id): return s +# Get stroke data belonging to C2 ID def get_c2_workout_strokes(user,c2id): r = Rower.objects.get(user=user) if (r.c2token == '') or (r.c2token is None): @@ -459,6 +411,8 @@ def get_c2_workout_strokes(user,c2id): return s +# Get list of C2 workouts. We load only the first page, +# assuming that users don't want to import their old workouts def get_c2_workout_list(user): r = Rower.objects.get(user=user) if (r.c2token == '') or (r.c2token is None): @@ -479,7 +433,8 @@ def get_c2_workout_list(user): return s - +# Get username, having access token. +# Handy for checking if the API access is working def get_username(access_token): authorizationstring = str('Bearer ' + access_token) headers = {'Authorization': authorizationstring, @@ -495,6 +450,8 @@ def get_username(access_token): return me_json['data']['username'] +# Get user id, having access token +# Handy for checking if the API access is working def get_userid(access_token): authorizationstring = str('Bearer ' + access_token) headers = {'Authorization': authorizationstring, @@ -510,6 +467,7 @@ def get_userid(access_token): return me_json['data']['id'] +# For debugging purposes def process_callback(request): # need error handling @@ -521,6 +479,7 @@ def process_callback(request): return HttpResponse("got a user name: %s" % username) +# Uploading workout def workout_c2_upload(user,w): response = 'trying C2 upload' r = Rower.objects.get(user=user) @@ -535,8 +494,6 @@ def workout_c2_upload(user,w): if (checkworkoutuser(user,w)): c2userid = get_userid(r.c2token) data = createc2workoutdata(w) - # if (w.workouttype=='water'): - # data = createc2workoutdata_as_splits(w) authorizationstring = str('Bearer ' + r.c2token) headers = {'Authorization': authorizationstring, 'user-agent': 'sanderroosendaal', @@ -554,6 +511,7 @@ def workout_c2_upload(user,w): return response +# This is token refresh. Looks for tokens in our database, then refreshes def rower_c2_token_refresh(user): r = Rower.objects.get(user=user) res = do_refresh_token(r.c2refreshtoken) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 482e525d..816f08cf 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1,3 +1,5 @@ +# All the data preparation, data cleaning and data mangling should +# be defined here from rowers.models import Workout, User, Rower from rowingdata import rowingdata as rrdata @@ -37,11 +39,13 @@ database_url = 'mysql://{user}:{password}@{host}:{port}/{database_name}'.format( port=port, ) +# Use SQLite local database when we're in debug mode if settings.DEBUG or user=='': # database_url = 'sqlite:///db.sqlite3' database_url = 'sqlite:///'+database_name +# mapping the DB column names to the CSV file column names columndict = { 'time':'TimeStamp (sec)', 'hr':' HRCur (bpm)', @@ -63,6 +67,7 @@ from scipy.signal import savgol_filter import datetime +# A string representation for time deltas def niceformat(values): out = [] for v in values: @@ -71,6 +76,7 @@ def niceformat(values): return out +# A nice printable format for time delta values def strfdelta(tdelta): try: minutes,seconds = divmod(tdelta.seconds,60) @@ -87,6 +93,7 @@ def strfdelta(tdelta): return res +# A nice printable format for pace values def nicepaceformat(values): out = [] for v in values: @@ -96,6 +103,7 @@ def nicepaceformat(values): return out +# Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace def timedeltaconv(x): if not np.isnan(x): dt = datetime.timedelta(seconds=x) @@ -105,6 +113,9 @@ def timedeltaconv(x): return dt +# Create new workout from file and store it in the database +# This routine should be used everywhere in views.py and mailprocessing.pu +# Currently there is code duplication def new_workout_from_file(r,f2, workouttype='rower', title='Workout', @@ -263,6 +274,9 @@ def new_workout_from_file(r,f2, return True +# Compare the data from the CSV file and the database +# Currently only calculates number of strokes. To be expanded with +# more elaborate testing if needed def compare_data(id): row = Workout.objects.get(id=id) f1 = row.csvfilename @@ -288,6 +302,8 @@ def compare_data(id): ldb = l2 return l1==l2,ldb,lfile +# Repair data for workouts where the CSV file is lost (or the DB entries +# don't exist) def repair_data(verbose=False): ws = Workout.objects.all() for w in ws: @@ -319,6 +335,7 @@ def repair_data(verbose=False): print str(sys.exc_info()[0]) pass +# A wrapper around the rowingdata class, with some error catching def rdata(file,rower=rrower()): try: res = rrdata(file,rower=rower) @@ -330,6 +347,7 @@ def rdata(file,rower=rrower()): return res +# Remove all stroke data for workout ID from database def delete_strokedata(id): engine = create_engine(database_url, echo=False) query = sa.text('DELETE FROM strokedata WHERE workoutid={id};'.format( @@ -343,10 +361,12 @@ def delete_strokedata(id): conn.close() engine.dispose() +# Replace stroke data in DB with data from CSV file def update_strokedata(id,df): delete_strokedata(id) rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True) - + +# Test that all data are of a numerical time def testdata(time,distance,pace,spm): t1 = np.issubdtype(time,np.number) t2 = np.issubdtype(distance,np.number) @@ -355,6 +375,8 @@ def testdata(time,distance,pace,spm): return t1 and t2 and t3 and t4 +# Get data from DB for one workout (fetches all data). If data +# is not in DB, read from CSV file (and create DB entry) def getrowdata_db(id=0): data = read_df_sql(id) data['x_right'] = data['x_right']/1.0e6 @@ -369,12 +391,14 @@ def getrowdata_db(id=0): return data,row +# Fetch a subset of the data from the DB def getsmallrowdata_db(columns,ids=[]): prepmultipledata(ids) data = read_cols_df_sql(ids,columns) return data - + +# Fetch both the workout and the workout stroke data (from CSV file) def getrowdata(id=0): # check if valid ID exists (workout exists) @@ -395,7 +419,12 @@ def getrowdata(id=0): return rowdata,row - +# Checks if all rows for a list of workout IDs have entries in the +# stroke_data table. If this is not the case, it creates the stroke +# data +# In theory, this should never yield any work, but it's a good +# safety net for programming errors elsewhere in the app +# Also used heavily when I moved from CSV file only to CSV+Stroke data def prepmultipledata(ids,verbose=False): query = sa.text('SELECT DISTINCT workoutid FROM strokedata') engine = create_engine(database_url, echo=False) @@ -420,6 +449,8 @@ def prepmultipledata(ids,verbose=False): data = dataprep(rowdata.df,id=id,bands=True,barchart=True,otwpower=True) return res +# Read a set of columns for a set of workout ids, returns data as a +# pandas dataframe def read_cols_df_sql(ids,columns): columns = list(columns)+['distance','spm'] columns = [x for x in columns if x != 'None'] @@ -450,7 +481,7 @@ def read_cols_df_sql(ids,columns): engine.dispose() return df - +# Read stroke data from the DB for a Workout ID. Returns a pandas dataframe def read_df_sql(id): engine = create_engine(database_url, echo=False) @@ -460,10 +491,8 @@ def read_df_sql(id): engine.dispose() return df - - - - +# Get the necessary data from the strokedata table in the DB. +# For the flex plot def smalldataprep(therows,xparam,yparam1,yparam2): df = pd.DataFrame() if yparam2 == 'None': @@ -503,7 +532,10 @@ def smalldataprep(therows,xparam,yparam1,yparam2): return df - +# This is the main routine. +# it reindexes, sorts, filters, and smooths the data, then +# saves it to the stroke_data table in the database +# Takes a rowingdata object's DataFrame as input def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, empower=True): rowdatadf.set_index([range(len(rowdatadf))],inplace=True) diff --git a/rowers/temp.py b/rowers/temp.py index 8f3b3203..71fc7a26 100644 --- a/rowers/temp.py +++ b/rowers/temp.py @@ -1,6 +1,58 @@ # This is just a scratch pad to temporarily park code, just in case I need # it later. Hardly used since i have proper versioning +# +def createc2workoutdata_grouped(w): + filename = w.csvfilename + row = rowingdata(filename) + + # resize per minute + df = row.df.groupby(lambda x:x/10).mean() + + averagehr = int(df[' HRCur (bpm)'].mean()) + maxhr = int(df[' HRCur (bpm)'].max()) + + # adding diff, trying to see if this is valid + t = 10*df.ix[:,' ElapsedTime (sec)'].values + t[0] = t[1] + d = df.ix[:,' Horizontal (meters)'].values + d[0] = d[1] + p = 10*df.ix[:,' Stroke500mPace (sec/500m)'].values + t = t.astype(int) + d = d.astype(int) + p = p.astype(int) + spm = df[' Cadence (stokes/min)'].astype(int) + spm[0] = spm[1] + hr = df[' HRCur (bpm)'].astype(int) + stroke_data = [] + for i in range(len(t)): + thisrecord = {"t":t[i],"d":d[i],"p":p[i],"spm":spm[i],"hr":hr[i]} + stroke_data.append(thisrecord) + + + try: + durationstr = datetime.strptime(str(w.duration),"%H:%M:%S.%f") + except ValueError: + durationstr = datetime.strptime(str(w.duration),"%H:%M:%S") + + + data = { + "type": w.workouttype, + "date": w.startdatetime.isoformat(), + "distance": int(w.distance), + "time": int(10*makeseconds(durationstr)), + "weight_class": c2wc(w.weightcategory), + "timezone": "Etc/UTC", + "comments": w.notes, + "heart_rate": { + "average": averagehr, + "max": maxhr, + }, + "stroke_data": stroke_data, + } + + + return data @login_required() def workout_edit_view(request,id=0): diff --git a/rowers/tests.py b/rowers/tests.py index 7b47e031..9b2310f4 100644 --- a/rowers/tests.py +++ b/rowers/tests.py @@ -1037,8 +1037,7 @@ class subroutinetests(TestCase): jsond = json.dumps(data) data = c2stuff.createc2workoutdata_as_splits(w) jsond = json.dumps(data) - data = c2stuff.createc2workoutdata_as_grouped(w) - jsond = json.dumps(data) + class PlotTests(TestCase):