from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals # All the data preparation, data cleaning and data mangling should # be defined here from __future__ import unicode_literals, absolute_import from rowers.models import ( Workout, Team, CalcAgePerformance,C2WorldClassAgePerformance, User ) import pytz import collections from rowingdata import rowingdata as rrdata from rowingdata import rower as rrower import yaml import shutil from shutil import copyfile from rowingdata import ( get_file_type, get_empower_rigging,get_empower_firmware ) from rowers.tasks import ( handle_sendemail_unrecognized,handle_setcp, handle_getagegrouprecords ) from rowers.tasks import handle_zip_file from pandas import DataFrame, Series import dask.dataframe as dd from dask.delayed import delayed import pyarrow.parquet as pq import pyarrow as pa from pyarrow.lib import ArrowInvalid from django.utils import timezone from django.utils.timezone import get_current_timezone from django_mailbox.models import Message,Mailbox,MessageAttachment from django.core.exceptions import ValidationError from time import strftime import arrow thetimezone = get_current_timezone() from rowingdata import ( TCXParser, RowProParser, ErgDataParser, CoxMateParser, BoatCoachParser, RowPerfectParser, BoatCoachAdvancedParser, ETHParser, MysteryParser, BoatCoachOTWParser,QuiskeParser, painsledDesktopParser, speedcoachParser, ErgStickParser, SpeedCoach2Parser, FITParser, fitsummarydata, RitmoTimeParser,KinoMapParser,NKLiNKLogbookParser, make_cumvalues,cumcpdata,ExcelTemplate, summarydata, get_file_type, ) from rowingdata.csvparsers import HumonParser from rowers.metrics import axes,calc_trimp,rowingmetrics,dtypes,metricsgroups from rowers.models import strokedatafields #allowedcolumns = [item[0] for item in rowingmetrics] allowedcolumns = [key for key,value in strokedatafields.items()] #from async_messages import messages as a_messages import os import zipfile import pandas as pd import numpy as np import itertools from fitparse import FitFile from fitparse.base import FitHeaderError import math from rowers.tasks import ( handle_sendemail_unrecognized, handle_sendemail_breakthrough, handle_sendemail_hard, handle_updatecp,handle_updateergcp, handle_calctrimp, ) from django.conf import settings from sqlalchemy import create_engine import sqlalchemy as sa import sys import rowers.utils as utils import rowers.datautils as datautils from rowers.utils import lbstoN,myqueue,wavg from timezonefinder import TimezoneFinder import django_rq queue = django_rq.get_queue('default') queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') from rowsandall_app.settings import SITE_URL from rowers.mytypes import otwtypes,otetypes,rowtypes from rowers import mytypes from rowers.database import * from rowers.opaque import encoder # mapping the DB column names to the CSV file column names columndict = { 'time': 'TimeStamp (sec)', 'hr': ' HRCur (bpm)', 'velo': ' AverageBoatSpeed (m/s)', 'pace': ' Stroke500mPace (sec/500m)', 'spm': ' Cadence (stokes/min)', 'power': ' Power (watts)', 'averageforce': ' AverageDriveForce (lbs)', 'drivelength': ' DriveLength (meters)', 'peakforce': ' PeakDriveForce (lbs)', 'distance': ' Horizontal (meters)', 'catch': 'catch', 'finish': 'finish', 'peakforceangle': 'peakforceangle', 'wash': 'wash', 'slip': 'slip', 'workoutstate': ' WorkoutState', 'cumdist': 'cum_dist', } from scipy.signal import savgol_filter import datetime def get_video_data(w,groups=['basic'],mode='water'): modes = [mode,'both','basic'] columns = ['time','velo','spm'] columns += [name for name,d in rowingmetrics if d['group'] in groups and d['mode'] in modes] columns = list(set(columns)) df = getsmallrowdata_db(columns,ids=[w.id], workstrokesonly=False,doclean=False,compute=False) df['time'] = (df['time']-df['time'].min())/1000. df.sort_values(by='time',inplace=True) df.set_index(pd.to_timedelta(df['time'],unit='s'),inplace=True) df2 = df.resample('1s').first().fillna(method='ffill') df2['time'] = df2.index.total_seconds() if 'pace' in columns: df2['pace'] = df2['pace']/1000. p = df2['pace'] p = p.apply(lambda x:timedeltaconv(x)) p = nicepaceformat(p) df2['pace'] = p #mask = df2['time'] < delay #df2 = df2.mask(mask).dropna() df2['time'] = (df2['time']-df2['time'].min()) df2 = df2.round(decimals=2) boatspeed = (100*df2['velo']).astype(int)/100. try: coordinates = get_latlon_time(w.id) except KeyError: # pragma: no cover nulseries = df['time']*0 coordinates = pd.DataFrame({ 'time': df['time'], 'latitude': nulseries, 'longitude': nulseries, }) coordinates.set_index(pd.to_timedelta(coordinates['time'],unit='s'),inplace=True) coordinates = coordinates.resample('1s').mean().interpolate() #mask = coordinates['time'] < delay #coordinates = coordinates.mask(mask).dropna() coordinates['time'] = coordinates['time']-coordinates['time'].min() latitude = coordinates['latitude'] longitude = coordinates['longitude'] # bundle data data = { 'boatspeed':boatspeed.values.tolist(), 'latitude':latitude.values.tolist(), 'longitude':longitude.values.tolist(), } metrics = {} for c in columns: if c != 'time': try: if dict(rowingmetrics)[c]['numtype'] == 'integer': # pragma: no cover data[c] = df2[c].astype(int).tolist() else: sigfigs = dict(rowingmetrics)[c]['sigfigs'] if (c != 'pace'): da = ((10**sigfigs)*df2[c]).astype(int)/(10**sigfigs) else: da = df2[c] data[c] = da.values.tolist() metrics[c] = { 'name': dict(rowingmetrics)[c]['verbose_name'], 'metric': c, 'unit': '' } except KeyError: # pragma: no cover pass metrics['boatspeed'] = metrics.pop('velo') # metrics['workperstroke'] = metrics.pop('driveenergy') metrics = collections.OrderedDict(sorted(metrics.items())) maxtime = coordinates['time'].max() return data, metrics, maxtime def polarization_index(df,rower): df['dt'] = df['time'].diff()/6.e4 # remove rest (spm<15) df.dropna(axis=0,inplace=True) df['dt'] = df['dt'].clip(upper=4,lower=0) masklow = (df['power']>0) & (df['power']=rower.pw_at) & (df['power']rower.pw_an) time_low_pw = df.loc[masklow,'dt'].sum() time_mid_pw = df.loc[maskmid,'dt'].sum() time_high_pw = df.loc[maskhigh,'dt'].sum() frac_low = time_low_pw/(time_low_pw+time_mid_pw+time_high_pw) frac_mid = time_mid_pw/(time_low_pw+time_mid_pw+time_high_pw) frac_high = time_high_pw/(time_low_pw+time_mid_pw+time_high_pw) index = math.log10(frac_high*100.*frac_low/frac_mid) return index def get_latlon(id): try: w = Workout.objects.get(id=id) except Workout.DoesNotExist: # pragma: no cover return False rowdata = rdata(w.csvfilename) if rowdata.df.empty: # pragma: no cover return [pd.Series([],dtype='float'), pd.Series([],dtype='float')] try: try: latitude = rowdata.df.loc[:, ' latitude'] longitude = rowdata.df.loc[:, ' longitude'] except KeyError: latitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] longitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] return [latitude, longitude] except AttributeError: # pragma: no cover return [pd.Series([],dtype='float'), pd.Series([],dtype='float')] return [pd.Series([],dtype='float'), pd.Series([],dtype='float')] # pragma: no cover def get_latlon_time(id): try: w = Workout.objects.get(id=id) except Workout.DoesNotExist: # pragma: no cover return False rowdata = rdata(w.csvfilename) if rowdata.df.empty: # pragma: no cover return [pd.Series([],dtype='float'), pd.Series([],dtype='float')] try: try: latitude = rowdata.df.loc[:, ' latitude'] longitude = rowdata.df.loc[:, ' longitude'] except KeyError: # pragma: no cover latitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] longitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] except AttributeError: # pragma: no cover return pd.DataFrame() df = pd.DataFrame({ 'time': rowdata.df['TimeStamp (sec)']-rowdata.df['TimeStamp (sec)'].min(), 'latitude': rowdata.df[' latitude'], 'longitude': rowdata.df[' longitude'] }) return df def workout_summary_to_df( rower, startdate=datetime.datetime(1970,1,1), enddate=timezone.now()+timezone.timedelta(days=1)): ws = Workout.objects.filter( user=rower,date__gte=startdate,date__lte=enddate, duplicate=False ).order_by("startdatetime") types = [] names = [] ids = [] startdatetimes = [] timezones = [] distances = [] durations = [] weightcategories = [] adaptivetypes = [] weightvalues = [] notes = [] tcx_links = [] csv_links = [] workout_links = [] goldstandards = [] goldstandarddurations = [] rscores = [] hrtss = [] trimps = [] rankingpieces = [] boattypes = [] counter1 = 0 counter2 = len(ws) for w in ws: counter1 += 1 if counter1 % 10 == 0: # pragma: no cover print(counter1,'/',counter2) types.append(w.workouttype) names.append(w.name) ids.append(encoder.encode_hex(w.id)) startdatetimes.append(w.startdatetime) timezones.append(w.timezone) distances.append(w.distance) durations.append(w.duration) weightcategories.append(w.weightcategory) adaptivetypes.append(w.adaptiveclass) weightvalues.append(w.weightvalue) boattypes.append(w.boattype) notes.append(w.notes) tcx_link = SITE_URL+'/rowers/workout/{id}/emailtcx'.format( id=encoder.encode_hex(w.id) ) tcx_links.append(tcx_link) csv_link = SITE_URL+'/rowers/workout/{id}/emailcsv'.format( id=encoder.encode_hex(w.id) ) csv_links.append(csv_link) workout_link = SITE_URL+'/rowers/workout/{id}/'.format( id=encoder.encode_hex(w.id) ) workout_links.append(workout_link) trimps.append(workout_trimp(w)[0]) rscore = workout_rscore(w) rscores.append(int(rscore[0])) hrtss.append(int(w.hrtss)) goldstandard,goldstandardduration = workout_goldmedalstandard(w) goldstandards.append(int(goldstandard)) goldstandarddurations.append(int(goldstandardduration)) rankingpieces.append(w.rankingpiece) df = pd.DataFrame({ 'ID': ids, 'date':startdatetimes, 'name':names, 'link':workout_links, 'timezone':timezones, 'type':types, 'boat type':boattypes, 'distance (m)':distances, 'duration ':durations, 'ranking piece':rankingpieces, 'weight category':weightcategories, 'adaptive classification':adaptivetypes, 'weight (kg)':weightvalues, 'Stroke Data TCX':tcx_links, 'Stroke Data CSV':csv_links, 'TRIMP Training Load':trimps, 'TSS Training Load':rscores, 'hrTSS Training Load':hrtss, 'GS':goldstandards, 'GS_secs':goldstandarddurations, 'notes':notes, }) return df def get_workouts(ids, userid): # pragma: no cover goodids = [] for id in ids: w = Workout.objects.get(id=id) if int(w.user.user.id) == int(userid): goodids.append(id) return [Workout.objects.get(id=id) for id in goodids] def filter_df(datadf, fieldname, value, largerthan=True): try: x = datadf[fieldname] except KeyError: return datadf try: if largerthan: mask = datadf[fieldname] < value else: mask = datadf[fieldname] >= value datadf.loc[mask, fieldname] = np.nan except TypeError: pass return datadf # joins workouts def join_workouts(r,ids,title='Joined Workout', parent=None, setprivate=False, forceunit='lbs',killparents=False): message = None summary = '' if parent: # pragma: no cover oarlength = parent.oarlength inboard = parent.inboard workouttype = parent.workouttype notes = parent.notes summary = parent.summary if parent.privacy == 'hidden': makeprivate = True else: makeprivate = False startdatetime = parent.startdatetime else: oarlength = 2.89 inboard = 0.88 workouttype = 'rower' notes = '' summary = '' makeprivate = False startdatetime = timezone.now() if setprivate == True and makeprivate == False: # pragma: no cover makeprivate = True elif setprivate == False and makeprivate == True: # pragma: no cover makeprivate = False # reorder in chronological order ws = Workout.objects.filter(id__in=ids).order_by("startdatetime") if not parent: parent = ws[0] oarlength = parent.oarlength inboard = parent.inboard workouttype = parent.workouttype notes = parent.notes summary = parent.summary files = [w.csvfilename for w in ws] row = rdata(files[0]) files = files[1:] while len(files): row2 = rdata(files[0]) if row2 != 0: row = row+row2 files = files[1:] timestr = strftime("%Y%m%d-%H%M%S") csvfilename = 'media/df_' + timestr + '.csv' row.write_csv(csvfilename,gzip=True) id, message = save_workout_database(csvfilename, r, workouttype=workouttype, title=title, notes=notes, oarlength=oarlength, inboard=inboard, makeprivate=makeprivate, dosmooth=False, consistencychecks=False) if killparents: # pragma: no cover for w in ws: w.delete() w = Workout.objects.get(id=id) w.duplicate = False w.save() if message is not None and "duplicate" in message: message = "" return (id, message) def df_resample(datadf): # time stamps must be in seconds timestamps = datadf['TimeStamp (sec)'].astype('int') datadf['timestamps'] = timestamps newdf = datadf.groupby(['timestamps']).mean() return newdf def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, ignoreadvanced=False): # clean data remove zeros and negative values try: workoutids = datadf['workoutid'].unique() except KeyError: datadf['workoutid'] = 0 before = {} for workoutid in datadf['workoutid'].unique(): before[workoutid] = len(datadf[datadf['workoutid']==workoutid]) data_orig = datadf.copy() # bring metrics which have negative values to positive domain if len(datadf)==0: return datadf try: datadf['catch'] = -datadf['catch'] except (KeyError,TypeError): pass try: datadf['peakforceangle'] = datadf['peakforceangle'] + 1000 except (KeyError,TypeError): pass try: datadf['hr'] = datadf['hr'] + 10 except (KeyError,TypeError): pass # protect 0 spm values from being nulled try: datadf['spm'] = datadf['spm'] + 1.0 except (KeyError,TypeError) as e: pass # protect 0 workoutstate values from being nulled try: datadf['workoutstate'] = datadf['workoutstate'] + 1 except (KeyError,TypeError) as e: pass try: datadf = datadf.clip(lower=0) except TypeError: pass # protect advanced metrics columns advancedcols = [ 'rhythm', 'power', 'drivelength', 'forceratio', 'drivespeed', 'driveenergy', 'catch', 'finish', 'averageforce', 'peakforce', 'slip', 'wash', 'peakforceangle', 'effectiveangle', ] datadf.replace(to_replace=0, value=np.nan, inplace=True) # datadf = datadf.map_partitions(lambda df:df.replace(to_replace=0,value=np.nan)) # bring spm back to real values try: datadf['spm'] = datadf['spm'] - 1 except (TypeError,KeyError) as e: pass # bring workoutstate back to real values try: datadf['workoutstate'] = datadf['workoutstate'] - 1 except (TypeError,KeyError) as e: pass # return from positive domain to negative try: datadf['catch'] = -datadf['catch'] except (KeyError,TypeError): pass try: datadf['peakforceangle'] = datadf['peakforceangle'] - 1000 except (KeyError,TypeError): pass try: datadf['hr'] = datadf['hr'] - 10 except (KeyError,TypeError): pass # clean data for useful ranges per column if not ignorehr: try: mask = datadf['hr'] < 30 datadf.mask(mask,inplace=True) except (KeyError,TypeError): # pragma: no cover pass try: mask = datadf['spm'] < 0 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['efficiency'] > 200. datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['spm'] < 10 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['pace'] / 1000. > 300. datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['efficiency'] < 0. datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['pace'] / 1000. < 60. datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['power'] > 5000 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['spm'] > 120 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['wash'] < 1 datadf.loc[mask, 'wash'] = np.nan except (KeyError,TypeError): pass # try to guess ignoreadvanced if not ignoreadvanced: for metric in advancedcols: try: sum = datadf[metric].std() if sum == 0 or np.isnan(sum): ignoreadvanced = True except KeyError: pass if not ignoreadvanced: try: mask = datadf['rhythm'] < 0 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['rhythm'] > 70 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['power'] < 20 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['drivelength'] < 0.5 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['forceratio'] < 0.2 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['forceratio'] > 1.0 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['drivespeed'] < 0.5 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['drivespeed'] > 4 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['driveenergy'] > 2000 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['driveenergy'] < 100 datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass try: mask = datadf['catch'] > -30. datadf.mask(mask,inplace=True) except (KeyError,TypeError): pass workoutstateswork = [1, 4, 5, 8, 9, 6, 7] workoutstatesrest = [3] workoutstatetransition = [0, 2, 10, 11, 12, 13] if workstrokesonly == 'True' or workstrokesonly == True: try: datadf = datadf[~datadf['workoutstate'].isin(workoutstatesrest)] except: pass after = {} for workoutid in data_orig['workoutid'].unique(): after[workoutid] = len(datadf[datadf['workoutid']==workoutid].dropna()) ratio = float(after[workoutid])/float(before[workoutid]) if ratio < 0.01 or after[workoutid] < 2: return data_orig return datadf def getpartofday(row,r): workoutstartdatetime = row.rowdatetime try: latavg = row.df[' latitude'].mean() lonavg = row.df[' longitude'].mean() tf = TimezoneFinder() try: timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) except (ValueError,OverflowError): # pragma: no cover timezone_str = 'UTC' if timezone_str == None: # pragma: no cover timezone_str = tf.closest_timezone_at(lng=lonavg, lat=latavg) if timezone_str == None: timezone_str = r.defaulttimezone try: workoutstartdatetime = pytz.timezone(timezone_str).localize( row.rowdatetime ) except ValueError: workoutstartdatetime = row.rowdatetime except KeyError: timezone_str = r.defaulttimezone workoutstartdatetime = row.rowdatetime h = workoutstartdatetime.astimezone(pytz.timezone(timezone_str)).hour if h < 12: # pragma: no cover return "Morning" elif h < 18: # pragma: no cover return "Afternoon" elif h < 22: # pragma: no cover return "Evening" else: # pragma: no cover return "Night" return None # pragma: no cover def getstatsfields(): fielddict = {name:d['verbose_name'] for name,d in rowingmetrics} # fielddict.pop('ergpace') # fielddict.pop('hr_an') # fielddict.pop('hr_tr') # fielddict.pop('hr_at') # fielddict.pop('hr_ut2') # fielddict.pop('hr_ut1') fielddict.pop('time') fielddict.pop('distance') # fielddict.pop('nowindpace') # fielddict.pop('fnowindpace') # fielddict.pop('fergpace') # fielddict.pop('equivergpower') # fielddict.pop('workoutstate') # fielddict.pop('fpace') # fielddict.pop('pace') # fielddict.pop('id') # fielddict.pop('ftime') # fielddict.pop('x_right') # fielddict.pop('hr_max') # fielddict.pop('hr_bottom') fielddict.pop('cumdist') try: fieldlist = [field for field, value in fielddict.iteritems()] except AttributeError: fieldlist = [field for field, value in fielddict.items()] return fieldlist, fielddict # A string representation for time deltas def niceformat(values): out = [] for v in values: formattedv = strfdelta(v) out.append(formattedv) return out # A nice printable format for time delta values def strfdelta(tdelta): try: minutes, seconds = divmod(tdelta.seconds, 60) tenths = int(tdelta.microseconds / 1e5) except AttributeError: # pragma: no cover minutes, seconds = divmod(tdelta.view(np.int64), 60e9) seconds, rest = divmod(seconds, 1e9) tenths = int(rest / 1e8) res = "{minutes:0>2}:{seconds:0>2}.{tenths:0>1}".format( minutes=minutes, seconds=seconds, tenths=tenths, ) return res def timedelta_to_seconds(tdelta): # pragma: no cover return 60.*tdelta.minute+tdelta.second # A nice printable format for pace values def nicepaceformat(values): out = [] for v in values: formattedv = strfdelta(v) out.append(formattedv) return out # Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace def timedeltaconv(x): if np.isfinite(x) and x != 0 and x > 0 and x < 175000: dt = datetime.timedelta(seconds=x) else: dt = datetime.timedelta(seconds=350.) return dt def paceformatsecs(values): out = [] for v in values: td = timedeltaconv(v) formattedv = strfdelta(td) out.append(formattedv) return out def update_c2id_sql(id,c2id): engine = create_engine(database_url, echo=False) table = 'rowers_workout' query = "UPDATE %s SET uploadedtoc2 = %s WHERE `id` = %s;" % (table,c2id,id) with engine.connect() as conn, conn.begin(): result = conn.execute(query) conn.close() engine.dispose() return 1 def getcpdata_sql(rower_id,table='cpdata'): engine = create_engine(database_url, echo=False) query = sa.text('SELECT * from {table} WHERE user={rower_id};'.format( rower_id=rower_id, table=table, )) connection = engine.raw_connection() df = pd.read_sql_query(query, engine) return df def deletecpdata_sql(rower_id,table='cpdata'): # pragma: no cover engine = create_engine(database_url, echo=False) query = sa.text('DELETE from {table} WHERE user={rower_id};'.format( rower_id=rower_id, table=table, )) with engine.connect() as conn, conn.begin(): try: result = conn.execute(query) except: print("Database locked") conn.close() engine.dispose() def updatecpdata_sql(rower_id,delta,cp,table='cpdata',distance=[]): # pragma: no cover deletecpdata_sql(rower_id) df = pd.DataFrame( { 'delta':delta, 'cp':cp, 'user':rower_id } ) if not distance.empty: df['distance'] = distance engine = create_engine(database_url, echo=False) with engine.connect() as conn, conn.begin(): df.to_sql(table, engine, if_exists='append', index=False) conn.close() engine.dispose() def fetchcperg(rower,theworkouts): theids = [int(w.id) for w in theworkouts] thefilenames = [w.csvfilename for w in theworkouts] cpdf = getcpdata_sql(rower.id,table='ergcpdata') job = myqueue( queuelow, handle_updateergcp, rower.id, thefilenames) return cpdf from rowers.datautils import p0 from rowers.utils import calculate_age from scipy import optimize def get_workoutsummaries(userid,startdate): # pragma: no cover u = User.objects.get(id=userid) r = u.rower df = workout_summary_to_df(r,startdate=startdate) df.drop(['Stroke Data TCX','Stroke Data CSV'],axis=1,inplace=True) df = df.sort_values('date',ascending=False) return df def workout_goldmedalstandard(workout,reset=False): if workout.goldmedalstandard > 0 and not reset: return workout.goldmedalstandard,workout.goldmedalseconds if workout.workouttype in rowtypes: goldmedalstandard,goldmedalseconds = calculate_goldmedalstandard(workout.user,workout) if workout.workouttype in otwtypes: factor = 100./(100.-workout.user.otwslack) goldmedalstandard = goldmedalstandard*factor workout.goldmedalstandard = goldmedalstandard workout.goldmedalseconds = goldmedalseconds workout.save() return goldmedalstandard, goldmedalseconds else: return 0,0 def check_marker(workout): r = workout.user gmstandard,gmseconds = workout_goldmedalstandard(workout) if gmseconds<60: return None dd = arrow.get(workout.date).datetime-datetime.timedelta(days=r.kfit) ws = Workout.objects.filter(date__gte=dd, date__lte=workout.date, user=r,duplicate=False, workouttype__in=mytypes.rowtypes, ).order_by("date") ids = [] gms = [] for w in ws: gmstandard,gmseconds = workout_goldmedalstandard(w) if gmseconds>60: ids.append(w.id) gms.append(gmstandard) df = pd.DataFrame({ 'id':ids, 'gms':gms, }) if df.empty: # pragma: no cover workout.ranking = True workout.save() return workout indexmax = df['gms'].idxmax() theid = df.loc[indexmax,'id'] wmax = Workout.objects.get(id=theid) gms_max = wmax.goldmedalstandard # check if equal, bigger, or smaller than previous if not wmax.rankingpiece: rankingworkouts = ws.filter(rankingpiece=True) if len(rankingworkouts) == 0: wmax.rankingpiece = True wmax.save() return wmax lastranking = rankingworkouts[len(rankingworkouts)-1] if lastranking.goldmedalstandard+0.2 < wmax.goldmedalstandard: # pragma: no cover wmax.rankingpiece = True wmax.save() return wmax else: # pragma: no cover return wmax return None def calculate_goldmedalstandard(rower,workout,recurrance=True): cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) try: df = pd.read_parquet(cpfile) except: background = True if settings.TESTING: background = False df, delta, cpvalues = setcp(workout,background=background) if df.empty: return 0,0 if df.empty and recurrance: # pragma: no cover df, delta, cpvalues = setcp(workout,recurrance=False,background=True) if df.empty: return 0,0 age = calculate_age(rower.birthdate,today=workout.date) agerecords = CalcAgePerformance.objects.filter( age=age, sex=rower.sex, weightcategory = rower.weightcategory ) wcdurations = [] wcpower = [] getrecords = False if not settings.TESTING: # pragma: no cover if len(agerecords) == 0: # pragma: no cover getrecords = True for record in agerecords: # pragma: no cover if record.power > 0: wcdurations.append(record.duration) wcpower.append(record.power) else: getrecords = True if getrecords: # pragma: no cover durations = [1,4,30,60] distances = [100,500,1000,2000,5000,6000,10000,21097,42195] df2 = pd.DataFrame( list( C2WorldClassAgePerformance.objects.filter( sex=rower.sex, weightcategory=rower.weightcategory ).values() ) ) jsondf = df2.to_json() job = myqueue(queuelow,handle_getagegrouprecords, jsondf,distances,durations,age,rower.sex,rower.weightcategory) wcpower = pd.Series(wcpower,dtype='float') wcdurations = pd.Series(wcdurations,dtype='float') fitfunc = lambda pars,x: pars[0]/(1+(x/pars[2])) + pars[1]/(1+(x/pars[3])) errfunc = lambda pars,x,y: fitfunc(pars,x)-y if len(wcdurations)>=4: # pragma: no cover p1wc, success = optimize.leastsq(errfunc, p0[:],args=(wcdurations,wcpower)) else: factor = fitfunc(p0,wcdurations.mean()/wcpower.mean()) p1wc = [p0[0]/factor,p0[1]/factor,p0[2],p0[3]] success = 0 return 0,0 times = df['delta'] powers = df['cp'] wcpowers = fitfunc(p1wc,times) scores = 100.*powers/wcpowers try: indexmax = scores.idxmax() delta = int(df.loc[indexmax,'delta']) maxvalue = scores.max() except (ValueError,TypeError): # pragma: no cover indexmax = 0 delta = 0 maxvalue = 0 return maxvalue,delta def fetchcp_new(rower,workouts): data = [] for workout in workouts: cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) try: df = pd.read_parquet(cpfile) df['workout'] = str(workout) df['url'] = workout.url() data.append(df) except: # CP data file doesn't exist yet. has to be created df, delta, cpvalues = setcp(workout) df['workout'] = str(workout) df['url'] = workout.url() data.append(df) if len(data) == 0: return pd.Series(dtype='float'),pd.Series(dtype='float'),0,pd.Series(dtype='float'),pd.Series(dtype='float') if len(data)>1: df = pd.concat(data,axis=0) try: df = df[df['cp'] == df.groupby(['delta'])['cp'].transform('max')] except KeyError: # pragma: no cover return pd.Series(dtype='float'),pd.Series(dtype='float'),0,pd.Series(dtype='float'),pd.Series(dtype='float') df = df.sort_values(['delta']).reset_index() return df['delta'],df['cp'],0,df['workout'],df['url'] def setcp(workout,background=False,recurrance=True): filename = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) strokesdf = getsmallrowdata_db(['power','workoutid','time'],ids = [workout.id]) try: if strokesdf['power'].std()==0: return pd.DataFrame(),pd.Series(dtype='float'),pd.Series(dtype='float') except KeyError: return pd.DataFrame(),pd.Series(dtype='float'),pd.Series(dtype='float') if background: # pragma: no cover job = myqueue(queuelow,handle_setcp,strokesdf,filename,workout.id) return pd.DataFrame({'delta':[],'cp':[]}),pd.Series(dtype='float'),pd.Series(dtype='float') if not strokesdf.empty: totaltime = strokesdf['time'].max() try: powermean = strokesdf['power'].mean() except KeyError: # pragma: no cover powermean = 0 if powermean != 0: thesecs = totaltime maxt = 1.05 * thesecs if maxt > 0: logarr = datautils.getlogarr(maxt) dfgrouped = strokesdf.groupby(['workoutid']) delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) df = pd.DataFrame({ 'delta':delta, 'cp':cpvalues, 'id':workout.id, }) df.to_parquet(filename,engine='fastparquet',compression='GZIP') if recurrance: goldmedalstandard, goldmedalduration = calculate_goldmedalstandard(workout.user,workout) workout.goldmedalstandard = goldmedalstandard workout.goldmedalduration = goldmedalduration workout.save() return df,delta,cpvalues return pd.DataFrame({'delta':[],'cp':[]}),pd.Series(dtype='float'),pd.Series(dtype='float') def update_rolling_cp(r,types,mode='water'): firstdate = datetime.date.today()-datetime.timedelta(days=r.cprange) workouts = Workout.objects.filter( date__gte=firstdate, workouttype__in=types, user = r ) delta, cp, avgpower, workoutnames,urls = fetchcp_new(r,workouts) powerdf = pd.DataFrame({ 'Delta':delta, 'CP':cp, }) powerdf = powerdf[powerdf['CP']>0] powerdf.dropna(axis=0,inplace=True) powerdf.sort_values(['Delta','CP'],ascending=[1,0],inplace=True) powerdf.drop_duplicates(subset='Delta',keep='first',inplace=True) res2 = datautils.cpfit(powerdf) if len(powerdf) != 0: if mode == 'water': p1 = res2[0] r.p0 = p1[0] r.p1 = p1[1] r.p2 = p1[2] r.p3 = p1[3] r.cpratio = res2[3] r.save() else: p1 = res2[0] r.ep0 = p1[0] r.ep1 = p1[1] r.ep2 = p1[2] r.ep3 = p1[3] r.ecpratio = res2[3] r.save() return True return False def fetchcp(rower,theworkouts,table='cpdata'): # pragma: no cover # get all power data from database (plus workoutid) theids = [int(w.id) for w in theworkouts] columns = ['power','workoutid','time'] df = getsmallrowdata_db(columns,ids=theids) df.dropna(inplace=True,axis=0) if df.empty: avgpower2 = {} for id in theids: avgpower2[id] = 0 return pd.Series([],dtype='float'),pd.Series([],dtype='float'),avgpower2 try: dfgrouped = df.groupby(['workoutid']) except KeyError: avgpower2 = {} return pd.Series([],dtype='float'),pd.Series([],dtype='float'),avgpower2 try: avgpower2 = dict(dfgrouped.mean()['power'].astype(int)) except KeyError: avgpower2 = {} for id in theids: avgpower2[id] = 0 return pd.Series([],dtype='float'),pd.Series([],dtype='float'),avgpower2 cpdf = getcpdata_sql(rower.id,table=table) if not cpdf.empty: return cpdf['delta'],cpdf['cp'],avgpower2 else: job = myqueue(queuelow, handle_updatecp, rower.id, theids, table=table) return pd.Series([],dtype='float'),pd.Series([],dtype='float'),avgpower2 return pd.Series([],dtype='float'),pd.Series([],dtype='float'),avgpower2 # create a new workout from manually entered data def create_row_df(r,distance,duration,startdatetime,workouttype='rower', avghr=None,avgpwr=None,avgspm=None, rankingpiece = False, duplicate=False,rpe=-1, title='Manual entry',notes='',weightcategory='hwt', adaptiveclass='None'): if duration is not None: totalseconds = duration.hour*3600. totalseconds += duration.minute*60. totalseconds += duration.second totalseconds += duration.microsecond/1.e6 else: # pragma: no cover totalseconds = 60. if distance is None: # pragma: no cover distance = 0 try: nr_strokes = int(distance/10.) except TypeError: # pragma: no cover nr_strokes = int(20.*totalseconds) if nr_strokes == 0: # pragma: no cover nr_strokes = 100 unixstarttime = arrow.get(startdatetime).timestamp() if not avgspm: # pragma: no cover try: spm = 60.*nr_strokes/totalseconds except ZeroDivisionError: spm = 20. else: spm = avgspm step = totalseconds/float(nr_strokes) elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) dstep = distance/float(nr_strokes) d = np.arange(nr_strokes)*distance/(float(nr_strokes-1)) unixtime = unixstarttime + elapsed try: pace = 500.*totalseconds/distance except ZeroDivisionError: # pragma: no cover pace = 240. if workouttype in ['rower','slides','dynamic']: try: velo = distance/totalseconds except ZeroDivisionError: # pragma: no cover velo = 2.4 power = 2.8*velo**3 elif avgpwr is not None: # pragma: no cover power = avgpwr else: # pragma: no cover power = 0 if avghr is not None: hr = avghr else: # pragma: no cover hr = 0 df = pd.DataFrame({ 'TimeStamp (sec)': unixtime, ' Horizontal (meters)': d, ' Cadence (stokes/min)': spm, ' Stroke500mPace (sec/500m)':pace, ' ElapsedTime (sec)':elapsed, ' Power (watts)':power, ' HRCur (bpm)':hr, }) timestr = strftime("%Y%m%d-%H%M%S") csvfilename = 'media/df_' + timestr + '.csv' df[' ElapsedTime (sec)'] = df['TimeStamp (sec)'] row = rrdata(df=df) row.write_csv(csvfilename, gzip = True) id, message = save_workout_database(csvfilename, r, title=title, notes=notes, rankingpiece=rankingpiece, duplicate=duplicate, dosmooth=False, workouttype=workouttype, consistencychecks=False, weightcategory=weightcategory, adaptiveclass=adaptiveclass, totaltime=totalseconds) return (id, message) from rowers.utils import totaltime_sec_to_string def checkbreakthrough(w, r): isbreakthrough = False ishard = False workouttype = w.workouttype if workouttype in rowtypes: cpdf,delta,cpvalues = setcp(w) if not cpdf.empty: if workouttype in otwtypes: res, btvalues, res2 = utils.isbreakthrough( delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) success = update_rolling_cp(r,otwtypes,'water') elif workouttype in otetypes: res, btvalues, res2 = utils.isbreakthrough( delta, cpvalues, r.ep0, r.ep1, r.ep2, r.ep3, r.ecpratio) success = update_rolling_cp(r,otetypes,'erg') else: # pragma: no cover res = 0 res2 = 0 if res: isbreakthrough = True if res2 and not isbreakthrough: # pragma: no cover ishard = True # submit email task to send email about breakthrough workout if isbreakthrough: if not w.duplicate: w.rankingpiece = True w.save() if r.getemailnotifications and not r.emailbounced: # pragma: no cover job = myqueue(queuehigh,handle_sendemail_breakthrough, w.id, r.user.email, r.user.first_name, r.user.last_name, btvalues=btvalues.to_json()) # submit email task to send email about breakthrough workout if ishard: # pragma: no cover if not w.duplicate: w.rankingpiece = True w.save() if r.getemailnotifications and not r.emailbounced: job = myqueue(queuehigh,handle_sendemail_hard, w.id, r.user.email, r.user.first_name, r.user.last_name, btvalues=btvalues.to_json()) return isbreakthrough, ishard def checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime): duplicate = False ws = Workout.objects.filter(user=r,date=workoutdate,duplicate=False).exclude( startdatetime__gt=workoutenddatetime ) ws2 = [] for ww in ws: t = ww.duration delta = datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) enddatetime = ww.startdatetime+delta if enddatetime > workoutstartdatetime: ws2.append(ww) if (len(ws2) != 0): message = "Warning: This workout overlaps with an existing one and was marked as a duplicate" duplicate = True return duplicate return duplicate # Processes painsled CSV file to database def save_workout_database(f2, r, dosmooth=True, workouttype='rower', boattype='1x', adaptiveclass='None', weightcategory='hwt', dosummary=True, title='Workout', workoutsource='unknown', notes='', totaldist=0, totaltime=0, rankingpiece=False, rpe=-1, duplicate=False, summary='', makeprivate=False, oarlength=2.89, inboard=0.88, forceunit='lbs', consistencychecks=False, startdatetime='', impeller=False): message = None powerperc = 100 * np.array([r.pw_ut2, r.pw_ut1, r.pw_at, r.pw_tr, r.pw_an]) / r.ftp # make workout and put in database rr = rrower(hrmax=r.max, hrut2=r.ut2, hrut1=r.ut1, hrat=r.at, hrtr=r.tr, hran=r.an, ftp=r.ftp, powerperc=powerperc, powerzones=r.powerzones) row = rdata(f2, rower=rr) if startdatetime != '': # pragma: no cover row.rowdatetime = arrow.get(startdatetime).datetime if title is None or title == '': title = 'Workout' partofday = getpartofday(row,r) if partofday is not None: title = '{partofday} {workouttype}'.format( partofday=partofday, workouttype=workouttype, ) if row.df.empty: # pragma: no cover return (0, 'Error: CSV data file was empty') dtavg = row.df['TimeStamp (sec)'].diff().mean() if dtavg < 1: newdf = df_resample(row.df) try: os.remove(f2) except: pass return new_workout_from_df(r, newdf, title=title,boattype=boattype, workouttype=workouttype, workoutsource=workoutsource,startdatetime=startdatetime) try: checks = row.check_consistency() allchecks = 1 for key, value in checks.items(): if not value: allchecks = 0 except ZeroDivisionError: # pragma: no cover pass if not allchecks and consistencychecks: # row.repair() pass if row == 0: # pragma: no cover return (0, 'Error: CSV data file not found') try: lat = row.df[' latitude'] if lat.mean() != 0 and lat.std() != 0 and workouttype == 'rower': workouttype = 'water' except KeyError: pass if dosmooth: # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values velo = 500. / pace f = row.df['TimeStamp (sec)'].diff().mean() if f != 0 and not np.isnan(f): windowsize = 2 * (int(10. / (f))) + 1 else: # pragma: no cover windowsize = 1 if not 'originalvelo' in row.df: row.df['originalvelo'] = velo if windowsize > 3 and windowsize < len(velo): velo2 = savgol_filter(velo, windowsize, 3) else: # pragma: no cover velo2 = velo velo3 = pd.Series(velo2,dtype='float') velo3 = velo3.replace([-np.inf, np.inf], np.nan) velo3 = velo3.fillna(method='ffill') pace2 = 500. / abs(velo3) row.df[' Stroke500mPace (sec/500m)'] = pace2 row.df = row.df.fillna(0) row.write_csv(f2, gzip=True) try: os.remove(f2) except: pass # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: if r.erg_recalculatepower: row.erg_recalculatepower() row.write_csv(f2, gzip=True) except: pass averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() if totaldist == 0: totaldist = row.df['cum_dist'].max() if totaltime == 0: totaltime = row.df['TimeStamp (sec)'].max( ) - row.df['TimeStamp (sec)'].min() try: totaltime = totaltime + row.df.loc[:, ' ElapsedTime (sec)'].iloc[0] except KeyError: # pragma: no cover pass if np.isnan(totaltime): # pragma: no cover totaltime = 0 if dosummary: summary = row.allstats() if startdatetime != '': # pragma: no cover workoutstartdatetime = arrow.get(startdatetime).datetime else: workoutstartdatetime = row.rowdatetime timezone_str = 'UTC' try: workoutstartdatetime = timezone.make_aware(workoutstartdatetime) except ValueError: pass try: latavg = row.df[' latitude'].mean() lonavg = row.df[' longitude'].mean() tf = TimezoneFinder() try: timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) except (ValueError,OverflowError): # pragma: no cover timezone_str = 'UTC' if timezone_str == None: # pragma: no cover timezone_str = tf.closest_timezone_at(lng=lonavg, lat=latavg) if timezone_str == None: # pragma: no cover timezone_str = r.defaulttimezone try: workoutstartdatetime = pytz.timezone(timezone_str).localize( row.rowdatetime ) except ValueError: workoutstartdatetime = workoutstartdatetime.astimezone( pytz.timezone(timezone_str) ) except KeyError: timezone_str = r.defaulttimezone duration = totaltime_sec_to_string(totaltime) workoutdate = workoutstartdatetime.astimezone( pytz.timezone(timezone_str) ).strftime('%Y-%m-%d') workoutstarttime = workoutstartdatetime.astimezone( pytz.timezone(timezone_str) ).strftime('%H:%M:%S') if makeprivate: # pragma: no cover privacy = 'hidden' else: privacy = 'visible' # checking for inf values totaldist = np.nan_to_num(totaldist) maxhr = np.nan_to_num(maxhr) averagehr = np.nan_to_num(averagehr) dragfactor = 0 if workouttype in otetypes: dragfactor = row.dragfactor t = datetime.datetime.strptime(duration,"%H:%M:%S.%f") delta = datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) workoutenddatetime = workoutstartdatetime+delta # check for duplicate start times and duration duplicate = checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime) if duplicate: rankingpiece = False # test title length if title is not None and len(title)>140: # pragma: no cover title = title[0:140] w = Workout(user=r, name=title, date=workoutdate, workouttype=workouttype, boattype=boattype, dragfactor=dragfactor, duration=duration, distance=totaldist, weightcategory=weightcategory, adaptiveclass=adaptiveclass, starttime=workoutstarttime, duplicate=duplicate, workoutsource=workoutsource, rankingpiece=rankingpiece, forceunit=forceunit, rpe=rpe, csvfilename=f2, notes=notes, summary=summary, maxhr=maxhr, averagehr=averagehr, startdatetime=workoutstartdatetime, inboard=inboard, oarlength=oarlength, timezone=timezone_str, privacy=privacy, impeller=impeller) try: w.save() except ValidationError: # pragma: no cover try: w.startdatetime = timezone.now() w.save() except ValidationError: return (0,'Unable to create your workout') if privacy == 'visible': ts = Team.objects.filter(rower=r) for t in ts: w.team.add(t) # put stroke data in database res = dataprep(row.df, id=w.id, bands=True, barchart=True, otwpower=True, empower=True, inboard=inboard) isbreakthrough, ishard = checkbreakthrough(w, r) marker = check_marker(w) job = myqueue(queuehigh,handle_calctrimp,w.id,f2,r.ftp,r.sex,r.hrftp,r.max,r.rest) return (w.id, message) parsers = { 'kinomap': KinoMapParser, 'xls': ExcelTemplate, 'rp': RowProParser, 'tcx':TCXParser, 'mystery':MysteryParser, 'ritmotime':RitmoTimeParser, 'quiske': QuiskeParser, 'rowperfect3': RowPerfectParser, 'coxmate': CoxMateParser, 'bcmike': BoatCoachAdvancedParser, 'boatcoach': BoatCoachParser, 'boatcoachotw': BoatCoachOTWParser, 'painsleddesktop': painsledDesktopParser, 'speedcoach': speedcoachParser, 'speedcoach2': SpeedCoach2Parser, 'ergstick': ErgStickParser, 'fit': FITParser, 'ergdata': ErgDataParser, 'humon': HumonParser, 'eth': ETHParser, 'nklinklogbook': NKLiNKLogbookParser, } def parsenonpainsled(fileformat,f2,summary,startdatetime='',empowerfirmware=None,inboard=None,oarlength=None): try: if fileformat == 'nklinklogbook' and empowerfirmware is not None: # pragma: no cover if inboard is not None and oarlength is not None: row = NKLiNKLogbookParser(f2,firmware=empowerfirmware,inboard=inboard,oarlength=oarlength) else: row = NKLiNKLogbookParser(f2) else: row = parsers[fileformat](f2) if startdatetime != '': # pragma: no cover row.rowdatetime = arrow.get(startdatetime).datetime hasrecognized = True except (KeyError,IndexError,ValueError): # pragma: no cover hasrecognized = False return None, hasrecognized, '', 'unknown' # handle speed coach GPS 2 if (fileformat == 'speedcoach2'): oarlength, inboard = get_empower_rigging(f2) empowerfirmware = get_empower_firmware(f2) if empowerfirmware != '': fileformat = fileformat+'v'+str(empowerfirmware) else: # pragma: no cover fileformat = 'speedcoach2v0' try: summary = row.allstats() except ZeroDivisionError: # pragma: no cover summary = '' else: fileformat = fileformat+'v'+str(empowerfirmware) # handle FIT if (fileformat == 'fit'): # pragma: no cover try: s = fitsummarydata(f2) s.setsummary() summary = s.summarytext except: pass hasrecognized = True return row,hasrecognized,summary,fileformat def handle_nonpainsled(f2, fileformat, summary='',startdatetime='',empowerfirmware=None,impeller=False): oarlength = 2.89 inboard = 0.88 hasrecognized = False row,hasrecognized,summary,fileformat = parsenonpainsled(fileformat,f2,summary,startdatetime=startdatetime, empowerfirmware=empowerfirmware) # Handle c2log if (fileformat == 'c2log' or fileformat == 'rowprolog'): # pragma: no cover return (0,'',0,0,'',impeller) if not hasrecognized: # pragma: no cover return (0,'',0,0,'',impeller) f_to_be_deleted = f2 # should delete file f2 = f2[:-4] + 'o.csv' row2 = rrdata(df = row.df) if 'speedcoach2' in fileformat or 'nklinklogbook' in fileformat: # impeller consistency impellerdata, consistent, ratio = row.impellerconsistent(threshold=0.3) if impellerdata and consistent: impeller = True if impellerdata and not consistent: row2.use_gpsdata() if impeller: row2.use_impellerdata() row2.write_csv(f2, gzip=True) # os.remove(f2) try: os.remove(f_to_be_deleted) except: # pragma: no cover try: os.remove(f_to_be_deleted + '.gz') except: pass return (f2, summary, oarlength, inboard, fileformat, impeller) # Create new workout from file and store it in the database # This routine should be used everywhere in views.py and mailprocessing.py # Currently there is code duplication def get_workouttype_from_fit(filename,workouttype='water'): try: fitfile = FitFile(filename,check_crc=False) except FitHeaderError: # pragma: no cover return workouttype records = fitfile.messages fittype = 'rowing' for record in records: if record.name in ['sport','lap']: try: fittype = record.get_values()['sport'].lower() except (KeyError,AttributeError): # pragma: no cover return 'water' try: workouttype = mytypes.fitmappinginv[fittype] except KeyError: # pragma: no cover return workouttype return workouttype import rowingdata.tcxtools as tcxtools def get_workouttype_from_tcx(filename,workouttype='water'): tcxtype = 'rowing' if workouttype in mytypes.otwtypes: return workouttype try: # pragma: no cover d = tcxtools.tcx_getdict(filename) try: tcxtype = d['Activities']['Activity']['@Sport'].lower() if tcxtype == 'other': tcxtype = 'rowing' except KeyError: return workouttype except TypeError: # pragma: no cover pass try: # pragma: no cover workouttype = mytypes.garminmappinginv[tcxtype.upper()] except KeyError: # pragma: no cover return workouttype return workouttype # pragma: no cover def new_workout_from_file(r, f2, workouttype='rower', workoutsource=None, title='Workout', boattype='1x', rpe=-1, makeprivate=False, startdatetime='', notes='', oarlockfirmware='', inboard=None, oarlength=None, impeller=False, uploadoptions={'boattype':'1x','workouttype':'rower'}): message = "" try: fileformat = get_file_type(f2) except (IOError,UnicodeDecodeError): # pragma: no cover os.remove(f2) message = "Rowsandall could not process this file. The extension is supported but the file seems corrupt. Contact info@rowsandall.com if you think this is incorrect." return (0, message, f2) summary = '' oarlength = 2.89 inboard = 0.88 # Save zip files to email box for further processing if len(fileformat) == 3 and fileformat[0] == 'zip': # pragma: no cover uploadoptions['fromuploadform'] = True bodyyaml = yaml.safe_dump(uploadoptions,default_flow_style=False) f_to_be_deleted = f2 workoutsbox = Mailbox.objects.filter(name='workouts')[0] msg = Message(mailbox=workoutsbox, from_header=r.user.email, subject = title,body=bodyyaml) msg.save() f3 = 'media/mailbox_attachments/'+f2[6:] copyfile(f2,f3) f3 = f3[6:] a = MessageAttachment(message=msg,document=f3) a.save() message = "Zip file was stored for offline processing" return -1, message, f2 # Some people try to upload Concept2 logbook summaries if fileformat == 'c2log': os.remove(f2) message = "This summary does not contain stroke data. Use the files containing stroke by stroke data." return (0, message, f2) if fileformat == 'nostrokes': # pragma: no cover os.remove(f2) message = "It looks like this file doesn't contain stroke data." return (0, message, f2) if fileformat == 'kml': # pragma: no cover os.remove(f2) message = "KML files are not supported" return (0, message, f2) # Some people upload corrupted zip files if fileformat == 'notgzip': # pragma: no cover os.remove(f2) message = "Rowsandall could not process this file. The extension is supported but the file seems corrupt. Contact info@rowsandall.com if you think this is incorrect." return (0, message, f2) # Some people try to upload RowPro summary logs if fileformat == 'rowprolog': # pragma: no cover os.remove(f2) message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log." return (0, message, f2) # Sometimes people try an unsupported file type. # Send an email to info@rowsandall.com with the file attached # for me to check if it is a bug, or a new file type # worth supporting if fileformat == 'gpx': # pragma: no cover os.remove(f2) message = "GPX files support is on our roadmap. Check back soon." return (0, message, f2) if fileformat == 'unknown': # pragma: no cover message = "We couldn't recognize the file type" extension = os.path.splitext(f2)[1] filename = os.path.splitext(f2)[0] if extension == '.gz': filename = os.path.splitext(filename)[0] extension2 = os.path.splitext(filename)[1]+extension extension = extension2 f4 = filename+'a'+extension copyfile(f2,f4) job = myqueue(queuehigh, handle_sendemail_unrecognized, f4, r.user.email) return (0, message, f2) if fileformat == 'att': # pragma: no cover # email attachment which can safely be ignored return (0, '', f2) # Get workout type from fit & tcx if (fileformat == 'fit'): # pragma: no cover workouttype = get_workouttype_from_fit(f2,workouttype=workouttype) if (fileformat == 'tcx'): workouttype = get_workouttype_from_tcx(f2,workouttype=workouttype) # handle non-Painsled by converting it to painsled compatible CSV if (fileformat != 'csv'): f2, summary, oarlength, inboard, fileformat, impeller = handle_nonpainsled( f2, fileformat, startdatetime=startdatetime, summary=summary, empowerfirmware=oarlockfirmware, impeller=impeller, ) if not f2: # pragma: no cover message = 'Something went wrong' return (0, message, '') dosummary = (fileformat != 'fit' and 'speedcoach2' not in fileformat) dosummary = dosummary or summary == '' if 'speedcoach2' in fileformat and workouttype == 'rower': workouttype = 'water' if workoutsource is None: workoutsource = fileformat id, message = save_workout_database( f2, r, notes=notes, workouttype=workouttype, weightcategory=r.weightcategory, adaptiveclass=r.adaptiveclass, boattype=boattype, makeprivate=makeprivate, dosummary=dosummary, workoutsource=workoutsource, summary=summary, startdatetime=startdatetime, rpe=rpe, inboard=inboard, oarlength=oarlength, title=title, forceunit='N', impeller=impeller, ) return (id, message, f2) def split_workout(r, parent, splitsecond, splitmode): data, row = getrowdata_db(id=parent.id) latitude, longitude = get_latlon(parent.id) if not latitude.empty and not longitude.empty: data[' latitude'] = latitude data[' longitude'] = longitude data['time'] = data['time'] / 1000. data1 = data[data['time'] <= splitsecond].copy() data2 = data[data['time'] > splitsecond].copy() data1 = data1.sort_values(['time']) data1 = data1.interpolate(method='linear', axis=0, limit_direction='both', limit=10) data1.fillna(method='bfill', inplace=True) # Some new stuff to try out data1 = data1.groupby('time', axis=0).mean() data1['time'] = data1.index data1.reset_index(drop=True, inplace=True) data2 = data2.sort_values(['time']) data2 = data2.interpolate(method='linear', axis=0, limit_direction='both', limit=10) data2.fillna(method='bfill', inplace=True) # Some new stuff to try out data2 = data2.groupby('time', axis=0).mean() data2['time'] = data2.index data2.reset_index(drop=True, inplace=True) data1['pace'] = data1['pace'] / 1000. data2['pace'] = data2['pace'] / 1000. data1.drop_duplicates(subset='time', inplace=True) data2.drop_duplicates(subset='time', inplace=True) messages = [] ids = [] if 'keep first' in splitmode: if 'firstprivate' in splitmode: # pragma: no cover setprivate = True else: setprivate = False id, message = new_workout_from_df(r, data1, title=parent.name + ' (1)', parent=parent, setprivate=setprivate, forceunit='N') messages.append(message) ids.append(encoder.encode_hex(id)) if 'keep second' in splitmode: data2['cumdist'] = data2['cumdist'] - data2.iloc[ 0, data2.columns.get_loc('cumdist') ] data2['distance'] = data2['distance'] - data2.iloc[ 0, data2.columns.get_loc('distance') ] data2['time'] = data2['time'] - data2.iloc[ 0, data2.columns.get_loc('time') ] if 'secondprivate' in splitmode: # pragma: no cover setprivate = True else: setprivate = False dt = datetime.timedelta(seconds=splitsecond) id, message = new_workout_from_df(r, data2, title=parent.name + ' (2)', parent=parent, setprivate=setprivate, dt=dt, forceunit='N') messages.append(message) ids.append(encoder.encode_hex(id)) if not 'keep original' in splitmode: # pragma: no cover if 'keep second' in splitmode or 'keep first' in splitmode: parent.delete() messages.append('Deleted Workout: ' + parent.name) else: messages.append('That would delete your workout') ids.append(encoder.encode_hex(parent.id)) elif 'originalprivate' in splitmode: # pragma: no cover parent.privacy = 'hidden' parent.save() return ids, messages # Create new workout from data frame and store it in the database # This routine should be used everywhere in views.py and mailprocessing.py # Currently there is code duplication def new_workout_from_df(r, df, title='New Workout', workoutsource='unknown', boattype='1x', workouttype='rower', parent=None, startdatetime='', setprivate=False, forceunit='lbs', dt=datetime.timedelta()): message = None summary = '' if parent: oarlength = parent.oarlength inboard = parent.inboard workoutsource = parent.workoutsource workouttype = parent.workouttype boattype = parent.boattype notes = parent.notes summary = parent.summary rpe = parent.rpe if parent.privacy == 'hidden': # pragma: no cover makeprivate = True else: makeprivate = False startdatetime = parent.startdatetime + dt else: oarlength = 2.89 inboard = 0.88 notes = '' summary = '' makeprivate = False rpe = 0 if startdatetime == '': startdatetime = timezone.now() if setprivate: # pragma: no cover makeprivate = True timestr = strftime("%Y%m%d-%H%M%S") csvfilename = 'media/df_' + timestr + '.csv' if forceunit == 'N': # change to lbs for now df['peakforce'] /= lbstoN df['averageforce'] /= lbstoN df.rename(columns=columndict, inplace=True) #starttimeunix = mktime(startdatetime.utctimetuple()) starttimeunix = arrow.get(startdatetime).timestamp() df[' ElapsedTime (sec)'] = df['TimeStamp (sec)'] df['TimeStamp (sec)'] = df['TimeStamp (sec)'] + starttimeunix row = rrdata(df=df) row.write_csv(csvfilename, gzip=True) # res = df.to_csv(csvfilename+'.gz',index_label='index', # compression='gzip') id, message = save_workout_database(csvfilename, r, workouttype=workouttype, boattype=boattype, title=title, workoutsource=workoutsource, notes=notes, oarlength=oarlength, inboard=inboard, makeprivate=makeprivate, dosmooth=False, rpe=rpe, consistencychecks=False) job = myqueue(queuehigh,handle_calctrimp,id,csvfilename,r.ftp,r.sex,r.hrftp,r.max,r.rest) return (id, message) # A wrapper around the rowingdata class, with some error catching def rdata(file, rower=rrower()): try: res = rrdata(csvfile=file, rower=rower) except (IOError, IndexError): # pragma: no cover try: res = rrdata(csvfile=file + '.gz', rower=rower) except (IOError, IndexError): res = rrdata() except: res = rrdata() except EOFError: # pragma: no cover res = rrdata() except: # pragma: no cover res = rrdata() return res # Remove all stroke data for workout ID from database def delete_strokedata(id): dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id) try: shutil.rmtree(dirname) except OSError: try: os.remove(dirname) except FileNotFoundError: pass except FileNotFoundError: # pragma: no cover pass # Replace stroke data in DB with data from CSV file def update_strokedata(id, df): delete_strokedata(id) rowdata = dataprep(df, id=id, bands=True, barchart=True, otwpower=True) # Test that all data are of a numerical time def testdata(time, distance, pace, spm): # pragma: no cover t1 = np.issubdtype(time, np.number) t2 = np.issubdtype(distance, np.number) t3 = np.issubdtype(pace, np.number) t4 = np.issubdtype(spm, np.number) return t1 and t2 and t3 and t4 # Get data from DB for one workout (fetches all data). If data # is not in DB, read from CSV file (and create DB entry) def getrowdata_db(id=0, doclean=False, convertnewtons=True, checkefficiency=True): data = read_df_sql(id) try: data['deltat'] = data['time'].diff() except KeyError: # pragma: no cover data = pd.DataFrame() if data.empty: rowdata, row = getrowdata(id=id) if not rowdata.empty: data = dataprep(rowdata.df, id=id, bands=True, barchart=True, otwpower=True) else: data = pd.DataFrame() # returning empty dataframe else: row = Workout.objects.get(id=id) if checkefficiency==True and not data.empty: try: if data['efficiency'].mean() == 0 and data['power'].mean() != 0: # pragma: no cover data = add_efficiency(id=id) except KeyError: # pragma: no cover data = add_efficiency(id=id) if doclean: # pragma: no cover data = clean_df_stats(data, ignorehr=True) return data, row # Fetch a subset of the data from the DB def getsmallrowdata_db(columns, ids=[], doclean=True,workstrokesonly=True,compute=True): # prepmultipledata(ids) if ids: csvfilenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] else: return pd.DataFrame() data = [] columns = [c for c in columns if c != 'None'] columns = list(set(columns)) if len(ids)>1: for id,f in zip(ids,csvfilenames): try: #df = dd.read_parquet(f,columns=columns,engine='pyarrow') df = pd.read_parquet(f,columns=columns) data.append(df) except OSError: # pragma: no cover rowdata, row = getrowdata(id=id) if rowdata and len(rowdata.df): datadf = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) # df = dd.read_parquet(f,columns=columns,engine='pyarrow') df = pd.read_parquet(f,columns=columns) data.append(df) df = pd.concat(data,axis=0) # df = dd.concat(data,axis=0) else: try: df = pd.read_parquet(csvfilenames[0],columns=columns) except (OSError,ArrowInvalid,IndexError): rowdata,row = getrowdata(id=ids[0]) if rowdata and len(rowdata.df): # pragma: no cover data = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) df = pd.read_parquet(csvfilenames[0],columns=columns) # df = dd.read_parquet(csvfilenames[0], # column=columns,engine='pyarrow', # ) # df = df.loc[:,~df.columns.duplicated()] else: df = pd.DataFrame() if compute and len(df): data = df.copy() if doclean: data = clean_df_stats(data, ignorehr=True, workstrokesonly=workstrokesonly) data.dropna(axis=1,how='all',inplace=True) data.dropna(axis=0,how='any',inplace=True) return data return df # Fetch both the workout and the workout stroke data (from CSV file) def getrowdata(id=0): # check if valid ID exists (workout exists) try: row = Workout.objects.get(id=id) except Workout.DoesNotExist: # pragma: no cover return rrdata(),None f1 = row.csvfilename # get user r = row.user u = r.user rr = rrower(hrmax=r.max, hrut2=r.ut2, hrut1=r.ut1, hrat=r.at, hrtr=r.tr, hran=r.an, ftp=r.ftp) rowdata = rdata(f1, rower=rr) return rowdata, row # Checks if all rows for a list of workout IDs have entries in the # stroke_data table. If this is not the case, it creates the stroke # data # In theory, this should never yield any work, but it's a good # safety net for programming errors elsewhere in the app # Also used heavily when I moved from CSV file only to CSV+Stroke data import glob def prepmultipledata(ids, verbose=False): # pragma: no cover filenames = glob.glob('media/*.parquet') ids = [id for id in ids if 'media/strokedata_{id}.parquet.gz'.format(id=id) not in filenames] for id in ids: rowdata, row = getrowdata(id=id) if verbose: print(id) if rowdata and len(rowdata.df): data = dataprep(rowdata.df, id=id, bands=True, barchart=True, otwpower=True) return ids # Read a set of columns for a set of workout ids, returns data as a # pandas dataframe def read_cols_df_sql(ids, columns, convertnewtons=True): # drop columns that are not in offical list # axx = [ax[0] for ax in axes] extracols = [] columns = list(columns) + ['distance', 'spm', 'workoutid'] columns = [x for x in columns if x != 'None'] columns = list(set(columns)) ids = [int(id) for id in ids] df = pd.DataFrame() if len(ids) == 0: # pragma: no cover return pd.DataFrame(),extracols elif len(ids) == 1: # pragma: no cover try: filename = 'media/strokedata_{id}.parquet.gz'.format(id=ids[0]) df = pd.read_parquet(filename,columns=columns) except OSError: rowdata,row = getrowdata(id=ids[0]) if rowdata and len(rowdata.df): datadf = dataprep(rowdata.df,id=ids[0],bands=True,otwpower=True,barchart=True) df = pd.read_parquet(filename,columns=columns) else: data = [] filenames = ['media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] for id,f in zip(ids,filenames): try: df = pd.read_parquet(f,columns=columns) data.append(df) except (OSError,IndexError,ArrowInvalid): rowdata,row = getrowdata(id=id) if rowdata and len(rowdata.df): # pragma: no cover datadf = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) df = pd.read_parquet(f,columns=columns) data.append(df) try: df = pd.concat(data,axis=0) except ValueError: # pragma: no cover return pd.DataFrame(), extracols df = df.fillna(value=0) if 'peakforce' in columns: funits = ((w.id, w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id, u in funits: if u == 'lbs': mask = df['workoutid'] == id df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN if 'averageforce' in columns: funits = ((w.id, w.forceunit) for w in Workout.objects.filter(id__in=ids)) for id, u in funits: if u == 'lbs': mask = df['workoutid'] == id df.loc[mask, 'averageforce'] = df.loc[mask, 'averageforce'] * lbstoN return df,extracols def initiate_cp(r): success = update_rolling_cp(r,otwtypes,'water') success = update_rolling_cp(r,otetypes,'erg') # Read stroke data from the DB for a Workout ID. Returns a pandas dataframe def read_df_sql(id): try: f = 'media/strokedata_{id}.parquet.gz'.format(id=id) df = pd.read_parquet(f) except (OSError,ArrowInvalid,IndexError): # pragma: no cover rowdata,row = getrowdata(id=id) if rowdata and len(rowdata.df): data = dataprep(rowdata.df,id=id,bands=True,otwpower=True,barchart=True) try: df = pd.read_parquet(f) except OSError: df = data else: df = pd.DataFrame() df = df.fillna(value=0) return df # data fusion def datafusion(id1, id2, columns, offset): workout1 = Workout.objects.get(id=id1) workout2 = Workout.objects.get(id=id2) df1, w1 = getrowdata_db(id=id1) df1 = df1.drop([ # 'cumdist', 'hr_ut2', 'hr_ut1', 'hr_at', 'hr_tr', 'hr_an', 'hr_max', 'ftime', 'fpace', 'workoutid', 'id'], 1, errors='ignore') # Add coordinates to DataFrame latitude, longitude = get_latlon(id1) df1[' latitude'] = latitude df1[' longitude'] = longitude df2 = getsmallrowdata_db(['time'] + columns, ids=[id2], doclean=False) forceunit = 'N' offsetmillisecs = offset.seconds * 1000 + offset.microseconds / 1000. offsetmillisecs += offset.days * (3600 * 24 * 1000) df2['time'] = df2['time'] + offsetmillisecs keep1 = {c: c for c in set(df1.columns)} for c in columns: keep1.pop(c) for c in df1.columns: if not c in keep1: df1 = df1.drop(c, 1, errors='ignore') df = pd.concat([df1, df2], ignore_index=True) df = df.sort_values(['time']) df = df.interpolate(method='linear', axis=0, limit_direction='both', limit=10) df.fillna(method='bfill', inplace=True) # Some new stuff to try out df = df.groupby('time', axis=0).mean() df['time'] = df.index df.reset_index(drop=True, inplace=True) df['time'] = df['time'] / 1000. df['pace'] = df['pace'] / 1000. df['cum_dist'] = df['cumdist'] return df, forceunit def fix_newtons(id=0, limit=3000): # pragma: no cover # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False) try: #avgforce = rowdata['averageforce'] peakforce = rowdata['peakforce'] if peakforce.mean() > limit: w = Workout.objects.get(id=id) rowdata = rdata(w.csvfilename) if rowdata and len(rowdata.df): update_strokedata(w.id, rowdata.df) except KeyError: pass def remove_invalid_columns(df): # pragma: no cover for c in df.columns: if not c in allowedcolumns: df.drop(labels=c,axis=1,inplace=True) return df def add_efficiency(id=0): # pragma: no cover rowdata, row = getrowdata_db(id=id, doclean=False, convertnewtons=False, checkefficiency=False) power = rowdata['power'] pace = rowdata['pace'] / 1.0e3 velo = 500. / pace ergpw = 2.8 * velo**3 efficiency = 100. * ergpw / power efficiency = efficiency.replace([-np.inf, np.inf], np.nan) efficiency.fillna(method='ffill') rowdata['efficiency'] = efficiency rowdata = remove_invalid_columns(rowdata) rowdata = rowdata.replace([-np.inf, np.inf], np.nan) rowdata = rowdata.fillna(method='ffill') delete_strokedata(id) if id != 0: rowdata['workoutid'] = id filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) df = dd.from_pandas(rowdata,npartitions=1) df.to_parquet(filename,engine='fastparquet',compression='GZIP') return rowdata # This is the main routine. # it reindexes, sorts, filters, and smooths the data, then # saves it to the stroke_data table in the database # Takes a rowingdata object's DataFrame as input def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, empower=True, inboard=0.88, forceunit='lbs'): if rowdatadf.empty: return 0 #rowdatadf.set_index([range(len(rowdatadf))], inplace=True) t = rowdatadf.loc[:, 'TimeStamp (sec)'] t = pd.Series(t - rowdatadf.loc[:, 'TimeStamp (sec)'].iloc[0]) row_index = rowdatadf.loc[:, ' Stroke500mPace (sec/500m)'] > 3000 rowdatadf.loc[row_index, ' Stroke500mPace (sec/500m)'] = 3000. p = rowdatadf.loc[:, ' Stroke500mPace (sec/500m)'] try: velo = rowdatadf.loc[:,' AverageBoatSpeed (m/s)'] except KeyError: # pragma: no cover velo = 500./p hr = rowdatadf.loc[:, ' HRCur (bpm)'] spm = rowdatadf.loc[:, ' Cadence (stokes/min)'] cumdist = rowdatadf.loc[:, 'cum_dist'] power = rowdatadf.loc[:, ' Power (watts)'] averageforce = rowdatadf.loc[:, ' AverageDriveForce (lbs)'] drivelength = rowdatadf.loc[:, ' DriveLength (meters)'] try: workoutstate = rowdatadf.loc[:, ' WorkoutState'] except KeyError: # pragma: no cover workoutstate = 0 * hr peakforce = rowdatadf.loc[:, ' PeakDriveForce (lbs)'] forceratio = averageforce / peakforce forceratio = forceratio.fillna(value=0) try: drivetime = rowdatadf.loc[:, ' DriveTime (ms)'] recoverytime = rowdatadf.loc[:, ' StrokeRecoveryTime (ms)'] rhythm = 100. * drivetime / (recoverytime + drivetime) rhythm = rhythm.fillna(value=0) except: # pragma: no cover rhythm = 0.0 * forceratio f = rowdatadf['TimeStamp (sec)'].diff().mean() if f != 0 and not np.isinf(f): try: windowsize = 2 * (int(10. / (f))) + 1 except ValueError: # pragma: no cover windowsize = 1 else: windowsize = 1 if windowsize <= 3: windowsize = 5 if windowsize > 3 and windowsize < len(hr): spm = savgol_filter(spm, windowsize, 3) hr = savgol_filter(hr, windowsize, 3) drivelength = savgol_filter(drivelength, windowsize, 3) forceratio = savgol_filter(forceratio, windowsize, 3) try: t2 = t.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) except TypeError: # pragma: no cover t2 = 0 * t p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) try: drivespeed = drivelength / rowdatadf[' DriveTime (ms)'] * 1.0e3 except TypeError: # pragma: no cover drivespeed = 0.0 * rowdatadf['TimeStamp (sec)'] drivespeed = drivespeed.fillna(value=0) try: driveenergy = rowdatadf['driveenergy'] except KeyError: # pragma: no cover if forceunit == 'lbs': driveenergy = drivelength * averageforce * lbstoN else: drivenergy = drivelength * averageforce if forceunit == 'lbs': averageforce *= lbstoN peakforce *= lbstoN powerhr = 60.*power/hr powerhr = powerhr.fillna(value=0) if driveenergy.mean() == 0 and driveenergy.std() == 0: driveenergy = 0*driveenergy+100 distance = rowdatadf.loc[:, 'cum_dist'] velo = 500. / p distanceperstroke = 60. * velo / spm data = DataFrame( dict( time=t * 1e3, hr=hr, pace=p * 1e3, spm=spm, velo=velo, cumdist=cumdist, ftime=niceformat(t2), fpace=nicepaceformat(p2), driveenergy=driveenergy, power=power, workoutstate=workoutstate, averageforce=averageforce, drivelength=drivelength, peakforce=peakforce, forceratio=forceratio, distance=distance, drivespeed=drivespeed, rhythm=rhythm, distanceperstroke=distanceperstroke, # powerhr=powerhr, ) ) if bands: # HR bands data['hr_ut2'] = rowdatadf.loc[:, 'hr_ut2'] data['hr_ut1'] = rowdatadf.loc[:, 'hr_ut1'] data['hr_at'] = rowdatadf.loc[:, 'hr_at'] data['hr_tr'] = rowdatadf.loc[:, 'hr_tr'] data['hr_an'] = rowdatadf.loc[:, 'hr_an'] data['hr_max'] = rowdatadf.loc[:, 'hr_max'] data['hr_bottom'] = 0.0 * data['hr'] try: tel = rowdatadf.loc[:, ' ElapsedTime (sec)'] except KeyError: # pragma: no cover rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] if empower: try: wash = rowdatadf.loc[:, 'wash'] except KeyError: wash = 0 * power try: catch = rowdatadf.loc[:, 'catch'] except KeyError: catch = 0 * power try: finish = rowdatadf.loc[:, 'finish'] except KeyError: finish = 0 * power try: peakforceangle = rowdatadf.loc[:, 'peakforceangle'] except KeyError: peakforceangle = 0 * power if data['driveenergy'].mean() == 0: # pragma: no cover try: driveenergy = rowdatadf.loc[:, 'driveenergy'] except KeyError: driveenergy = power * 60 / spm else: driveenergy = data['driveenergy'] arclength = (inboard - 0.05) * (np.radians(finish) - np.radians(catch)) if arclength.mean() > 0: drivelength = arclength elif drivelength.mean() == 0: drivelength = driveenergy / (averageforce * 4.44822) try: slip = rowdatadf.loc[:, 'slip'] except KeyError: slip = 0 * power try: totalangle = finish - catch effectiveangle = finish - wash - catch - slip except ValueError: # pragma: no cover totalangle = 0 * power effectiveangle = 0 * power if windowsize > 3 and windowsize < len(slip): try: wash = savgol_filter(wash, windowsize, 3) except TypeError: # pragma: no cover pass try: slip = savgol_filter(slip, windowsize, 3) except TypeError: # pragma: no cover pass try: catch = savgol_filter(catch, windowsize, 3) except TypeError: # pragma: no cover pass try: finish = savgol_filter(finish, windowsize, 3) except TypeError: # pragma: no cover pass try: peakforceangle = savgol_filter(peakforceangle, windowsize, 3) except TypeError: # pragma: no cover pass try: driveenergy = savgol_filter(driveenergy, windowsize, 3) except TypeError: # pragma: no cover pass try: drivelength = savgol_filter(drivelength, windowsize, 3) except TypeError: # pragma: no cover pass try: totalangle = savgol_filter(totalangle, windowsize, 3) except TypeError: # pragma: no cover pass try: effectiveangle = savgol_filter(effectiveangle, windowsize, 3) except TypeError: # pragma: no cover pass velo = 500. / p ergpw = 2.8 * velo**3 efficiency = 100. * ergpw / power efficiency = efficiency.replace([-np.inf, np.inf], np.nan) efficiency.fillna(method='ffill') try: data['wash'] = wash data['catch'] = catch data['slip'] = slip data['finish'] = finish data['peakforceangle'] = peakforceangle data['driveenergy'] = driveenergy data['drivelength'] = drivelength data['totalangle'] = totalangle data['effectiveangle'] = effectiveangle data['efficiency'] = efficiency except ValueError: # pragma: no cover pass if otwpower: try: nowindpace = rowdatadf.loc[:, 'nowindpace'] except KeyError: nowindpace = p try: equivergpower = rowdatadf.loc[:, 'equivergpower'] except KeyError: equivergpower = 0 * p + 50. nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x)) ergvelo = (equivergpower / 2.8)**(1. / 3.) ergpace = 500. / ergvelo ergpace[ergpace == np.inf] = 240. ergpace2 = ergpace.apply(lambda x: timedeltaconv(x)) data['ergpace'] = ergpace * 1e3 data['nowindpace'] = nowindpace * 1e3 data['equivergpower'] = equivergpower data['fergpace'] = nicepaceformat(ergpace2) data['fnowindpace'] = nicepaceformat(nowindpace2) data = data.replace([-np.inf, np.inf], np.nan) data = data.fillna(method='ffill') # write data if id given if id != 0: data['workoutid'] = id data.fillna(0,inplace=True) for k, v in dtypes.items(): try: data[k] = data[k].astype(v) except KeyError: # pragma: no cover pass filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) df = dd.from_pandas(data,npartitions=1) df.to_parquet(filename,engine='fastparquet',compression='GZIP') return data def workout_trimp(w,reset=False): r = w.user if w.trimp > 0 and not reset: return w.trimp,w.hrtss r = w.user ftp = float(r.ftp) if w.workouttype in otwtypes: ftp = ftp*(100.-r.otwslack)/100. if r.hrftp == 0: hrftp = (r.an+r.tr)/2. r.hrftp = int(hrftp) r.save() if w.averagehr is None: rowdata = rdata(w.csvfilename) try: avghr = rowdata.df[' HRCur (bpm)'].mean() maxhr = rowdata.df[' HRCur (bpm)'].max() except KeyError: # pragma: no cover avghr = None maxhr = None w.averagehr = avghr w.maxhr = maxhr w.save() job = myqueue( queuehigh, handle_calctrimp, w.id, w.csvfilename, ftp, r.sex, r.hrftp, r.max, r.rest) return 0,0 def workout_rscore(w,reset=False): if w.rscore > 0 and not reset: return w.rscore,w.normp r = w.user ftp = float(r.ftp) if w.workouttype in otwtypes: ftp = ftp*(100.-r.otwslack)/100. if r.hrftp == 0: hrftp = (r.an+r.tr)/2. r.hrftp = int(hrftp) r.save() job = myqueue( queuehigh, handle_calctrimp, w.id, w.csvfilename, ftp, r.sex, r.hrftp, r.max, r.rest) return 0,0 def workout_normv(w,pp=4.0): if w.normv > 0: # pragma: no cover return w.normv,w.normw r = w.user ftp = float(r.ftp) if w.workouttype in otwtypes: ftp = ftp*(100.-r.otwslack)/100. if r.hrftp == 0: # pragma: no cover hrftp = (r.an+r.tr)/2. r.hrftp = int(hrftp) r.save() job = myqueue( queuehigh, handle_calctrimp, w.id, w.csvfilename, ftp, r.sex, r.hrftp, r.max, r.rest) return 0,0