diff --git a/rowers/dataprep.py b/rowers/dataprep.py index eb2e3c46..217baa68 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -79,6 +79,8 @@ from rowingdata import ( get_file_type, get_empower_rigging, get_empower_firmware ) +from rowers.dataroutines import * + from rowers.tasks import ( handle_sendemail_unrecognized, handle_setcp, handle_getagegrouprecords, handle_update_wps, @@ -116,203 +118,89 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') -# mapping the DB column names to the CSV file column names -columndict = { - 'time': 'TimeStamp (sec)', - 'hr': ' HRCur (bpm)', - 'velo': ' AverageBoatSpeed (m/s)', - 'pace': ' Stroke500mPace (sec/500m)', - 'spm': ' Cadence (stokes/min)', - 'power': ' Power (watts)', - 'averageforce': ' AverageDriveForce (lbs)', - 'drivelength': ' DriveLength (meters)', - 'peakforce': ' PeakDriveForce (lbs)', - 'distance': ' Horizontal (meters)', - 'catch': 'catch', - 'finish': 'finish', - 'peakforceangle': 'peakforceangle', - 'wash': 'wash', - 'slip': 'slip', - 'workoutstate': ' WorkoutState', - 'cumdist': 'cum_dist', -} + +def fetchcperg(rower, theworkouts): + thefilenames = [w.csvfilename for w in theworkouts] + cpdf = getcpdata_sql(rower.id, table='ergcpdata') + + _ = myqueue( + queuelow, + handle_updateergcp, + rower.id, + thefilenames) + + return cpdf + +def workout_goldmedalstandard(workout, reset=False): + if workout.goldmedalstandard > 0 and not reset: + return workout.goldmedalstandard, workout.goldmedalseconds + if workout.workouttype in rowtypes: + goldmedalstandard, goldmedalseconds = calculate_goldmedalstandard( + workout.user, workout) + if workout.workouttype in otwtypes: + factor = 100./(100.-workout.user.otwslack) + goldmedalstandard = goldmedalstandard*factor + workout.goldmedalstandard = goldmedalstandard + workout.goldmedalseconds = goldmedalseconds + workout.save() + return goldmedalstandard, goldmedalseconds + else: + return 0, 0 -def get_video_data(w, groups=['basic'], mode='water'): - modes = [mode, 'both', 'basic'] - columns = ['time', 'velo', 'spm'] - columns += [name for name, d in rowingmetrics if d['group'] - in groups and d['mode'] in modes] - columns = list(set(columns)) - df = getsmallrowdata_db(columns, ids=[w.id], - workstrokesonly=False, doclean=False, compute=False) +def check_marker(workout): + r = workout.user + gmstandard, gmseconds = workout_goldmedalstandard(workout) + if gmseconds < 60: + return None - df['time'] = (df['time']-df['time'].min())/1000. - - df.sort_values(by='time', inplace=True) - - df.set_index(pd.to_timedelta(df['time'], unit='s'), inplace=True) - df2 = df.resample('1s').first().fillna(method='ffill') - df2['time'] = df2.index.total_seconds() - - if 'pace' in columns: - df2['pace'] = df2['pace']/1000. - p = df2['pace'] - p = p.apply(lambda x: timedeltaconv(x)) - p = nicepaceformat(p) - df2['pace'] = p - - df2['time'] = (df2['time']-df2['time'].min()) - - df2 = df2.round(decimals=2) - - boatspeed = (100*df2['velo']).astype(int)/100. - - try: - coordinates = get_latlon_time(w.id) - except KeyError: # pragma: no cover - nulseries = df['time']*0 - coordinates = pd.DataFrame({ - 'time': df['time'], - 'latitude': nulseries, - 'longitude': nulseries, - }) - - coordinates.set_index(pd.to_timedelta( - coordinates['time'], unit='s'), inplace=True) - coordinates = coordinates.resample('1s').mean().interpolate() - coordinates['time'] = coordinates['time']-coordinates['time'].min() - latitude = coordinates['latitude'] - longitude = coordinates['longitude'] - - # bundle data - data = { - 'boatspeed': boatspeed.values.tolist(), - 'latitude': latitude.values.tolist(), - 'longitude': longitude.values.tolist(), - } - - metrics = {} - - for c in columns: - if c != 'time': - try: - if dict(rowingmetrics)[c]['numtype'] == 'integer': # pragma: no cover - data[c] = df2[c].astype(int).tolist() - else: - sigfigs = dict(rowingmetrics)[c]['sigfigs'] - if (c != 'pace'): - da = ((10**sigfigs)*df2[c]).astype(int)/(10**sigfigs) - else: - da = df2[c] - data[c] = da.values.tolist() - metrics[c] = { - 'name': dict(rowingmetrics)[c]['verbose_name'], - 'metric': c, - 'unit': '' - } - except KeyError: # pragma: no cover - pass - - metrics['boatspeed'] = metrics.pop('velo') - # metrics['workperstroke'] = metrics.pop('driveenergy') - metrics = collections.OrderedDict(sorted(metrics.items())) - - maxtime = coordinates['time'].max() - - return data, metrics, maxtime - - -def polarization_index(df, rower): - df['dt'] = df['time'].diff()/6.e4 - # remove rest (spm<15) - df.dropna(axis=0, inplace=True) - df['dt'] = df['dt'].clip(upper=4, lower=0) - - masklow = (df['power'] > 0) & (df['power'] < int(rower.pw_at)) - maskmid = (df['power'] >= rower.pw_at) & (df['power'] < int(rower.pw_an)) - maskhigh = (df['power'] > rower.pw_an) - - time_low_pw = df.loc[masklow, 'dt'].sum() - time_mid_pw = df.loc[maskmid, 'dt'].sum() - time_high_pw = df.loc[maskhigh, 'dt'].sum() - - frac_low = time_low_pw/(time_low_pw+time_mid_pw+time_high_pw) - frac_mid = time_mid_pw/(time_low_pw+time_mid_pw+time_high_pw) - frac_high = time_high_pw/(time_low_pw+time_mid_pw+time_high_pw) - - index = math.log10(frac_high*100.*frac_low/frac_mid) - - return index - - -def get_latlon(id): - try: - w = Workout.objects.get(id=id) - except Workout.DoesNotExist: # pragma: no cover - return False - - rowdata = rdata(w.csvfilename) - - if rowdata.df.empty: # pragma: no cover - return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] - - try: - try: - latitude = rowdata.df.loc[:, ' latitude'] - longitude = rowdata.df.loc[:, ' longitude'] - except KeyError: - latitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] - longitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] - return [latitude, longitude] - except AttributeError: # pragma: no cover - return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] - - return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] # pragma: no cover - - -def get_latlon_time(id): - try: - w = Workout.objects.get(id=id) - except Workout.DoesNotExist: # pragma: no cover - return False - - rowdata = rdata(w.csvfilename) - - if rowdata.df.empty: # pragma: no cover - return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] - - try: - try: - _ = rowdata.df.loc[:, ' latitude'] - _ = rowdata.df.loc[:, ' longitude'] - except KeyError: # pragma: no cover - rowdata.df['latitude'] = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] - rowdata.df['longitude'] = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] - except AttributeError: # pragma: no cover - return pd.DataFrame() + dd = arrow.get(workout.date).datetime-datetime.timedelta(days=r.kfit) + ws = Workout.objects.filter(date__gte=dd, + date__lte=workout.date, + user=r, duplicate=False, + workouttype__in=mytypes.rowtypes, + ).order_by("date") + ids = [] + gms = [] + for w in ws: + gmstandard, gmseconds = workout_goldmedalstandard(w) + if gmseconds > 60: + ids.append(w.id) + gms.append(gmstandard) df = pd.DataFrame({ - 'time': rowdata.df['TimeStamp (sec)']-rowdata.df['TimeStamp (sec)'].min(), - 'latitude': rowdata.df[' latitude'], - 'longitude': rowdata.df[' longitude'] + 'id': ids, + 'gms': gms, }) - return df + if df.empty: # pragma: no cover + workout.ranking = True + workout.save() + return workout + indexmax = df['gms'].idxmax() + theid = df.loc[indexmax, 'id'] -def workout_has_latlon(id): - latitude, longitude = get_latlon(id) - latmean = latitude.mean() - lonmean = longitude.mean() + wmax = Workout.objects.get(id=theid) + # gms_max = wmax.goldmedalstandard - if latmean == 0 and lonmean == 0: - return False, latmean, lonmean + # check if equal, bigger, or smaller than previous + if not wmax.rankingpiece: + rankingworkouts = ws.filter(rankingpiece=True) + if len(rankingworkouts) == 0: + wmax.rankingpiece = True + wmax.save() + return wmax - if latitude.std() > 0 and longitude.std() > 0: - return True, latmean, lonmean + lastranking = rankingworkouts[len(rankingworkouts)-1] + if lastranking.goldmedalstandard+0.2 < wmax.goldmedalstandard: # pragma: no cover + wmax.rankingpiece = True + wmax.save() + return wmax + else: # pragma: no cover + return wmax - return False, latmean, lonmean + return None def workout_summary_to_df( @@ -414,37 +302,238 @@ def workout_summary_to_df( return df -def get_workouts(ids, userid): # pragma: no cover - goodids = [] - for id in ids: - w = Workout.objects.get(id=id) - if int(w.user.user.id) == int(userid): - goodids.append(id) +def resample(id, r, parent, overwrite='copy'): + data, row = getrowdata_db(id=id) + messages = [] - return [Workout.objects.get(id=id) for id in goodids] + # resample + startdatetime = row.startdatetime + data['datetime'] = data['time'].apply( + lambda x: startdatetime+datetime.timedelta(seconds=x/1000.)) + + data = data.resample('S', on='datetime').mean() + data.interpolate(method='linear', inplace=True) + data.reset_index(drop=True, inplace=True) + + # data.drop('datetime',inplace=True) + data['pace'] = data['pace'] / 1000. + data['time'] = data['time'] / 1000. + + if overwrite == 'overwrite': + # remove CP data + try: + cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=parent.id) + os.remove(cpfile) + except FileNotFoundError: + pass + # save + data.rename(columns=columndict, inplace=True) + + starttimeunix = arrow.get(startdatetime).timestamp() + data[' ElapsedTime (sec)'] = data['TimeStamp (sec)'] + + data['TimeStamp (sec)'] = data['TimeStamp (sec)'] + starttimeunix + + row = rrdata(df=data) + + row.write_csv(parent.csvfilename, gzip=True) + + _ = dataprep(row.df, id=parent.id, bands=True, barchart=True, + otwpower=True, empower=True, inboard=parent.inboard) + isbreakthrough, ishard = checkbreakthrough(parent, r) + _ = check_marker(parent) + _ = update_wps(r, mytypes.otwtypes) + _ = update_wps(r, mytypes.otetypes) + + tss, normp = workout_rscore(parent) + goldmedalstandard, goldmedalseconds = workout_goldmedalstandard(parent) + else: + id, message = new_workout_from_df(r, data, title=parent.name + '(Resampled)', + parent=parent, forceunit='N') + messages.append(message) + + return data, id, messages -def filter_df(datadf, fieldname, value, largerthan=True): + +def calculate_goldmedalstandard(rower, workout, recurrance=True): + cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) try: - _ = datadf[fieldname] - except KeyError: - return datadf + df = pd.read_parquet(cpfile) + except: + background = True + if settings.TESTING: + background = False + df, delta, cpvalues = setcp(workout, background=background) + if df.empty: + return 0, 0 - try: - if largerthan: - mask = datadf[fieldname] < value + if df.empty and recurrance: # pragma: no cover + df, delta, cpvalues = setcp(workout, recurrance=False, background=True) + if df.empty: + return 0, 0 + + age = calculate_age(rower.birthdate, today=workout.date) + + agerecords = CalcAgePerformance.objects.filter( + age=age, + sex=rower.sex, + weightcategory=rower.weightcategory + ) + + wcdurations = [] + wcpower = [] + getrecords = False + if not settings.TESTING: # pragma: no cover + if len(agerecords) == 0: # pragma: no cover + getrecords = True + + for record in agerecords: # pragma: no cover + if record.power > 0: + wcdurations.append(record.duration) + wcpower.append(record.power) else: - mask = datadf[fieldname] >= value + getrecords = True - datadf.loc[mask, fieldname] = np.nan - except TypeError: + if getrecords: # pragma: no cover + durations = [1, 4, 30, 60] + distances = [100, 500, 1000, 2000, 5000, 6000, 10000, 21097, 42195] + df2 = pd.DataFrame( + list( + C2WorldClassAgePerformance.objects.filter( + sex=rower.sex, + weightcategory=rower.weightcategory + ).values() + ) + ) + jsondf = df2.to_json() + _ = myqueue(queuelow, handle_getagegrouprecords, + jsondf, distances, durations, age, rower.sex, rower.weightcategory) + + wcpower = pd.Series(wcpower, dtype='float') + wcdurations = pd.Series(wcdurations, dtype='float') + + def fitfunc(pars, x): + return pars[0] / (1+(x/pars[2])) + pars[1]/(1+(x/pars[3])) + + def errfunc(pars, x, y): + return fitfunc(pars, x)-y + + if len(wcdurations) >= 4: # pragma: no cover + p1wc, success = optimize.leastsq( + errfunc, p0[:], args=(wcdurations, wcpower)) + else: + factor = fitfunc(p0, wcdurations.mean()/wcpower.mean()) + p1wc = [p0[0]/factor, p0[1]/factor, p0[2], p0[3]] + + return 0, 0 + + times = df['delta'] + powers = df['cp'] + wcpowers = fitfunc(p1wc, times) + scores = 100.*powers/wcpowers + + try: + indexmax = scores.idxmax() + delta = int(df.loc[indexmax, 'delta']) + maxvalue = scores.max() + except (ValueError, TypeError): # pragma: no cover + indexmax = 0 + delta = 0 + maxvalue = 0 + + return maxvalue, delta + + + +def setcp(workout, background=False, recurrance=True): + filename = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) + + strokesdf = getsmallrowdata_db( + ['power', 'workoutid', 'time'], ids=[workout.id]) + + try: + if strokesdf['power'].std() == 0: + return pd.DataFrame(), pd.Series(dtype='float'), pd.Series(dtype='float') + except KeyError: + return pd.DataFrame(), pd.Series(dtype='float'), pd.Series(dtype='float') + + if background: # pragma: no cover + _ = myqueue(queuelow, handle_setcp, strokesdf, filename, workout.id) + return pd.DataFrame({'delta': [], 'cp': []}), pd.Series(dtype='float'), pd.Series(dtype='float') + + if not strokesdf.empty: + totaltime = strokesdf['time'].max() + try: + powermean = strokesdf['power'].mean() + except KeyError: # pragma: no cover + powermean = 0 + + if powermean != 0: + thesecs = totaltime + maxt = 1.05 * thesecs + + if maxt > 0: + logarr = datautils.getlogarr(maxt) + dfgrouped = strokesdf.groupby(['workoutid']) + delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) + + df = pd.DataFrame({ + 'delta': delta, + 'cp': cpvalues, + 'id': workout.id, + }) + df.to_parquet(filename, engine='fastparquet', + compression='GZIP') + if recurrance: + goldmedalstandard, goldmedalduration = calculate_goldmedalstandard( + workout.user, workout) + workout.goldmedalstandard = goldmedalstandard + workout.goldmedalduration = goldmedalduration + workout.save() + return df, delta, cpvalues + + return pd.DataFrame({'delta': [], 'cp': []}), pd.Series(dtype='float'), pd.Series(dtype='float') + + +def update_wps(r, types, mode='water', asynchron=True): + firstdate = timezone.now()-datetime.timedelta(days=r.cprange) + workouts = Workout.objects.filter( + date__gte=firstdate, + workouttype__in=types, + user=r + ) + + ids = [w.id for w in workouts] + if asynchron: + _ = myqueue( + queue, + handle_update_wps, + r.id, + types, + ids, + mode + ) + + df = getsmallrowdata_db(['time', 'driveenergy'], ids=ids) + + try: + mask = df['driveenergy'] > 100 + except (KeyError, TypeError): + return False + try: + wps_median = int(df.loc[mask, 'driveenergy'].median()) + if mode == 'water': + r.median_wps = wps_median + else: # pragma: no cover + r.median_wps_erg = wps_median + + r.save() + except ValueError: # pragma: no cover pass - return datadf - -# joins workouts - + return True def join_workouts(r, ids, title='Joined Workout', parent=None, @@ -537,712 +626,6 @@ def join_workouts(r, ids, title='Joined Workout', return (id, message) -def df_resample(datadf): - # time stamps must be in seconds - timestamps = datadf['TimeStamp (sec)'].astype('int') - datadf['timestamps'] = timestamps - newdf = datadf.groupby(['timestamps']).mean() - return newdf - - -def resample(id, r, parent, overwrite='copy'): - data, row = getrowdata_db(id=id) - messages = [] - - # resample - startdatetime = row.startdatetime - data['datetime'] = data['time'].apply( - lambda x: startdatetime+datetime.timedelta(seconds=x/1000.)) - - data = data.resample('S', on='datetime').mean() - data.interpolate(method='linear', inplace=True) - data.reset_index(drop=True, inplace=True) - - # data.drop('datetime',inplace=True) - data['pace'] = data['pace'] / 1000. - data['time'] = data['time'] / 1000. - - if overwrite == 'overwrite': - # remove CP data - try: - cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=parent.id) - os.remove(cpfile) - except FileNotFoundError: - pass - # save - data.rename(columns=columndict, inplace=True) - - starttimeunix = arrow.get(startdatetime).timestamp() - data[' ElapsedTime (sec)'] = data['TimeStamp (sec)'] - - data['TimeStamp (sec)'] = data['TimeStamp (sec)'] + starttimeunix - - row = rrdata(df=data) - - row.write_csv(parent.csvfilename, gzip=True) - - _ = dataprep(row.df, id=parent.id, bands=True, barchart=True, - otwpower=True, empower=True, inboard=parent.inboard) - isbreakthrough, ishard = checkbreakthrough(parent, r) - _ = check_marker(parent) - _ = update_wps(r, mytypes.otwtypes) - _ = update_wps(r, mytypes.otetypes) - - tss, normp = workout_rscore(parent) - goldmedalstandard, goldmedalseconds = workout_goldmedalstandard(parent) - else: - id, message = new_workout_from_df(r, data, title=parent.name + '(Resampled)', - parent=parent, forceunit='N') - messages.append(message) - - return data, id, messages - - -def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, - ignoreadvanced=False): - # clean data remove zeros and negative values - - try: - _ = datadf['workoutid'].unique() - except KeyError: - datadf['workoutid'] = 0 - - before = {} - for workoutid in datadf['workoutid'].unique(): - before[workoutid] = len(datadf[datadf['workoutid'] == workoutid]) - - data_orig = datadf.copy() - - # bring metrics which have negative values to positive domain - if len(datadf) == 0: - return datadf - try: - datadf['catch'] = -datadf['catch'] - except (KeyError, TypeError): - pass - - try: - datadf['peakforceangle'] = datadf['peakforceangle'] + 1000 - except (KeyError, TypeError): - pass - - try: - datadf['hr'] = datadf['hr'] + 10 - except (KeyError, TypeError): - pass - - # protect 0 spm values from being nulled - try: - datadf['spm'] = datadf['spm'] + 1.0 - except (KeyError, TypeError): - pass - - # protect 0 workoutstate values from being nulled - try: - datadf['workoutstate'] = datadf['workoutstate'] + 1 - except (KeyError, TypeError): - pass - - try: - datadf = datadf.clip(lower=0) - except TypeError: - pass - - # protect advanced metrics columns - advancedcols = [ - 'rhythm', - 'power', - 'drivelength', - 'forceratio', - 'drivespeed', - 'driveenergy', - 'catch', - 'finish', - 'averageforce', - 'peakforce', - 'slip', - 'wash', - 'peakforceangle', - 'effectiveangle', - ] - - datadf.replace(to_replace=0, value=np.nan, inplace=True) - # datadf = datadf.map_partitions(lambda df:df.replace(to_replace=0,value=np.nan)) - - # bring spm back to real values - try: - datadf['spm'] = datadf['spm'] - 1 - except (TypeError, KeyError): - pass - - # bring workoutstate back to real values - try: - datadf['workoutstate'] = datadf['workoutstate'] - 1 - except (TypeError, KeyError): - pass - - # return from positive domain to negative - try: - datadf['catch'] = -datadf['catch'] - except (KeyError, TypeError): - pass - - try: - datadf['peakforceangle'] = datadf['peakforceangle'] - 1000 - except (KeyError, TypeError): - pass - - try: - datadf['hr'] = datadf['hr'] - 10 - except (KeyError, TypeError): - pass - - # clean data for useful ranges per column - if not ignorehr: - try: - mask = datadf['hr'] < 30 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): # pragma: no cover - pass - - try: - mask = datadf['spm'] < 0 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['efficiency'] > 200. - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['spm'] < 10 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['pace'] / 1000. > 300. - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['efficiency'] < 0. - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['pace'] / 1000. < 60. - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['power'] > 5000 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['spm'] > 120 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['wash'] < 1 - datadf.loc[mask, 'wash'] = np.nan - except (KeyError, TypeError): - pass - - # try to guess ignoreadvanced - if not ignoreadvanced: - for metric in advancedcols: - try: - sum = datadf[metric].std() - if sum == 0 or np.isnan(sum): - ignoreadvanced = True - except KeyError: - pass - - if not ignoreadvanced: - try: - mask = datadf['rhythm'] < 0 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['rhythm'] > 70 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['power'] < 20 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['drivelength'] < 0.5 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['forceratio'] < 0.2 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['forceratio'] > 1.0 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['drivespeed'] < 0.5 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['drivespeed'] > 4 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['driveenergy'] > 2000 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['driveenergy'] < 100 - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - try: - mask = datadf['catch'] > -30. - datadf.mask(mask, inplace=True) - except (KeyError, TypeError): - pass - - # workoutstateswork = [1, 4, 5, 8, 9, 6, 7] - workoutstatesrest = [3] - # workoutstatetransition = [0, 2, 10, 11, 12, 13] - - if workstrokesonly == 'True' or workstrokesonly is True: - try: - datadf = datadf[~datadf['workoutstate'].isin(workoutstatesrest)] - except: - pass - - after = {} - for workoutid in data_orig['workoutid'].unique(): - after[workoutid] = len( - datadf[datadf['workoutid'] == workoutid].dropna()) - ratio = float(after[workoutid])/float(before[workoutid]) - if ratio < 0.01 or after[workoutid] < 2: - return data_orig - - return datadf - - -def getpartofday(row, r): - workoutstartdatetime = row.rowdatetime - try: # pragma: no cover - latavg = row.df[' latitude'].mean() - lonavg = row.df[' longitude'].mean() - - tf = TimezoneFinder() - try: - timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) - except (ValueError, OverflowError): # pragma: no cover - timezone_str = 'UTC' - if timezone_str is None: # pragma: no cover - timezone_str = tf.closest_timezone_at(lng=lonavg, - lat=latavg) - if timezone_str is None: - timezone_str = r.defaulttimezone - try: - workoutstartdatetime = pytz.timezone(timezone_str).localize( - row.rowdatetime - ) - except ValueError: - workoutstartdatetime = row.rowdatetime - except KeyError: - timezone_str = r.defaulttimezone - workoutstartdatetime = row.rowdatetime - - h = workoutstartdatetime.astimezone(pytz.timezone(timezone_str)).hour - - if h < 12: # pragma: no cover - return "Morning" - elif h < 18: # pragma: no cover - return "Afternoon" - elif h < 22: # pragma: no cover - return "Evening" - else: # pragma: no cover - return "Night" - - return None # pragma: no cover - - -def getstatsfields(): - fielddict = {name: d['verbose_name'] for name, d in rowingmetrics} - -# fielddict.pop('ergpace') -# fielddict.pop('hr_an') -# fielddict.pop('hr_tr') -# fielddict.pop('hr_at') -# fielddict.pop('hr_ut2') -# fielddict.pop('hr_ut1') - fielddict.pop('time') - fielddict.pop('distance') -# fielddict.pop('nowindpace') -# fielddict.pop('fnowindpace') -# fielddict.pop('fergpace') -# fielddict.pop('equivergpower') -# fielddict.pop('workoutstate') -# fielddict.pop('fpace') -# fielddict.pop('pace') -# fielddict.pop('id') -# fielddict.pop('ftime') -# fielddict.pop('x_right') -# fielddict.pop('hr_max') -# fielddict.pop('hr_bottom') - fielddict.pop('cumdist') - - try: - fieldlist = [field for field, value in fielddict.iteritems()] - except AttributeError: - fieldlist = [field for field, value in fielddict.items()] - - return fieldlist, fielddict - - -# A string representation for time deltas -def niceformat(values): - out = [] - for v in values: - formattedv = strfdelta(v) - out.append(formattedv) - - return out - -# A nice printable format for time delta values - - -def strfdelta(tdelta): - try: - minutes, seconds = divmod(tdelta.seconds, 60) - tenths = int(tdelta.microseconds / 1e5) - except AttributeError: # pragma: no cover - minutes, seconds = divmod(tdelta.view(np.int64), 60e9) - seconds, rest = divmod(seconds, 1e9) - tenths = int(rest / 1e8) - res = "{minutes:0>2}:{seconds:0>2}.{tenths:0>1}".format( - minutes=minutes, - seconds=seconds, - tenths=tenths, - ) - - return res - - -def timedelta_to_seconds(tdelta): # pragma: no cover - return 60.*tdelta.minute+tdelta.second - - -# A nice printable format for pace values - - -def nicepaceformat(values): - out = [] - for v in values: - formattedv = strfdelta(v) - out.append(formattedv) - - return out - -# Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace - - -def timedeltaconv(x): - if np.isfinite(x) and x != 0 and x > 0 and x < 175000: - dt = datetime.timedelta(seconds=x) - else: - dt = datetime.timedelta(seconds=350.) - - return dt - - -def paceformatsecs(values): - out = [] - for v in values: - td = timedeltaconv(v) - formattedv = strfdelta(td) - out.append(formattedv) - - return out - - -def update_c2id_sql(id, c2id): - engine = create_engine(database_url, echo=False) - table = 'rowers_workout' - - query = "UPDATE %s SET uploadedtoc2 = %s WHERE `id` = %s;" % ( - table, c2id, id) - - with engine.connect() as conn, conn.begin(): - _ = conn.execute(query) - - conn.close() - engine.dispose() - - return 1 - - -def getcpdata_sql(rower_id, table='cpdata'): - engine = create_engine(database_url, echo=False) - query = sa.text('SELECT * from {table} WHERE user={rower_id};'.format( - rower_id=rower_id, - table=table, - )) - - _ = engine.raw_connection() - df = pd.read_sql_query(query, engine) - - return df - - -def deletecpdata_sql(rower_id, table='cpdata'): # pragma: no cover - engine = create_engine(database_url, echo=False) - query = sa.text('DELETE from {table} WHERE user={rower_id};'.format( - rower_id=rower_id, - table=table, - )) - with engine.connect() as conn, conn.begin(): - try: - _ = conn.execute(query) - except: - print("Database locked") - conn.close() - engine.dispose() - - -def updatecpdata_sql(rower_id, delta, cp, table='cpdata', distance=[]): # pragma: no cover - deletecpdata_sql(rower_id) - df = pd.DataFrame( - { - 'delta': delta, - 'cp': cp, - 'user': rower_id - } - ) - - if not distance.empty: - df['distance'] = distance - - engine = create_engine(database_url, echo=False) - with engine.connect() as conn, conn.begin(): - df.to_sql(table, engine, if_exists='append', index=False) - conn.close() - engine.dispose() - - -def fetchcperg(rower, theworkouts): - thefilenames = [w.csvfilename for w in theworkouts] - cpdf = getcpdata_sql(rower.id, table='ergcpdata') - - _ = myqueue( - queuelow, - handle_updateergcp, - rower.id, - thefilenames) - - return cpdf - - -def get_workoutsummaries(userid, startdate): # pragma: no cover - u = User.objects.get(id=userid) - r = u.rower - df = workout_summary_to_df(r, startdate=startdate) - df.drop(['Stroke Data TCX', 'Stroke Data CSV'], axis=1, inplace=True) - df = df.sort_values('date', ascending=False) - - return df - - -def workout_goldmedalstandard(workout, reset=False): - if workout.goldmedalstandard > 0 and not reset: - return workout.goldmedalstandard, workout.goldmedalseconds - if workout.workouttype in rowtypes: - goldmedalstandard, goldmedalseconds = calculate_goldmedalstandard( - workout.user, workout) - if workout.workouttype in otwtypes: - factor = 100./(100.-workout.user.otwslack) - goldmedalstandard = goldmedalstandard*factor - workout.goldmedalstandard = goldmedalstandard - workout.goldmedalseconds = goldmedalseconds - workout.save() - return goldmedalstandard, goldmedalseconds - else: - return 0, 0 - - -def check_marker(workout): - r = workout.user - gmstandard, gmseconds = workout_goldmedalstandard(workout) - if gmseconds < 60: - return None - - dd = arrow.get(workout.date).datetime-datetime.timedelta(days=r.kfit) - ws = Workout.objects.filter(date__gte=dd, - date__lte=workout.date, - user=r, duplicate=False, - workouttype__in=mytypes.rowtypes, - ).order_by("date") - ids = [] - gms = [] - for w in ws: - gmstandard, gmseconds = workout_goldmedalstandard(w) - if gmseconds > 60: - ids.append(w.id) - gms.append(gmstandard) - - df = pd.DataFrame({ - 'id': ids, - 'gms': gms, - }) - - if df.empty: # pragma: no cover - workout.ranking = True - workout.save() - return workout - - indexmax = df['gms'].idxmax() - theid = df.loc[indexmax, 'id'] - - wmax = Workout.objects.get(id=theid) - # gms_max = wmax.goldmedalstandard - - # check if equal, bigger, or smaller than previous - if not wmax.rankingpiece: - rankingworkouts = ws.filter(rankingpiece=True) - if len(rankingworkouts) == 0: - wmax.rankingpiece = True - wmax.save() - return wmax - - lastranking = rankingworkouts[len(rankingworkouts)-1] - if lastranking.goldmedalstandard+0.2 < wmax.goldmedalstandard: # pragma: no cover - wmax.rankingpiece = True - wmax.save() - return wmax - else: # pragma: no cover - return wmax - - return None - - -def calculate_goldmedalstandard(rower, workout, recurrance=True): - cpfile = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) - try: - df = pd.read_parquet(cpfile) - except: - background = True - if settings.TESTING: - background = False - df, delta, cpvalues = setcp(workout, background=background) - if df.empty: - return 0, 0 - - if df.empty and recurrance: # pragma: no cover - df, delta, cpvalues = setcp(workout, recurrance=False, background=True) - if df.empty: - return 0, 0 - - age = calculate_age(rower.birthdate, today=workout.date) - - agerecords = CalcAgePerformance.objects.filter( - age=age, - sex=rower.sex, - weightcategory=rower.weightcategory - ) - - wcdurations = [] - wcpower = [] - getrecords = False - if not settings.TESTING: # pragma: no cover - if len(agerecords) == 0: # pragma: no cover - getrecords = True - - for record in agerecords: # pragma: no cover - if record.power > 0: - wcdurations.append(record.duration) - wcpower.append(record.power) - else: - getrecords = True - - if getrecords: # pragma: no cover - durations = [1, 4, 30, 60] - distances = [100, 500, 1000, 2000, 5000, 6000, 10000, 21097, 42195] - df2 = pd.DataFrame( - list( - C2WorldClassAgePerformance.objects.filter( - sex=rower.sex, - weightcategory=rower.weightcategory - ).values() - ) - ) - jsondf = df2.to_json() - _ = myqueue(queuelow, handle_getagegrouprecords, - jsondf, distances, durations, age, rower.sex, rower.weightcategory) - - wcpower = pd.Series(wcpower, dtype='float') - wcdurations = pd.Series(wcdurations, dtype='float') - - def fitfunc(pars, x): - return pars[0] / (1+(x/pars[2])) + pars[1]/(1+(x/pars[3])) - - def errfunc(pars, x, y): - return fitfunc(pars, x)-y - - if len(wcdurations) >= 4: # pragma: no cover - p1wc, success = optimize.leastsq( - errfunc, p0[:], args=(wcdurations, wcpower)) - else: - factor = fitfunc(p0, wcdurations.mean()/wcpower.mean()) - p1wc = [p0[0]/factor, p0[1]/factor, p0[2], p0[3]] - - return 0, 0 - - times = df['delta'] - powers = df['cp'] - wcpowers = fitfunc(p1wc, times) - scores = 100.*powers/wcpowers - - try: - indexmax = scores.idxmax() - delta = int(df.loc[indexmax, 'delta']) - maxvalue = scores.max() - except (ValueError, TypeError): # pragma: no cover - indexmax = 0 - delta = 0 - maxvalue = 0 - - return maxvalue, delta - def fetchcp_new(rower, workouts): @@ -1276,93 +659,45 @@ def fetchcp_new(rower, workouts): return df['delta'], df['cp'], 0, df['workout'], df['url'] -def setcp(workout, background=False, recurrance=True): - filename = 'media/cpdata_{id}.parquet.gz'.format(id=workout.id) - - strokesdf = getsmallrowdata_db( - ['power', 'workoutid', 'time'], ids=[workout.id]) +def fetchcp(rower, theworkouts, table='cpdata'): # pragma: no cover + # get all power data from database (plus workoutid) + theids = [int(w.id) for w in theworkouts] + columns = ['power', 'workoutid', 'time'] + df = getsmallrowdata_db(columns, ids=theids) + df.dropna(inplace=True, axis=0) + if df.empty: + avgpower2 = {} + for id in theids: + avgpower2[id] = 0 + return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 try: - if strokesdf['power'].std() == 0: - return pd.DataFrame(), pd.Series(dtype='float'), pd.Series(dtype='float') + dfgrouped = df.groupby(['workoutid']) except KeyError: - return pd.DataFrame(), pd.Series(dtype='float'), pd.Series(dtype='float') - - if background: # pragma: no cover - _ = myqueue(queuelow, handle_setcp, strokesdf, filename, workout.id) - return pd.DataFrame({'delta': [], 'cp': []}), pd.Series(dtype='float'), pd.Series(dtype='float') - - if not strokesdf.empty: - totaltime = strokesdf['time'].max() - try: - powermean = strokesdf['power'].mean() - except KeyError: # pragma: no cover - powermean = 0 - - if powermean != 0: - thesecs = totaltime - maxt = 1.05 * thesecs - - if maxt > 0: - logarr = datautils.getlogarr(maxt) - dfgrouped = strokesdf.groupby(['workoutid']) - delta, cpvalues, avgpower = datautils.getcp(dfgrouped, logarr) - - df = pd.DataFrame({ - 'delta': delta, - 'cp': cpvalues, - 'id': workout.id, - }) - df.to_parquet(filename, engine='fastparquet', - compression='GZIP') - if recurrance: - goldmedalstandard, goldmedalduration = calculate_goldmedalstandard( - workout.user, workout) - workout.goldmedalstandard = goldmedalstandard - workout.goldmedalduration = goldmedalduration - workout.save() - return df, delta, cpvalues - - return pd.DataFrame({'delta': [], 'cp': []}), pd.Series(dtype='float'), pd.Series(dtype='float') - - -def update_wps(r, types, mode='water', asynchron=True): - firstdate = timezone.now()-datetime.timedelta(days=r.cprange) - workouts = Workout.objects.filter( - date__gte=firstdate, - workouttype__in=types, - user=r - ) - - ids = [w.id for w in workouts] - if asynchron: - _ = myqueue( - queue, - handle_update_wps, - r.id, - types, - ids, - mode - ) - - df = getsmallrowdata_db(['time', 'driveenergy'], ids=ids) - + avgpower2 = {} + return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 try: - mask = df['driveenergy'] > 100 - except (KeyError, TypeError): - return False - try: - wps_median = int(df.loc[mask, 'driveenergy'].median()) - if mode == 'water': - r.median_wps = wps_median - else: # pragma: no cover - r.median_wps_erg = wps_median + avgpower2 = dict(dfgrouped.mean()['power'].astype(int)) + except KeyError: + avgpower2 = {} + for id in theids: + avgpower2[id] = 0 + return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 - r.save() - except ValueError: # pragma: no cover - pass + cpdf = getcpdata_sql(rower.id, table=table) - return True + if not cpdf.empty: + return cpdf['delta'], cpdf['cp'], avgpower2 + else: + _ = myqueue(queuelow, + handle_updatecp, + rower.id, + theids, + table=table) + + return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 + + return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 def update_rolling_cp(r, types, mode='water'): @@ -1407,47 +742,104 @@ def update_rolling_cp(r, types, mode='water'): return True return False +def initiate_cp(r): + _ = update_rolling_cp(r, otwtypes, 'water') + _ = update_rolling_cp(r, otetypes, 'erg') -def fetchcp(rower, theworkouts, table='cpdata'): # pragma: no cover - # get all power data from database (plus workoutid) - theids = [int(w.id) for w in theworkouts] - columns = ['power', 'workoutid', 'time'] - df = getsmallrowdata_db(columns, ids=theids) - df.dropna(inplace=True, axis=0) - if df.empty: - avgpower2 = {} - for id in theids: - avgpower2[id] = 0 - return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 +def split_workout(r, parent, splitsecond, splitmode): + data, row = getrowdata_db(id=parent.id) + latitude, longitude = get_latlon(parent.id) + if not latitude.empty and not longitude.empty: + data[' latitude'] = latitude + data[' longitude'] = longitude - try: - dfgrouped = df.groupby(['workoutid']) - except KeyError: - avgpower2 = {} - return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 - try: - avgpower2 = dict(dfgrouped.mean()['power'].astype(int)) - except KeyError: - avgpower2 = {} - for id in theids: - avgpower2[id] = 0 - return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 + data['time'] = data['time'] / 1000. - cpdf = getcpdata_sql(rower.id, table=table) + data1 = data[data['time'] <= splitsecond].copy() + data2 = data[data['time'] > splitsecond].copy() - if not cpdf.empty: - return cpdf['delta'], cpdf['cp'], avgpower2 - else: - _ = myqueue(queuelow, - handle_updatecp, - rower.id, - theids, - table=table) + data1 = data1.sort_values(['time']) + data1 = data1.interpolate(method='linear', axis=0, limit_direction='both', + limit=10) + data1.fillna(method='bfill', inplace=True) - return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 + # Some new stuff to try out + data1 = data1.groupby('time', axis=0).mean() + data1['time'] = data1.index + data1.reset_index(drop=True, inplace=True) - return pd.Series([], dtype='float'), pd.Series([], dtype='float'), avgpower2 + data2 = data2.sort_values(['time']) + data2 = data2.interpolate(method='linear', axis=0, limit_direction='both', + limit=10) + data2.fillna(method='bfill', inplace=True) + # Some new stuff to try out + data2 = data2.groupby('time', axis=0).mean() + data2['time'] = data2.index + data2.reset_index(drop=True, inplace=True) + + data1['pace'] = data1['pace'] / 1000. + data2['pace'] = data2['pace'] / 1000. + + data1.drop_duplicates(subset='time', inplace=True) + data2.drop_duplicates(subset='time', inplace=True) + + messages = [] + ids = [] + + if 'keep first' in splitmode: + if 'firstprivate' in splitmode: # pragma: no cover + setprivate = True + else: + setprivate = False + + id, message = new_workout_from_df(r, data1, + title=parent.name + ' (1)', + parent=parent, + setprivate=setprivate, + forceunit='N') + messages.append(message) + ids.append(encoder.encode_hex(id)) + if 'keep second' in splitmode: + data2['cumdist'] = data2['cumdist'] - data2.iloc[ + 0, + data2.columns.get_loc('cumdist') + ] + data2['distance'] = data2['distance'] - data2.iloc[ + 0, + data2.columns.get_loc('distance') + ] + data2['time'] = data2['time'] - data2.iloc[ + 0, + data2.columns.get_loc('time') + ] + if 'secondprivate' in splitmode: # pragma: no cover + setprivate = True + else: + setprivate = False + + dt = datetime.timedelta(seconds=splitsecond) + + id, message = new_workout_from_df(r, data2, + title=parent.name + ' (2)', + parent=parent, + setprivate=setprivate, + dt=dt, forceunit='N') + messages.append(message) + ids.append(encoder.encode_hex(id)) + + if 'keep original' not in splitmode: # pragma: no cover + if 'keep second' in splitmode or 'keep first' in splitmode: + parent.delete() + messages.append('Deleted Workout: ' + parent.name) + else: + messages.append('That would delete your workout') + ids.append(encoder.encode_hex(parent.id)) + elif 'originalprivate' in splitmode: # pragma: no cover + parent.privacy = 'hidden' + parent.save() + + return ids, messages # create a new workout from manually entered data def create_row_df(r, distance, duration, startdatetime, workouttype='rower', @@ -1551,6 +943,7 @@ def create_row_df(r, distance, duration, startdatetime, workouttype='rower', return (id, message) + def checkbreakthrough(w, r): isbreakthrough = False ishard = False @@ -1604,29 +997,6 @@ def checkbreakthrough(w, r): return isbreakthrough, ishard -def checkduplicates(r, workoutdate, workoutstartdatetime, workoutenddatetime): - duplicate = False - ws = Workout.objects.filter(user=r, date=workoutdate, duplicate=False).exclude( - startdatetime__gt=workoutenddatetime - ) - - ws2 = [] - - for ww in ws: - t = ww.duration - delta = datetime.timedelta( - hours=t.hour, minutes=t.minute, seconds=t.second) - enddatetime = ww.startdatetime+delta - if enddatetime > workoutstartdatetime: - ws2.append(ww) - - if (len(ws2) != 0): - duplicate = True - return duplicate - - return duplicate - - # Processes painsled CSV file to database def save_workout_database(f2, r, dosmooth=True, workouttype='rower', boattype='1x', @@ -1871,243 +1241,6 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', return (w.id, message) -parsers = { - 'kinomap': KinoMapParser, - 'xls': ExcelTemplate, - 'rp': RowProParser, - 'tcx': TCXParser, - 'mystery': MysteryParser, - 'ritmotime': RitmoTimeParser, - 'quiske': QuiskeParser, - 'rowperfect3': RowPerfectParser, - 'coxmate': CoxMateParser, - 'bcmike': BoatCoachAdvancedParser, - 'boatcoach': BoatCoachParser, - 'boatcoachotw': BoatCoachOTWParser, - 'painsleddesktop': painsledDesktopParser, - 'speedcoach': speedcoachParser, - 'speedcoach2': SpeedCoach2Parser, - 'ergstick': ErgStickParser, - 'fit': FITParser, - 'ergdata': ErgDataParser, - 'humon': HumonParser, - 'eth': ETHParser, - 'nklinklogbook': NKLiNKLogbookParser, - 'hero': HeroParser, - 'smartrow': SmartRowParser, -} - - -def get_startdate_time_zone(r, row, startdatetime=None): - if startdatetime is not None and startdatetime != '': - try: - timezone_str = pendulum.instance(startdatetime).timezone.name - except ValueError: # pragma: no cover - timezone_str = 'Ect/GMT' - elif startdatetime == '': - startdatetime = row.rowdatetime - else: - startdatetime = row.rowdatetime - - try: - _ = startdatetime.tzinfo - except AttributeError: # pragma: no cover - startdatetime = row.rowdatetime - - partofday = getpartofday(row, r) - - if startdatetime.tzinfo is None or str(startdatetime.tzinfo) in ['tzutc()', 'Ect/GMT']: - timezone_str = 'UTC' - try: - startdatetime = timezone.make_aware(startdatetime) - except ValueError: # pragma: no cover - pass - - try: - latavg = row.df[' latitude'].mean() - lonavg = row.df[' longitude'].mean() - - tf = TimezoneFinder() - if row.df[' latitude'].std() != 0: - try: - timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) - except (ValueError, OverflowError): # pragma: no cover - timezone_str = 'UTC' - if timezone_str is None: # pragma: no cover - timezone_str = tf.closest_timezone_at(lng=lonavg, - lat=latavg) - if timezone_str is None: # pragma: no cover - timezone_str = r.defaulttimezone - else: - timezone_str = r.defaulttimezone - try: - startdatetime = pytz.timezone(timezone_str).localize( - row.rowdatetime - ) - except ValueError: # pragma: no cover - startdatetime = startdatetime.astimezone( - pytz.timezone(timezone_str) - ) - except KeyError: # pragma: no cover - timezone_str = r.defaulttimezone - else: - timezone_str = str(startdatetime.tzinfo) - - startdatetime = startdatetime.astimezone(pytz.timezone(timezone_str)) - - startdate = startdatetime.strftime('%Y-%m-%d') - starttime = startdatetime.strftime('%H:%M:%S') - - if timezone_str == 'tzutc()': - timezone_str = 'UTC' # pragma: no cover - - return startdatetime, startdate, starttime, timezone_str, partofday - - -def parsenonpainsled(fileformat, f2, summary, startdatetime='', empowerfirmware=None, inboard=None, oarlength=None): - try: - if fileformat == 'nklinklogbook' and empowerfirmware is not None: # pragma: no cover - if inboard is not None and oarlength is not None: - row = NKLiNKLogbookParser( - f2, firmware=empowerfirmware, inboard=inboard, oarlength=oarlength) - else: - row = NKLiNKLogbookParser(f2) - else: - row = parsers[fileformat](f2) - if startdatetime != '': # pragma: no cover - row.rowdatetime = arrow.get(startdatetime).datetime - hasrecognized = True - except (KeyError, IndexError, ValueError): # pragma: no cover - hasrecognized = False - return None, hasrecognized, '', 'unknown' - - s = 'Parsenonpainsled, start date time = {startdatetime}'.format( - startdatetime=startdatetime, - ) - dologging('debuglog.log', s) - - # handle speed coach GPS 2 - if (fileformat == 'speedcoach2'): - oarlength, inboard = get_empower_rigging(f2) - empowerfirmware = get_empower_firmware(f2) - if empowerfirmware != '': - fileformat = fileformat+'v'+str(empowerfirmware) - else: # pragma: no cover - fileformat = 'speedcoach2v0' - try: - summary = row.allstats() - except ZeroDivisionError: # pragma: no cover - summary = '' - else: - fileformat = fileformat+'v'+str(empowerfirmware) - - # handle FIT - if (fileformat == 'fit'): # pragma: no cover - try: - s = fitsummarydata(f2) - s.setsummary() - summary = s.summarytext - except: - pass - hasrecognized = True - - return row, hasrecognized, summary, fileformat - - -def handle_nonpainsled(f2, fileformat, summary='', startdatetime='', empowerfirmware=None, impeller=False): - oarlength = 2.89 - inboard = 0.88 - hasrecognized = False - - row, hasrecognized, summary, fileformat = parsenonpainsled(fileformat, f2, summary, startdatetime=startdatetime, - empowerfirmware=empowerfirmware) - - # Handle c2log - if (fileformat == 'c2log' or fileformat == 'rowprolog'): # pragma: no cover - return (0, '', 0, 0, '', impeller) - - if not hasrecognized: # pragma: no cover - return (0, '', 0, 0, '', impeller) - - f_to_be_deleted = f2 - # should delete file - f2 = f2[:-4] + 'o.csv' - - row2 = rrdata(df=row.df) - - if 'speedcoach2' in fileformat or 'nklinklogbook' in fileformat: - # impeller consistency - impellerdata, consistent, ratio = row.impellerconsistent(threshold=0.3) - - if impellerdata and consistent: - impeller = True - if impellerdata and not consistent: - row2.use_gpsdata() - if impeller: - row2.use_impellerdata() - - row2.write_csv(f2, gzip=True) - - # os.remove(f2) - try: - os.remove(f_to_be_deleted) - except: # pragma: no cover - try: - os.remove(f_to_be_deleted + '.gz') - except: - pass - - return (f2, summary, oarlength, inboard, fileformat, impeller) - -# Create new workout from file and store it in the database -# This routine should be used everywhere in views.py - - -def get_workouttype_from_fit(filename, workouttype='water'): - try: - fitfile = FitFile(filename, check_crc=False) - except FitHeaderError: # pragma: no cover - return workouttype - - records = fitfile.messages - fittype = 'rowing' - for record in records: - if record.name in ['sport', 'lap']: - try: - fittype = record.get_values()['sport'].lower() - except (KeyError, AttributeError): # pragma: no cover - return 'water' - try: - workouttype = mytypes.fitmappinginv[fittype] - except KeyError: # pragma: no cover - return workouttype - - return workouttype - - -def get_workouttype_from_tcx(filename, workouttype='water'): - tcxtype = 'rowing' - if workouttype in mytypes.otwtypes: - return workouttype - try: # pragma: no cover - d = tcxtools.tcx_getdict(filename) - try: - tcxtype = d['Activities']['Activity']['@Sport'].lower() - if tcxtype == 'other': - tcxtype = 'rowing' - except KeyError: - return workouttype - - except TypeError: # pragma: no cover - pass - - try: # pragma: no cover - workouttype = mytypes.garminmappinginv[tcxtype.upper()] - except KeyError: # pragma: no cover - return workouttype - - return workouttype # pragma: no cover - def new_workout_from_file(r, f2, workouttype='rower', @@ -2290,106 +1423,6 @@ def new_workout_from_file(r, f2, return (id, message, f2) -def split_workout(r, parent, splitsecond, splitmode): - data, row = getrowdata_db(id=parent.id) - latitude, longitude = get_latlon(parent.id) - if not latitude.empty and not longitude.empty: - data[' latitude'] = latitude - data[' longitude'] = longitude - - data['time'] = data['time'] / 1000. - - data1 = data[data['time'] <= splitsecond].copy() - data2 = data[data['time'] > splitsecond].copy() - - data1 = data1.sort_values(['time']) - data1 = data1.interpolate(method='linear', axis=0, limit_direction='both', - limit=10) - data1.fillna(method='bfill', inplace=True) - - # Some new stuff to try out - data1 = data1.groupby('time', axis=0).mean() - data1['time'] = data1.index - data1.reset_index(drop=True, inplace=True) - - data2 = data2.sort_values(['time']) - data2 = data2.interpolate(method='linear', axis=0, limit_direction='both', - limit=10) - data2.fillna(method='bfill', inplace=True) - - # Some new stuff to try out - data2 = data2.groupby('time', axis=0).mean() - data2['time'] = data2.index - data2.reset_index(drop=True, inplace=True) - - data1['pace'] = data1['pace'] / 1000. - data2['pace'] = data2['pace'] / 1000. - - data1.drop_duplicates(subset='time', inplace=True) - data2.drop_duplicates(subset='time', inplace=True) - - messages = [] - ids = [] - - if 'keep first' in splitmode: - if 'firstprivate' in splitmode: # pragma: no cover - setprivate = True - else: - setprivate = False - - id, message = new_workout_from_df(r, data1, - title=parent.name + ' (1)', - parent=parent, - setprivate=setprivate, - forceunit='N') - messages.append(message) - ids.append(encoder.encode_hex(id)) - if 'keep second' in splitmode: - data2['cumdist'] = data2['cumdist'] - data2.iloc[ - 0, - data2.columns.get_loc('cumdist') - ] - data2['distance'] = data2['distance'] - data2.iloc[ - 0, - data2.columns.get_loc('distance') - ] - data2['time'] = data2['time'] - data2.iloc[ - 0, - data2.columns.get_loc('time') - ] - if 'secondprivate' in splitmode: # pragma: no cover - setprivate = True - else: - setprivate = False - - dt = datetime.timedelta(seconds=splitsecond) - - id, message = new_workout_from_df(r, data2, - title=parent.name + ' (2)', - parent=parent, - setprivate=setprivate, - dt=dt, forceunit='N') - messages.append(message) - ids.append(encoder.encode_hex(id)) - - if 'keep original' not in splitmode: # pragma: no cover - if 'keep second' in splitmode or 'keep first' in splitmode: - parent.delete() - messages.append('Deleted Workout: ' + parent.name) - else: - messages.append('That would delete your workout') - ids.append(encoder.encode_hex(parent.id)) - elif 'originalprivate' in splitmode: # pragma: no cover - parent.privacy = 'hidden' - parent.save() - - return ids, messages - -# Create new workout from data frame and store it in the database -# This routine should be used everywhere in views.py and mailprocessing.py -# Currently there is code duplication - - def new_workout_from_df(r, df, title='New Workout', workoutsource='unknown', @@ -2475,701 +1508,6 @@ def new_workout_from_df(r, df, # A wrapper around the rowingdata class, with some error catching -def rdata(file, rower=rrower()): - try: - res = rrdata(csvfile=file, rower=rower) - except (IOError, IndexError): # pragma: no cover - try: - res = rrdata(csvfile=file + '.gz', rower=rower) - except (IOError, IndexError): - res = rrdata() - except: - res = rrdata() - except EOFError: # pragma: no cover - res = rrdata() - except: # pragma: no cover - res = rrdata() - - return res - -# Remove all stroke data for workout ID from database - - -def delete_strokedata(id): - dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id) - try: - shutil.rmtree(dirname) - except OSError: - try: - os.remove(dirname) - except FileNotFoundError: - pass - except FileNotFoundError: # pragma: no cover - pass - -# Replace stroke data in DB with data from CSV file - - -def update_strokedata(id, df): - delete_strokedata(id) - _ = dataprep(df, id=id, bands=True, barchart=True, otwpower=True) - -# Test that all data are of a numerical time - - -def testdata(time, distance, pace, spm): # pragma: no cover - t1 = np.issubdtype(time, np.number) - t2 = np.issubdtype(distance, np.number) - t3 = np.issubdtype(pace, np.number) - t4 = np.issubdtype(spm, np.number) - - return t1 and t2 and t3 and t4 - -# Get data from DB for one workout (fetches all data). If data -# is not in DB, read from CSV file (and create DB entry) - - -def getrowdata_db(id=0, doclean=False, convertnewtons=True, - checkefficiency=True): - data = read_df_sql(id) - try: - data['deltat'] = data['time'].diff() - except KeyError: # pragma: no cover - data = pd.DataFrame() - - if data.empty: - rowdata, row = getrowdata(id=id) - if not rowdata.empty: - data = dataprep(rowdata.df, id=id, bands=True, - barchart=True, otwpower=True) - else: - data = pd.DataFrame() # returning empty dataframe - else: - row = Workout.objects.get(id=id) - - if checkefficiency is True and not data.empty: - try: - if data['efficiency'].mean() == 0 and data['power'].mean() != 0: # pragma: no cover - data = add_efficiency(id=id) - except KeyError: # pragma: no cover - data = add_efficiency(id=id) - - if doclean: # pragma: no cover - data = clean_df_stats(data, ignorehr=True) - - return data, row - -# Fetch a subset of the data from the DB - - -def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True, compute=True): - # prepmultipledata(ids) - - if ids: - csvfilenames = [ - 'media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] - else: - return pd.DataFrame() - - data = [] - columns = [c for c in columns if c != 'None'] - columns = list(set(columns)) - - if len(ids) > 1: - for id, f in zip(ids, csvfilenames): - try: - df = pd.read_parquet(f, columns=columns) - data.append(df) - except (OSError, ArrowInvalid, IndexError): # pragma: no cover - rowdata, row = getrowdata(id=id) - if rowdata and len(rowdata.df): - _ = dataprep(rowdata.df, id=id, - bands=True, otwpower=True, barchart=True) - df = pd.read_parquet(f, columns=columns) - data.append(df) - - try: - df = pd.concat(data, axis=0) - except ValueError: # pragma: no cover - return pd.DataFrame() - # df = dd.concat(data,axis=0) - - else: - try: - df = pd.read_parquet(csvfilenames[0], columns=columns) - rowdata, row = getrowdata(id=ids[0]) - except (OSError, ArrowInvalid, IndexError): - rowdata, row = getrowdata(id=ids[0]) - if rowdata and len(rowdata.df): # pragma: no cover - data = dataprep( - rowdata.df, id=ids[0], bands=True, otwpower=True, barchart=True) - df = pd.read_parquet(csvfilenames[0], columns=columns) - # df = dd.read_parquet(csvfilenames[0], - # column=columns,engine='pyarrow', - # ) - - # df = df.loc[:,~df.columns.duplicated()] - else: - df = pd.DataFrame() - - if compute and len(df): - data = df.copy() - if doclean: - data = clean_df_stats(data, ignorehr=True, - workstrokesonly=workstrokesonly) - data.dropna(axis=1, how='all', inplace=True) - data.dropna(axis=0, how='any', inplace=True) - return data - - return df - -# Fetch both the workout and the workout stroke data (from CSV file) - - -def getrowdata(id=0): - - # check if valid ID exists (workout exists) - try: - row = Workout.objects.get(id=id) - except Workout.DoesNotExist: # pragma: no cover - return rrdata(), None - - f1 = row.csvfilename - - # get user - - r = row.user - - rr = rrower(hrmax=r.max, hrut2=r.ut2, - hrut1=r.ut1, hrat=r.at, - hrtr=r.tr, hran=r.an, ftp=r.ftp) - - rowdata = rdata(f1, rower=rr) - - return rowdata, row - -# Checks if all rows for a list of workout IDs have entries in the -# stroke_data table. If this is not the case, it creates the stroke -# data -# In theory, this should never yield any work, but it's a good -# safety net for programming errors elsewhere in the app -# Also used heavily when I moved from CSV file only to CSV+Stroke data - - -def prepmultipledata(ids, verbose=False): # pragma: no cover - filenames = glob.glob('media/*.parquet') - ids = [ - id for id in ids if 'media/strokedata_{id}.parquet.gz'.format(id=id) not in filenames] - - for id in ids: - rowdata, row = getrowdata(id=id) - if verbose: - print(id) - if rowdata and len(rowdata.df): - _ = dataprep(rowdata.df, id=id, bands=True, - barchart=True, otwpower=True) - return ids - -# Read a set of columns for a set of workout ids, returns data as a -# pandas dataframe - - -def read_cols_df_sql(ids, columns, convertnewtons=True): - # drop columns that are not in offical list - # axx = [ax[0] for ax in axes] - - extracols = [] - - columns = list(columns) + ['distance', 'spm', 'workoutid'] - columns = [x for x in columns if x != 'None'] - columns = list(set(columns)) - ids = [int(id) for id in ids] - - df = pd.DataFrame() - - if len(ids) == 0: # pragma: no cover - return pd.DataFrame(), extracols - elif len(ids) == 1: # pragma: no cover - try: - filename = 'media/strokedata_{id}.parquet.gz'.format(id=ids[0]) - df = pd.read_parquet(filename, columns=columns) - except OSError: - rowdata, row = getrowdata(id=ids[0]) - if rowdata and len(rowdata.df): - _ = dataprep(rowdata.df, - id=ids[0], bands=True, otwpower=True, barchart=True) - df = pd.read_parquet(filename, columns=columns) - else: - data = [] - filenames = [ - 'media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] - for id, f in zip(ids, filenames): - try: - df = pd.read_parquet(f, columns=columns) - data.append(df) - except (OSError, IndexError, ArrowInvalid): - rowdata, row = getrowdata(id=id) - if rowdata and len(rowdata.df): # pragma: no cover - _ = dataprep(rowdata.df, id=id, - bands=True, otwpower=True, barchart=True) - df = pd.read_parquet(f, columns=columns) - data.append(df) - - try: - df = pd.concat(data, axis=0) - except ValueError: # pragma: no cover - return pd.DataFrame(), extracols - - df = df.fillna(value=0) - - if 'peakforce' in columns: - funits = ((w.id, w.forceunit) - for w in Workout.objects.filter(id__in=ids)) - for id, u in funits: - if u == 'lbs': - mask = df['workoutid'] == id - df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN - if 'averageforce' in columns: - funits = ((w.id, w.forceunit) - for w in Workout.objects.filter(id__in=ids)) - for id, u in funits: - if u == 'lbs': - mask = df['workoutid'] == id - df.loc[mask, 'averageforce'] = df.loc[mask, - 'averageforce'] * lbstoN - - return df, extracols - - -def initiate_cp(r): - _ = update_rolling_cp(r, otwtypes, 'water') - _ = update_rolling_cp(r, otetypes, 'erg') - -# Read stroke data from the DB for a Workout ID. Returns a pandas dataframe - - -def read_df_sql(id): - try: - f = 'media/strokedata_{id}.parquet.gz'.format(id=id) - df = pd.read_parquet(f) - except (OSError, ArrowInvalid, IndexError): # pragma: no cover - rowdata, row = getrowdata(id=id) - if rowdata and len(rowdata.df): - data = dataprep(rowdata.df, id=id, bands=True, - otwpower=True, barchart=True) - try: - df = pd.read_parquet(f) - except OSError: - df = data - else: - df = pd.DataFrame() - - df = df.fillna(value=0) - - return df - - -# data fusion - - -def datafusion(id1, id2, columns, offset): - df1, w1 = getrowdata_db(id=id1) - df1 = df1.drop([ # 'cumdist', - 'hr_ut2', - 'hr_ut1', - 'hr_at', - 'hr_tr', - 'hr_an', - 'hr_max', - 'ftime', - 'fpace', - 'workoutid', - 'id'], - 1, errors='ignore') - - # Add coordinates to DataFrame - latitude, longitude = get_latlon(id1) - - df1[' latitude'] = latitude - df1[' longitude'] = longitude - - df2 = getsmallrowdata_db(['time'] + columns, ids=[id2], doclean=False) - - forceunit = 'N' - - offsetmillisecs = offset.seconds * 1000 + offset.microseconds / 1000. - offsetmillisecs += offset.days * (3600 * 24 * 1000) - df2['time'] = df2['time'] + offsetmillisecs - - keep1 = {c: c for c in set(df1.columns)} - - for c in columns: - keep1.pop(c) - - for c in df1.columns: - if c not in keep1: - df1 = df1.drop(c, 1, errors='ignore') - - df = pd.concat([df1, df2], ignore_index=True) - df = df.sort_values(['time']) - df = df.interpolate(method='linear', axis=0, limit_direction='both', - limit=10) - df.fillna(method='bfill', inplace=True) - - # Some new stuff to try out - df = df.groupby('time', axis=0).mean() - df['time'] = df.index - df.reset_index(drop=True, inplace=True) - - df['time'] = df['time'] / 1000. - df['pace'] = df['pace'] / 1000. - df['cum_dist'] = df['cumdist'] - - return df, forceunit - - -def fix_newtons(id=0, limit=3000): # pragma: no cover - # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) - rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False) - try: - peakforce = rowdata['peakforce'] - if peakforce.mean() > limit: - w = Workout.objects.get(id=id) - - rowdata = rdata(w.csvfilename) - if rowdata and len(rowdata.df): - update_strokedata(w.id, rowdata.df) - except KeyError: - pass - - -def remove_invalid_columns(df): # pragma: no cover - for c in df.columns: - if c not in allowedcolumns: - df.drop(labels=c, axis=1, inplace=True) - - return df - - -def add_efficiency(id=0): # pragma: no cover - rowdata, row = getrowdata_db(id=id, - doclean=False, - convertnewtons=False, - checkefficiency=False) - power = rowdata['power'] - pace = rowdata['pace'] / 1.0e3 - velo = 500. / pace - ergpw = 2.8 * velo**3 - efficiency = 100. * ergpw / power - - efficiency = efficiency.replace([-np.inf, np.inf], np.nan) - efficiency.fillna(method='ffill') - rowdata['efficiency'] = efficiency - - rowdata = remove_invalid_columns(rowdata) - rowdata = rowdata.replace([-np.inf, np.inf], np.nan) - rowdata = rowdata.fillna(method='ffill') - - delete_strokedata(id) - - if id != 0: - rowdata['workoutid'] = id - filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) - df = dd.from_pandas(rowdata, npartitions=1) - df.to_parquet(filename, engine='fastparquet', compression='GZIP') - - return rowdata - -# This is the main routine. -# it reindexes, sorts, filters, and smooths the data, then -# saves it to the stroke_data table in the database -# Takes a rowingdata object's DataFrame as input - - -def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, - empower=True, inboard=0.88, forceunit='lbs'): - - if rowdatadf.empty: - return 0 - - t = rowdatadf.loc[:, 'TimeStamp (sec)'] - t = pd.Series(t - rowdatadf.loc[:, 'TimeStamp (sec)'].iloc[0]) - - row_index = rowdatadf.loc[:, ' Stroke500mPace (sec/500m)'] > 3000 - rowdatadf.loc[row_index, ' Stroke500mPace (sec/500m)'] = 3000. - - p = rowdatadf.loc[:, ' Stroke500mPace (sec/500m)'] - try: - velo = rowdatadf.loc[:, ' AverageBoatSpeed (m/s)'] - except KeyError: # pragma: no cover - velo = 500./p - - hr = rowdatadf.loc[:, ' HRCur (bpm)'] - spm = rowdatadf.loc[:, ' Cadence (stokes/min)'] - cumdist = rowdatadf.loc[:, 'cum_dist'] - power = rowdatadf.loc[:, ' Power (watts)'] - averageforce = rowdatadf.loc[:, ' AverageDriveForce (lbs)'] - drivelength = rowdatadf.loc[:, ' DriveLength (meters)'] - try: - workoutstate = rowdatadf.loc[:, ' WorkoutState'] - except KeyError: # pragma: no cover - workoutstate = 0 * hr - - peakforce = rowdatadf.loc[:, ' PeakDriveForce (lbs)'] - - forceratio = averageforce / peakforce - forceratio = forceratio.fillna(value=0) - - try: - drivetime = rowdatadf.loc[:, ' DriveTime (ms)'] - recoverytime = rowdatadf.loc[:, ' StrokeRecoveryTime (ms)'] - rhythm = 100. * drivetime / (recoverytime + drivetime) - rhythm = rhythm.fillna(value=0) - except: # pragma: no cover - rhythm = 0.0 * forceratio - - f = rowdatadf['TimeStamp (sec)'].diff().mean() - if f != 0 and not np.isinf(f): - try: - windowsize = 2 * (int(10. / (f))) + 1 - except ValueError: # pragma: no cover - windowsize = 1 - else: - windowsize = 1 - if windowsize <= 3: - windowsize = 5 - - if windowsize > 3 and windowsize < len(hr): - spm = savgol_filter(spm, windowsize, 3) - hr = savgol_filter(hr, windowsize, 3) - drivelength = savgol_filter(drivelength, windowsize, 3) - forceratio = savgol_filter(forceratio, windowsize, 3) - - try: - t2 = t.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) - except TypeError: # pragma: no cover - t2 = 0 * t - - p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) - - try: - drivespeed = drivelength / rowdatadf[' DriveTime (ms)'] * 1.0e3 - except TypeError: # pragma: no cover - drivespeed = 0.0 * rowdatadf['TimeStamp (sec)'] - - drivespeed = drivespeed.fillna(value=0) - - try: - driveenergy = rowdatadf['driveenergy'] - except KeyError: # pragma: no cover - if forceunit == 'lbs': - driveenergy = drivelength * averageforce * lbstoN - else: - driveenergy = drivelength * averageforce - - if forceunit == 'lbs': - averageforce *= lbstoN - peakforce *= lbstoN - - powerhr = 60.*power/hr - powerhr = powerhr.fillna(value=0) - - if driveenergy.mean() == 0 and driveenergy.std() == 0: - driveenergy = 0*driveenergy+100 - - distance = rowdatadf.loc[:, 'cum_dist'] - velo = 500. / p - - distanceperstroke = 60. * velo / spm - - data = DataFrame( - dict( - time=t * 1e3, - hr=hr, - pace=p * 1e3, - spm=spm, - velo=velo, - cumdist=cumdist, - ftime=niceformat(t2), - fpace=nicepaceformat(p2), - driveenergy=driveenergy, - power=power, - workoutstate=workoutstate, - averageforce=averageforce, - drivelength=drivelength, - peakforce=peakforce, - forceratio=forceratio, - distance=distance, - drivespeed=drivespeed, - rhythm=rhythm, - distanceperstroke=distanceperstroke, - # powerhr=powerhr, - ) - ) - - if bands: - # HR bands - data['hr_ut2'] = rowdatadf.loc[:, 'hr_ut2'] - data['hr_ut1'] = rowdatadf.loc[:, 'hr_ut1'] - data['hr_at'] = rowdatadf.loc[:, 'hr_at'] - data['hr_tr'] = rowdatadf.loc[:, 'hr_tr'] - data['hr_an'] = rowdatadf.loc[:, 'hr_an'] - data['hr_max'] = rowdatadf.loc[:, 'hr_max'] - data['hr_bottom'] = 0.0 * data['hr'] - - try: - _ = rowdatadf.loc[:, ' ElapsedTime (sec)'] - except KeyError: # pragma: no cover - rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] - - if empower: - try: - wash = rowdatadf.loc[:, 'wash'] - except KeyError: - wash = 0 * power - - try: - catch = rowdatadf.loc[:, 'catch'] - except KeyError: - catch = 0 * power - - try: - finish = rowdatadf.loc[:, 'finish'] - except KeyError: - finish = 0 * power - - try: - peakforceangle = rowdatadf.loc[:, 'peakforceangle'] - except KeyError: - peakforceangle = 0 * power - - if data['driveenergy'].mean() == 0: # pragma: no cover - try: - driveenergy = rowdatadf.loc[:, 'driveenergy'] - except KeyError: - driveenergy = power * 60 / spm - else: - driveenergy = data['driveenergy'] - - arclength = (inboard - 0.05) * (np.radians(finish) - np.radians(catch)) - if arclength.mean() > 0: - drivelength = arclength - elif drivelength.mean() == 0: - drivelength = driveenergy / (averageforce * 4.44822) - - try: - slip = rowdatadf.loc[:, 'slip'] - except KeyError: - slip = 0 * power - - try: - totalangle = finish - catch - effectiveangle = finish - wash - catch - slip - except ValueError: # pragma: no cover - totalangle = 0 * power - effectiveangle = 0 * power - - if windowsize > 3 and windowsize < len(slip): - try: - wash = savgol_filter(wash, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - slip = savgol_filter(slip, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - catch = savgol_filter(catch, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - finish = savgol_filter(finish, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - peakforceangle = savgol_filter(peakforceangle, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - driveenergy = savgol_filter(driveenergy, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - drivelength = savgol_filter(drivelength, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - totalangle = savgol_filter(totalangle, windowsize, 3) - except TypeError: # pragma: no cover - pass - try: - effectiveangle = savgol_filter(effectiveangle, windowsize, 3) - except TypeError: # pragma: no cover - pass - - velo = 500. / p - - ergpw = 2.8 * velo**3 - efficiency = 100. * ergpw / power - - efficiency = efficiency.replace([-np.inf, np.inf], np.nan) - efficiency.fillna(method='ffill') - - try: - data['wash'] = wash - data['catch'] = catch - data['slip'] = slip - data['finish'] = finish - data['peakforceangle'] = peakforceangle - data['driveenergy'] = driveenergy - data['drivelength'] = drivelength - data['totalangle'] = totalangle - data['effectiveangle'] = effectiveangle - data['efficiency'] = efficiency - except ValueError: # pragma: no cover - pass - - if otwpower: - try: - nowindpace = rowdatadf.loc[:, 'nowindpace'] - except KeyError: - nowindpace = p - try: - equivergpower = rowdatadf.loc[:, 'equivergpower'] - except KeyError: - equivergpower = 0 * p + 50. - - nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x)) - ergvelo = (equivergpower / 2.8)**(1. / 3.) - - ergpace = 500. / ergvelo - ergpace[ergpace == np.inf] = 240. - ergpace2 = ergpace.apply(lambda x: timedeltaconv(x)) - - data['ergpace'] = ergpace * 1e3 - data['nowindpace'] = nowindpace * 1e3 - data['equivergpower'] = equivergpower - data['fergpace'] = nicepaceformat(ergpace2) - data['fnowindpace'] = nicepaceformat(nowindpace2) - - data = data.replace([-np.inf, np.inf], np.nan) - data = data.fillna(method='ffill') - - # write data if id given - if id != 0: - data['workoutid'] = id - data.fillna(0, inplace=True) - for k, v in dtypes.items(): - try: - data[k] = data[k].astype(v) - except KeyError: # pragma: no cover - pass - - filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) - df = dd.from_pandas(data, npartitions=1) - df.to_parquet(filename, engine='fastparquet', compression='GZIP') - - return data def workout_trimp(w, reset=False): diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index e3713b7a..e59582ac 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -160,161 +160,10 @@ def rdata(file, rower=rrower()): # pragma: no cover return res -# Creates C2 stroke data -def create_c2_stroke_data_db( - distance, duration, workouttype, - workoutid, starttimeunix, csvfilename, debug=False): # pragma: no cover - - nr_strokes = int(distance/10.) - - totalseconds = duration.hour*3600. - totalseconds += duration.minute*60. - totalseconds += duration.second - totalseconds += duration.microsecond/1.e6 - - try: - spm = 60.*nr_strokes/totalseconds - except ZeroDivisionError: - spm = 20*np.zeros(nr_strokes) - - try: - _ = totalseconds/float(nr_strokes) - except ZeroDivisionError: - return 0 - - elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) - - d = np.arange(nr_strokes)*distance/(float(nr_strokes-1)) - - unixtime = starttimeunix + elapsed - - pace = 500.*totalseconds/distance - - if workouttype in ['rower', 'slides', 'dynamic']: - try: - velo = distance/totalseconds - except ZeroDivisionError: - velo = 0 - power = 2.8*velo**3 - else: - power = 0 - - df = pd.DataFrame({ - 'TimeStamp (sec)': unixtime, - ' Horizontal (meters)': d, - ' Cadence (stokes/min)': spm, - ' Stroke500mPace (sec/500m)': pace, - ' ElapsedTime (sec)': elapsed, - ' Power (watts)': power, - ' HRCur (bpm)': np.zeros(nr_strokes), - ' longitude': np.zeros(nr_strokes), - ' latitude': np.zeros(nr_strokes), - ' DragFactor': np.zeros(nr_strokes), - ' DriveLength (meters)': np.zeros(nr_strokes), - ' StrokeDistance (meters)': np.zeros(nr_strokes), - ' DriveTime (ms)': np.zeros(nr_strokes), - ' StrokeRecoveryTime (ms)': np.zeros(nr_strokes), - ' AverageDriveForce (lbs)': np.zeros(nr_strokes), - ' PeakDriveForce (lbs)': np.zeros(nr_strokes), - ' lapIdx': np.zeros(nr_strokes), - 'cum_dist': d - }) - - df[' ElapsedTime (sec)'] = df['TimeStamp (sec)'] - - _ = df.to_csv(csvfilename, index_label='index', compression='gzip') - - data = dataprep(df, id=workoutid, bands=False, debug=debug) - - return data # Saves C2 stroke data to CSV and database -def add_c2_stroke_data_db(strokedata, workoutid, starttimeunix, csvfilename, - debug=False, workouttype='rower'): - - res = make_cumvalues(0.1*strokedata['t']) - cum_time = res[0] - lapidx = res[1] - - unixtime = cum_time+starttimeunix - # unixtime[0] = starttimeunix - seconds = 0.1*strokedata.loc[:, 't'] - - nr_rows = len(unixtime) - - try: # pragma: no cover - latcoord = strokedata.loc[:, 'lat'] - loncoord = strokedata.loc[:, 'lon'] - except: - latcoord = np.zeros(nr_rows) - loncoord = np.zeros(nr_rows) - - try: - strokelength = strokedata.loc[:, 'strokelength'] - except: - strokelength = np.zeros(nr_rows) - - dist2 = 0.1*strokedata.loc[:, 'd'] - - try: - spm = strokedata.loc[:, 'spm'] - except KeyError: # pragma: no cover - spm = 0*dist2 - - try: - hr = strokedata.loc[:, 'hr'] - except KeyError: # pragma: no cover - hr = 0*spm - - pace = strokedata.loc[:, 'p']/10. - pace = np.clip(pace, 0, 1e4) - pace = pace.replace(0, 300) - - velo = 500./pace - power = 2.8*velo**3 - if workouttype == 'bike': # pragma: no cover - velo = 1000./pace - - # save csv - # Create data frame with all necessary data to write to csv - df = pd.DataFrame({'TimeStamp (sec)': unixtime, - ' Horizontal (meters)': dist2, - ' Cadence (stokes/min)': spm, - ' HRCur (bpm)': hr, - ' longitude': loncoord, - ' latitude': latcoord, - ' Stroke500mPace (sec/500m)': pace, - ' Power (watts)': power, - ' DragFactor': np.zeros(nr_rows), - ' DriveLength (meters)': np.zeros(nr_rows), - ' StrokeDistance (meters)': strokelength, - ' DriveTime (ms)': np.zeros(nr_rows), - ' StrokeRecoveryTime (ms)': np.zeros(nr_rows), - ' AverageDriveForce (lbs)': np.zeros(nr_rows), - ' PeakDriveForce (lbs)': np.zeros(nr_rows), - ' lapIdx': lapidx, - ' WorkoutState': 4, - ' ElapsedTime (sec)': seconds, - 'cum_dist': dist2 - }) - - df.sort_values(by='TimeStamp (sec)', ascending=True) - - # Create CSV file name and save data to CSV file - - res = df.to_csv(csvfilename, index_label='index', - compression='gzip') - - try: - data = dataprep(df, id=workoutid, bands=False, debug=debug) - except: # pragma: no cover - return 0 - - return data - - def handle_nonpainsled(f2, fileformat, summary=''): # pragma: no cover oarlength = 2.89 inboard = 0.88 @@ -413,47 +262,6 @@ def update_strokedata(id, df, debug=False): return rowdata -def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): # pragma: no cover - - corr_factor = 1.0 - if 'x' in boattype: - # sweep - a = 0.06 - b = 0.275 - else: - # scull - a = 0.15 - b = 0.275 - - corr_factor = empower_bug_correction(oarlength, inboard, a, b) - - success = False - - try: - df['power empower old'] = df[' Power (watts)'] - df[' Power (watts)'] = df[' Power (watts)'] * corr_factor - df['driveenergy empower old'] = df['driveenergy'] - df['driveenergy'] = df['driveenergy'] * corr_factor - success = True - except KeyError: - pass - - if success: - delete_strokedata(id, debug=debug) - if debug: # pragma: no cover - print("updated ", id) - print("correction ", corr_factor) - else: - if debug: # pragma: no cover - print("not updated ", id) - - _ = dataprep(df, id=id, bands=True, barchart=True, otwpower=True, debug=debug) - - row = rrdata(df=df) - row.write_csv(f1, gzip=True) - - return success - def testdata(time, distance, pace, spm): # pragma: no cover t1 = np.issubdtype(time, np.number) @@ -633,42 +441,11 @@ def delete_agegroup_db(age, sex, weightcategory, debug=False): engine.dispose() -def update_agegroup_db(age, sex, weightcategory, wcdurations, wcpower, - debug=False): - - delete_agegroup_db(age, sex, weightcategory, debug=debug) - - wcdurations = [None if type(y) is float and np.isnan( - y) else y for y in wcdurations] - wcpower = [None if type(y) is float and np.isnan(y) - else y for y in wcpower] - - df = pd.DataFrame( - { - 'duration': wcdurations, - 'power': wcpower, - } - ) - - df['sex'] = sex - df['age'] = age - df['weightcategory'] = weightcategory - df.replace([np.inf, -np.inf], np.nan, inplace=True) - df.dropna(axis=0, inplace=True) - - if debug: # pragma: no cover # pragma: no cover - engine = create_engine(database_url_debug, echo=False) - else: - engine = create_engine(database_url, echo=False) - - table = 'calcagegrouprecords' - with engine.connect() as conn, conn.begin(): - df.to_sql(table, engine, if_exists='append', index=False) - conn.close() - engine.dispose() -def updatecpdata_sql(rower_id, delta, cp, table='cpdata', distance=pd.Series([], dtype='float'), debug=False): + +def updatecpdata_sql(rower_id, delta, cp, table='cpdata', + distance=pd.Series([], dtype='float'), debug=False): deletecpdata_sql(rower_id, table=table, debug=debug) df = pd.DataFrame( { diff --git a/rowers/dataroutines.py b/rowers/dataroutines.py new file mode 100644 index 00000000..028fc100 --- /dev/null +++ b/rowers/dataroutines.py @@ -0,0 +1,2050 @@ +from rowers.metrics import axes, calc_trimp, rowingmetrics, dtypes, metricsgroups +from rowers.utils import lbstoN, wavg, dologging +from rowers.mytypes import otwtypes, otetypes, rowtypes +import glob +import rowingdata.tcxtools as tcxtools +from rowers.utils import totaltime_sec_to_string +from rowers.datautils import p0 +from scipy import optimize +from rowers.utils import calculate_age +import datetime +from scipy.signal import savgol_filter +from rowers.opaque import encoder +from rowers.database import * +from rowers import mytypes +from rowsandall_app.settings import SITE_URL +import django_rq +from timezonefinder import TimezoneFinder +import rowers.datautils as datautils +import rowers.utils as utils +import sys +import sqlalchemy as sa +from sqlalchemy import create_engine +from django.conf import settings +import math +from fitparse.base import FitHeaderError +from fitparse import FitFile +import itertools +import numpy as np +import pandas as pd +from zipfile import BadZipFile +import zipfile +import os +from rowers.models import strokedatafields + +from rowingdata import ( + KinoMapParser, + ExcelTemplate, + TCXParser, + MysteryParser, + RowProParser, + RitmoTimeParser, + QuiskeParser, + RowPerfectParser, + CoxMateParser, + BoatCoachParser, + BoatCoachOTWParser, + BoatCoachAdvancedParser, + painsledDesktopParser, + speedcoachParser, + SpeedCoach2Parser, + ErgStickParser, + FITParser, + ErgDataParser, + HumonParser, + ETHParser, + NKLiNKLogbookParser, + HeroParser, + SmartRowParser,) + +from rowingdata import make_cumvalues + +# All the data preparation, data cleaning and data mangling should +# be defined here +from rowers.models import ( + Workout, Team, CalcAgePerformance, C2WorldClassAgePerformance, + User +) + +import pytz +import collections +import pendulum +from rowingdata import rowingdata as rrdata + +from rowingdata import rower as rrower + +import yaml +import shutil +from shutil import copyfile + +from rowingdata import ( + get_file_type, get_empower_rigging, get_empower_firmware +) + + +from pandas import DataFrame, Series +import dask.dataframe as dd +from dask.delayed import delayed +import pyarrow.parquet as pq +import pyarrow as pa + +from pyarrow.lib import ArrowInvalid + +from django.utils import timezone +from django.utils.timezone import get_current_timezone +from django.urls import reverse +import requests + +from django.core.exceptions import ValidationError + +from time import strftime +import arrow + +thetimezone = get_current_timezone() + +allowedcolumns = [key for key, value in strokedatafields.items()] + + + +# mapping the DB column names to the CSV file column names +columndict = { + 'time': 'TimeStamp (sec)', + 'hr': ' HRCur (bpm)', + 'velo': ' AverageBoatSpeed (m/s)', + 'pace': ' Stroke500mPace (sec/500m)', + 'spm': ' Cadence (stokes/min)', + 'power': ' Power (watts)', + 'averageforce': ' AverageDriveForce (lbs)', + 'drivelength': ' DriveLength (meters)', + 'peakforce': ' PeakDriveForce (lbs)', + 'distance': ' Horizontal (meters)', + 'catch': 'catch', + 'finish': 'finish', + 'peakforceangle': 'peakforceangle', + 'wash': 'wash', + 'slip': 'slip', + 'workoutstate': ' WorkoutState', + 'cumdist': 'cum_dist', +} + + +def get_video_data(w, groups=['basic'], mode='water'): + modes = [mode, 'both', 'basic'] + columns = ['time', 'velo', 'spm'] + columns += [name for name, d in rowingmetrics if d['group'] + in groups and d['mode'] in modes] + columns = list(set(columns)) + df = getsmallrowdata_db(columns, ids=[w.id], + workstrokesonly=False, doclean=False, compute=False) + + df['time'] = (df['time']-df['time'].min())/1000. + + df.sort_values(by='time', inplace=True) + + df.set_index(pd.to_timedelta(df['time'], unit='s'), inplace=True) + df2 = df.resample('1s').first().fillna(method='ffill') + df2['time'] = df2.index.total_seconds() + + if 'pace' in columns: + df2['pace'] = df2['pace']/1000. + p = df2['pace'] + p = p.apply(lambda x: timedeltaconv(x)) + p = nicepaceformat(p) + df2['pace'] = p + + df2['time'] = (df2['time']-df2['time'].min()) + + df2 = df2.round(decimals=2) + + boatspeed = (100*df2['velo']).astype(int)/100. + + try: + coordinates = get_latlon_time(w.id) + except KeyError: # pragma: no cover + nulseries = df['time']*0 + coordinates = pd.DataFrame({ + 'time': df['time'], + 'latitude': nulseries, + 'longitude': nulseries, + }) + + coordinates.set_index(pd.to_timedelta( + coordinates['time'], unit='s'), inplace=True) + coordinates = coordinates.resample('1s').mean().interpolate() + coordinates['time'] = coordinates['time']-coordinates['time'].min() + latitude = coordinates['latitude'] + longitude = coordinates['longitude'] + + # bundle data + data = { + 'boatspeed': boatspeed.values.tolist(), + 'latitude': latitude.values.tolist(), + 'longitude': longitude.values.tolist(), + } + + metrics = {} + + for c in columns: + if c != 'time': + try: + if dict(rowingmetrics)[c]['numtype'] == 'integer': # pragma: no cover + data[c] = df2[c].astype(int).tolist() + else: + sigfigs = dict(rowingmetrics)[c]['sigfigs'] + if (c != 'pace'): + da = ((10**sigfigs)*df2[c]).astype(int)/(10**sigfigs) + else: + da = df2[c] + data[c] = da.values.tolist() + metrics[c] = { + 'name': dict(rowingmetrics)[c]['verbose_name'], + 'metric': c, + 'unit': '' + } + except KeyError: # pragma: no cover + pass + + metrics['boatspeed'] = metrics.pop('velo') + # metrics['workperstroke'] = metrics.pop('driveenergy') + metrics = collections.OrderedDict(sorted(metrics.items())) + + maxtime = coordinates['time'].max() + + return data, metrics, maxtime + + +def polarization_index(df, rower): + df['dt'] = df['time'].diff()/6.e4 + # remove rest (spm<15) + df.dropna(axis=0, inplace=True) + df['dt'] = df['dt'].clip(upper=4, lower=0) + + masklow = (df['power'] > 0) & (df['power'] < int(rower.pw_at)) + maskmid = (df['power'] >= rower.pw_at) & (df['power'] < int(rower.pw_an)) + maskhigh = (df['power'] > rower.pw_an) + + time_low_pw = df.loc[masklow, 'dt'].sum() + time_mid_pw = df.loc[maskmid, 'dt'].sum() + time_high_pw = df.loc[maskhigh, 'dt'].sum() + + frac_low = time_low_pw/(time_low_pw+time_mid_pw+time_high_pw) + frac_mid = time_mid_pw/(time_low_pw+time_mid_pw+time_high_pw) + frac_high = time_high_pw/(time_low_pw+time_mid_pw+time_high_pw) + + index = math.log10(frac_high*100.*frac_low/frac_mid) + + return index + + +def get_latlon(id): + try: + w = Workout.objects.get(id=id) + except Workout.DoesNotExist: # pragma: no cover + return False + + rowdata = rdata(w.csvfilename) + + if rowdata.df.empty: # pragma: no cover + return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] + + try: + try: + latitude = rowdata.df.loc[:, ' latitude'] + longitude = rowdata.df.loc[:, ' longitude'] + except KeyError: + latitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] + longitude = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] + return [latitude, longitude] + except AttributeError: # pragma: no cover + return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] + + return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] # pragma: no cover + + +def get_latlon_time(id): + try: + w = Workout.objects.get(id=id) + except Workout.DoesNotExist: # pragma: no cover + return False + + rowdata = rdata(w.csvfilename) + + if rowdata.df.empty: # pragma: no cover + return [pd.Series([], dtype='float'), pd.Series([], dtype='float')] + + try: + try: + _ = rowdata.df.loc[:, ' latitude'] + _ = rowdata.df.loc[:, ' longitude'] + except KeyError: # pragma: no cover + rowdata.df['latitude'] = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] + rowdata.df['longitude'] = 0 * rowdata.df.loc[:, 'TimeStamp (sec)'] + except AttributeError: # pragma: no cover + return pd.DataFrame() + + df = pd.DataFrame({ + 'time': rowdata.df['TimeStamp (sec)']-rowdata.df['TimeStamp (sec)'].min(), + 'latitude': rowdata.df[' latitude'], + 'longitude': rowdata.df[' longitude'] + }) + + return df + + +def workout_has_latlon(id): + latitude, longitude = get_latlon(id) + latmean = latitude.mean() + lonmean = longitude.mean() + + if latmean == 0 and lonmean == 0: + return False, latmean, lonmean + + if latitude.std() > 0 and longitude.std() > 0: + return True, latmean, lonmean + + return False, latmean, lonmean + + + +def get_workouts(ids, userid): # pragma: no cover + goodids = [] + for id in ids: + w = Workout.objects.get(id=id) + if int(w.user.user.id) == int(userid): + goodids.append(id) + + return [Workout.objects.get(id=id) for id in goodids] + + +def filter_df(datadf, fieldname, value, largerthan=True): + + try: + _ = datadf[fieldname] + except KeyError: + return datadf + + try: + if largerthan: + mask = datadf[fieldname] < value + else: + mask = datadf[fieldname] >= value + + datadf.loc[mask, fieldname] = np.nan + except TypeError: + pass + + return datadf + +# joins workouts + + + +def df_resample(datadf): + # time stamps must be in seconds + timestamps = datadf['TimeStamp (sec)'].astype('int') + datadf['timestamps'] = timestamps + newdf = datadf.groupby(['timestamps']).mean() + return newdf + + + +def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True, + ignoreadvanced=False): + # clean data remove zeros and negative values + + try: + _ = datadf['workoutid'].unique() + except KeyError: + datadf['workoutid'] = 0 + + before = {} + for workoutid in datadf['workoutid'].unique(): + before[workoutid] = len(datadf[datadf['workoutid'] == workoutid]) + + data_orig = datadf.copy() + + # bring metrics which have negative values to positive domain + if len(datadf) == 0: + return datadf + try: + datadf['catch'] = -datadf['catch'] + except (KeyError, TypeError): + pass + + try: + datadf['peakforceangle'] = datadf['peakforceangle'] + 1000 + except (KeyError, TypeError): + pass + + try: + datadf['hr'] = datadf['hr'] + 10 + except (KeyError, TypeError): + pass + + # protect 0 spm values from being nulled + try: + datadf['spm'] = datadf['spm'] + 1.0 + except (KeyError, TypeError): + pass + + # protect 0 workoutstate values from being nulled + try: + datadf['workoutstate'] = datadf['workoutstate'] + 1 + except (KeyError, TypeError): + pass + + try: + datadf = datadf.clip(lower=0) + except TypeError: + pass + + # protect advanced metrics columns + advancedcols = [ + 'rhythm', + 'power', + 'drivelength', + 'forceratio', + 'drivespeed', + 'driveenergy', + 'catch', + 'finish', + 'averageforce', + 'peakforce', + 'slip', + 'wash', + 'peakforceangle', + 'effectiveangle', + ] + + datadf.replace(to_replace=0, value=np.nan, inplace=True) + # datadf = datadf.map_partitions(lambda df:df.replace(to_replace=0,value=np.nan)) + + # bring spm back to real values + try: + datadf['spm'] = datadf['spm'] - 1 + except (TypeError, KeyError): + pass + + # bring workoutstate back to real values + try: + datadf['workoutstate'] = datadf['workoutstate'] - 1 + except (TypeError, KeyError): + pass + + # return from positive domain to negative + try: + datadf['catch'] = -datadf['catch'] + except (KeyError, TypeError): + pass + + try: + datadf['peakforceangle'] = datadf['peakforceangle'] - 1000 + except (KeyError, TypeError): + pass + + try: + datadf['hr'] = datadf['hr'] - 10 + except (KeyError, TypeError): + pass + + # clean data for useful ranges per column + if not ignorehr: + try: + mask = datadf['hr'] < 30 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): # pragma: no cover + pass + + try: + mask = datadf['spm'] < 0 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['efficiency'] > 200. + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['spm'] < 10 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['pace'] / 1000. > 300. + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['efficiency'] < 0. + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['pace'] / 1000. < 60. + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['power'] > 5000 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['spm'] > 120 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['wash'] < 1 + datadf.loc[mask, 'wash'] = np.nan + except (KeyError, TypeError): + pass + + # try to guess ignoreadvanced + if not ignoreadvanced: + for metric in advancedcols: + try: + sum = datadf[metric].std() + if sum == 0 or np.isnan(sum): + ignoreadvanced = True + except KeyError: + pass + + if not ignoreadvanced: + try: + mask = datadf['rhythm'] < 0 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['rhythm'] > 70 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['power'] < 20 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['drivelength'] < 0.5 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['forceratio'] < 0.2 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['forceratio'] > 1.0 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['drivespeed'] < 0.5 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['drivespeed'] > 4 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['driveenergy'] > 2000 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['driveenergy'] < 100 + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + try: + mask = datadf['catch'] > -30. + datadf.mask(mask, inplace=True) + except (KeyError, TypeError): + pass + + # workoutstateswork = [1, 4, 5, 8, 9, 6, 7] + workoutstatesrest = [3] + # workoutstatetransition = [0, 2, 10, 11, 12, 13] + + if workstrokesonly == 'True' or workstrokesonly is True: + try: + datadf = datadf[~datadf['workoutstate'].isin(workoutstatesrest)] + except: + pass + + after = {} + for workoutid in data_orig['workoutid'].unique(): + after[workoutid] = len( + datadf[datadf['workoutid'] == workoutid].dropna()) + ratio = float(after[workoutid])/float(before[workoutid]) + if ratio < 0.01 or after[workoutid] < 2: + return data_orig + + return datadf + + +def getpartofday(row, r): + workoutstartdatetime = row.rowdatetime + try: # pragma: no cover + latavg = row.df[' latitude'].mean() + lonavg = row.df[' longitude'].mean() + + tf = TimezoneFinder() + try: + timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) + except (ValueError, OverflowError): # pragma: no cover + timezone_str = 'UTC' + if timezone_str is None: # pragma: no cover + timezone_str = tf.closest_timezone_at(lng=lonavg, + lat=latavg) + if timezone_str is None: + timezone_str = r.defaulttimezone + try: + workoutstartdatetime = pytz.timezone(timezone_str).localize( + row.rowdatetime + ) + except ValueError: + workoutstartdatetime = row.rowdatetime + except KeyError: + timezone_str = r.defaulttimezone + workoutstartdatetime = row.rowdatetime + + h = workoutstartdatetime.astimezone(pytz.timezone(timezone_str)).hour + + if h < 12: # pragma: no cover + return "Morning" + elif h < 18: # pragma: no cover + return "Afternoon" + elif h < 22: # pragma: no cover + return "Evening" + else: # pragma: no cover + return "Night" + + return None # pragma: no cover + + +def getstatsfields(): + fielddict = {name: d['verbose_name'] for name, d in rowingmetrics} + +# fielddict.pop('ergpace') +# fielddict.pop('hr_an') +# fielddict.pop('hr_tr') +# fielddict.pop('hr_at') +# fielddict.pop('hr_ut2') +# fielddict.pop('hr_ut1') + fielddict.pop('time') + fielddict.pop('distance') +# fielddict.pop('nowindpace') +# fielddict.pop('fnowindpace') +# fielddict.pop('fergpace') +# fielddict.pop('equivergpower') +# fielddict.pop('workoutstate') +# fielddict.pop('fpace') +# fielddict.pop('pace') +# fielddict.pop('id') +# fielddict.pop('ftime') +# fielddict.pop('x_right') +# fielddict.pop('hr_max') +# fielddict.pop('hr_bottom') + fielddict.pop('cumdist') + + try: + fieldlist = [field for field, value in fielddict.iteritems()] + except AttributeError: + fieldlist = [field for field, value in fielddict.items()] + + return fieldlist, fielddict + + +# A string representation for time deltas +def niceformat(values): + out = [] + for v in values: + formattedv = strfdelta(v) + out.append(formattedv) + + return out + +# A nice printable format for time delta values + + +def strfdelta(tdelta): + try: + minutes, seconds = divmod(tdelta.seconds, 60) + tenths = int(tdelta.microseconds / 1e5) + except AttributeError: # pragma: no cover + minutes, seconds = divmod(tdelta.view(np.int64), 60e9) + seconds, rest = divmod(seconds, 1e9) + tenths = int(rest / 1e8) + res = "{minutes:0>2}:{seconds:0>2}.{tenths:0>1}".format( + minutes=minutes, + seconds=seconds, + tenths=tenths, + ) + + return res + + +def timedelta_to_seconds(tdelta): # pragma: no cover + return 60.*tdelta.minute+tdelta.second + + +# A nice printable format for pace values + + +def nicepaceformat(values): + out = [] + for v in values: + formattedv = strfdelta(v) + out.append(formattedv) + + return out + +# Convert seconds to a Time Delta value, replacing NaN with a 5:50 pace + + +def timedeltaconv(x): + if np.isfinite(x) and x != 0 and x > 0 and x < 175000: + dt = datetime.timedelta(seconds=x) + else: + dt = datetime.timedelta(seconds=350.) + + return dt + + +def paceformatsecs(values): + out = [] + for v in values: + td = timedeltaconv(v) + formattedv = strfdelta(td) + out.append(formattedv) + + return out + + +def update_c2id_sql(id, c2id): + engine = create_engine(database_url, echo=False) + table = 'rowers_workout' + + query = "UPDATE %s SET uploadedtoc2 = %s WHERE `id` = %s;" % ( + table, c2id, id) + + with engine.connect() as conn, conn.begin(): + _ = conn.execute(query) + + conn.close() + engine.dispose() + + return 1 + + +def getcpdata_sql(rower_id, table='cpdata'): + engine = create_engine(database_url, echo=False) + query = sa.text('SELECT * from {table} WHERE user={rower_id};'.format( + rower_id=rower_id, + table=table, + )) + + _ = engine.raw_connection() + df = pd.read_sql_query(query, engine) + + return df + + +def deletecpdata_sql(rower_id, table='cpdata'): # pragma: no cover + engine = create_engine(database_url, echo=False) + query = sa.text('DELETE from {table} WHERE user={rower_id};'.format( + rower_id=rower_id, + table=table, + )) + with engine.connect() as conn, conn.begin(): + try: + _ = conn.execute(query) + except: + print("Database locked") + conn.close() + engine.dispose() + + +def updatecpdata_sql(rower_id, delta, cp, table='cpdata', distance=pd.Series([], dtype='float'), + debug=False): # pragma: no cover + deletecpdata_sql(rower_id) + df = pd.DataFrame( + { + 'delta': delta, + 'cp': cp, + 'user': rower_id + } + ) + + if not distance.empty: + df['distance'] = distance + + engine = create_engine(database_url, echo=False) + with engine.connect() as conn, conn.begin(): + df.to_sql(table, engine, if_exists='append', index=False) + conn.close() + engine.dispose() + + + +def get_workoutsummaries(userid, startdate): # pragma: no cover + u = User.objects.get(id=userid) + r = u.rower + df = workout_summary_to_df(r, startdate=startdate) + df.drop(['Stroke Data TCX', 'Stroke Data CSV'], axis=1, inplace=True) + df = df.sort_values('date', ascending=False) + + return df + + + + + + + + +def checkduplicates(r, workoutdate, workoutstartdatetime, workoutenddatetime): + duplicate = False + ws = Workout.objects.filter(user=r, date=workoutdate, duplicate=False).exclude( + startdatetime__gt=workoutenddatetime + ) + + ws2 = [] + + for ww in ws: + t = ww.duration + delta = datetime.timedelta( + hours=t.hour, minutes=t.minute, seconds=t.second) + enddatetime = ww.startdatetime+delta + if enddatetime > workoutstartdatetime: + ws2.append(ww) + + if (len(ws2) != 0): + duplicate = True + return duplicate + + return duplicate + + + +parsers = { + 'kinomap': KinoMapParser, + 'xls': ExcelTemplate, + 'rp': RowProParser, + 'tcx': TCXParser, + 'mystery': MysteryParser, + 'ritmotime': RitmoTimeParser, + 'quiske': QuiskeParser, + 'rowperfect3': RowPerfectParser, + 'coxmate': CoxMateParser, + 'bcmike': BoatCoachAdvancedParser, + 'boatcoach': BoatCoachParser, + 'boatcoachotw': BoatCoachOTWParser, + 'painsleddesktop': painsledDesktopParser, + 'speedcoach': speedcoachParser, + 'speedcoach2': SpeedCoach2Parser, + 'ergstick': ErgStickParser, + 'fit': FITParser, + 'ergdata': ErgDataParser, + 'humon': HumonParser, + 'eth': ETHParser, + 'nklinklogbook': NKLiNKLogbookParser, + 'hero': HeroParser, + 'smartrow': SmartRowParser, +} + + +def get_startdate_time_zone(r, row, startdatetime=None): + if startdatetime is not None and startdatetime != '': + try: + timezone_str = pendulum.instance(startdatetime).timezone.name + except ValueError: # pragma: no cover + timezone_str = 'Ect/GMT' + elif startdatetime == '': + startdatetime = row.rowdatetime + else: + startdatetime = row.rowdatetime + + try: + _ = startdatetime.tzinfo + except AttributeError: # pragma: no cover + startdatetime = row.rowdatetime + + partofday = getpartofday(row, r) + + if startdatetime.tzinfo is None or str(startdatetime.tzinfo) in ['tzutc()', 'Ect/GMT']: + timezone_str = 'UTC' + try: + startdatetime = timezone.make_aware(startdatetime) + except ValueError: # pragma: no cover + pass + + try: + latavg = row.df[' latitude'].mean() + lonavg = row.df[' longitude'].mean() + + tf = TimezoneFinder() + if row.df[' latitude'].std() != 0: + try: + timezone_str = tf.timezone_at(lng=lonavg, lat=latavg) + except (ValueError, OverflowError): # pragma: no cover + timezone_str = 'UTC' + if timezone_str is None: # pragma: no cover + timezone_str = tf.closest_timezone_at(lng=lonavg, + lat=latavg) + if timezone_str is None: # pragma: no cover + timezone_str = r.defaulttimezone + else: + timezone_str = r.defaulttimezone + try: + startdatetime = pytz.timezone(timezone_str).localize( + row.rowdatetime + ) + except ValueError: # pragma: no cover + startdatetime = startdatetime.astimezone( + pytz.timezone(timezone_str) + ) + except KeyError: # pragma: no cover + timezone_str = r.defaulttimezone + else: + timezone_str = str(startdatetime.tzinfo) + + startdatetime = startdatetime.astimezone(pytz.timezone(timezone_str)) + + startdate = startdatetime.strftime('%Y-%m-%d') + starttime = startdatetime.strftime('%H:%M:%S') + + if timezone_str == 'tzutc()': + timezone_str = 'UTC' # pragma: no cover + + return startdatetime, startdate, starttime, timezone_str, partofday + + +def parsenonpainsled(fileformat, f2, summary, startdatetime='', empowerfirmware=None, inboard=None, oarlength=None): + try: + if fileformat == 'nklinklogbook' and empowerfirmware is not None: # pragma: no cover + if inboard is not None and oarlength is not None: + row = NKLiNKLogbookParser( + f2, firmware=empowerfirmware, inboard=inboard, oarlength=oarlength) + else: + row = NKLiNKLogbookParser(f2) + else: + row = parsers[fileformat](f2) + if startdatetime != '': # pragma: no cover + row.rowdatetime = arrow.get(startdatetime).datetime + hasrecognized = True + except (KeyError, IndexError, ValueError): # pragma: no cover + hasrecognized = False + return None, hasrecognized, '', 'unknown' + + s = 'Parsenonpainsled, start date time = {startdatetime}'.format( + startdatetime=startdatetime, + ) + dologging('debuglog.log', s) + + # handle speed coach GPS 2 + if (fileformat == 'speedcoach2'): + oarlength, inboard = get_empower_rigging(f2) + empowerfirmware = get_empower_firmware(f2) + if empowerfirmware != '': + fileformat = fileformat+'v'+str(empowerfirmware) + else: # pragma: no cover + fileformat = 'speedcoach2v0' + try: + summary = row.allstats() + except ZeroDivisionError: # pragma: no cover + summary = '' + else: + fileformat = fileformat+'v'+str(empowerfirmware) + + # handle FIT + if (fileformat == 'fit'): # pragma: no cover + try: + s = fitsummarydata(f2) + s.setsummary() + summary = s.summarytext + except: + pass + hasrecognized = True + + return row, hasrecognized, summary, fileformat + + +def handle_nonpainsled(f2, fileformat, summary='', startdatetime='', empowerfirmware=None, impeller=False): + oarlength = 2.89 + inboard = 0.88 + hasrecognized = False + + row, hasrecognized, summary, fileformat = parsenonpainsled(fileformat, f2, summary, startdatetime=startdatetime, + empowerfirmware=empowerfirmware) + + # Handle c2log + if (fileformat == 'c2log' or fileformat == 'rowprolog'): # pragma: no cover + return (0, '', 0, 0, '', impeller) + + if not hasrecognized: # pragma: no cover + return (0, '', 0, 0, '', impeller) + + f_to_be_deleted = f2 + # should delete file + f2 = f2[:-4] + 'o.csv' + + row2 = rrdata(df=row.df) + + if 'speedcoach2' in fileformat or 'nklinklogbook' in fileformat: + # impeller consistency + impellerdata, consistent, ratio = row.impellerconsistent(threshold=0.3) + + if impellerdata and consistent: + impeller = True + if impellerdata and not consistent: + row2.use_gpsdata() + if impeller: + row2.use_impellerdata() + + row2.write_csv(f2, gzip=True) + + # os.remove(f2) + try: + os.remove(f_to_be_deleted) + except: # pragma: no cover + try: + os.remove(f_to_be_deleted + '.gz') + except: + pass + + return (f2, summary, oarlength, inboard, fileformat, impeller) + +# Create new workout from file and store it in the database +# This routine should be used everywhere in views.py + + +def get_workouttype_from_fit(filename, workouttype='water'): + try: + fitfile = FitFile(filename, check_crc=False) + except FitHeaderError: # pragma: no cover + return workouttype + + records = fitfile.messages + fittype = 'rowing' + for record in records: + if record.name in ['sport', 'lap']: + try: + fittype = record.get_values()['sport'].lower() + except (KeyError, AttributeError): # pragma: no cover + return 'water' + try: + workouttype = mytypes.fitmappinginv[fittype] + except KeyError: # pragma: no cover + return workouttype + + return workouttype + + +def get_workouttype_from_tcx(filename, workouttype='water'): + tcxtype = 'rowing' + if workouttype in mytypes.otwtypes: + return workouttype + try: # pragma: no cover + d = tcxtools.tcx_getdict(filename) + try: + tcxtype = d['Activities']['Activity']['@Sport'].lower() + if tcxtype == 'other': + tcxtype = 'rowing' + except KeyError: + return workouttype + + except TypeError: # pragma: no cover + pass + + try: # pragma: no cover + workouttype = mytypes.garminmappinginv[tcxtype.upper()] + except KeyError: # pragma: no cover + return workouttype + + return workouttype # pragma: no cover + + + + +# Create new workout from data frame and store it in the database +# This routine should be used everywhere in views.py and mailprocessing.py +# Currently there is code duplication + + +# A wrapper around the rowingdata class, with some error catching + + +def rdata(file, rower=rrower()): + try: + res = rrdata(csvfile=file, rower=rower) + except (IOError, IndexError): # pragma: no cover + try: + res = rrdata(csvfile=file + '.gz', rower=rower) + except (IOError, IndexError): + res = rrdata() + except: + res = rrdata() + except EOFError: # pragma: no cover + res = rrdata() + except: # pragma: no cover + res = rrdata() + + return res + +# Remove all stroke data for workout ID from database + + +def delete_strokedata(id, debug=False): + dirname = 'media/strokedata_{id}.parquet.gz'.format(id=id) + try: + shutil.rmtree(dirname) + except OSError: + try: + os.remove(dirname) + except FileNotFoundError: + pass + except FileNotFoundError: # pragma: no cover + pass + +# Replace stroke data in DB with data from CSV file + + +def update_strokedata(id, df, debug=False): + delete_strokedata(id, debug=debug) + _ = dataprep(df, id=id, bands=True, barchart=True, otwpower=True) + +# Test that all data are of a numerical time + + +def testdata(time, distance, pace, spm): # pragma: no cover + t1 = np.issubdtype(time, np.number) + t2 = np.issubdtype(distance, np.number) + t3 = np.issubdtype(pace, np.number) + t4 = np.issubdtype(spm, np.number) + + return t1 and t2 and t3 and t4 + +# Get data from DB for one workout (fetches all data). If data +# is not in DB, read from CSV file (and create DB entry) + + +def getrowdata_db(id=0, doclean=False, convertnewtons=True, + checkefficiency=True): + data = read_df_sql(id) + try: + data['deltat'] = data['time'].diff() + except KeyError: # pragma: no cover + data = pd.DataFrame() + + if data.empty: + rowdata, row = getrowdata(id=id) + if not rowdata.empty: + data = dataprep(rowdata.df, id=id, bands=True, + barchart=True, otwpower=True) + else: + data = pd.DataFrame() # returning empty dataframe + else: + row = Workout.objects.get(id=id) + + if checkefficiency is True and not data.empty: + try: + if data['efficiency'].mean() == 0 and data['power'].mean() != 0: # pragma: no cover + data = add_efficiency(id=id) + except KeyError: # pragma: no cover + data = add_efficiency(id=id) + + if doclean: # pragma: no cover + data = clean_df_stats(data, ignorehr=True) + + return data, row + +# Fetch a subset of the data from the DB + + +def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True, compute=True, + debug=False): + # prepmultipledata(ids) + + if ids: + csvfilenames = [ + 'media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] + else: + return pd.DataFrame() + + data = [] + columns = [c for c in columns if c != 'None'] + columns = list(set(columns)) + + if len(ids) > 1: + for id, f in zip(ids, csvfilenames): + try: + df = pd.read_parquet(f, columns=columns) + data.append(df) + except (OSError, ArrowInvalid, IndexError): # pragma: no cover + rowdata, row = getrowdata(id=id) + if rowdata and len(rowdata.df): + _ = dataprep(rowdata.df, id=id, + bands=True, otwpower=True, barchart=True) + df = pd.read_parquet(f, columns=columns) + data.append(df) + + try: + df = pd.concat(data, axis=0) + except ValueError: # pragma: no cover + return pd.DataFrame() + # df = dd.concat(data,axis=0) + + else: + try: + df = pd.read_parquet(csvfilenames[0], columns=columns) + rowdata, row = getrowdata(id=ids[0]) + except (OSError, ArrowInvalid, IndexError): + rowdata, row = getrowdata(id=ids[0]) + if rowdata and len(rowdata.df): # pragma: no cover + data = dataprep( + rowdata.df, id=ids[0], bands=True, otwpower=True, barchart=True) + df = pd.read_parquet(csvfilenames[0], columns=columns) + # df = dd.read_parquet(csvfilenames[0], + # column=columns,engine='pyarrow', + # ) + + # df = df.loc[:,~df.columns.duplicated()] + else: + df = pd.DataFrame() + + if compute and len(df): + data = df.copy() + if doclean: + data = clean_df_stats(data, ignorehr=True, + workstrokesonly=workstrokesonly) + data.dropna(axis=1, how='all', inplace=True) + data.dropna(axis=0, how='any', inplace=True) + return data + + return df + +# Fetch both the workout and the workout stroke data (from CSV file) + + +def getrowdata(id=0): + + # check if valid ID exists (workout exists) + try: + row = Workout.objects.get(id=id) + except Workout.DoesNotExist: # pragma: no cover + return rrdata(), None + + f1 = row.csvfilename + + # get user + + r = row.user + + rr = rrower(hrmax=r.max, hrut2=r.ut2, + hrut1=r.ut1, hrat=r.at, + hrtr=r.tr, hran=r.an, ftp=r.ftp) + + rowdata = rdata(f1, rower=rr) + + return rowdata, row + +# Checks if all rows for a list of workout IDs have entries in the +# stroke_data table. If this is not the case, it creates the stroke +# data +# In theory, this should never yield any work, but it's a good +# safety net for programming errors elsewhere in the app +# Also used heavily when I moved from CSV file only to CSV+Stroke data + + +def prepmultipledata(ids, verbose=False): # pragma: no cover + filenames = glob.glob('media/*.parquet') + ids = [ + id for id in ids if 'media/strokedata_{id}.parquet.gz'.format(id=id) not in filenames] + + for id in ids: + rowdata, row = getrowdata(id=id) + if verbose: + print(id) + if rowdata and len(rowdata.df): + _ = dataprep(rowdata.df, id=id, bands=True, + barchart=True, otwpower=True) + return ids + +# Read a set of columns for a set of workout ids, returns data as a +# pandas dataframe + + +def read_cols_df_sql(ids, columns, convertnewtons=True): + # drop columns that are not in offical list + # axx = [ax[0] for ax in axes] + + extracols = [] + + columns = list(columns) + ['distance', 'spm', 'workoutid'] + columns = [x for x in columns if x != 'None'] + columns = list(set(columns)) + ids = [int(id) for id in ids] + + df = pd.DataFrame() + + if len(ids) == 0: # pragma: no cover + return pd.DataFrame(), extracols + elif len(ids) == 1: # pragma: no cover + try: + filename = 'media/strokedata_{id}.parquet.gz'.format(id=ids[0]) + df = pd.read_parquet(filename, columns=columns) + except OSError: + rowdata, row = getrowdata(id=ids[0]) + if rowdata and len(rowdata.df): + _ = dataprep(rowdata.df, + id=ids[0], bands=True, otwpower=True, barchart=True) + df = pd.read_parquet(filename, columns=columns) + else: + data = [] + filenames = [ + 'media/strokedata_{id}.parquet.gz'.format(id=id) for id in ids] + for id, f in zip(ids, filenames): + try: + df = pd.read_parquet(f, columns=columns) + data.append(df) + except (OSError, IndexError, ArrowInvalid): + rowdata, row = getrowdata(id=id) + if rowdata and len(rowdata.df): # pragma: no cover + _ = dataprep(rowdata.df, id=id, + bands=True, otwpower=True, barchart=True) + df = pd.read_parquet(f, columns=columns) + data.append(df) + + try: + df = pd.concat(data, axis=0) + except ValueError: # pragma: no cover + return pd.DataFrame(), extracols + + df = df.fillna(value=0) + + if 'peakforce' in columns: + funits = ((w.id, w.forceunit) + for w in Workout.objects.filter(id__in=ids)) + for id, u in funits: + if u == 'lbs': + mask = df['workoutid'] == id + df.loc[mask, 'peakforce'] = df.loc[mask, 'peakforce'] * lbstoN + if 'averageforce' in columns: + funits = ((w.id, w.forceunit) + for w in Workout.objects.filter(id__in=ids)) + for id, u in funits: + if u == 'lbs': + mask = df['workoutid'] == id + df.loc[mask, 'averageforce'] = df.loc[mask, + 'averageforce'] * lbstoN + + return df, extracols + + + +# Read stroke data from the DB for a Workout ID. Returns a pandas dataframe + + +def read_df_sql(id): + try: + f = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = pd.read_parquet(f) + except (OSError, ArrowInvalid, IndexError): # pragma: no cover + rowdata, row = getrowdata(id=id) + if rowdata and len(rowdata.df): + data = dataprep(rowdata.df, id=id, bands=True, + otwpower=True, barchart=True) + try: + df = pd.read_parquet(f) + except OSError: + df = data + else: + df = pd.DataFrame() + + df = df.fillna(value=0) + + return df + + +# data fusion + + +def datafusion(id1, id2, columns, offset): + df1, w1 = getrowdata_db(id=id1) + df1 = df1.drop([ # 'cumdist', + 'hr_ut2', + 'hr_ut1', + 'hr_at', + 'hr_tr', + 'hr_an', + 'hr_max', + 'ftime', + 'fpace', + 'workoutid', + 'id'], + 1, errors='ignore') + + # Add coordinates to DataFrame + latitude, longitude = get_latlon(id1) + + df1[' latitude'] = latitude + df1[' longitude'] = longitude + + df2 = getsmallrowdata_db(['time'] + columns, ids=[id2], doclean=False) + + forceunit = 'N' + + offsetmillisecs = offset.seconds * 1000 + offset.microseconds / 1000. + offsetmillisecs += offset.days * (3600 * 24 * 1000) + df2['time'] = df2['time'] + offsetmillisecs + + keep1 = {c: c for c in set(df1.columns)} + + for c in columns: + keep1.pop(c) + + for c in df1.columns: + if c not in keep1: + df1 = df1.drop(c, 1, errors='ignore') + + df = pd.concat([df1, df2], ignore_index=True) + df = df.sort_values(['time']) + df = df.interpolate(method='linear', axis=0, limit_direction='both', + limit=10) + df.fillna(method='bfill', inplace=True) + + # Some new stuff to try out + df = df.groupby('time', axis=0).mean() + df['time'] = df.index + df.reset_index(drop=True, inplace=True) + + df['time'] = df['time'] / 1000. + df['pace'] = df['pace'] / 1000. + df['cum_dist'] = df['cumdist'] + + return df, forceunit + + +def fix_newtons(id=0, limit=3000): # pragma: no cover + # rowdata,row = getrowdata_db(id=id,doclean=False,convertnewtons=False) + rowdata = getsmallrowdata_db(['peakforce'], ids=[id], doclean=False) + try: + peakforce = rowdata['peakforce'] + if peakforce.mean() > limit: + w = Workout.objects.get(id=id) + + rowdata = rdata(w.csvfilename) + if rowdata and len(rowdata.df): + update_strokedata(w.id, rowdata.df) + except KeyError: + pass + + +def remove_invalid_columns(df): # pragma: no cover + for c in df.columns: + if c not in allowedcolumns: + df.drop(labels=c, axis=1, inplace=True) + + return df + + +def add_efficiency(id=0): # pragma: no cover + rowdata, row = getrowdata_db(id=id, + doclean=False, + convertnewtons=False, + checkefficiency=False) + power = rowdata['power'] + pace = rowdata['pace'] / 1.0e3 + velo = 500. / pace + ergpw = 2.8 * velo**3 + efficiency = 100. * ergpw / power + + efficiency = efficiency.replace([-np.inf, np.inf], np.nan) + efficiency.fillna(method='ffill') + rowdata['efficiency'] = efficiency + + rowdata = remove_invalid_columns(rowdata) + rowdata = rowdata.replace([-np.inf, np.inf], np.nan) + rowdata = rowdata.fillna(method='ffill') + + delete_strokedata(id) + + if id != 0: + rowdata['workoutid'] = id + filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = dd.from_pandas(rowdata, npartitions=1) + df.to_parquet(filename, engine='fastparquet', compression='GZIP') + + return rowdata + +# This is the main routine. +# it reindexes, sorts, filters, and smooths the data, then +# saves it to the stroke_data table in the database +# Takes a rowingdata object's DataFrame as input + + +def dataprep(rowdatadf, id=0, bands=True, barchart=True, otwpower=True, + empower=True, inboard=0.88, forceunit='lbs'): + + if rowdatadf.empty: + return 0 + + t = rowdatadf.loc[:, 'TimeStamp (sec)'] + t = pd.Series(t - rowdatadf.loc[:, 'TimeStamp (sec)'].iloc[0]) + + row_index = rowdatadf.loc[:, ' Stroke500mPace (sec/500m)'] > 3000 + rowdatadf.loc[row_index, ' Stroke500mPace (sec/500m)'] = 3000. + + p = rowdatadf.loc[:, ' Stroke500mPace (sec/500m)'] + try: + velo = rowdatadf.loc[:, ' AverageBoatSpeed (m/s)'] + except KeyError: # pragma: no cover + velo = 500./p + + hr = rowdatadf.loc[:, ' HRCur (bpm)'] + spm = rowdatadf.loc[:, ' Cadence (stokes/min)'] + cumdist = rowdatadf.loc[:, 'cum_dist'] + power = rowdatadf.loc[:, ' Power (watts)'] + averageforce = rowdatadf.loc[:, ' AverageDriveForce (lbs)'] + drivelength = rowdatadf.loc[:, ' DriveLength (meters)'] + try: + workoutstate = rowdatadf.loc[:, ' WorkoutState'] + except KeyError: # pragma: no cover + workoutstate = 0 * hr + + peakforce = rowdatadf.loc[:, ' PeakDriveForce (lbs)'] + + forceratio = averageforce / peakforce + forceratio = forceratio.fillna(value=0) + + try: + drivetime = rowdatadf.loc[:, ' DriveTime (ms)'] + recoverytime = rowdatadf.loc[:, ' StrokeRecoveryTime (ms)'] + rhythm = 100. * drivetime / (recoverytime + drivetime) + rhythm = rhythm.fillna(value=0) + except: # pragma: no cover + rhythm = 0.0 * forceratio + + f = rowdatadf['TimeStamp (sec)'].diff().mean() + if f != 0 and not np.isinf(f): + try: + windowsize = 2 * (int(10. / (f))) + 1 + except ValueError: # pragma: no cover + windowsize = 1 + else: + windowsize = 1 + if windowsize <= 3: + windowsize = 5 + + if windowsize > 3 and windowsize < len(hr): + spm = savgol_filter(spm, windowsize, 3) + hr = savgol_filter(hr, windowsize, 3) + drivelength = savgol_filter(drivelength, windowsize, 3) + forceratio = savgol_filter(forceratio, windowsize, 3) + + try: + t2 = t.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) + except TypeError: # pragma: no cover + t2 = 0 * t + + p2 = p.fillna(method='ffill').apply(lambda x: timedeltaconv(x)) + + try: + drivespeed = drivelength / rowdatadf[' DriveTime (ms)'] * 1.0e3 + except TypeError: # pragma: no cover + drivespeed = 0.0 * rowdatadf['TimeStamp (sec)'] + + drivespeed = drivespeed.fillna(value=0) + + try: + driveenergy = rowdatadf['driveenergy'] + except KeyError: # pragma: no cover + if forceunit == 'lbs': + driveenergy = drivelength * averageforce * lbstoN + else: + driveenergy = drivelength * averageforce + + if forceunit == 'lbs': + averageforce *= lbstoN + peakforce *= lbstoN + + powerhr = 60.*power/hr + powerhr = powerhr.fillna(value=0) + + if driveenergy.mean() == 0 and driveenergy.std() == 0: + driveenergy = 0*driveenergy+100 + + distance = rowdatadf.loc[:, 'cum_dist'] + velo = 500. / p + + distanceperstroke = 60. * velo / spm + + data = DataFrame( + dict( + time=t * 1e3, + hr=hr, + pace=p * 1e3, + spm=spm, + velo=velo, + cumdist=cumdist, + ftime=niceformat(t2), + fpace=nicepaceformat(p2), + driveenergy=driveenergy, + power=power, + workoutstate=workoutstate, + averageforce=averageforce, + drivelength=drivelength, + peakforce=peakforce, + forceratio=forceratio, + distance=distance, + drivespeed=drivespeed, + rhythm=rhythm, + distanceperstroke=distanceperstroke, + # powerhr=powerhr, + ) + ) + + if bands: + # HR bands + data['hr_ut2'] = rowdatadf.loc[:, 'hr_ut2'] + data['hr_ut1'] = rowdatadf.loc[:, 'hr_ut1'] + data['hr_at'] = rowdatadf.loc[:, 'hr_at'] + data['hr_tr'] = rowdatadf.loc[:, 'hr_tr'] + data['hr_an'] = rowdatadf.loc[:, 'hr_an'] + data['hr_max'] = rowdatadf.loc[:, 'hr_max'] + data['hr_bottom'] = 0.0 * data['hr'] + + try: + _ = rowdatadf.loc[:, ' ElapsedTime (sec)'] + except KeyError: # pragma: no cover + rowdatadf[' ElapsedTime (sec)'] = rowdatadf['TimeStamp (sec)'] + + if empower: + try: + wash = rowdatadf.loc[:, 'wash'] + except KeyError: + wash = 0 * power + + try: + catch = rowdatadf.loc[:, 'catch'] + except KeyError: + catch = 0 * power + + try: + finish = rowdatadf.loc[:, 'finish'] + except KeyError: + finish = 0 * power + + try: + peakforceangle = rowdatadf.loc[:, 'peakforceangle'] + except KeyError: + peakforceangle = 0 * power + + if data['driveenergy'].mean() == 0: # pragma: no cover + try: + driveenergy = rowdatadf.loc[:, 'driveenergy'] + except KeyError: + driveenergy = power * 60 / spm + else: + driveenergy = data['driveenergy'] + + arclength = (inboard - 0.05) * (np.radians(finish) - np.radians(catch)) + if arclength.mean() > 0: + drivelength = arclength + elif drivelength.mean() == 0: + drivelength = driveenergy / (averageforce * 4.44822) + + try: + slip = rowdatadf.loc[:, 'slip'] + except KeyError: + slip = 0 * power + + try: + totalangle = finish - catch + effectiveangle = finish - wash - catch - slip + except ValueError: # pragma: no cover + totalangle = 0 * power + effectiveangle = 0 * power + + if windowsize > 3 and windowsize < len(slip): + try: + wash = savgol_filter(wash, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + slip = savgol_filter(slip, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + catch = savgol_filter(catch, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + finish = savgol_filter(finish, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + peakforceangle = savgol_filter(peakforceangle, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + driveenergy = savgol_filter(driveenergy, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + drivelength = savgol_filter(drivelength, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + totalangle = savgol_filter(totalangle, windowsize, 3) + except TypeError: # pragma: no cover + pass + try: + effectiveangle = savgol_filter(effectiveangle, windowsize, 3) + except TypeError: # pragma: no cover + pass + + velo = 500. / p + + ergpw = 2.8 * velo**3 + efficiency = 100. * ergpw / power + + efficiency = efficiency.replace([-np.inf, np.inf], np.nan) + efficiency.fillna(method='ffill') + + try: + data['wash'] = wash + data['catch'] = catch + data['slip'] = slip + data['finish'] = finish + data['peakforceangle'] = peakforceangle + data['driveenergy'] = driveenergy + data['drivelength'] = drivelength + data['totalangle'] = totalangle + data['effectiveangle'] = effectiveangle + data['efficiency'] = efficiency + except ValueError: # pragma: no cover + pass + + if otwpower: + try: + nowindpace = rowdatadf.loc[:, 'nowindpace'] + except KeyError: + nowindpace = p + try: + equivergpower = rowdatadf.loc[:, 'equivergpower'] + except KeyError: + equivergpower = 0 * p + 50. + + nowindpace2 = nowindpace.apply(lambda x: timedeltaconv(x)) + ergvelo = (equivergpower / 2.8)**(1. / 3.) + + ergpace = 500. / ergvelo + ergpace[ergpace == np.inf] = 240. + ergpace2 = ergpace.apply(lambda x: timedeltaconv(x)) + + data['ergpace'] = ergpace * 1e3 + data['nowindpace'] = nowindpace * 1e3 + data['equivergpower'] = equivergpower + data['fergpace'] = nicepaceformat(ergpace2) + data['fnowindpace'] = nicepaceformat(nowindpace2) + + data = data.replace([-np.inf, np.inf], np.nan) + data = data.fillna(method='ffill') + + # write data if id given + if id != 0: + data['workoutid'] = id + data.fillna(0, inplace=True) + for k, v in dtypes.items(): + try: + data[k] = data[k].astype(v) + except KeyError: # pragma: no cover + pass + + filename = 'media/strokedata_{id}.parquet.gz'.format(id=id) + df = dd.from_pandas(data, npartitions=1) + df.to_parquet(filename, engine='fastparquet', compression='GZIP') + + return data + + + +def delete_agegroup_db(age, sex, weightcategory, debug=False): + if debug: # pragma: no cover + engine = create_engine(database_url_debug, echo=False) + else: # pragma: no cover + engine = create_engine(database_url, echo=False) + + query = sa.text('DELETE from {table} WHERE age={age} and weightcategory = {weightcategory} and sex={sex};'.format( + sex=sex, + age=age, + weightcategory=weightcategory, + table='calcagegrouprecords' + )) + with engine.connect() as conn, conn.begin(): + try: + _ = conn.execute(query) + except: # pragma: no cover + print("Database locked") + conn.close() + engine.dispose() + + + + + +def update_agegroup_db(age, sex, weightcategory, wcdurations, wcpower, + debug=False): + + delete_agegroup_db(age, sex, weightcategory, debug=debug) + + wcdurations = [None if type(y) is float and np.isnan( + y) else y for y in wcdurations] + wcpower = [None if type(y) is float and np.isnan(y) + else y for y in wcpower] + + df = pd.DataFrame( + { + 'duration': wcdurations, + 'power': wcpower, + } + ) + + df['sex'] = sex + df['age'] = age + df['weightcategory'] = weightcategory + df.replace([np.inf, -np.inf], np.nan, inplace=True) + df.dropna(axis=0, inplace=True) + + if debug: # pragma: no cover # pragma: no cover + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + + table = 'calcagegrouprecords' + with engine.connect() as conn, conn.begin(): + df.to_sql(table, engine, if_exists='append', index=False) + conn.close() + engine.dispose() + + + +def add_c2_stroke_data_db(strokedata, workoutid, starttimeunix, csvfilename, + debug=False, workouttype='rower'): + + res = make_cumvalues(0.1*strokedata['t']) + cum_time = res[0] + lapidx = res[1] + + unixtime = cum_time+starttimeunix + # unixtime[0] = starttimeunix + seconds = 0.1*strokedata.loc[:, 't'] + + nr_rows = len(unixtime) + + try: # pragma: no cover + latcoord = strokedata.loc[:, 'lat'] + loncoord = strokedata.loc[:, 'lon'] + except: + latcoord = np.zeros(nr_rows) + loncoord = np.zeros(nr_rows) + + try: + strokelength = strokedata.loc[:, 'strokelength'] + except: + strokelength = np.zeros(nr_rows) + + dist2 = 0.1*strokedata.loc[:, 'd'] + + try: + spm = strokedata.loc[:, 'spm'] + except KeyError: # pragma: no cover + spm = 0*dist2 + + try: + hr = strokedata.loc[:, 'hr'] + except KeyError: # pragma: no cover + hr = 0*spm + + pace = strokedata.loc[:, 'p']/10. + pace = np.clip(pace, 0, 1e4) + pace = pace.replace(0, 300) + + velo = 500./pace + power = 2.8*velo**3 + if workouttype == 'bike': # pragma: no cover + velo = 1000./pace + + # save csv + # Create data frame with all necessary data to write to csv + df = pd.DataFrame({'TimeStamp (sec)': unixtime, + ' Horizontal (meters)': dist2, + ' Cadence (stokes/min)': spm, + ' HRCur (bpm)': hr, + ' longitude': loncoord, + ' latitude': latcoord, + ' Stroke500mPace (sec/500m)': pace, + ' Power (watts)': power, + ' DragFactor': np.zeros(nr_rows), + ' DriveLength (meters)': np.zeros(nr_rows), + ' StrokeDistance (meters)': strokelength, + ' DriveTime (ms)': np.zeros(nr_rows), + ' StrokeRecoveryTime (ms)': np.zeros(nr_rows), + ' AverageDriveForce (lbs)': np.zeros(nr_rows), + ' PeakDriveForce (lbs)': np.zeros(nr_rows), + ' lapIdx': lapidx, + ' WorkoutState': 4, + ' ElapsedTime (sec)': seconds, + 'cum_dist': dist2 + }) + + df.sort_values(by='TimeStamp (sec)', ascending=True) + + # Create CSV file name and save data to CSV file + + res = df.to_csv(csvfilename, index_label='index', + compression='gzip') + + try: + data = dataprep(df, id=workoutid, bands=False, debug=debug) + except: # pragma: no cover + return 0 + + return data + +# Creates C2 stroke data +def create_c2_stroke_data_db( + distance, duration, workouttype, + workoutid, starttimeunix, csvfilename, debug=False): # pragma: no cover + + nr_strokes = int(distance/10.) + + totalseconds = duration.hour*3600. + totalseconds += duration.minute*60. + totalseconds += duration.second + totalseconds += duration.microsecond/1.e6 + + try: + spm = 60.*nr_strokes/totalseconds + except ZeroDivisionError: + spm = 20*np.zeros(nr_strokes) + + try: + _ = totalseconds/float(nr_strokes) + except ZeroDivisionError: + return 0 + + elapsed = np.arange(nr_strokes)*totalseconds/(float(nr_strokes-1)) + + d = np.arange(nr_strokes)*distance/(float(nr_strokes-1)) + + unixtime = starttimeunix + elapsed + + pace = 500.*totalseconds/distance + + if workouttype in ['rower', 'slides', 'dynamic']: + try: + velo = distance/totalseconds + except ZeroDivisionError: + velo = 0 + power = 2.8*velo**3 + else: + power = 0 + + df = pd.DataFrame({ + 'TimeStamp (sec)': unixtime, + ' Horizontal (meters)': d, + ' Cadence (stokes/min)': spm, + ' Stroke500mPace (sec/500m)': pace, + ' ElapsedTime (sec)': elapsed, + ' Power (watts)': power, + ' HRCur (bpm)': np.zeros(nr_strokes), + ' longitude': np.zeros(nr_strokes), + ' latitude': np.zeros(nr_strokes), + ' DragFactor': np.zeros(nr_strokes), + ' DriveLength (meters)': np.zeros(nr_strokes), + ' StrokeDistance (meters)': np.zeros(nr_strokes), + ' DriveTime (ms)': np.zeros(nr_strokes), + ' StrokeRecoveryTime (ms)': np.zeros(nr_strokes), + ' AverageDriveForce (lbs)': np.zeros(nr_strokes), + ' PeakDriveForce (lbs)': np.zeros(nr_strokes), + ' lapIdx': np.zeros(nr_strokes), + 'cum_dist': d + }) + + df[' ElapsedTime (sec)'] = df['TimeStamp (sec)'] + + _ = df.to_csv(csvfilename, index_label='index', compression='gzip') + + data = dataprep(df, id=workoutid, bands=False, debug=debug) + + return data + + +def update_empower(id, inboard, oarlength, boattype, df, f1, debug=False): # pragma: no cover + + corr_factor = 1.0 + if 'x' in boattype: + # sweep + a = 0.06 + b = 0.275 + else: + # scull + a = 0.15 + b = 0.275 + + corr_factor = empower_bug_correction(oarlength, inboard, a, b) + + success = False + + try: + df['power empower old'] = df[' Power (watts)'] + df[' Power (watts)'] = df[' Power (watts)'] * corr_factor + df['driveenergy empower old'] = df['driveenergy'] + df['driveenergy'] = df['driveenergy'] * corr_factor + success = True + except KeyError: + pass + + if success: + delete_strokedata(id, debug=debug) + if debug: # pragma: no cover + print("updated ", id) + print("correction ", corr_factor) + else: + if debug: # pragma: no cover + print("not updated ", id) + + _ = dataprep(df, id=id, bands=True, barchart=True, otwpower=True, debug=debug) + + row = rrdata(df=df) + row.write_csv(f1, gzip=True) + + return success diff --git a/rowers/middleware.py b/rowers/middleware.py index 7dc0d2d1..5f7d9c43 100644 --- a/rowers/middleware.py +++ b/rowers/middleware.py @@ -2,7 +2,7 @@ from django.shortcuts import redirect from django.http import HttpResponse from django.contrib import messages from rowers.mytypes import otwtypes -from rowers.tasks import handle_sendemail_expired +#from rowers.tasks import handle_sendemail_expired from django.utils import timezone from rowers.models import Workout, PowerTimeFitnessMetric, Rower, PaidPlan import datetime @@ -99,6 +99,7 @@ class RowerPlanMiddleWare(object): # remove from Free Coach groups # send email + from rowers.tasks import handle_sendemail_expired _ = myqueue(queue, handle_sendemail_expired, r.user.email, diff --git a/rowers/tasks.py b/rowers/tasks.py index 45f806ea..56201f88 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -1,3 +1,18 @@ +import os +os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" +from YamJam import yamjam +CFG = yamjam()['rowsandallapp'] + +try: + os.environ.setdefault("DJANGO_SETTINGS_MODULE",CFG['settings_name']) +except KeyError: + os.environ.setdefault("DJANGO_SETTINGS_MODULE","rowsandall_app.settings") + +from django.core.wsgi import get_wsgi_application + +application = get_wsgi_application() +from rowers.models import * + import math from rowers.courseutils import ( coursetime_paths, coursetime_first, time_in_path, @@ -16,8 +31,7 @@ import rowers.longtask as longtask import requests import rowers.datautils as datautils -""" Background tasks done by Celery (develop) or QR (production) """ -import os +""" Background tasks done by QR (production) """ import time import gc import gzip @@ -61,13 +75,15 @@ import rowers.otw_power_calculator_pb2_grpc as calculator_pb2_grpc import rowers.rowing_workout_metrics_pb2 as metrics_pb2 import rowers.rowing_workout_metrics_pb2_grpc as metrics_pb2_grpc -from rowsandall_app.settings import SITE_URL -from rowsandall_app.settings_dev import SITE_URL as SITE_URL_DEV -from rowsandall_app.settings import PROGRESS_CACHE_SECRET -from rowsandall_app.settings import SETTINGS_NAME -from rowsandall_app.settings import workoutemailbox -from rowsandall_app.settings import UPLOAD_SERVICE_SECRET, UPLOAD_SERVICE_URL -from rowsandall_app.settings import NK_API_LOCATION +from django.conf import settings +SITE_URL = settings.SITE_URL +SITE_URL_DEV = settings.SITE_URL +PROGRESS_CACHE_SECRET = settings.PROGRESS_CACHE_SECRET +SETTINGS_NAME = settings.SETTINGS_NAME + +UPLOAD_SERVICE_URL = settings.UPLOAD_SERVICE_URL +UPLOAD_SERVICE_SECRET = settings.UPLOAD_SERVICE_SECRET +NK_API_LOCATION = settings.NK_API_LOCATION from requests_oauthlib import OAuth1, OAuth1Session @@ -83,16 +99,19 @@ from rowers.emails import htmlstrip from rowers import mytypes -from rowers.dataprepnodjango import ( +from rowers.dataroutines import ( getsmallrowdata_db, updatecpdata_sql, update_c2id_sql, - update_workout_field_sql, + #update_workout_field_sql, update_agegroup_db, update_strokedata, add_c2_stroke_data_db, totaltime_sec_to_string, create_c2_stroke_data_db, update_empower, - database_url_debug, database_url, dataprep, + # database_url_debug, + database_url, dataprep, # create_strava_stroke_data_db ) +database_url_debug = database_url + from rowers.opaque import encoder @@ -297,10 +316,11 @@ def handle_c2_sync(workoutid, url, headers, data, debug=False, **kwargs): s = response.json() c2id = s['data']['id'] - res = update_workout_field_sql( - workoutid, 'uploadedtoc2', c2id, debug=debug) + workout = Workout.objects.get(id=workoutid) + workout.uploadedtoc2 = c2id + workout.save() - return res + return 1 @app.task @@ -316,8 +336,10 @@ def handle_sporttracks_sync(workoutid, url, headers, data, debug=False, **kwargs id = int(m) - _ = update_workout_field_sql( - workoutid, 'uploadedtosporttracks', id, debug=debug) + workout = Workout.objects.get(id=workoutid) + workout.uploadedtosporttracks = id + workout.save() + return 1 @@ -378,8 +400,9 @@ def handle_strava_sync(stravatoken, workoutid, filename, name, activity_type, de failed = True if not failed: - _ = update_workout_field_sql( - workoutid, 'uploadedtostrava', res.id, debug=debug) + workout = Workout.objects.get(id=workoutid) + workout.uploadedtostrava = res.id + workout.save() try: act = client.update_activity(res.id, activity_type=activity_type, description=description, device_name='Rowsandall.com')