works but runs into email rate limiting
This commit is contained in:
Sander Roosendaal
1
rowers/.#dataprep.py
Normal file
1
rowers/.#dataprep.py
Normal file
@@ -0,0 +1 @@
|
||||
E408191@CZ27LT9RCGN72.13020:1492098105
|
||||
@@ -4,6 +4,7 @@ from rowers.models import Workout, User, Rower,StrokeData
|
||||
from rowingdata import rowingdata as rrdata
|
||||
|
||||
from rowers.tasks import handle_sendemail_unrecognized
|
||||
from rowers.tasks import handle_zip_file
|
||||
|
||||
from rowingdata import rower as rrower
|
||||
from rowingdata import main as rmain
|
||||
@@ -44,7 +45,7 @@ import sys
|
||||
import django_rq
|
||||
queue = django_rq.get_queue('default')
|
||||
queuelow = django_rq.get_queue('low')
|
||||
queuehigh = django_rq.get_queue('low')
|
||||
queuehigh = django_rq.get_queue('default')
|
||||
|
||||
|
||||
user = settings.DATABASES['default']['USER']
|
||||
@@ -573,17 +574,26 @@ def new_workout_from_file(r,f2,
|
||||
if len(fileformat)==3 and fileformat[0]=='zip':
|
||||
f_to_be_deleted = f2
|
||||
with zipfile.ZipFile(f2) as z:
|
||||
# for now, we're getting only the first file
|
||||
# from the NK zip file (issue #69 on bitbucket)
|
||||
for fname in z.namelist():
|
||||
f3 = z.extract(fname,path='media/')
|
||||
id,message,f2 = new_workout_from_file(r,f3,
|
||||
workouttype=workouttype,
|
||||
makeprivate=makeprivate,
|
||||
title = title,
|
||||
notes='')
|
||||
print f3,r.user.email,title
|
||||
if settings.DEBUG:
|
||||
res = handle_zip_file.delay(
|
||||
r.user.email,title,f3
|
||||
)
|
||||
|
||||
else:
|
||||
res = queuelow.enqueue(
|
||||
handle_zip_file,
|
||||
r.user.email,
|
||||
title,
|
||||
f3
|
||||
)
|
||||
|
||||
|
||||
|
||||
os.remove(f_to_be_deleted)
|
||||
return id,message,f2
|
||||
return -1,message,f2
|
||||
|
||||
# Some people try to upload Concept2 logbook summaries
|
||||
if fileformat == 'c2log':
|
||||
|
||||
@@ -91,6 +91,311 @@ def rdata(file,rower=rrower()):
|
||||
|
||||
return res
|
||||
|
||||
# Processes painsled CSV file to database
|
||||
def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
|
||||
dosummary=True,title='Workout',
|
||||
notes='',totaldist=0,totaltime=0,
|
||||
summary='',
|
||||
makeprivate=False,
|
||||
oarlength=2.89,inboard=0.88):
|
||||
message = None
|
||||
powerperc = 100*np.array([r.pw_ut2,
|
||||
r.pw_ut1,
|
||||
r.pw_at,
|
||||
r.pw_tr,r.pw_an])/r.ftp
|
||||
|
||||
# make workout and put in database
|
||||
rr = rrower(hrmax=r.max,hrut2=r.ut2,
|
||||
hrut1=r.ut1,hrat=r.at,
|
||||
hrtr=r.tr,hran=r.an,ftp=r.ftp,
|
||||
powerperc=powerperc,powerzones=r.powerzones)
|
||||
row = rdata(f2,rower=rr)
|
||||
if row == 0:
|
||||
return (0,'Error: CSV data file not found')
|
||||
|
||||
if dosmooth:
|
||||
# auto smoothing
|
||||
pace = row.df[' Stroke500mPace (sec/500m)'].values
|
||||
velo = 500./pace
|
||||
|
||||
f = row.df['TimeStamp (sec)'].diff().mean()
|
||||
if f !=0:
|
||||
windowsize = 2*(int(10./(f)))+1
|
||||
else:
|
||||
windowsize = 1
|
||||
if not 'originalvelo' in row.df:
|
||||
row.df['originalvelo'] = velo
|
||||
|
||||
if windowsize > 3 and windowsize<len(velo):
|
||||
velo2 = savgol_filter(velo,windowsize,3)
|
||||
else:
|
||||
velo2 = velo
|
||||
|
||||
velo3 = pd.Series(velo2)
|
||||
velo3 = velo3.replace([-np.inf,np.inf],np.nan)
|
||||
velo3 = velo3.fillna(method='ffill')
|
||||
|
||||
pace2 = 500./abs(velo3)
|
||||
|
||||
row.df[' Stroke500mPace (sec/500m)'] = pace2
|
||||
|
||||
row.df = row.df.fillna(0)
|
||||
|
||||
row.write_csv(f2,gzip=True)
|
||||
try:
|
||||
os.remove(f2)
|
||||
except:
|
||||
pass
|
||||
|
||||
# recalculate power data
|
||||
if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
|
||||
try:
|
||||
row.erg_recalculatepower()
|
||||
row.write_csv(f2,gzip=True)
|
||||
except:
|
||||
pass
|
||||
|
||||
averagehr = row.df[' HRCur (bpm)'].mean()
|
||||
maxhr = row.df[' HRCur (bpm)'].max()
|
||||
|
||||
if totaldist == 0:
|
||||
totaldist = row.df['cum_dist'].max()
|
||||
if totaltime == 0:
|
||||
totaltime = row.df['TimeStamp (sec)'].max()-row.df['TimeStamp (sec)'].min()
|
||||
totaltime = totaltime+row.df.ix[0,' ElapsedTime (sec)']
|
||||
|
||||
|
||||
|
||||
hours = int(totaltime/3600.)
|
||||
if hours>23:
|
||||
message = 'Warning: The workout duration was longer than 23 hours. '
|
||||
hours = 23
|
||||
|
||||
minutes = int((totaltime - 3600.*hours)/60.)
|
||||
if minutes>59:
|
||||
minutes = 59
|
||||
if not message:
|
||||
message = 'Warning: there is something wrong with the workout duration'
|
||||
|
||||
seconds = int(totaltime - 3600.*hours - 60.*minutes)
|
||||
if seconds > 59:
|
||||
seconds = 59
|
||||
if not message:
|
||||
message = 'Warning: there is something wrong with the workout duration'
|
||||
|
||||
tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds))
|
||||
if tenths > 9:
|
||||
tenths = 9
|
||||
if not message:
|
||||
message = 'Warning: there is something wrong with the workout duration'
|
||||
|
||||
duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths)
|
||||
|
||||
if dosummary:
|
||||
summary = row.summary()
|
||||
summary += '\n'
|
||||
summary += row.intervalstats()
|
||||
|
||||
workoutdate = row.rowdatetime.strftime('%Y-%m-%d')
|
||||
workoutstarttime = row.rowdatetime.strftime('%H:%M:%S')
|
||||
workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc)
|
||||
|
||||
if makeprivate:
|
||||
privacy = 'private'
|
||||
else:
|
||||
privacy = 'visible'
|
||||
|
||||
# check for duplicate start times
|
||||
ws = Workout.objects.filter(startdatetime=workoutstartdatetime,
|
||||
user=r)
|
||||
if (len(ws) != 0):
|
||||
message = "Warning: This workout probably already exists in the database"
|
||||
|
||||
|
||||
w = Workout(user=r,name=title,date=workoutdate,
|
||||
workouttype=workouttype,
|
||||
duration=duration,distance=totaldist,
|
||||
weightcategory=r.weightcategory,
|
||||
starttime=workoutstarttime,
|
||||
csvfilename=f2,notes=notes,summary=summary,
|
||||
maxhr=maxhr,averagehr=averagehr,
|
||||
startdatetime=workoutstartdatetime,
|
||||
inboard=inboard,oarlength=oarlength,
|
||||
privacy=privacy)
|
||||
|
||||
|
||||
w.save()
|
||||
|
||||
if privacy == 'visible':
|
||||
ts = Team.objects.filter(rower=r)
|
||||
for t in ts:
|
||||
w.team.add(t)
|
||||
|
||||
# put stroke data in database
|
||||
res = dataprep(row.df,id=w.id,bands=True,
|
||||
barchart=True,otwpower=True,empower=True,inboard=inboard)
|
||||
|
||||
return (w.id,message)
|
||||
|
||||
def handle_nonpainsled(f2,fileformat,summary=''):
|
||||
oarlength = 2.89
|
||||
inboard = 0.88
|
||||
# handle RowPro:
|
||||
if (fileformat == 'rp'):
|
||||
row = RowProParser(f2)
|
||||
# handle TCX
|
||||
if (fileformat == 'tcx'):
|
||||
row = TCXParser(f2)
|
||||
|
||||
# handle Mystery
|
||||
if (fileformat == 'mystery'):
|
||||
row = MysteryParser(f2)
|
||||
|
||||
# handle TCX no HR
|
||||
if (fileformat == 'tcxnohr'):
|
||||
row = TCXParserNoHR(f2)
|
||||
|
||||
# handle RowPerfect
|
||||
if (fileformat == 'rowperfect3'):
|
||||
row = RowPerfectParser(f2)
|
||||
|
||||
# handle ErgData
|
||||
if (fileformat == 'ergdata'):
|
||||
row = ErgDataParser(f2)
|
||||
|
||||
# handle Mike
|
||||
if (fileformat == 'bcmike'):
|
||||
row = BoatCoachAdvancedParser(f2)
|
||||
|
||||
# handle BoatCoach
|
||||
if (fileformat == 'boatcoach'):
|
||||
row = BoatCoachParser(f2)
|
||||
|
||||
# handle painsled desktop
|
||||
if (fileformat == 'painsleddesktop'):
|
||||
row = painsledDesktopParser(f2)
|
||||
|
||||
# handle speed coach GPS
|
||||
if (fileformat == 'speedcoach'):
|
||||
row = speedcoachParser(f2)
|
||||
|
||||
# handle speed coach GPS 2
|
||||
if (fileformat == 'speedcoach2'):
|
||||
row = SpeedCoach2Parser(f2)
|
||||
try:
|
||||
oarlength,inboard = get_empower_rigging(f2)
|
||||
summary = row.allstats()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
# handle ErgStick
|
||||
if (fileformat == 'ergstick'):
|
||||
row = ErgStickParser(f2)
|
||||
|
||||
# handle FIT
|
||||
if (fileformat == 'fit'):
|
||||
row = FITParser(f2)
|
||||
s = fitsummarydata(f2)
|
||||
s.setsummary()
|
||||
summary = s.summarytext
|
||||
|
||||
|
||||
f_to_be_deleted = f2
|
||||
# should delete file
|
||||
f2 = f2[:-4]+'o.csv'
|
||||
row.write_csv(f2,gzip=True)
|
||||
|
||||
#os.remove(f2)
|
||||
try:
|
||||
os.remove(f_to_be_deleted)
|
||||
except:
|
||||
os.remove(f_to_be_deleted+'.gz')
|
||||
|
||||
return (f2,summary,oarlength,inboard)
|
||||
|
||||
# Create new workout from file and store it in the database
|
||||
# This routine should be used everywhere in views.py and mailprocessing.py
|
||||
# Currently there is code duplication
|
||||
def new_workout_from_file(r,f2,
|
||||
workouttype='rower',
|
||||
title='Workout',
|
||||
makeprivate=False,
|
||||
notes=''):
|
||||
message = None
|
||||
fileformat = get_file_type(f2)
|
||||
summary = ''
|
||||
oarlength = 2.89
|
||||
inboard = 0.88
|
||||
if len(fileformat)==3 and fileformat[0]=='zip':
|
||||
f_to_be_deleted = f2
|
||||
with zipfile.ZipFile(f2) as z:
|
||||
for fname in z.namelist():
|
||||
f3 = z.extract(fname,path='media/')
|
||||
id,message,f2 = new_workout_from_file(r,f3,
|
||||
workouttype=workouttype,
|
||||
makeprivate=makeprivate,
|
||||
title = title,
|
||||
notes='')
|
||||
os.remove(f_to_be_deleted)
|
||||
return id,message,f2
|
||||
|
||||
# Some people try to upload Concept2 logbook summaries
|
||||
if fileformat == 'c2log':
|
||||
os.remove(f2)
|
||||
message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook."
|
||||
return (0,message,f2)
|
||||
|
||||
if fileformat == 'nostrokes':
|
||||
os.remove(f2)
|
||||
message = "It looks like this file doesn't contain stroke data."
|
||||
return (0,message,f2)
|
||||
|
||||
# Some people try to upload RowPro summary logs
|
||||
if fileformat == 'rowprolog':
|
||||
os.remove(f2)
|
||||
message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log."
|
||||
return (0,message,f2)
|
||||
|
||||
# Sometimes people try an unsupported file type.
|
||||
# Send an email to info@rowsandall.com with the file attached
|
||||
# for me to check if it is a bug, or a new file type
|
||||
# worth supporting
|
||||
if fileformat == 'unknown':
|
||||
message = "We couldn't recognize the file type"
|
||||
if settings.DEBUG:
|
||||
res = handle_sendemail_unrecognized.delay(f2,
|
||||
r.user.email)
|
||||
|
||||
else:
|
||||
res = queuehigh.enqueue(handle_sendemail_unrecognized,
|
||||
f2,r.user.email)
|
||||
return (0,message,f2)
|
||||
|
||||
# handle non-Painsled by converting it to painsled compatible CSV
|
||||
if (fileformat != 'csv'):
|
||||
try:
|
||||
f2,summary,oarlength,inboard = handle_nonpainsled(f2,
|
||||
fileformat,
|
||||
summary=summary)
|
||||
except:
|
||||
errorstring = str(sys.exc_info()[0])
|
||||
message = 'Something went wrong: '+errorstring
|
||||
return (0,message,'')
|
||||
|
||||
|
||||
|
||||
dosummary = (fileformat != 'fit')
|
||||
id,message = save_workout_database(f2,r,
|
||||
workouttype=workouttype,
|
||||
makeprivate=makeprivate,
|
||||
dosummary=dosummary,
|
||||
summary=summary,
|
||||
inboard=inboard,oarlength=oarlength,
|
||||
title=title)
|
||||
|
||||
return (id,message,f2)
|
||||
|
||||
def delete_strokedata(id,debug=True):
|
||||
if debug:
|
||||
engine = create_engine(database_url_debug, echo=False)
|
||||
|
||||
@@ -21,7 +21,7 @@ import stravalib
|
||||
from utils import serialize_list,deserialize_list
|
||||
|
||||
from rowers.dataprepnodjango import update_strokedata
|
||||
|
||||
from rowers.dataprepnodjango import new_workout_from_file
|
||||
|
||||
from django.core.mail import send_mail, BadHeaderError,EmailMessage
|
||||
|
||||
@@ -30,6 +30,16 @@ from django.core.mail import send_mail, BadHeaderError,EmailMessage
|
||||
def add(x, y):
|
||||
return x + y
|
||||
|
||||
# create workout
|
||||
@app.task
|
||||
def handle_new_workout_from_file(r,f2,
|
||||
workouttype='rower',
|
||||
title='Workout',
|
||||
makeprivate=False,
|
||||
notes=''):
|
||||
return new_workout_from_file(r,f2,workouttype,
|
||||
title,makeprivate,notes)
|
||||
|
||||
# send email to me when an unrecognized file is uploaded
|
||||
@app.task
|
||||
def handle_sendemail_unrecognized(unrecognizedfile,useremail):
|
||||
@@ -80,6 +90,16 @@ def handle_sendemailtcx(first_name,last_name,email,tcxfile):
|
||||
os.remove(tcxfile)
|
||||
return 1
|
||||
|
||||
@app.task
|
||||
def handle_zip_file(emailfrom,subject,file):
|
||||
message = "... zip processing ... "
|
||||
email = EmailMessage(subject,message,
|
||||
emailfrom,
|
||||
['workouts@rowsandall.com'])
|
||||
email.attach_file(file)
|
||||
res = email.send()
|
||||
return 1
|
||||
|
||||
# Send email with CSV attachment
|
||||
@app.task
|
||||
def handle_sendemailcsv(first_name,last_name,email,csvfile):
|
||||
|
||||
@@ -5230,7 +5230,12 @@ def workout_upload_view(request,message="",
|
||||
args=[str(message)])
|
||||
response = HttpResponseRedirect(url)
|
||||
return response
|
||||
|
||||
elif id == -1:
|
||||
message = 'Zip files will be processed in the background'
|
||||
url = reverse(workout_upload_view,
|
||||
args=[str(message)])
|
||||
response = HttpResponseRedirect(url)
|
||||
return response
|
||||
else:
|
||||
if message:
|
||||
url = reverse(workout_edit_view,
|
||||
|
||||
Reference in New Issue
Block a user