diff --git a/rowers/dataprep.py b/rowers/dataprep.py index acb1ff77..c648468d 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -4,6 +4,7 @@ from rowers.models import Workout, User, Rower,StrokeData from rowingdata import rowingdata as rrdata from rowers.tasks import handle_sendemail_unrecognized +from rowers.tasks import handle_zip_file from rowingdata import rower as rrower from rowingdata import main as rmain @@ -44,7 +45,7 @@ import sys import django_rq queue = django_rq.get_queue('default') queuelow = django_rq.get_queue('low') -queuehigh = django_rq.get_queue('low') +queuehigh = django_rq.get_queue('default') user = settings.DATABASES['default']['USER'] @@ -362,7 +363,10 @@ def save_workout_database(f2,r,dosmooth=True,workouttype='rower', velo = 500./pace f = row.df['TimeStamp (sec)'].diff().mean() - windowsize = 2*(int(10./(f)))+1 + if f !=0: + windowsize = 2*(int(10./(f)))+1 + else: + windowsize = 1 if not 'originalvelo' in row.df: row.df['originalvelo'] = velo @@ -569,18 +573,21 @@ def new_workout_from_file(r,f2, inboard = 0.88 if len(fileformat)==3 and fileformat[0]=='zip': f_to_be_deleted = f2 - with zipfile.ZipFile(f2) as z: - # for now, we're getting only the first file - # from the NK zip file (issue #69 on bitbucket) - for fname in z.namelist(): - f3 = z.extract(fname,path='media/') - id,message,f2 = new_workout_from_file(r,f3, - workouttype=workouttype, - makeprivate=makeprivate, - title = title, - notes='') - os.remove(f_to_be_deleted) - return id,message,f2 + title = os.path.basename(f2) + if settings.DEBUG: + res = handle_zip_file.delay( + r.user.email,title,f2 + ) + + else: + res = queuelow.enqueue( + handle_zip_file, + r.user.email, + title, + f2 + ) + + return -1,message,f2 # Some people try to upload Concept2 logbook summaries if fileformat == 'c2log': @@ -1053,7 +1060,10 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, rhythm = 0.0*forceratio f = rowdatadf['TimeStamp (sec)'].diff().mean() - windowsize = 2*(int(10./(f)))+1 + if f != 0: + windowsize = 2*(int(10./(f)))+1 + else: + windowsize = 1 if windowsize <= 3: windowsize = 5 diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index c42855e8..0cd3fc55 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -91,6 +91,311 @@ def rdata(file,rower=rrower()): return res +# Processes painsled CSV file to database +def save_workout_database(f2,r,dosmooth=True,workouttype='rower', + dosummary=True,title='Workout', + notes='',totaldist=0,totaltime=0, + summary='', + makeprivate=False, + oarlength=2.89,inboard=0.88): + message = None + powerperc = 100*np.array([r.pw_ut2, + r.pw_ut1, + r.pw_at, + r.pw_tr,r.pw_an])/r.ftp + + # make workout and put in database + rr = rrower(hrmax=r.max,hrut2=r.ut2, + hrut1=r.ut1,hrat=r.at, + hrtr=r.tr,hran=r.an,ftp=r.ftp, + powerperc=powerperc,powerzones=r.powerzones) + row = rdata(f2,rower=rr) + if row == 0: + return (0,'Error: CSV data file not found') + + if dosmooth: + # auto smoothing + pace = row.df[' Stroke500mPace (sec/500m)'].values + velo = 500./pace + + f = row.df['TimeStamp (sec)'].diff().mean() + if f !=0: + windowsize = 2*(int(10./(f)))+1 + else: + windowsize = 1 + if not 'originalvelo' in row.df: + row.df['originalvelo'] = velo + + if windowsize > 3 and windowsize23: + message = 'Warning: The workout duration was longer than 23 hours. ' + hours = 23 + + minutes = int((totaltime - 3600.*hours)/60.) + if minutes>59: + minutes = 59 + if not message: + message = 'Warning: there is something wrong with the workout duration' + + seconds = int(totaltime - 3600.*hours - 60.*minutes) + if seconds > 59: + seconds = 59 + if not message: + message = 'Warning: there is something wrong with the workout duration' + + tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds)) + if tenths > 9: + tenths = 9 + if not message: + message = 'Warning: there is something wrong with the workout duration' + + duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) + + if dosummary: + summary = row.summary() + summary += '\n' + summary += row.intervalstats() + + workoutdate = row.rowdatetime.strftime('%Y-%m-%d') + workoutstarttime = row.rowdatetime.strftime('%H:%M:%S') + workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc) + + if makeprivate: + privacy = 'private' + else: + privacy = 'visible' + + # check for duplicate start times + ws = Workout.objects.filter(startdatetime=workoutstartdatetime, + user=r) + if (len(ws) != 0): + message = "Warning: This workout probably already exists in the database" + + + w = Workout(user=r,name=title,date=workoutdate, + workouttype=workouttype, + duration=duration,distance=totaldist, + weightcategory=r.weightcategory, + starttime=workoutstarttime, + csvfilename=f2,notes=notes,summary=summary, + maxhr=maxhr,averagehr=averagehr, + startdatetime=workoutstartdatetime, + inboard=inboard,oarlength=oarlength, + privacy=privacy) + + + w.save() + + if privacy == 'visible': + ts = Team.objects.filter(rower=r) + for t in ts: + w.team.add(t) + + # put stroke data in database + res = dataprep(row.df,id=w.id,bands=True, + barchart=True,otwpower=True,empower=True,inboard=inboard) + + return (w.id,message) + +def handle_nonpainsled(f2,fileformat,summary=''): + oarlength = 2.89 + inboard = 0.88 + # handle RowPro: + if (fileformat == 'rp'): + row = RowProParser(f2) + # handle TCX + if (fileformat == 'tcx'): + row = TCXParser(f2) + + # handle Mystery + if (fileformat == 'mystery'): + row = MysteryParser(f2) + + # handle TCX no HR + if (fileformat == 'tcxnohr'): + row = TCXParserNoHR(f2) + + # handle RowPerfect + if (fileformat == 'rowperfect3'): + row = RowPerfectParser(f2) + + # handle ErgData + if (fileformat == 'ergdata'): + row = ErgDataParser(f2) + + # handle Mike + if (fileformat == 'bcmike'): + row = BoatCoachAdvancedParser(f2) + + # handle BoatCoach + if (fileformat == 'boatcoach'): + row = BoatCoachParser(f2) + + # handle painsled desktop + if (fileformat == 'painsleddesktop'): + row = painsledDesktopParser(f2) + + # handle speed coach GPS + if (fileformat == 'speedcoach'): + row = speedcoachParser(f2) + + # handle speed coach GPS 2 + if (fileformat == 'speedcoach2'): + row = SpeedCoach2Parser(f2) + try: + oarlength,inboard = get_empower_rigging(f2) + summary = row.allstats() + except: + pass + + + # handle ErgStick + if (fileformat == 'ergstick'): + row = ErgStickParser(f2) + + # handle FIT + if (fileformat == 'fit'): + row = FITParser(f2) + s = fitsummarydata(f2) + s.setsummary() + summary = s.summarytext + + + f_to_be_deleted = f2 + # should delete file + f2 = f2[:-4]+'o.csv' + row.write_csv(f2,gzip=True) + + #os.remove(f2) + try: + os.remove(f_to_be_deleted) + except: + os.remove(f_to_be_deleted+'.gz') + + return (f2,summary,oarlength,inboard) + +# Create new workout from file and store it in the database +# This routine should be used everywhere in views.py and mailprocessing.py +# Currently there is code duplication +def new_workout_from_file(r,f2, + workouttype='rower', + title='Workout', + makeprivate=False, + notes=''): + message = None + fileformat = get_file_type(f2) + summary = '' + oarlength = 2.89 + inboard = 0.88 + if len(fileformat)==3 and fileformat[0]=='zip': + f_to_be_deleted = f2 + with zipfile.ZipFile(f2) as z: + for fname in z.namelist(): + f3 = z.extract(fname,path='media/') + id,message,f2 = new_workout_from_file(r,f3, + workouttype=workouttype, + makeprivate=makeprivate, + title = title, + notes='') + os.remove(f_to_be_deleted) + return id,message,f2 + + # Some people try to upload Concept2 logbook summaries + if fileformat == 'c2log': + os.remove(f2) + message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook." + return (0,message,f2) + + if fileformat == 'nostrokes': + os.remove(f2) + message = "It looks like this file doesn't contain stroke data." + return (0,message,f2) + + # Some people try to upload RowPro summary logs + if fileformat == 'rowprolog': + os.remove(f2) + message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log." + return (0,message,f2) + + # Sometimes people try an unsupported file type. + # Send an email to info@rowsandall.com with the file attached + # for me to check if it is a bug, or a new file type + # worth supporting + if fileformat == 'unknown': + message = "We couldn't recognize the file type" + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(f2, + r.user.email) + + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + f2,r.user.email) + return (0,message,f2) + + # handle non-Painsled by converting it to painsled compatible CSV + if (fileformat != 'csv'): + try: + f2,summary,oarlength,inboard = handle_nonpainsled(f2, + fileformat, + summary=summary) + except: + errorstring = str(sys.exc_info()[0]) + message = 'Something went wrong: '+errorstring + return (0,message,'') + + + + dosummary = (fileformat != 'fit') + id,message = save_workout_database(f2,r, + workouttype=workouttype, + makeprivate=makeprivate, + dosummary=dosummary, + summary=summary, + inboard=inboard,oarlength=oarlength, + title=title) + + return (id,message,f2) + def delete_strokedata(id,debug=True): if debug: engine = create_engine(database_url_debug, echo=False) diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 918602ec..53a755d7 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -63,12 +63,14 @@ class Command(BaseCommand): z = zipfile.ZipFile(a.document) for f in z.namelist(): f2 = z.extract(f,path='media/') + title = os.path.basename(f2) wid = [ - make_new_workout_from_email(rr,f2[6:],name) + make_new_workout_from_email(rr,f2[6:],title) ] res += wid link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - dd = send_confirm(rr.user,name,link) + dd = send_confirm(rr.user,title,link) + time.sleep(10) else: # move attachment and make workout diff --git a/rowers/tasks.py b/rowers/tasks.py index f37fe6ab..6f4a2a01 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -21,7 +21,7 @@ import stravalib from utils import serialize_list,deserialize_list from rowers.dataprepnodjango import update_strokedata - +from rowers.dataprepnodjango import new_workout_from_file from django.core.mail import send_mail, BadHeaderError,EmailMessage @@ -30,6 +30,16 @@ from django.core.mail import send_mail, BadHeaderError,EmailMessage def add(x, y): return x + y +# create workout +@app.task +def handle_new_workout_from_file(r,f2, + workouttype='rower', + title='Workout', + makeprivate=False, + notes=''): + return new_workout_from_file(r,f2,workouttype, + title,makeprivate,notes) + # send email to me when an unrecognized file is uploaded @app.task def handle_sendemail_unrecognized(unrecognizedfile,useremail): @@ -80,6 +90,17 @@ def handle_sendemailtcx(first_name,last_name,email,tcxfile): os.remove(tcxfile) return 1 +@app.task +def handle_zip_file(emailfrom,subject,file): + message = "... zip processing ... " + email = EmailMessage(subject,message, + emailfrom, + ['workouts@rowsandall.com']) + email.attach_file(file) + res = email.send() + time.sleep(60) + return 1 + # Send email with CSV attachment @app.task def handle_sendemailcsv(first_name,last_name,email,csvfile): diff --git a/rowers/views.py b/rowers/views.py index d80126b9..bdf96a25 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -5287,7 +5287,12 @@ def workout_upload_view(request,message="", args=[str(message)]) response = HttpResponseRedirect(url) return response - + elif id == -1: + message = 'The zip archive will be processed in the background. The files in the archive will only be uploaded without the extra actions. You will receive email when the workouts are ready.' + url = reverse(workout_upload_view, + args=[str(message)]) + response = HttpResponseRedirect(url) + return response else: if message: url = reverse(workout_edit_view,