diff --git a/rowers/.#dataprep.py b/rowers/.#dataprep.py new file mode 100644 index 00000000..61086939 --- /dev/null +++ b/rowers/.#dataprep.py @@ -0,0 +1 @@ +E408191@CZ27LT9RCGN72.13020:1492098105 \ No newline at end of file diff --git a/rowers/dataprep.py b/rowers/dataprep.py index fa47dec4..82878440 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -4,6 +4,7 @@ from rowers.models import Workout, User, Rower,StrokeData from rowingdata import rowingdata as rrdata from rowers.tasks import handle_sendemail_unrecognized +from rowers.tasks import handle_zip_file from rowingdata import rower as rrower from rowingdata import main as rmain @@ -44,7 +45,7 @@ import sys import django_rq queue = django_rq.get_queue('default') queuelow = django_rq.get_queue('low') -queuehigh = django_rq.get_queue('low') +queuehigh = django_rq.get_queue('default') user = settings.DATABASES['default']['USER'] @@ -573,17 +574,26 @@ def new_workout_from_file(r,f2, if len(fileformat)==3 and fileformat[0]=='zip': f_to_be_deleted = f2 with zipfile.ZipFile(f2) as z: - # for now, we're getting only the first file - # from the NK zip file (issue #69 on bitbucket) for fname in z.namelist(): f3 = z.extract(fname,path='media/') - id,message,f2 = new_workout_from_file(r,f3, - workouttype=workouttype, - makeprivate=makeprivate, - title = title, - notes='') + print f3,r.user.email,title + if settings.DEBUG: + res = handle_zip_file.delay( + r.user.email,title,f3 + ) + + else: + res = queuelow.enqueue( + handle_zip_file, + r.user.email, + title, + f3 + ) + + + os.remove(f_to_be_deleted) - return id,message,f2 + return -1,message,f2 # Some people try to upload Concept2 logbook summaries if fileformat == 'c2log': diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index c42855e8..0cd3fc55 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -91,6 +91,311 @@ def rdata(file,rower=rrower()): return res +# Processes painsled CSV file to database +def save_workout_database(f2,r,dosmooth=True,workouttype='rower', + dosummary=True,title='Workout', + notes='',totaldist=0,totaltime=0, + summary='', + makeprivate=False, + oarlength=2.89,inboard=0.88): + message = None + powerperc = 100*np.array([r.pw_ut2, + r.pw_ut1, + r.pw_at, + r.pw_tr,r.pw_an])/r.ftp + + # make workout and put in database + rr = rrower(hrmax=r.max,hrut2=r.ut2, + hrut1=r.ut1,hrat=r.at, + hrtr=r.tr,hran=r.an,ftp=r.ftp, + powerperc=powerperc,powerzones=r.powerzones) + row = rdata(f2,rower=rr) + if row == 0: + return (0,'Error: CSV data file not found') + + if dosmooth: + # auto smoothing + pace = row.df[' Stroke500mPace (sec/500m)'].values + velo = 500./pace + + f = row.df['TimeStamp (sec)'].diff().mean() + if f !=0: + windowsize = 2*(int(10./(f)))+1 + else: + windowsize = 1 + if not 'originalvelo' in row.df: + row.df['originalvelo'] = velo + + if windowsize > 3 and windowsize23: + message = 'Warning: The workout duration was longer than 23 hours. ' + hours = 23 + + minutes = int((totaltime - 3600.*hours)/60.) + if minutes>59: + minutes = 59 + if not message: + message = 'Warning: there is something wrong with the workout duration' + + seconds = int(totaltime - 3600.*hours - 60.*minutes) + if seconds > 59: + seconds = 59 + if not message: + message = 'Warning: there is something wrong with the workout duration' + + tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds)) + if tenths > 9: + tenths = 9 + if not message: + message = 'Warning: there is something wrong with the workout duration' + + duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths) + + if dosummary: + summary = row.summary() + summary += '\n' + summary += row.intervalstats() + + workoutdate = row.rowdatetime.strftime('%Y-%m-%d') + workoutstarttime = row.rowdatetime.strftime('%H:%M:%S') + workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc) + + if makeprivate: + privacy = 'private' + else: + privacy = 'visible' + + # check for duplicate start times + ws = Workout.objects.filter(startdatetime=workoutstartdatetime, + user=r) + if (len(ws) != 0): + message = "Warning: This workout probably already exists in the database" + + + w = Workout(user=r,name=title,date=workoutdate, + workouttype=workouttype, + duration=duration,distance=totaldist, + weightcategory=r.weightcategory, + starttime=workoutstarttime, + csvfilename=f2,notes=notes,summary=summary, + maxhr=maxhr,averagehr=averagehr, + startdatetime=workoutstartdatetime, + inboard=inboard,oarlength=oarlength, + privacy=privacy) + + + w.save() + + if privacy == 'visible': + ts = Team.objects.filter(rower=r) + for t in ts: + w.team.add(t) + + # put stroke data in database + res = dataprep(row.df,id=w.id,bands=True, + barchart=True,otwpower=True,empower=True,inboard=inboard) + + return (w.id,message) + +def handle_nonpainsled(f2,fileformat,summary=''): + oarlength = 2.89 + inboard = 0.88 + # handle RowPro: + if (fileformat == 'rp'): + row = RowProParser(f2) + # handle TCX + if (fileformat == 'tcx'): + row = TCXParser(f2) + + # handle Mystery + if (fileformat == 'mystery'): + row = MysteryParser(f2) + + # handle TCX no HR + if (fileformat == 'tcxnohr'): + row = TCXParserNoHR(f2) + + # handle RowPerfect + if (fileformat == 'rowperfect3'): + row = RowPerfectParser(f2) + + # handle ErgData + if (fileformat == 'ergdata'): + row = ErgDataParser(f2) + + # handle Mike + if (fileformat == 'bcmike'): + row = BoatCoachAdvancedParser(f2) + + # handle BoatCoach + if (fileformat == 'boatcoach'): + row = BoatCoachParser(f2) + + # handle painsled desktop + if (fileformat == 'painsleddesktop'): + row = painsledDesktopParser(f2) + + # handle speed coach GPS + if (fileformat == 'speedcoach'): + row = speedcoachParser(f2) + + # handle speed coach GPS 2 + if (fileformat == 'speedcoach2'): + row = SpeedCoach2Parser(f2) + try: + oarlength,inboard = get_empower_rigging(f2) + summary = row.allstats() + except: + pass + + + # handle ErgStick + if (fileformat == 'ergstick'): + row = ErgStickParser(f2) + + # handle FIT + if (fileformat == 'fit'): + row = FITParser(f2) + s = fitsummarydata(f2) + s.setsummary() + summary = s.summarytext + + + f_to_be_deleted = f2 + # should delete file + f2 = f2[:-4]+'o.csv' + row.write_csv(f2,gzip=True) + + #os.remove(f2) + try: + os.remove(f_to_be_deleted) + except: + os.remove(f_to_be_deleted+'.gz') + + return (f2,summary,oarlength,inboard) + +# Create new workout from file and store it in the database +# This routine should be used everywhere in views.py and mailprocessing.py +# Currently there is code duplication +def new_workout_from_file(r,f2, + workouttype='rower', + title='Workout', + makeprivate=False, + notes=''): + message = None + fileformat = get_file_type(f2) + summary = '' + oarlength = 2.89 + inboard = 0.88 + if len(fileformat)==3 and fileformat[0]=='zip': + f_to_be_deleted = f2 + with zipfile.ZipFile(f2) as z: + for fname in z.namelist(): + f3 = z.extract(fname,path='media/') + id,message,f2 = new_workout_from_file(r,f3, + workouttype=workouttype, + makeprivate=makeprivate, + title = title, + notes='') + os.remove(f_to_be_deleted) + return id,message,f2 + + # Some people try to upload Concept2 logbook summaries + if fileformat == 'c2log': + os.remove(f2) + message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook." + return (0,message,f2) + + if fileformat == 'nostrokes': + os.remove(f2) + message = "It looks like this file doesn't contain stroke data." + return (0,message,f2) + + # Some people try to upload RowPro summary logs + if fileformat == 'rowprolog': + os.remove(f2) + message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log." + return (0,message,f2) + + # Sometimes people try an unsupported file type. + # Send an email to info@rowsandall.com with the file attached + # for me to check if it is a bug, or a new file type + # worth supporting + if fileformat == 'unknown': + message = "We couldn't recognize the file type" + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(f2, + r.user.email) + + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + f2,r.user.email) + return (0,message,f2) + + # handle non-Painsled by converting it to painsled compatible CSV + if (fileformat != 'csv'): + try: + f2,summary,oarlength,inboard = handle_nonpainsled(f2, + fileformat, + summary=summary) + except: + errorstring = str(sys.exc_info()[0]) + message = 'Something went wrong: '+errorstring + return (0,message,'') + + + + dosummary = (fileformat != 'fit') + id,message = save_workout_database(f2,r, + workouttype=workouttype, + makeprivate=makeprivate, + dosummary=dosummary, + summary=summary, + inboard=inboard,oarlength=oarlength, + title=title) + + return (id,message,f2) + def delete_strokedata(id,debug=True): if debug: engine = create_engine(database_url_debug, echo=False) diff --git a/rowers/tasks.py b/rowers/tasks.py index f37fe6ab..137cc718 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -21,7 +21,7 @@ import stravalib from utils import serialize_list,deserialize_list from rowers.dataprepnodjango import update_strokedata - +from rowers.dataprepnodjango import new_workout_from_file from django.core.mail import send_mail, BadHeaderError,EmailMessage @@ -30,6 +30,16 @@ from django.core.mail import send_mail, BadHeaderError,EmailMessage def add(x, y): return x + y +# create workout +@app.task +def handle_new_workout_from_file(r,f2, + workouttype='rower', + title='Workout', + makeprivate=False, + notes=''): + return new_workout_from_file(r,f2,workouttype, + title,makeprivate,notes) + # send email to me when an unrecognized file is uploaded @app.task def handle_sendemail_unrecognized(unrecognizedfile,useremail): @@ -80,6 +90,16 @@ def handle_sendemailtcx(first_name,last_name,email,tcxfile): os.remove(tcxfile) return 1 +@app.task +def handle_zip_file(emailfrom,subject,file): + message = "... zip processing ... " + email = EmailMessage(subject,message, + emailfrom, + ['workouts@rowsandall.com']) + email.attach_file(file) + res = email.send() + return 1 + # Send email with CSV attachment @app.task def handle_sendemailcsv(first_name,last_name,email,csvfile): diff --git a/rowers/views.py b/rowers/views.py index 2a04bddd..7e422794 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -5230,7 +5230,12 @@ def workout_upload_view(request,message="", args=[str(message)]) response = HttpResponseRedirect(url) return response - + elif id == -1: + message = 'Zip files will be processed in the background' + url = reverse(workout_upload_view, + args=[str(message)]) + response = HttpResponseRedirect(url) + return response else: if message: url = reverse(workout_edit_view,