works but runs into email rate limiting

rowers/.#dataprep.py

@@ -0,0 +1 @@
+E408191@CZ27LT9RCGN72.13020:1492098105

rowers/dataprep.py

@@ -4,6 +4,7 @@ from rowers.models import Workout, User, Rower,StrokeData
 from rowingdata import rowingdata as rrdata
 
 from rowers.tasks import handle_sendemail_unrecognized
+from rowers.tasks import handle_zip_file
 
 from rowingdata import rower as rrower
 from rowingdata import main as rmain
@@ -44,7 +45,7 @@ import sys
 import django_rq
 queue = django_rq.get_queue('default')
 queuelow = django_rq.get_queue('low')
-queuehigh = django_rq.get_queue('low')
+queuehigh = django_rq.get_queue('default')
 
 
 user = settings.DATABASES['default']['USER']
@@ -573,17 +574,26 @@ def new_workout_from_file(r,f2,
     if len(fileformat)==3 and fileformat[0]=='zip':
         f_to_be_deleted = f2
         with zipfile.ZipFile(f2) as z:
-            # for now, we're getting only the first file
-            # from the NK zip file (issue #69 on bitbucket)
             for fname in z.namelist():
                 f3 = z.extract(fname,path='media/')
-                id,message,f2 = new_workout_from_file(r,f3,
+                print f3,r.user.email,title
-                                                      workouttype=workouttype,                                                
+                if settings.DEBUG:	    
-                                                      makeprivate=makeprivate,
+	            res = handle_zip_file.delay(
-                                                      title = title,
+                        r.user.email,title,f3
-                                                      notes='')
+                    )
+
+	        else:
+	            res = queuelow.enqueue(
+                        handle_zip_file,
+                        r.user.email,
+                        title,
+                        f3
+                    )
+
+
+
         os.remove(f_to_be_deleted)
-        return id,message,f2
+        return -1,message,f2
 
     # Some people try to upload Concept2 logbook summaries
     if fileformat == 'c2log':

rowers/dataprepnodjango.py

@@ -91,6 +91,311 @@ def rdata(file,rower=rrower()):
 
     return res
 
+# Processes painsled CSV file to database
+def save_workout_database(f2,r,dosmooth=True,workouttype='rower',
+                          dosummary=True,title='Workout',
+                          notes='',totaldist=0,totaltime=0,
+                          summary='',
+                          makeprivate=False,
+                          oarlength=2.89,inboard=0.88):
+    message = None
+    powerperc = 100*np.array([r.pw_ut2,
+                              r.pw_ut1,
+                              r.pw_at,
+                              r.pw_tr,r.pw_an])/r.ftp
+
+    # make workout and put in database
+    rr = rrower(hrmax=r.max,hrut2=r.ut2,
+		hrut1=r.ut1,hrat=r.at,
+		hrtr=r.tr,hran=r.an,ftp=r.ftp,
+                powerperc=powerperc,powerzones=r.powerzones)
+    row = rdata(f2,rower=rr)
+    if row == 0:
+	return (0,'Error: CSV data file not found')
+
+    if dosmooth:
+        # auto smoothing
+        pace = row.df[' Stroke500mPace (sec/500m)'].values
+        velo = 500./pace
+                
+        f = row.df['TimeStamp (sec)'].diff().mean()
+        if f !=0:
+            windowsize = 2*(int(10./(f)))+1
+        else:
+            windowsize = 1
+        if not 'originalvelo' in row.df:
+	    row.df['originalvelo'] = velo
+
+        if windowsize > 3 and windowsize<len(velo):
+	    velo2 = savgol_filter(velo,windowsize,3)
+        else:
+	    velo2 = velo
+
+        velo3 = pd.Series(velo2)
+        velo3 = velo3.replace([-np.inf,np.inf],np.nan)
+        velo3 = velo3.fillna(method='ffill')
+                
+        pace2 = 500./abs(velo3)
+        
+        row.df[' Stroke500mPace (sec/500m)'] = pace2
+
+        row.df = row.df.fillna(0)
+
+        row.write_csv(f2,gzip=True)
+        try:
+            os.remove(f2)
+        except:
+            pass
+                
+    # recalculate power data
+    if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
+	try:
+	    row.erg_recalculatepower()
+            row.write_csv(f2,gzip=True)
+	except:
+	    pass
+            
+    averagehr = row.df[' HRCur (bpm)'].mean()
+    maxhr = row.df[' HRCur (bpm)'].max()
+    
+    if totaldist == 0:
+        totaldist = row.df['cum_dist'].max()
+    if totaltime == 0:
+        totaltime = row.df['TimeStamp (sec)'].max()-row.df['TimeStamp (sec)'].min()
+        totaltime = totaltime+row.df.ix[0,' ElapsedTime (sec)']
+
+
+
+    hours = int(totaltime/3600.)
+    if hours>23:
+        message = 'Warning: The workout duration was longer than 23 hours. '
+        hours = 23
+
+    minutes = int((totaltime - 3600.*hours)/60.)
+    if minutes>59:
+        minutes = 59
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+    
+    seconds = int(totaltime - 3600.*hours - 60.*minutes)
+    if seconds > 59:
+        seconds = 59
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+
+    tenths = int(10*(totaltime - 3600.*hours - 60.*minutes - seconds))
+    if tenths > 9:
+        tenths = 9
+        if not message:
+            message = 'Warning: there is something wrong with the workout duration'
+    
+    duration = "%s:%s:%s.%s" % (hours,minutes,seconds,tenths)
+
+    if dosummary:
+        summary = row.summary()
+        summary += '\n'
+        summary += row.intervalstats()
+
+    workoutdate = row.rowdatetime.strftime('%Y-%m-%d')
+    workoutstarttime = row.rowdatetime.strftime('%H:%M:%S')
+    workoutstartdatetime = thetimezone.localize(row.rowdatetime).astimezone(utc)
+
+    if makeprivate:
+        privacy = 'private'
+    else:
+        privacy = 'visible'
+
+    # check for duplicate start times
+    ws = Workout.objects.filter(startdatetime=workoutstartdatetime,
+				    user=r)
+    if (len(ws) != 0):
+	message =  "Warning: This workout probably already exists in the database"
+
+        
+    w = Workout(user=r,name=title,date=workoutdate,
+		workouttype=workouttype,
+		duration=duration,distance=totaldist,
+		weightcategory=r.weightcategory,
+		starttime=workoutstarttime,
+		csvfilename=f2,notes=notes,summary=summary,
+		maxhr=maxhr,averagehr=averagehr,
+		startdatetime=workoutstartdatetime,
+                inboard=inboard,oarlength=oarlength,
+                privacy=privacy)
+
+
+    w.save()
+
+    if privacy == 'visible':
+        ts = Team.objects.filter(rower=r)
+        for t in ts:
+            w.team.add(t)
+
+    # put stroke data in database
+    res = dataprep(row.df,id=w.id,bands=True,
+                   barchart=True,otwpower=True,empower=True,inboard=inboard)
+
+    return (w.id,message)
+
+def handle_nonpainsled(f2,fileformat,summary=''):
+    oarlength = 2.89
+    inboard = 0.88
+    # handle RowPro:
+    if (fileformat == 'rp'):
+	row = RowProParser(f2)
+	# handle TCX
+    if (fileformat ==  'tcx'):
+	row = TCXParser(f2)
+                    
+    # handle Mystery
+    if (fileformat == 'mystery'):
+	row = MysteryParser(f2)
+
+    # handle TCX no HR
+    if (fileformat == 'tcxnohr'):
+	row = TCXParserNoHR(f2)
+		    
+    # handle RowPerfect
+    if (fileformat == 'rowperfect3'):
+        row = RowPerfectParser(f2)
+            
+    # handle ErgData
+    if (fileformat == 'ergdata'):
+	row = ErgDataParser(f2)
+
+    # handle Mike
+    if (fileformat == 'bcmike'):
+        row = BoatCoachAdvancedParser(f2)
+            
+    # handle BoatCoach
+    if (fileformat == 'boatcoach'):
+	row = BoatCoachParser(f2)
+
+    # handle painsled desktop
+    if (fileformat == 'painsleddesktop'):
+	row = painsledDesktopParser(f2)
+
+    # handle speed coach GPS
+    if (fileformat == 'speedcoach'):
+	row = speedcoachParser(f2)
+
+    # handle speed coach GPS 2 
+    if (fileformat == 'speedcoach2'):
+	row = SpeedCoach2Parser(f2)
+        try:
+            oarlength,inboard = get_empower_rigging(f2)
+            summary = row.allstats()
+        except:
+            pass
+
+
+    # handle ErgStick
+    if (fileformat == 'ergstick'):
+	row = ErgStickParser(f2)
+		    
+    # handle FIT
+    if (fileformat == 'fit'):
+	row = FITParser(f2)
+	s = fitsummarydata(f2)
+	s.setsummary()
+	summary = s.summarytext
+
+
+    f_to_be_deleted = f2
+    # should delete file
+    f2 = f2[:-4]+'o.csv'
+    row.write_csv(f2,gzip=True)
+    
+    #os.remove(f2)
+    try:
+	os.remove(f_to_be_deleted)
+    except:
+        os.remove(f_to_be_deleted+'.gz')
+
+    return (f2,summary,oarlength,inboard)
+
+# Create new workout from file and store it in the database
+# This routine should be used everywhere in views.py and mailprocessing.py
+# Currently there is code duplication
+def new_workout_from_file(r,f2,
+                          workouttype='rower',
+                          title='Workout',
+                          makeprivate=False,
+                          notes=''):
+    message = None
+    fileformat = get_file_type(f2)
+    summary = ''
+    oarlength = 2.89
+    inboard = 0.88
+    if len(fileformat)==3 and fileformat[0]=='zip':
+        f_to_be_deleted = f2
+        with zipfile.ZipFile(f2) as z:
+            for fname in z.namelist():
+                f3 = z.extract(fname,path='media/')
+                id,message,f2 = new_workout_from_file(r,f3,
+                                                      workouttype=workouttype,                                                
+                                                      makeprivate=makeprivate,
+                                                      title = title,
+                                                      notes='')
+        os.remove(f_to_be_deleted)
+        return id,message,f2
+
+    # Some people try to upload Concept2 logbook summaries
+    if fileformat == 'c2log':
+        os.remove(f2)
+        message = "This C2 logbook summary does not contain stroke data. Please download the Export Stroke Data file from the workout details on the C2 logbook."
+        return (0,message,f2)
+
+    if fileformat == 'nostrokes':
+        os.remove(f2)
+        message = "It looks like this file doesn't contain stroke data."
+        return (0,message,f2)
+    
+    # Some people try to upload RowPro summary logs
+    if fileformat == 'rowprolog':
+        os.remove(f2)
+        message = "This RowPro logbook summary does not contain stroke data. Please use the Stroke Data CSV file for the individual workout in your log."
+        return (0,message,f2)
+
+    # Sometimes people try an unsupported file type.
+    # Send an email to info@rowsandall.com with the file attached
+    # for me to check if it is a bug, or a new file type
+    # worth supporting
+    if fileformat == 'unknown':
+	message = "We couldn't recognize the file type"
+	if settings.DEBUG:	    
+	    res = handle_sendemail_unrecognized.delay(f2,
+						      r.user.email)
+
+	else:
+	    res = queuehigh.enqueue(handle_sendemail_unrecognized,
+				    f2,r.user.email)
+        return (0,message,f2)
+        
+    # handle non-Painsled by converting it to painsled compatible CSV
+    if (fileformat != 'csv'):
+        try:
+            f2,summary,oarlength,inboard = handle_nonpainsled(f2,
+                                                              fileformat,
+                                                              summary=summary)
+        except:
+            errorstring = str(sys.exc_info()[0])
+            message = 'Something went wrong: '+errorstring
+            return (0,message,'')
+
+
+
+    dosummary = (fileformat != 'fit')
+    id,message = save_workout_database(f2,r,
+                                       workouttype=workouttype,
+                                       makeprivate=makeprivate,
+                                       dosummary=dosummary,
+                                       summary=summary,
+                                       inboard=inboard,oarlength=oarlength,
+                                       title=title)
+
+    return (id,message,f2)
+
 def delete_strokedata(id,debug=True):
     if debug:
         engine = create_engine(database_url_debug, echo=False)

rowers/tasks.py

@@ -21,7 +21,7 @@ import stravalib
 from utils import serialize_list,deserialize_list
 
 from rowers.dataprepnodjango import update_strokedata
-
+from rowers.dataprepnodjango import new_workout_from_file
 
 from django.core.mail import send_mail, BadHeaderError,EmailMessage
 
@@ -30,6 +30,16 @@ from django.core.mail import send_mail, BadHeaderError,EmailMessage
 def add(x, y):
     return x + y
 
+# create workout
+@app.task
+def handle_new_workout_from_file(r,f2,
+                          workouttype='rower',
+                          title='Workout',
+                          makeprivate=False,
+                          notes=''):
+    return new_workout_from_file(r,f2,workouttype,
+                                 title,makeprivate,notes)
+
 # send email to me when an unrecognized file is uploaded
 @app.task
 def handle_sendemail_unrecognized(unrecognizedfile,useremail):
@@ -80,6 +90,16 @@ def handle_sendemailtcx(first_name,last_name,email,tcxfile):
     os.remove(tcxfile)
     return 1
 
+@app.task
+def handle_zip_file(emailfrom,subject,file):
+    message = "... zip processing ... "
+    email = EmailMessage(subject,message,
+                         emailfrom,
+                         ['workouts@rowsandall.com'])
+    email.attach_file(file)
+    res = email.send()
+    return 1
+
 # Send email with CSV attachment
 @app.task
 def handle_sendemailcsv(first_name,last_name,email,csvfile):

rowers/views.py

@@ -5230,7 +5230,12 @@ def workout_upload_view(request,message="",
 			      args=[str(message)])
 		response = HttpResponseRedirect(url)
                 return response
-
+            elif id == -1:
+                message = 'Zip files will be processed in the background'
+                url = reverse(workout_upload_view,
+                               args=[str(message)])
+                response = HttpResponseRedirect(url)
+                return response
             else:
                 if message:
                     url = reverse(workout_edit_view,