Merge branch 'feature/emailprocessing' into develop

rowers/mailprocessing.py

@@ -1,24 +1,19 @@
 # Processes emails sent to workouts@rowsandall.com
+""" Processes emails sent to workouts@rowsandall.com """
+import shutil
 import time
 from django.conf import settings
 from rowers.tasks import handle_sendemail_unrecognized
-from django_mailbox.models import Mailbox,Message,MessageAttachment
+from django_mailbox.models import Message, MessageAttachment
-from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
+from rowers.models import User, Rower, RowerForm
-from django.core.files.base import ContentFile
+
-from django.core.mail import send_mail, BadHeaderError,EmailMessage
+from django.core.mail import EmailMessage
-from rowsandall_app.settings import BASE_DIR
 
 from rowingdata import rower as rrower
-from rowingdata import main as rmain
-from rowingdata import rowingdata as rrdata
-from rowingdata import TCXParser,RowProParser,ErgDataParser
-from rowingdata import MysteryParser,BoatCoachParser
-from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser
-from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata
-from rowingdata import make_cumvalues
-from rowingdata import summarydata,get_file_type
 
-from scipy.signal import savgol_filter
+from rowingdata import rowingdata as rrdata
+
+from rowingdata import get_file_type
 
 import zipfile
 import os
@@ -30,19 +25,21 @@ queuelow = django_rq.get_queue('low')
 queuehigh = django_rq.get_queue('default')
 
 # Sends a confirmation with a link to the workout
-def send_confirm(u,name,link,options):
+
-    fullemail = u.email
+
-    subject = 'Workout added: '+name
+def send_confirm(user, name, link, options):
-    message = 'Dear '+u.first_name+',\n\n'
+    """ Send confirmation email to user when email has been processed """
+    fullemail = user.email
+    subject = 'Workout added: ' + name
+    message = 'Dear ' + user.first_name + ',\n\n'
     message += "Your workout has been added to Rowsandall.com.\n"
-    message += "Link to workout: "+link+"\n\n"
+    message += "Link to workout: " + link + "\n\n"
     message += "Best Regards, the Rowsandall Team"
 
     if options:
-        message += "\n\n"+str(options)
+        message += "\n\n" + str(options)
 
-
+    email = EmailMessage(subject, message,
-    email = EmailMessage(subject,message,
                          'Rowsandall <info@rowsandall.com>',
                          [fullemail])
 
@@ -51,203 +48,102 @@ def send_confirm(u,name,link,options):
     return 1
 
 # Reads a "rowingdata" object, plus some error protections
-def rdata(file,rower=rrower()):
+
+
+def rdata(file, rower=rrower()):
+    """ Reads rowingdata data or returns 0 on Error """
     try:
-	res = rrdata(file,rower=rower)
+        result = rrdata(file, rower=rower)
     except IOError:
         try:
-            res = rrdata(file+'.gz',rower=rower)
+            result = rrdata(file + '.gz', rower=rower)
         except IOError:
-	    res = 0
+            result = 0
 
-    return res
+    return result
-
-# Some error protection around process attachments
-def safeprocessattachments():
-    try:
-	return processattachments()
-    except:
-	return [0]
-
-# This is duplicated in management/commands/processemail
-# Need to double check the code there, update here, and only
-# use the code here.
-def processattachments():
-    # in res, we store the ids of the new workouts
-    res = []
-    attachments = MessageAttachment.objects.all()
-    for a in attachments:
-	donotdelete = 0
-	m = Message.objects.get(id=a.message_id)
-	from_address = m.from_address[0]
-	name = m.subject
-	
-	# get a list of users
-	theusers = User.objects.filter(email=from_address)
-	for u in theusers:
-	    try:
-		rr = Rower.objects.get(user=u.id)
-		# move attachment and make workout
-		try:
-		    wid = [make_new_workout_from_email(rr,a.document,name)]
-                    res += wid
-                    link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
-                    if wid != 1:
-                        dd = send_confirm(u,name,link)
-		except:
-		    # replace with code to process error
-		    res += ['fail: '+name]
-		    donotdelete = 1
-	    except Rower.DoesNotExist:
-		pass
-	
-	    
-	# remove attachment
-	if donotdelete == 0:
-	    a.delete()
-
-	if m.attachments.exists()==False:
-	    # no attachments, so can be deleted
-	    m.delete()
-
-    # Delete remaining messages (which should not have attachments)
-    mm = Message.objects.all()
-    for m in mm:
-	if m.attachments.exists()==False:
-	    m.delete()
-
-    return res
-
-# As above, but with some print commands for debugging purposes
-def processattachments_debug():
-    res = []
-    attachments = MessageAttachment.objects.all()
-    for a in attachments:
-	donotdelete = 1
-	m = Message.objects.get(id=a.message_id)
-	from_address = m.from_address[0]
-	name = m.subject
-	
-	# get a list of users
-	theusers = User.objects.filter(email=from_address)
-        print theusers
-	for u in theusers:
-            try:
-	        rr = Rower.objects.get(user=u.id)
-                doorgaan = 1
-            except:
-                doorgaan = 0
-            if doorgaan:
-	        # move attachment and make workout
-                print a.document
-                print name
-	        wid = [make_new_workout_from_email(rr,a.document,name)]
-                res += wid
-                link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
-                if wid != 1:
-                    dd = send_confirm(u,name,link)
 
 
 
-	# remove attachment
+def make_new_workout_from_email(rower, datafile, name, cntr=0):
-	if donotdelete == 0:
+    """ This one is used in processemail """
-	    a.delete()
-
-	if m.attachments.exists()==False:
-	    # no attachments, so can be deleted
-	    m.delete()
-
-    mm = Message.objects.all()
-    for m in mm:
-	if m.attachments.exists()==False:
-	    m.delete()
-
-    return res
-
-# Process the attachment file, create new workout
-# The code here is duplication of the code in views.py (workout_upload_view)
-# Need to move the code to a subroutine used both in views.py and here
-def make_new_workout_from_email(rr,f2,name,cntr=0):
     workouttype = 'rower'
 
     try:
-        f2 = f2.name
+        datafilename = datafile.name
-        fileformat = get_file_type('media/'+f2)
+        fileformat = get_file_type('media/' + datafilename)
     except IOError:
-        f2 = f2.name+'.gz'
+        datafilename = datafile.name + '.gz'
-        fileformat = get_file_type('media/'+f2)
+        fileformat = get_file_type('media/' + datafilename)
     except AttributeError:
-        fileformat = get_file_type('media/'+f2)
+        datafilename = datafile
+        fileformat = get_file_type('media/' + datafile)
 
-    if len(fileformat)==3 and fileformat[0]=='zip':
+    if len(fileformat) == 3 and fileformat[0] == 'zip':
-        f_to_be_deleted = f2
+        with zipfile.ZipFile('media/' + datafilename) as zip_file:
-        with zipfile.ZipFile('media/'+f2) as z:
+            datafilename = zip_file.extract(
-            f2 = z.extract(z.namelist()[0],path='media/')[6:]
+                zip_file.namelist()[0],
+                path='media/')[6:]
             fileformat = fileformat[2]
 
     if fileformat == 'unknown':
-	if settings.DEBUG:	    
+#        extension = datafilename[-4:].lower()
-	    res = handle_sendemail_unrecognized.delay(f2,
+#        fcopy = "media/"+datafilename[:-4]+"_copy"+extension
-						      "roosendaalsander@gmail.com")
+#        with open('media/'+datafilename, 'r') as f_in, open(fcopy, 'w') as f_out:
+#            shutil.copyfileobj(f_in,f_out)
+        fcopy = "media/"+datafilename
+        if settings.DEBUG:
+            res = handle_sendemail_unrecognized.delay(fcopy,
+                                                      rower.user.email)
 
-	else:
+        else:
-	    res = queuehigh.enqueue(handle_sendemail_unrecognized,
+            res = queuehigh.enqueue(handle_sendemail_unrecognized,
-				    f2,"roosendaalsander@gmail.com")
+                                    fcopy,
+                                    rower.user.email)
 
-	return 1
+        return 0
 
     summary = ''
     # handle non-Painsled
     if fileformat != 'csv':
-        f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary)
+        filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled(
+            'media/' + datafilename, fileformat, summary)
     else:
-        f3 = 'media/'+f2
+        filename_mediadir = 'media/' + datafilename
         inboard = 0.88
         oarlength = 2.89
 
-
+    row = rdata(filename_mediadir)  
-    
-    
-    # make workout and put in database
-    #r = rrower(hrmax=rr.max,hrut2=rr.ut2,
-	#       hrut1=rr.ut1,hrat=rr.at,
-	#       hrtr=rr.tr,hran=rr.an,ftp=r.ftp)
-    row = rdata(f3) #,rower=r)
     if row == 0:
-	return 0
+        return 0
-
 
     # change filename
-    if f2[:5] != 'media':
+    if datafilename[:5] != 'media':
-	timestr = time.strftime("%Y%m%d-%H%M%S")
+        timestr = time.strftime("%Y%m%d-%H%M%S")
-	f2 = 'media/'+timestr+str(cntr)+'o.csv'
+        datafilename = 'media/' + timestr + str(cntr) + 'o.csv'
 
     try:
-	avglat = row.df[' latitude'].mean()
+        avglat = row.df[' latitude'].mean()
-	avglon = row.df[' longitude'].mean()
+        avglon = row.df[' longitude'].mean()
         if avglat != 0 or avglon != 0:
             workouttype = 'water'
     except KeyError:
         pass
 
-    row.write_csv(f2,gzip=True)
+    row.write_csv(datafilename, gzip=True)
     dosummary = (fileformat != 'fit')
 
     if name == '':
-	name = 'imported through email'
+        name = 'imported through email'
-
-    id,message = dataprep.save_workout_database(f2,rr,
-                                                workouttype=workouttype,
-                                                dosummary=dosummary,
-                                                inboard=inboard,
-                                                oarlength=oarlength,
-                                                title=name,
-                                                workoutsource=fileformat,
-                                                notes='imported through email')
 
+    id, message = dataprep.save_workout_database(
+        datafilename, rower,
+        workouttype=workouttype,
+        dosummary=dosummary,
+        inboard=inboard,
+        oarlength=oarlength,
+        title=name,
+        workoutsource=fileformat,
+        notes='imported through email'
+    )
 
     return id
-	    
-
-
-

rowers/management/commands/processemail.py

@@ -1,162 +1,148 @@
 #!/srv/venv/bin/python
+""" Process emails """
 import sys
 import os
+
+import zipfile
+
+import time
+from time import strftime
+
+from django.core.management.base import BaseCommand
+from django_mailbox.models import Message, MessageAttachment
+from django.core.urlresolvers import reverse
+from django.conf import settings
+
+from rowers.models import Workout, Rower
+
+from rowingdata import rower as rrower
+from rowingdata import rowingdata as rrdata
+
+import rowers.uploads as uploads
+from rowers.mailprocessing import make_new_workout_from_email, send_confirm
+
 # If you find a solution that does not need the two paths, please comment!
 sys.path.append('$path_to_root_of_project$')
 sys.path.append('$path_to_root_of_project$/$project_name$')
 
 os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings'
 
-import zipfile
+if not getattr(__builtins__, "WindowsError", None):
+    class WindowsError(OSError): pass
 
-from django.core.management.base import BaseCommand, CommandError
+def rdata(file_obj, rower=rrower()):
-from django.conf import settings
+    """ Read rowing data file and return 0 if file doesn't exist"""
-#from rowers.mailprocessing import processattachments
-import time
-from time import strftime
-from django.conf import settings
-from rowers.tasks import handle_sendemail_unrecognized
-from django_mailbox.models import Mailbox,Message,MessageAttachment
-from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
-from django.core.files.base import ContentFile
-
-from rowsandall_app.settings import BASE_DIR
-
-from rowingdata import rower as rrower
-from rowingdata import main as rmain
-from rowingdata import rowingdata as rrdata
-
-from rowingdata import make_cumvalues
-from rowingdata import summarydata,get_file_type
-
-from scipy.signal import savgol_filter
-from rowers.mailprocessing import make_new_workout_from_email,send_confirm
-import rowers.uploads as uploads
-
-def rdata(file,rower=rrower()):
     try:
-	res = rrdata(file,rower=rower)
+        result = rrdata(file_obj, rower=rower)
     except IOError:
-	res = 0
+        result = 0
 
-    return res
+    return result
 
+def processattachment(rower, fileobj, title, uploadoptions):
+    try:
+        filename = fileobj.name
+    except AttributeError:
+        filename = fileobj[6:]
 
+    workoutid = [
+        make_new_workout_from_email(rower, filename, title)
+    ]
+    if workoutid[0]:
+        link = settings.SITE_URL+reverse(
+            rower.defaultlandingpage,
+            kwargs = {
+                'id':workoutid[0],
+            }
+        )
+
+        if uploadoptions and not 'error' in uploadoptions:
+            workout = Workout.objects.get(id=workoutid[0])
+            uploads.do_sync(workout, uploadoptions)
+            uploads.make_private(workout, uploadoptions)
+        if 'make_plot' in uploadoptions:
+            plottype = uploadoptions['plottype']
+            workoutcsvfilename = workout.csvfilename[6:-4]
+            timestr = strftime("%Y%m%d-%H%M%S")
+            imagename = workoutcsvfilename + timestr + '.png'
+            result = uploads.make_plot(
+                workout.user, workout, workoutcsvfilename,
+                workout.csvfilename,
+                plottype, title,
+                imagename=imagename
+            )
+        try:
+            if workoutid:
+                email_sent = send_confirm(
+                    rower.user, title, link,
+                    uploadoptions
+                )
+                time.sleep(10)
+        except:
+            try:
+                time.sleep(10)
+                if workoutid:
+                    email_sent = send_confirm(
+                        rower.user, title, link,
+                        uploadoptions
+                    )
+            except:
+                pass
+
+    return workoutid
 
 class Command(BaseCommand):
+    """Run the Email processing command """
     def handle(self, *args, **options):
-	res = []
+        attachments = MessageAttachment.objects.all()
-	attachments = MessageAttachment.objects.all()
+        cntr = 0
-	cntr = 0
+        for attachment in attachments:
-	for a in attachments:
+            extension = attachment.document.name[-3:].lower()
-            extension = a.document.name[-3:].lower()
+            message = Message.objects.get(id=attachment.message_id)
-	    donotdelete = 0
+            body = "\n".join(message.text.splitlines())
-	    m = Message.objects.get(id=a.message_id)
-            body = "\n".join(m.text.splitlines())
             uploadoptions = uploads.upload_options(body)
-	    from_address = m.from_address[0].lower()
+            from_address = message.from_address[0].lower()
-	    name = m.subject
+            name = message.subject
-	    cntr += 1
+            # get a list of users
-	    # get a list of users
+            # theusers = User.objects.filter(email=from_address)
-	    # theusers = User.objects.filter(email=from_address)
+            rowers = [
-            ther = [
                 r for r in Rower.objects.all() if r.user.email.lower() == from_address
             ]
-	    for rr in ther:
+            for rower in rowers:
                 if extension == 'zip':
-                    z = zipfile.ZipFile(a.document)
+                    zip_file = zipfile.ZipFile(attachment.document)
-                    for f in z.namelist():
+                    for filename in zip_file.namelist():
-                        f2 = z.extract(f,path='media/')
+                        datafile = zip_file.extract(filename, path='media/')
-                        title = os.path.basename(f2)
+                        title = os.path.basename(datafile)
-                        wid = [
+                        workoutid = processattachment(
-                            make_new_workout_from_email(rr,f2[6:],title)
+                            rower, datafile, title, uploadoptions
-                        ]
+                        )
-                        res += wid
-                        link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
-                        if uploadoptions and not 'error' in uploadoptions:
-                            w = Workout.objects.get(id=wid[0])
-                            r = w.user
-                            uploads.do_sync(w,uploadoptions)
-                            uploads.make_private(w,uploadoptions)
-                        if 'make_plot' in uploadoptions:
-                            plottype = uploadoptions['plottype']
-                            f1 = w.csvfilename[6:-4] 
-                            timestr = strftime("%Y%m%d-%H%M%S")
-	                    imagename = f1+timestr+'.png'
-                            resu = uploads.make_plot(r,w,f1,
-                                                     w.csvfilename,
-                                                     plottype,name,
-                                                     imagename=imagename)
-                        try:
-                            if wid != 1:
-                                dd = send_confirm(rr.user,title,link,
-                                                  uploadoptions)
-                            time.sleep(10)
-                        except:
-                            try:
-                                time.sleep(10)
-                                if wid != 1:
-                                    dd = send_confirm(rr.user,title,link,
-                                                      uploadoptions)
-                            except:
-                                pass
-
                 else:
-		# move attachment and make workout
+                    # move attachment and make workout
-		    try:
+                    workoutid = processattachment(
-		        wid = [
+                        rower, attachment.document, name, uploadoptions
-                            make_new_workout_from_email(rr,
+                    )
-                                                        a.document,
-                                                        name)
-                        ]
-                        res += wid
-                        link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
-                        if uploadoptions:
-                            w = Workout.objects.get(id=wid[0])
-                            r = w.user
-                            uploads.do_sync(w,uploadoptions)
-                            uploads.make_private(w,uploadoptions)
-                        if 'make_plot' in uploadoptions:
-                            plottype = uploadoptions['plottype']
-                            f1 = w.csvfilename[6:-4] 
-                            timestr = strftime("%Y%m%d-%H%M%S")
-	                    imagename = f1+timestr+'.png'
-                            resu = uploads.make_plot(r,w,f1,
-                                                     w.csvfilename,
-                                                     plottype,name,
-                                                     imagename=imagename)
-
-		    except:
-			# replace with code to process error
-			res += ['fail: '+name]
-			donotdelete = 1
-                        wid = 1
-                    try:
-                        if wid != 1:
-                            dd = send_confirm(rr.user,name,link,
-                                              uploadoptions)
-                        time.sleep(10)
-                    except:
-                        pass
 
+            # We're done with the attachment. It can be deleted
             try:
-                a.delete()
+                attachment.delete()
             except IOError:
                 pass
-            # remove attachment
+            except WindowsError:
-	    #if donotdelete == 0:
+                time.sleep(2)
-
+                try:
-	    if m.attachments.exists()==False:
+                    attachment.delete()
-		# no attachments, so can be deleted
+                except WindowsError:
-		m.delete()
+                    pass
-
-	mm = Message.objects.all()
-	for m in mm:
-	    if m.attachments.exists()==False:
-		m.delete()
-
-	self.stdout.write(self.style.SUCCESS('Successfully processed email attachments'))
-	
 
+            if message.attachments.exists() is False:
+                # no attachments, so can be deleted
+                message.delete()
 
+        messages = Message.objects.all()
+        for message in messages:
+            if message.attachments.exists() is False:
+                message.delete()
 
+        self.stdout.write(self.style.SUCCESS(
+            'Successfully processed email attachments'))

rowsandall_app/settings.py

@@ -261,6 +261,9 @@ TP_CLIENT_SECRET = CFG["tp_client_secret"]
 TP_REDIRECT_URI = CFG["tp_redirect_uri"]
 TP_CLIENT_KEY = TP_CLIENT_ID
 
+# Full Site URL
+SITE_URL = "https://rowsandall.com"
+
 # RQ stuff
 
 RQ_QUEUES = {

rowsandall_app/settings_dev.py

@@ -76,6 +76,8 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/'
 
 SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"
 
+SITE_URL = "http://localhost:8000"
+
 #EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
 #EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'