From a866b253a19e3da1e4e39201052fdf556c9e44c1 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 11:49:47 +0200 Subject: [PATCH 1/4] cleaned up processemail --- rowers/mailprocessing.py | 250 +++++++++++---------- rowers/management/commands/processemail.py | 157 +++++++------ 2 files changed, 202 insertions(+), 205 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index eefd0031..df6760c7 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -2,21 +2,21 @@ import time from django.conf import settings from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm +from django_mailbox.models import Mailbox, Message, MessageAttachment +from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm from django.core.files.base import ContentFile -from django.core.mail import send_mail, BadHeaderError,EmailMessage +from django.core.mail import send_mail, BadHeaderError, EmailMessage from rowsandall_app.settings import BASE_DIR from rowingdata import rower as rrower from rowingdata import main as rmain from rowingdata import rowingdata as rrdata -from rowingdata import TCXParser,RowProParser,ErgDataParser -from rowingdata import MysteryParser,BoatCoachParser -from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser -from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata +from rowingdata import TCXParser, RowProParser, ErgDataParser +from rowingdata import MysteryParser, BoatCoachParser +from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser +from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type +from rowingdata import summarydata, get_file_type from scipy.signal import savgol_filter @@ -30,19 +30,20 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u,name,link,options): + + +def send_confirm(u, name, link, options): fullemail = u.email - subject = 'Workout added: '+name - message = 'Dear '+u.first_name+',\n\n' + subject = 'Workout added: ' + name + message = 'Dear ' + u.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" - message += "Link to workout: "+link+"\n\n" + message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" if options: - message += "\n\n"+str(options) + message += "\n\n" + str(options) - - email = EmailMessage(subject,message, + email = EmailMessage(subject, message, 'Rowsandall ', [fullemail]) @@ -51,203 +52,204 @@ def send_confirm(u,name,link,options): return 1 # Reads a "rowingdata" object, plus some error protections -def rdata(file,rower=rrower()): + + +def rdata(file, rower=rrower()): try: - res = rrdata(file,rower=rower) + res = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file+'.gz',rower=rower) + res = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + res = 0 return res # Some error protection around process attachments + + def safeprocessattachments(): try: - return processattachments() + return processattachments() except: - return [0] + return [0] # This is duplicated in management/commands/processemail # Need to double check the code there, update here, and only # use the code here. + + def processattachments(): # in res, we store the ids of the new workouts res = [] attachments = MessageAttachment.objects.all() for a in attachments: - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - # move attachment and make workout - try: - wid = [make_new_workout_from_email(rr,a.document,name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if wid != 1: - dd = send_confirm(u,name,link) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 - except Rower.DoesNotExist: - pass - - - # remove attachment - if donotdelete == 0: - a.delete() + donotdelete = 0 + m = Message.objects.get(id=a.message_id) + from_address = m.from_address[0] + name = m.subject - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + # get a list of users + theusers = User.objects.filter(email=from_address) + for u in theusers: + try: + rr = Rower.objects.get(user=u.id) + # move attachment and make workout + try: + wid = [make_new_workout_from_email(rr, a.document, name)] + res += wid + link = 'https://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' + if wid != 1: + dd = send_confirm(u, name, link) + except: + # replace with code to process error + res += ['fail: ' + name] + donotdelete = 1 + except Rower.DoesNotExist: + pass + + # remove attachment + if donotdelete == 0: + a.delete() + + if m.attachments.exists() == False: + # no attachments, so can be deleted + m.delete() # Delete remaining messages (which should not have attachments) mm = Message.objects.all() for m in mm: - if m.attachments.exists()==False: - m.delete() + if m.attachments.exists() == False: + m.delete() return res # As above, but with some print commands for debugging purposes + + def processattachments_debug(): res = [] attachments = MessageAttachment.objects.all() for a in attachments: - donotdelete = 1 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) + donotdelete = 1 + m = Message.objects.get(id=a.message_id) + from_address = m.from_address[0] + name = m.subject + + # get a list of users + theusers = User.objects.filter(email=from_address) print theusers - for u in theusers: + for u in theusers: try: - rr = Rower.objects.get(user=u.id) + rr = Rower.objects.get(user=u.id) doorgaan = 1 except: doorgaan = 0 if doorgaan: - # move attachment and make workout + # move attachment and make workout print a.document print name - wid = [make_new_workout_from_email(rr,a.document,name)] + wid = [make_new_workout_from_email(rr, a.document, name)] res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'https://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if wid != 1: - dd = send_confirm(u,name,link) - - - - # remove attachment - if donotdelete == 0: - a.delete() + dd = send_confirm(u, name, link) - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + # remove attachment + if donotdelete == 0: + a.delete() + + if m.attachments.exists() == False: + # no attachments, so can be deleted + m.delete() mm = Message.objects.all() for m in mm: - if m.attachments.exists()==False: - m.delete() + if m.attachments.exists() == False: + m.delete() return res # Process the attachment file, create new workout # The code here is duplication of the code in views.py (workout_upload_view) # Need to move the code to a subroutine used both in views.py and here -def make_new_workout_from_email(rr,f2,name,cntr=0): + + +def make_new_workout_from_email(rr, f2, name, cntr=0): workouttype = 'rower' try: f2 = f2.name - fileformat = get_file_type('media/'+f2) + fileformat = get_file_type('media/' + f2) except IOError: - f2 = f2.name+'.gz' - fileformat = get_file_type('media/'+f2) + f2 = f2.name + '.gz' + fileformat = get_file_type('media/' + f2) except AttributeError: - fileformat = get_file_type('media/'+f2) + fileformat = get_file_type('media/' + f2) - if len(fileformat)==3 and fileformat[0]=='zip': + if len(fileformat) == 3 and fileformat[0] == 'zip': f_to_be_deleted = f2 - with zipfile.ZipFile('media/'+f2) as z: - f2 = z.extract(z.namelist()[0],path='media/')[6:] + with zipfile.ZipFile('media/' + f2) as z: + f2 = z.extract(z.namelist()[0], path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, - "roosendaalsander@gmail.com") + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(f2, + "roosendaalsander@gmail.com") - else: - res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2,"roosendaalsander@gmail.com") + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + f2, "roosendaalsander@gmail.com") + + return 1 - return 1 - summary = '' # handle non-Painsled if fileformat != 'csv': - f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary) + f3, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + f2, fileformat, summary) else: - f3 = 'media/'+f2 + f3 = 'media/' + f2 inboard = 0.88 oarlength = 2.89 - - - # make workout and put in database - #r = rrower(hrmax=rr.max,hrut2=rr.ut2, - # hrut1=rr.ut1,hrat=rr.at, - # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) - row = rdata(f3) #,rower=r) + # r = rrower(hrmax=rr.max,hrut2=rr.ut2, + # hrut1=rr.ut1,hrat=rr.at, + # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) + row = rdata(f3) # ,rower=r) if row == 0: - return 0 - + return 0 # change filename if f2[:5] != 'media': - timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/'+timestr+str(cntr)+'o.csv' + timestr = time.strftime("%Y%m%d-%H%M%S") + f2 = 'media/' + timestr + str(cntr) + 'o.csv' try: - avglat = row.df[' latitude'].mean() - avglon = row.df[' longitude'].mean() + avglat = row.df[' latitude'].mean() + avglon = row.df[' longitude'].mean() if avglat != 0 or avglon != 0: workouttype = 'water' except KeyError: pass - row.write_csv(f2,gzip=True) + row.write_csv(f2, gzip=True) dosummary = (fileformat != 'fit') if name == '': - name = 'imported through email' - - id,message = dataprep.save_workout_database(f2,rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + name = 'imported through email' + id, message = dataprep.save_workout_database(f2, rr, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email') return id - - - - diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 6d06150a..0a7cf1ca 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -1,139 +1,137 @@ #!/srv/venv/bin/python +""" Process emails """ import sys import os + +import zipfile + +from django.core.management.base import BaseCommand + +import time +from time import strftime +from django.conf import settings + +from django_mailbox.models import Message, MessageAttachment +from rowers.models import Workout, Rower + + +from rowingdata import rower as rrower + +from rowingdata import rowingdata as rrdata + +from rowers.mailprocessing import make_new_workout_from_email, send_confirm +import rowers.uploads as uploads + # If you find a solution that does not need the two paths, please comment! sys.path.append('$path_to_root_of_project$') sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' -import zipfile - -from django.core.management.base import BaseCommand, CommandError -from django.conf import settings -#from rowers.mailprocessing import processattachments -import time -from time import strftime -from django.conf import settings -from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm -from django.core.files.base import ContentFile - -from rowsandall_app.settings import BASE_DIR - -from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata - -from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type - -from scipy.signal import savgol_filter -from rowers.mailprocessing import make_new_workout_from_email,send_confirm -import rowers.uploads as uploads - -def rdata(file,rower=rrower()): +def rdata(file, rower=rrower()): + """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file,rower=rower) + res = rrdata(file, rower=rower) except IOError: - res = 0 + res = 0 return res - class Command(BaseCommand): + """Run the Email processing command """ def handle(self, *args, **options): - res = [] - attachments = MessageAttachment.objects.all() - cntr = 0 - for a in attachments: + res = [] + attachments = MessageAttachment.objects.all() + cntr = 0 + for a in attachments: extension = a.document.name[-3:].lower() - donotdelete = 0 - m = Message.objects.get(id=a.message_id) + donotdelete = 0 + m = Message.objects.get(id=a.message_id) body = "\n".join(m.text.splitlines()) uploadoptions = uploads.upload_options(body) - from_address = m.from_address[0].lower() - name = m.subject - cntr += 1 - # get a list of users - # theusers = User.objects.filter(email=from_address) + from_address = m.from_address[0].lower() + name = m.subject + cntr += 1 + # get a list of users + # theusers = User.objects.filter(email=from_address) ther = [ r for r in Rower.objects.all() if r.user.email.lower() == from_address ] - for rr in ther: + for rr in ther: if extension == 'zip': z = zipfile.ZipFile(a.document) for f in z.namelist(): - f2 = z.extract(f,path='media/') + f2 = z.extract(f, path='media/') title = os.path.basename(f2) wid = [ - make_new_workout_from_email(rr,f2[6:],title) + make_new_workout_from_email(rr, f2[6:], title) ] res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if uploadoptions and not 'error' in uploadoptions: w = Workout.objects.get(id=wid[0]) r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + f1 = w.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, + imagename = f1 + timestr + '.png' + resu = uploads.make_plot(r, w, f1, w.csvfilename, - plottype,name, + plottype, name, imagename=imagename) try: if wid != 1: - dd = send_confirm(rr.user,title,link, + dd = send_confirm(rr.user, title, link, uploadoptions) time.sleep(10) except: try: time.sleep(10) if wid != 1: - dd = send_confirm(rr.user,title,link, + dd = send_confirm(rr.user, title, link, uploadoptions) except: pass else: - # move attachment and make workout - try: - wid = [ + # move attachment and make workout + try: + wid = [ make_new_workout_from_email(rr, a.document, name) ] res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if uploadoptions: w = Workout.objects.get(id=wid[0]) r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + f1 = w.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, + imagename = f1 + timestr + '.png' + resu = uploads.make_plot(r, w, f1, w.csvfilename, - plottype,name, + plottype, name, imagename=imagename) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 + except: + # replace with code to process error + res += ['fail: ' + name] + donotdelete = 1 wid = 1 try: if wid != 1: - dd = send_confirm(rr.user,name,link, + dd = send_confirm(rr.user, name, link, uploadoptions) time.sleep(10) except: @@ -144,19 +142,16 @@ class Command(BaseCommand): except IOError: pass # remove attachment - #if donotdelete == 0: + # if donotdelete == 0: - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + if m.attachments.exists() is False: + # no attachments, so can be deleted + m.delete() - mm = Message.objects.all() - for m in mm: - if m.attachments.exists()==False: - m.delete() + mm = Message.objects.all() + for m in mm: + if m.attachments.exists() is False: + m.delete() - self.stdout.write(self.style.SUCCESS('Successfully processed email attachments')) - - - - + self.stdout.write(self.style.SUCCESS( + 'Successfully processed email attachments')) From f7ab21560d941565b5331cefd227faaac7e66b13 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 13:01:16 +0200 Subject: [PATCH 2/4] refactored, not tested --- rowers/mailprocessing.py | 151 +++------------------ rowers/management/commands/processemail.py | 130 +++++++----------- 2 files changed, 69 insertions(+), 212 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index df6760c7..db3bccd0 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -1,24 +1,19 @@ # Processes emails sent to workouts@rowsandall.com +""" Processes emails sent to workouts@rowsandall.com """ +import shutil import time from django.conf import settings from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox, Message, MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm -from django.core.files.base import ContentFile -from django.core.mail import send_mail, BadHeaderError, EmailMessage -from rowsandall_app.settings import BASE_DIR +from django_mailbox.models import Message, MessageAttachment +from rowers.models import User, Rower, RowerForm + +from django.core.mail import EmailMessage from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata -from rowingdata import TCXParser, RowProParser, ErgDataParser -from rowingdata import MysteryParser, BoatCoachParser -from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser -from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata -from rowingdata import make_cumvalues -from rowingdata import summarydata, get_file_type -from scipy.signal import savgol_filter +from rowingdata import rowingdata as rrdata + +from rowingdata import get_file_type import zipfile import os @@ -33,6 +28,7 @@ queuehigh = django_rq.get_queue('default') def send_confirm(u, name, link, options): + """ Send confirmation email to user when email has been processed """ fullemail = u.email subject = 'Workout added: ' + name message = 'Dear ' + u.first_name + ',\n\n' @@ -55,6 +51,7 @@ def send_confirm(u, name, link, options): def rdata(file, rower=rrower()): + """ Reads rowingdata data or returns 0 on Error """ try: res = rrdata(file, rower=rower) except IOError: @@ -65,119 +62,10 @@ def rdata(file, rower=rrower()): return res -# Some error protection around process attachments - - -def safeprocessattachments(): - try: - return processattachments() - except: - return [0] - -# This is duplicated in management/commands/processemail -# Need to double check the code there, update here, and only -# use the code here. - - -def processattachments(): - # in res, we store the ids of the new workouts - res = [] - attachments = MessageAttachment.objects.all() - for a in attachments: - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - # move attachment and make workout - try: - wid = [make_new_workout_from_email(rr, a.document, name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if wid != 1: - dd = send_confirm(u, name, link) - except: - # replace with code to process error - res += ['fail: ' + name] - donotdelete = 1 - except Rower.DoesNotExist: - pass - - # remove attachment - if donotdelete == 0: - a.delete() - - if m.attachments.exists() == False: - # no attachments, so can be deleted - m.delete() - - # Delete remaining messages (which should not have attachments) - mm = Message.objects.all() - for m in mm: - if m.attachments.exists() == False: - m.delete() - - return res - -# As above, but with some print commands for debugging purposes - - -def processattachments_debug(): - res = [] - attachments = MessageAttachment.objects.all() - for a in attachments: - donotdelete = 1 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - print theusers - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - doorgaan = 1 - except: - doorgaan = 0 - if doorgaan: - # move attachment and make workout - print a.document - print name - wid = [make_new_workout_from_email(rr, a.document, name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if wid != 1: - dd = send_confirm(u, name, link) - - # remove attachment - if donotdelete == 0: - a.delete() - - if m.attachments.exists() == False: - # no attachments, so can be deleted - m.delete() - - mm = Message.objects.all() - for m in mm: - if m.attachments.exists() == False: - m.delete() - - return res - -# Process the attachment file, create new workout -# The code here is duplication of the code in views.py (workout_upload_view) -# Need to move the code to a subroutine used both in views.py and here def make_new_workout_from_email(rr, f2, name, cntr=0): + """ This one is used in processemail """ workouttype = 'rower' try: @@ -196,15 +84,18 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): fileformat = fileformat[2] if fileformat == 'unknown': + fcopy = "copy_of_"+f2 + with open(f2, 'r') as f_in, open(fcopy, 'w') as f_out: + shutil.copyfileobj(f_in,f_out) if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, + res = handle_sendemail_unrecognized.delay(fcopy, "roosendaalsander@gmail.com") else: res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2, "roosendaalsander@gmail.com") + fcopy, "roosendaalsander@gmail.com") - return 1 + return 0 summary = '' # handle non-Painsled @@ -216,11 +107,7 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): inboard = 0.88 oarlength = 2.89 - # make workout and put in database - # r = rrower(hrmax=rr.max,hrut2=rr.ut2, - # hrut1=rr.ut1,hrat=rr.at, - # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) - row = rdata(f3) # ,rower=r) + row = rdata(f3) if row == 0: return 0 diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 0a7cf1ca..db17b354 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -5,22 +5,19 @@ import os import zipfile -from django.core.management.base import BaseCommand - import time from time import strftime -from django.conf import settings +from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment from rowers.models import Workout, Rower from rowingdata import rower as rrower - from rowingdata import rowingdata as rrdata -from rowers.mailprocessing import make_new_workout_from_email, send_confirm import rowers.uploads as uploads +from rowers.mailprocessing import make_new_workout_from_email, send_confirm # If you find a solution that does not need the two paths, please comment! sys.path.append('$path_to_root_of_project$') @@ -28,15 +25,58 @@ sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' -def rdata(file, rower=rrower()): +def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file, rower=rower) + res = rrdata(file_obj, rower=rower) except IOError: res = 0 return res +def processattachment(rr, f2, title, uploadoptions): + wid = [ + make_new_workout_from_email(rr, f2[6:], title) + ] + if wid: + res += wid + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' + if uploadoptions and not 'error' in uploadoptions: + w = Workout.objects.get(id=wid[0]) + r = w.user + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) + if 'make_plot' in uploadoptions: + plottype = uploadoptions['plottype'] + f1 = w.csvfilename[6:-4] + timestr = strftime("%Y%m%d-%H%M%S") + imagename = f1 + timestr + '.png' + resu = uploads.make_plot( + r, w, f1, + w.csvfilename, + plottype, name, + imagename=imagename + ) + try: + if wid: + dd = send_confirm( + rr.user, title, link, + uploadoptions + ) + time.sleep(10) + except: + try: + time.sleep(10) + if wid: + dd = send_confirm( + rr.user, title, link, + uploadoptions + ) + except: + pass + + return wid class Command(BaseCommand): """Run the Email processing command """ @@ -46,7 +86,6 @@ class Command(BaseCommand): cntr = 0 for a in attachments: extension = a.document.name[-3:].lower() - donotdelete = 0 m = Message.objects.get(id=a.message_id) body = "\n".join(m.text.splitlines()) uploadoptions = uploads.upload_options(body) @@ -64,85 +103,16 @@ class Command(BaseCommand): for f in z.namelist(): f2 = z.extract(f, path='media/') title = os.path.basename(f2) - wid = [ - make_new_workout_from_email(rr, f2[6:], title) - ] - res += wid - link = 'http://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if uploadoptions and not 'error' in uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w, uploadoptions) - uploads.make_private(w, uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1 + timestr + '.png' - resu = uploads.make_plot(r, w, f1, - w.csvfilename, - plottype, name, - imagename=imagename) - try: - if wid != 1: - dd = send_confirm(rr.user, title, link, - uploadoptions) - time.sleep(10) - except: - try: - time.sleep(10) - if wid != 1: - dd = send_confirm(rr.user, title, link, - uploadoptions) - except: - pass - + wid = processattachment(rr, f2, title, uploadoptions) else: # move attachment and make workout - try: - wid = [ - make_new_workout_from_email(rr, - a.document, - name) - ] - res += wid - link = 'http://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' - if uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w, uploadoptions) - uploads.make_private(w, uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1 + timestr + '.png' - resu = uploads.make_plot(r, w, f1, - w.csvfilename, - plottype, name, - imagename=imagename) - - except: - # replace with code to process error - res += ['fail: ' + name] - donotdelete = 1 - wid = 1 - try: - if wid != 1: - dd = send_confirm(rr.user, name, link, - uploadoptions) - time.sleep(10) - except: - pass + wid = processattachment(rr, a.document, name, uploadoptions) + # We're done with the attachment. It can be deleted try: a.delete() except IOError: pass - # remove attachment - # if donotdelete == 0: if m.attachments.exists() is False: # no attachments, so can be deleted From f36911d73bc4221da1a8949a3cb4db1e5b713ff8 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 14:38:37 +0200 Subject: [PATCH 3/4] clarify variable naming I have made variable names longer to make it easier to follow the logic of the code. Also removed a few unused ones. Not tested for functionality, yet. --- rowers/management/commands/processemail.py | 108 +++++++++++---------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index db17b354..bc2ff090 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -10,8 +10,9 @@ from time import strftime from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment -from rowers.models import Workout, Rower +from django.core.urlresolvers import reverse +from rowers.models import Workout, Rower from rowingdata import rower as rrower from rowingdata import rowingdata as rrdata @@ -28,100 +29,105 @@ os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file_obj, rower=rower) + result = rrdata(file_obj, rower=rower) except IOError: - res = 0 + result = 0 - return res + return result -def processattachment(rr, f2, title, uploadoptions): - wid = [ - make_new_workout_from_email(rr, f2[6:], title) +def processattachment(rower, filename, title, uploadoptions): + workoutid = [ + make_new_workout_from_email(rower, filename[6:], title) ] - if wid: - res += wid - link = 'http://rowsandall.com/rowers/workout/' + \ - str(wid[0]) + '/edit' + if workoutid: + link = 'https://rowsandall.com'+reverse( + rower.defaultlandingpage, + kwargs = { + 'id':workoutid, + } + ) + if uploadoptions and not 'error' in uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w, uploadoptions) - uploads.make_private(w, uploadoptions) + workout = Workout.objects.get(id=workoutid[0]) + uploads.do_sync(workout, uploadoptions) + uploads.make_private(workout, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + workoutcsvfilename = workout.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1 + timestr + '.png' - resu = uploads.make_plot( - r, w, f1, - w.csvfilename, - plottype, name, + imagename = workoutcsvfilename + timestr + '.png' + result = uploads.make_plot( + workout.user, workout, workoutcsvfilename, + workout.csvfilename, + plottype, title, imagename=imagename ) try: - if wid: - dd = send_confirm( - rr.user, title, link, + if workoutid: + email_sent = send_confirm( + rower.user, title, link, uploadoptions ) time.sleep(10) except: try: time.sleep(10) - if wid: - dd = send_confirm( - rr.user, title, link, + if workoutid: + email_sent = send_confirm( + rower.user, title, link, uploadoptions ) except: pass - return wid + return workoutid class Command(BaseCommand): """Run the Email processing command """ def handle(self, *args, **options): - res = [] attachments = MessageAttachment.objects.all() cntr = 0 - for a in attachments: - extension = a.document.name[-3:].lower() - m = Message.objects.get(id=a.message_id) - body = "\n".join(m.text.splitlines()) + for attachment in attachments: + extension = attachment.document.name[-3:].lower() + message = Message.objects.get(id=attachment.message_id) + body = "\n".join(message.text.splitlines()) uploadoptions = uploads.upload_options(body) - from_address = m.from_address[0].lower() - name = m.subject - cntr += 1 + from_address = message.from_address[0].lower() + name = message.subject # get a list of users # theusers = User.objects.filter(email=from_address) - ther = [ + rowers = [ r for r in Rower.objects.all() if r.user.email.lower() == from_address ] - for rr in ther: + for rower in rowers: if extension == 'zip': - z = zipfile.ZipFile(a.document) - for f in z.namelist(): - f2 = z.extract(f, path='media/') - title = os.path.basename(f2) - wid = processattachment(rr, f2, title, uploadoptions) + zip_file = zipfile.ZipFile(attachment.document) + for filename in zip_file.namelist(): + datafile = zip_file.extract(filename, path='media/') + title = os.path.basename(datafile) + workoutid = processattachment( + rower, datafile, title, uploadoptions + ) else: # move attachment and make workout - wid = processattachment(rr, a.document, name, uploadoptions) + workoutid = processattachment( + rower, attachment.document, name, uploadoptions + ) # We're done with the attachment. It can be deleted try: - a.delete() + attachment.delete() except IOError: pass - if m.attachments.exists() is False: + if message.attachments.exists() is False: # no attachments, so can be deleted - m.delete() + message.delete() - mm = Message.objects.all() - for m in mm: - if m.attachments.exists() is False: - m.delete() + messages = Message.objects.all() + for message in messages: + if message.attachments.exists() is False: + message.delete() self.stdout.write(self.style.SUCCESS( 'Successfully processed email attachments')) From 973bbb9f0bd38cb9ac3b4942d36f3e4f09620f27 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 16:04:03 +0200 Subject: [PATCH 4/4] Complete and tested email processing Processed successfully individual files, zip files, unrecognized files, zips containing unrecognized files --- rowers/mailprocessing.py | 79 ++++++++++++---------- rowers/management/commands/processemail.py | 45 ++++++++---- rowsandall_app/settings.py | 3 + rowsandall_app/settings_dev.py | 2 + 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index db3bccd0..282fedbe 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -27,11 +27,11 @@ queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u, name, link, options): +def send_confirm(user, name, link, options): """ Send confirmation email to user when email has been processed """ - fullemail = u.email + fullemail = user.email subject = 'Workout added: ' + name - message = 'Dear ' + u.first_name + ',\n\n' + message = 'Dear ' + user.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" @@ -53,68 +53,73 @@ def send_confirm(u, name, link, options): def rdata(file, rower=rrower()): """ Reads rowingdata data or returns 0 on Error """ try: - res = rrdata(file, rower=rower) + result = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file + '.gz', rower=rower) + result = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + result = 0 - return res + return result -def make_new_workout_from_email(rr, f2, name, cntr=0): +def make_new_workout_from_email(rower, datafile, name, cntr=0): """ This one is used in processemail """ workouttype = 'rower' try: - f2 = f2.name - fileformat = get_file_type('media/' + f2) + datafilename = datafile.name + fileformat = get_file_type('media/' + datafilename) except IOError: - f2 = f2.name + '.gz' - fileformat = get_file_type('media/' + f2) + datafilename = datafile.name + '.gz' + fileformat = get_file_type('media/' + datafilename) except AttributeError: - fileformat = get_file_type('media/' + f2) + datafilename = datafile + fileformat = get_file_type('media/' + datafile) if len(fileformat) == 3 and fileformat[0] == 'zip': - f_to_be_deleted = f2 - with zipfile.ZipFile('media/' + f2) as z: - f2 = z.extract(z.namelist()[0], path='media/')[6:] + with zipfile.ZipFile('media/' + datafilename) as zip_file: + datafilename = zip_file.extract( + zip_file.namelist()[0], + path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - fcopy = "copy_of_"+f2 - with open(f2, 'r') as f_in, open(fcopy, 'w') as f_out: - shutil.copyfileobj(f_in,f_out) +# extension = datafilename[-4:].lower() +# fcopy = "media/"+datafilename[:-4]+"_copy"+extension +# with open('media/'+datafilename, 'r') as f_in, open(fcopy, 'w') as f_out: +# shutil.copyfileobj(f_in,f_out) + fcopy = "media/"+datafilename if settings.DEBUG: res = handle_sendemail_unrecognized.delay(fcopy, - "roosendaalsander@gmail.com") + rower.user.email) else: res = queuehigh.enqueue(handle_sendemail_unrecognized, - fcopy, "roosendaalsander@gmail.com") + fcopy, + rower.user.email) return 0 summary = '' # handle non-Painsled if fileformat != 'csv': - f3, summary, oarlength, inboard = dataprep.handle_nonpainsled( - 'media/' + f2, fileformat, summary) + filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + datafilename, fileformat, summary) else: - f3 = 'media/' + f2 + filename_mediadir = 'media/' + datafilename inboard = 0.88 oarlength = 2.89 - row = rdata(f3) + row = rdata(filename_mediadir) if row == 0: return 0 # change filename - if f2[:5] != 'media': + if datafilename[:5] != 'media': timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/' + timestr + str(cntr) + 'o.csv' + datafilename = 'media/' + timestr + str(cntr) + 'o.csv' try: avglat = row.df[' latitude'].mean() @@ -124,19 +129,21 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): except KeyError: pass - row.write_csv(f2, gzip=True) + row.write_csv(datafilename, gzip=True) dosummary = (fileformat != 'fit') if name == '': name = 'imported through email' - id, message = dataprep.save_workout_database(f2, rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + id, message = dataprep.save_workout_database( + datafilename, rower, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email' + ) return id diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index bc2ff090..5bd2d6f1 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -11,6 +11,7 @@ from time import strftime from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment from django.core.urlresolvers import reverse +from django.conf import settings from rowers.models import Workout, Rower @@ -26,6 +27,9 @@ sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' +if not getattr(__builtins__, "WindowsError", None): + class WindowsError(OSError): pass + def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: @@ -35,15 +39,20 @@ def rdata(file_obj, rower=rrower()): return result -def processattachment(rower, filename, title, uploadoptions): +def processattachment(rower, fileobj, title, uploadoptions): + try: + filename = fileobj.name + except AttributeError: + filename = fileobj[6:] + workoutid = [ - make_new_workout_from_email(rower, filename[6:], title) + make_new_workout_from_email(rower, filename, title) ] - if workoutid: - link = 'https://rowsandall.com'+reverse( + if workoutid[0]: + link = settings.SITE_URL+reverse( rower.defaultlandingpage, kwargs = { - 'id':workoutid, + 'id':workoutid[0], } ) @@ -62,23 +71,23 @@ def processattachment(rower, filename, title, uploadoptions): plottype, title, imagename=imagename ) + try: + if workoutid: + email_sent = send_confirm( + rower.user, title, link, + uploadoptions + ) + time.sleep(10) + except: try: + time.sleep(10) if workoutid: email_sent = send_confirm( rower.user, title, link, uploadoptions ) - time.sleep(10) except: - try: - time.sleep(10) - if workoutid: - email_sent = send_confirm( - rower.user, title, link, - uploadoptions - ) - except: - pass + pass return workoutid @@ -119,6 +128,12 @@ class Command(BaseCommand): attachment.delete() except IOError: pass + except WindowsError: + time.sleep(2) + try: + attachment.delete() + except WindowsError: + pass if message.attachments.exists() is False: # no attachments, so can be deleted diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index a7f25d88..beadcf86 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -261,6 +261,9 @@ TP_CLIENT_SECRET = CFG["tp_client_secret"] TP_REDIRECT_URI = CFG["tp_redirect_uri"] TP_CLIENT_KEY = TP_CLIENT_ID +# Full Site URL +SITE_URL = "https://rowsandall.com" + # RQ stuff RQ_QUEUES = { diff --git a/rowsandall_app/settings_dev.py b/rowsandall_app/settings_dev.py index f851d022..b37aaed8 100644 --- a/rowsandall_app/settings_dev.py +++ b/rowsandall_app/settings_dev.py @@ -76,6 +76,8 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/' SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" +SITE_URL = "http://localhost:8000" + #EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' #EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'