diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index eefd0031..df6760c7 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -2,21 +2,21 @@ import time from django.conf import settings from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm +from django_mailbox.models import Mailbox, Message, MessageAttachment +from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm from django.core.files.base import ContentFile -from django.core.mail import send_mail, BadHeaderError,EmailMessage +from django.core.mail import send_mail, BadHeaderError, EmailMessage from rowsandall_app.settings import BASE_DIR from rowingdata import rower as rrower from rowingdata import main as rmain from rowingdata import rowingdata as rrdata -from rowingdata import TCXParser,RowProParser,ErgDataParser -from rowingdata import MysteryParser,BoatCoachParser -from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser -from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata +from rowingdata import TCXParser, RowProParser, ErgDataParser +from rowingdata import MysteryParser, BoatCoachParser +from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser +from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type +from rowingdata import summarydata, get_file_type from scipy.signal import savgol_filter @@ -30,19 +30,20 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u,name,link,options): + + +def send_confirm(u, name, link, options): fullemail = u.email - subject = 'Workout added: '+name - message = 'Dear '+u.first_name+',\n\n' + subject = 'Workout added: ' + name + message = 'Dear ' + u.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" - message += "Link to workout: "+link+"\n\n" + message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" if options: - message += "\n\n"+str(options) + message += "\n\n" + str(options) - - email = EmailMessage(subject,message, + email = EmailMessage(subject, message, 'Rowsandall ', [fullemail]) @@ -51,203 +52,204 @@ def send_confirm(u,name,link,options): return 1 # Reads a "rowingdata" object, plus some error protections -def rdata(file,rower=rrower()): + + +def rdata(file, rower=rrower()): try: - res = rrdata(file,rower=rower) + res = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file+'.gz',rower=rower) + res = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + res = 0 return res # Some error protection around process attachments + + def safeprocessattachments(): try: - return processattachments() + return processattachments() except: - return [0] + return [0] # This is duplicated in management/commands/processemail # Need to double check the code there, update here, and only # use the code here. + + def processattachments(): # in res, we store the ids of the new workouts res = [] attachments = MessageAttachment.objects.all() for a in attachments: - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - # move attachment and make workout - try: - wid = [make_new_workout_from_email(rr,a.document,name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if wid != 1: - dd = send_confirm(u,name,link) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 - except Rower.DoesNotExist: - pass - - - # remove attachment - if donotdelete == 0: - a.delete() + donotdelete = 0 + m = Message.objects.get(id=a.message_id) + from_address = m.from_address[0] + name = m.subject - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + # get a list of users + theusers = User.objects.filter(email=from_address) + for u in theusers: + try: + rr = Rower.objects.get(user=u.id) + # move attachment and make workout + try: + wid = [make_new_workout_from_email(rr, a.document, name)] + res += wid + link = 'https://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' + if wid != 1: + dd = send_confirm(u, name, link) + except: + # replace with code to process error + res += ['fail: ' + name] + donotdelete = 1 + except Rower.DoesNotExist: + pass + + # remove attachment + if donotdelete == 0: + a.delete() + + if m.attachments.exists() == False: + # no attachments, so can be deleted + m.delete() # Delete remaining messages (which should not have attachments) mm = Message.objects.all() for m in mm: - if m.attachments.exists()==False: - m.delete() + if m.attachments.exists() == False: + m.delete() return res # As above, but with some print commands for debugging purposes + + def processattachments_debug(): res = [] attachments = MessageAttachment.objects.all() for a in attachments: - donotdelete = 1 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) + donotdelete = 1 + m = Message.objects.get(id=a.message_id) + from_address = m.from_address[0] + name = m.subject + + # get a list of users + theusers = User.objects.filter(email=from_address) print theusers - for u in theusers: + for u in theusers: try: - rr = Rower.objects.get(user=u.id) + rr = Rower.objects.get(user=u.id) doorgaan = 1 except: doorgaan = 0 if doorgaan: - # move attachment and make workout + # move attachment and make workout print a.document print name - wid = [make_new_workout_from_email(rr,a.document,name)] + wid = [make_new_workout_from_email(rr, a.document, name)] res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'https://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if wid != 1: - dd = send_confirm(u,name,link) - - - - # remove attachment - if donotdelete == 0: - a.delete() + dd = send_confirm(u, name, link) - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + # remove attachment + if donotdelete == 0: + a.delete() + + if m.attachments.exists() == False: + # no attachments, so can be deleted + m.delete() mm = Message.objects.all() for m in mm: - if m.attachments.exists()==False: - m.delete() + if m.attachments.exists() == False: + m.delete() return res # Process the attachment file, create new workout # The code here is duplication of the code in views.py (workout_upload_view) # Need to move the code to a subroutine used both in views.py and here -def make_new_workout_from_email(rr,f2,name,cntr=0): + + +def make_new_workout_from_email(rr, f2, name, cntr=0): workouttype = 'rower' try: f2 = f2.name - fileformat = get_file_type('media/'+f2) + fileformat = get_file_type('media/' + f2) except IOError: - f2 = f2.name+'.gz' - fileformat = get_file_type('media/'+f2) + f2 = f2.name + '.gz' + fileformat = get_file_type('media/' + f2) except AttributeError: - fileformat = get_file_type('media/'+f2) + fileformat = get_file_type('media/' + f2) - if len(fileformat)==3 and fileformat[0]=='zip': + if len(fileformat) == 3 and fileformat[0] == 'zip': f_to_be_deleted = f2 - with zipfile.ZipFile('media/'+f2) as z: - f2 = z.extract(z.namelist()[0],path='media/')[6:] + with zipfile.ZipFile('media/' + f2) as z: + f2 = z.extract(z.namelist()[0], path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, - "roosendaalsander@gmail.com") + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(f2, + "roosendaalsander@gmail.com") - else: - res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2,"roosendaalsander@gmail.com") + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + f2, "roosendaalsander@gmail.com") + + return 1 - return 1 - summary = '' # handle non-Painsled if fileformat != 'csv': - f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary) + f3, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + f2, fileformat, summary) else: - f3 = 'media/'+f2 + f3 = 'media/' + f2 inboard = 0.88 oarlength = 2.89 - - - # make workout and put in database - #r = rrower(hrmax=rr.max,hrut2=rr.ut2, - # hrut1=rr.ut1,hrat=rr.at, - # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) - row = rdata(f3) #,rower=r) + # r = rrower(hrmax=rr.max,hrut2=rr.ut2, + # hrut1=rr.ut1,hrat=rr.at, + # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) + row = rdata(f3) # ,rower=r) if row == 0: - return 0 - + return 0 # change filename if f2[:5] != 'media': - timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/'+timestr+str(cntr)+'o.csv' + timestr = time.strftime("%Y%m%d-%H%M%S") + f2 = 'media/' + timestr + str(cntr) + 'o.csv' try: - avglat = row.df[' latitude'].mean() - avglon = row.df[' longitude'].mean() + avglat = row.df[' latitude'].mean() + avglon = row.df[' longitude'].mean() if avglat != 0 or avglon != 0: workouttype = 'water' except KeyError: pass - row.write_csv(f2,gzip=True) + row.write_csv(f2, gzip=True) dosummary = (fileformat != 'fit') if name == '': - name = 'imported through email' - - id,message = dataprep.save_workout_database(f2,rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + name = 'imported through email' + id, message = dataprep.save_workout_database(f2, rr, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email') return id - - - - diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 6d06150a..0a7cf1ca 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -1,139 +1,137 @@ #!/srv/venv/bin/python +""" Process emails """ import sys import os + +import zipfile + +from django.core.management.base import BaseCommand + +import time +from time import strftime +from django.conf import settings + +from django_mailbox.models import Message, MessageAttachment +from rowers.models import Workout, Rower + + +from rowingdata import rower as rrower + +from rowingdata import rowingdata as rrdata + +from rowers.mailprocessing import make_new_workout_from_email, send_confirm +import rowers.uploads as uploads + # If you find a solution that does not need the two paths, please comment! sys.path.append('$path_to_root_of_project$') sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' -import zipfile - -from django.core.management.base import BaseCommand, CommandError -from django.conf import settings -#from rowers.mailprocessing import processattachments -import time -from time import strftime -from django.conf import settings -from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm -from django.core.files.base import ContentFile - -from rowsandall_app.settings import BASE_DIR - -from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata - -from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type - -from scipy.signal import savgol_filter -from rowers.mailprocessing import make_new_workout_from_email,send_confirm -import rowers.uploads as uploads - -def rdata(file,rower=rrower()): +def rdata(file, rower=rrower()): + """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file,rower=rower) + res = rrdata(file, rower=rower) except IOError: - res = 0 + res = 0 return res - class Command(BaseCommand): + """Run the Email processing command """ def handle(self, *args, **options): - res = [] - attachments = MessageAttachment.objects.all() - cntr = 0 - for a in attachments: + res = [] + attachments = MessageAttachment.objects.all() + cntr = 0 + for a in attachments: extension = a.document.name[-3:].lower() - donotdelete = 0 - m = Message.objects.get(id=a.message_id) + donotdelete = 0 + m = Message.objects.get(id=a.message_id) body = "\n".join(m.text.splitlines()) uploadoptions = uploads.upload_options(body) - from_address = m.from_address[0].lower() - name = m.subject - cntr += 1 - # get a list of users - # theusers = User.objects.filter(email=from_address) + from_address = m.from_address[0].lower() + name = m.subject + cntr += 1 + # get a list of users + # theusers = User.objects.filter(email=from_address) ther = [ r for r in Rower.objects.all() if r.user.email.lower() == from_address ] - for rr in ther: + for rr in ther: if extension == 'zip': z = zipfile.ZipFile(a.document) for f in z.namelist(): - f2 = z.extract(f,path='media/') + f2 = z.extract(f, path='media/') title = os.path.basename(f2) wid = [ - make_new_workout_from_email(rr,f2[6:],title) + make_new_workout_from_email(rr, f2[6:], title) ] res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if uploadoptions and not 'error' in uploadoptions: w = Workout.objects.get(id=wid[0]) r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + f1 = w.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, + imagename = f1 + timestr + '.png' + resu = uploads.make_plot(r, w, f1, w.csvfilename, - plottype,name, + plottype, name, imagename=imagename) try: if wid != 1: - dd = send_confirm(rr.user,title,link, + dd = send_confirm(rr.user, title, link, uploadoptions) time.sleep(10) except: try: time.sleep(10) if wid != 1: - dd = send_confirm(rr.user,title,link, + dd = send_confirm(rr.user, title, link, uploadoptions) except: pass else: - # move attachment and make workout - try: - wid = [ + # move attachment and make workout + try: + wid = [ make_new_workout_from_email(rr, a.document, name) ] res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' + link = 'http://rowsandall.com/rowers/workout/' + \ + str(wid[0]) + '/edit' if uploadoptions: w = Workout.objects.get(id=wid[0]) r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) + uploads.do_sync(w, uploadoptions) + uploads.make_private(w, uploadoptions) if 'make_plot' in uploadoptions: plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] + f1 = w.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, + imagename = f1 + timestr + '.png' + resu = uploads.make_plot(r, w, f1, w.csvfilename, - plottype,name, + plottype, name, imagename=imagename) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 + except: + # replace with code to process error + res += ['fail: ' + name] + donotdelete = 1 wid = 1 try: if wid != 1: - dd = send_confirm(rr.user,name,link, + dd = send_confirm(rr.user, name, link, uploadoptions) time.sleep(10) except: @@ -144,19 +142,16 @@ class Command(BaseCommand): except IOError: pass # remove attachment - #if donotdelete == 0: + # if donotdelete == 0: - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + if m.attachments.exists() is False: + # no attachments, so can be deleted + m.delete() - mm = Message.objects.all() - for m in mm: - if m.attachments.exists()==False: - m.delete() + mm = Message.objects.all() + for m in mm: + if m.attachments.exists() is False: + m.delete() - self.stdout.write(self.style.SUCCESS('Successfully processed email attachments')) - - - - + self.stdout.write(self.style.SUCCESS( + 'Successfully processed email attachments'))