diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index eefd0031..282fedbe 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -1,24 +1,19 @@ # Processes emails sent to workouts@rowsandall.com +""" Processes emails sent to workouts@rowsandall.com """ +import shutil import time from django.conf import settings from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm -from django.core.files.base import ContentFile -from django.core.mail import send_mail, BadHeaderError,EmailMessage -from rowsandall_app.settings import BASE_DIR +from django_mailbox.models import Message, MessageAttachment +from rowers.models import User, Rower, RowerForm + +from django.core.mail import EmailMessage from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata -from rowingdata import TCXParser,RowProParser,ErgDataParser -from rowingdata import MysteryParser,BoatCoachParser -from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser -from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata -from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type -from scipy.signal import savgol_filter +from rowingdata import rowingdata as rrdata + +from rowingdata import get_file_type import zipfile import os @@ -30,19 +25,21 @@ queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u,name,link,options): - fullemail = u.email - subject = 'Workout added: '+name - message = 'Dear '+u.first_name+',\n\n' + + +def send_confirm(user, name, link, options): + """ Send confirmation email to user when email has been processed """ + fullemail = user.email + subject = 'Workout added: ' + name + message = 'Dear ' + user.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" - message += "Link to workout: "+link+"\n\n" + message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" if options: - message += "\n\n"+str(options) + message += "\n\n" + str(options) - - email = EmailMessage(subject,message, + email = EmailMessage(subject, message, 'Rowsandall ', [fullemail]) @@ -51,203 +48,102 @@ def send_confirm(u,name,link,options): return 1 # Reads a "rowingdata" object, plus some error protections -def rdata(file,rower=rrower()): + + +def rdata(file, rower=rrower()): + """ Reads rowingdata data or returns 0 on Error """ try: - res = rrdata(file,rower=rower) + result = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file+'.gz',rower=rower) + result = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + result = 0 - return res + return result -# Some error protection around process attachments -def safeprocessattachments(): - try: - return processattachments() - except: - return [0] -# This is duplicated in management/commands/processemail -# Need to double check the code there, update here, and only -# use the code here. -def processattachments(): - # in res, we store the ids of the new workouts - res = [] - attachments = MessageAttachment.objects.all() - for a in attachments: - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - # move attachment and make workout - try: - wid = [make_new_workout_from_email(rr,a.document,name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if wid != 1: - dd = send_confirm(u,name,link) - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 - except Rower.DoesNotExist: - pass - - - # remove attachment - if donotdelete == 0: - a.delete() - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() - - # Delete remaining messages (which should not have attachments) - mm = Message.objects.all() - for m in mm: - if m.attachments.exists()==False: - m.delete() - - return res - -# As above, but with some print commands for debugging purposes -def processattachments_debug(): - res = [] - attachments = MessageAttachment.objects.all() - for a in attachments: - donotdelete = 1 - m = Message.objects.get(id=a.message_id) - from_address = m.from_address[0] - name = m.subject - - # get a list of users - theusers = User.objects.filter(email=from_address) - print theusers - for u in theusers: - try: - rr = Rower.objects.get(user=u.id) - doorgaan = 1 - except: - doorgaan = 0 - if doorgaan: - # move attachment and make workout - print a.document - print name - wid = [make_new_workout_from_email(rr,a.document,name)] - res += wid - link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if wid != 1: - dd = send_confirm(u,name,link) - - - - # remove attachment - if donotdelete == 0: - a.delete() - - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() - - mm = Message.objects.all() - for m in mm: - if m.attachments.exists()==False: - m.delete() - - return res - -# Process the attachment file, create new workout -# The code here is duplication of the code in views.py (workout_upload_view) -# Need to move the code to a subroutine used both in views.py and here -def make_new_workout_from_email(rr,f2,name,cntr=0): +def make_new_workout_from_email(rower, datafile, name, cntr=0): + """ This one is used in processemail """ workouttype = 'rower' try: - f2 = f2.name - fileformat = get_file_type('media/'+f2) + datafilename = datafile.name + fileformat = get_file_type('media/' + datafilename) except IOError: - f2 = f2.name+'.gz' - fileformat = get_file_type('media/'+f2) + datafilename = datafile.name + '.gz' + fileformat = get_file_type('media/' + datafilename) except AttributeError: - fileformat = get_file_type('media/'+f2) + datafilename = datafile + fileformat = get_file_type('media/' + datafile) - if len(fileformat)==3 and fileformat[0]=='zip': - f_to_be_deleted = f2 - with zipfile.ZipFile('media/'+f2) as z: - f2 = z.extract(z.namelist()[0],path='media/')[6:] + if len(fileformat) == 3 and fileformat[0] == 'zip': + with zipfile.ZipFile('media/' + datafilename) as zip_file: + datafilename = zip_file.extract( + zip_file.namelist()[0], + path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - if settings.DEBUG: - res = handle_sendemail_unrecognized.delay(f2, - "roosendaalsander@gmail.com") +# extension = datafilename[-4:].lower() +# fcopy = "media/"+datafilename[:-4]+"_copy"+extension +# with open('media/'+datafilename, 'r') as f_in, open(fcopy, 'w') as f_out: +# shutil.copyfileobj(f_in,f_out) + fcopy = "media/"+datafilename + if settings.DEBUG: + res = handle_sendemail_unrecognized.delay(fcopy, + rower.user.email) - else: - res = queuehigh.enqueue(handle_sendemail_unrecognized, - f2,"roosendaalsander@gmail.com") + else: + res = queuehigh.enqueue(handle_sendemail_unrecognized, + fcopy, + rower.user.email) + + return 0 - return 1 - summary = '' # handle non-Painsled if fileformat != 'csv': - f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary) + filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + datafilename, fileformat, summary) else: - f3 = 'media/'+f2 + filename_mediadir = 'media/' + datafilename inboard = 0.88 oarlength = 2.89 - - - - # make workout and put in database - #r = rrower(hrmax=rr.max,hrut2=rr.ut2, - # hrut1=rr.ut1,hrat=rr.at, - # hrtr=rr.tr,hran=rr.an,ftp=r.ftp) - row = rdata(f3) #,rower=r) + row = rdata(filename_mediadir) if row == 0: - return 0 - + return 0 # change filename - if f2[:5] != 'media': - timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/'+timestr+str(cntr)+'o.csv' + if datafilename[:5] != 'media': + timestr = time.strftime("%Y%m%d-%H%M%S") + datafilename = 'media/' + timestr + str(cntr) + 'o.csv' try: - avglat = row.df[' latitude'].mean() - avglon = row.df[' longitude'].mean() + avglat = row.df[' latitude'].mean() + avglon = row.df[' longitude'].mean() if avglat != 0 or avglon != 0: workouttype = 'water' except KeyError: pass - row.write_csv(f2,gzip=True) + row.write_csv(datafilename, gzip=True) dosummary = (fileformat != 'fit') if name == '': - name = 'imported through email' - - id,message = dataprep.save_workout_database(f2,rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + name = 'imported through email' + id, message = dataprep.save_workout_database( + datafilename, rower, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email' + ) return id - - - - diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index 6d06150a..5bd2d6f1 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -1,162 +1,148 @@ #!/srv/venv/bin/python +""" Process emails """ import sys import os + +import zipfile + +import time +from time import strftime + +from django.core.management.base import BaseCommand +from django_mailbox.models import Message, MessageAttachment +from django.core.urlresolvers import reverse +from django.conf import settings + +from rowers.models import Workout, Rower + +from rowingdata import rower as rrower +from rowingdata import rowingdata as rrdata + +import rowers.uploads as uploads +from rowers.mailprocessing import make_new_workout_from_email, send_confirm + # If you find a solution that does not need the two paths, please comment! sys.path.append('$path_to_root_of_project$') sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' -import zipfile +if not getattr(__builtins__, "WindowsError", None): + class WindowsError(OSError): pass -from django.core.management.base import BaseCommand, CommandError -from django.conf import settings -#from rowers.mailprocessing import processattachments -import time -from time import strftime -from django.conf import settings -from rowers.tasks import handle_sendemail_unrecognized -from django_mailbox.models import Mailbox,Message,MessageAttachment -from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm -from django.core.files.base import ContentFile - -from rowsandall_app.settings import BASE_DIR - -from rowingdata import rower as rrower -from rowingdata import main as rmain -from rowingdata import rowingdata as rrdata - -from rowingdata import make_cumvalues -from rowingdata import summarydata,get_file_type - -from scipy.signal import savgol_filter -from rowers.mailprocessing import make_new_workout_from_email,send_confirm -import rowers.uploads as uploads - -def rdata(file,rower=rrower()): +def rdata(file_obj, rower=rrower()): + """ Read rowing data file and return 0 if file doesn't exist""" try: - res = rrdata(file,rower=rower) + result = rrdata(file_obj, rower=rower) except IOError: - res = 0 + result = 0 - return res + return result +def processattachment(rower, fileobj, title, uploadoptions): + try: + filename = fileobj.name + except AttributeError: + filename = fileobj[6:] + workoutid = [ + make_new_workout_from_email(rower, filename, title) + ] + if workoutid[0]: + link = settings.SITE_URL+reverse( + rower.defaultlandingpage, + kwargs = { + 'id':workoutid[0], + } + ) + + if uploadoptions and not 'error' in uploadoptions: + workout = Workout.objects.get(id=workoutid[0]) + uploads.do_sync(workout, uploadoptions) + uploads.make_private(workout, uploadoptions) + if 'make_plot' in uploadoptions: + plottype = uploadoptions['plottype'] + workoutcsvfilename = workout.csvfilename[6:-4] + timestr = strftime("%Y%m%d-%H%M%S") + imagename = workoutcsvfilename + timestr + '.png' + result = uploads.make_plot( + workout.user, workout, workoutcsvfilename, + workout.csvfilename, + plottype, title, + imagename=imagename + ) + try: + if workoutid: + email_sent = send_confirm( + rower.user, title, link, + uploadoptions + ) + time.sleep(10) + except: + try: + time.sleep(10) + if workoutid: + email_sent = send_confirm( + rower.user, title, link, + uploadoptions + ) + except: + pass + + return workoutid class Command(BaseCommand): + """Run the Email processing command """ def handle(self, *args, **options): - res = [] - attachments = MessageAttachment.objects.all() - cntr = 0 - for a in attachments: - extension = a.document.name[-3:].lower() - donotdelete = 0 - m = Message.objects.get(id=a.message_id) - body = "\n".join(m.text.splitlines()) + attachments = MessageAttachment.objects.all() + cntr = 0 + for attachment in attachments: + extension = attachment.document.name[-3:].lower() + message = Message.objects.get(id=attachment.message_id) + body = "\n".join(message.text.splitlines()) uploadoptions = uploads.upload_options(body) - from_address = m.from_address[0].lower() - name = m.subject - cntr += 1 - # get a list of users - # theusers = User.objects.filter(email=from_address) - ther = [ + from_address = message.from_address[0].lower() + name = message.subject + # get a list of users + # theusers = User.objects.filter(email=from_address) + rowers = [ r for r in Rower.objects.all() if r.user.email.lower() == from_address ] - for rr in ther: + for rower in rowers: if extension == 'zip': - z = zipfile.ZipFile(a.document) - for f in z.namelist(): - f2 = z.extract(f,path='media/') - title = os.path.basename(f2) - wid = [ - make_new_workout_from_email(rr,f2[6:],title) - ] - res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if uploadoptions and not 'error' in uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, - w.csvfilename, - plottype,name, - imagename=imagename) - try: - if wid != 1: - dd = send_confirm(rr.user,title,link, - uploadoptions) - time.sleep(10) - except: - try: - time.sleep(10) - if wid != 1: - dd = send_confirm(rr.user,title,link, - uploadoptions) - except: - pass - + zip_file = zipfile.ZipFile(attachment.document) + for filename in zip_file.namelist(): + datafile = zip_file.extract(filename, path='media/') + title = os.path.basename(datafile) + workoutid = processattachment( + rower, datafile, title, uploadoptions + ) else: - # move attachment and make workout - try: - wid = [ - make_new_workout_from_email(rr, - a.document, - name) - ] - res += wid - link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' - if uploadoptions: - w = Workout.objects.get(id=wid[0]) - r = w.user - uploads.do_sync(w,uploadoptions) - uploads.make_private(w,uploadoptions) - if 'make_plot' in uploadoptions: - plottype = uploadoptions['plottype'] - f1 = w.csvfilename[6:-4] - timestr = strftime("%Y%m%d-%H%M%S") - imagename = f1+timestr+'.png' - resu = uploads.make_plot(r,w,f1, - w.csvfilename, - plottype,name, - imagename=imagename) - - except: - # replace with code to process error - res += ['fail: '+name] - donotdelete = 1 - wid = 1 - try: - if wid != 1: - dd = send_confirm(rr.user,name,link, - uploadoptions) - time.sleep(10) - except: - pass + # move attachment and make workout + workoutid = processattachment( + rower, attachment.document, name, uploadoptions + ) + # We're done with the attachment. It can be deleted try: - a.delete() + attachment.delete() except IOError: pass - # remove attachment - #if donotdelete == 0: + except WindowsError: + time.sleep(2) + try: + attachment.delete() + except WindowsError: + pass - if m.attachments.exists()==False: - # no attachments, so can be deleted - m.delete() + if message.attachments.exists() is False: + # no attachments, so can be deleted + message.delete() - mm = Message.objects.all() - for m in mm: - if m.attachments.exists()==False: - m.delete() + messages = Message.objects.all() + for message in messages: + if message.attachments.exists() is False: + message.delete() - self.stdout.write(self.style.SUCCESS('Successfully processed email attachments')) - - - - + self.stdout.write(self.style.SUCCESS( + 'Successfully processed email attachments')) diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index a7f25d88..beadcf86 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -261,6 +261,9 @@ TP_CLIENT_SECRET = CFG["tp_client_secret"] TP_REDIRECT_URI = CFG["tp_redirect_uri"] TP_CLIENT_KEY = TP_CLIENT_ID +# Full Site URL +SITE_URL = "https://rowsandall.com" + # RQ stuff RQ_QUEUES = { diff --git a/rowsandall_app/settings_dev.py b/rowsandall_app/settings_dev.py index f851d022..b37aaed8 100644 --- a/rowsandall_app/settings_dev.py +++ b/rowsandall_app/settings_dev.py @@ -76,6 +76,8 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/' SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" +SITE_URL = "http://localhost:8000" + #EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' #EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'