From 973bbb9f0bd38cb9ac3b4942d36f3e4f09620f27 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Oct 2017 16:04:03 +0200 Subject: [PATCH] Complete and tested email processing Processed successfully individual files, zip files, unrecognized files, zips containing unrecognized files --- rowers/mailprocessing.py | 79 ++++++++++++---------- rowers/management/commands/processemail.py | 45 ++++++++---- rowsandall_app/settings.py | 3 + rowsandall_app/settings_dev.py | 2 + 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index db3bccd0..282fedbe 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -27,11 +27,11 @@ queuehigh = django_rq.get_queue('default') # Sends a confirmation with a link to the workout -def send_confirm(u, name, link, options): +def send_confirm(user, name, link, options): """ Send confirmation email to user when email has been processed """ - fullemail = u.email + fullemail = user.email subject = 'Workout added: ' + name - message = 'Dear ' + u.first_name + ',\n\n' + message = 'Dear ' + user.first_name + ',\n\n' message += "Your workout has been added to Rowsandall.com.\n" message += "Link to workout: " + link + "\n\n" message += "Best Regards, the Rowsandall Team" @@ -53,68 +53,73 @@ def send_confirm(u, name, link, options): def rdata(file, rower=rrower()): """ Reads rowingdata data or returns 0 on Error """ try: - res = rrdata(file, rower=rower) + result = rrdata(file, rower=rower) except IOError: try: - res = rrdata(file + '.gz', rower=rower) + result = rrdata(file + '.gz', rower=rower) except IOError: - res = 0 + result = 0 - return res + return result -def make_new_workout_from_email(rr, f2, name, cntr=0): +def make_new_workout_from_email(rower, datafile, name, cntr=0): """ This one is used in processemail """ workouttype = 'rower' try: - f2 = f2.name - fileformat = get_file_type('media/' + f2) + datafilename = datafile.name + fileformat = get_file_type('media/' + datafilename) except IOError: - f2 = f2.name + '.gz' - fileformat = get_file_type('media/' + f2) + datafilename = datafile.name + '.gz' + fileformat = get_file_type('media/' + datafilename) except AttributeError: - fileformat = get_file_type('media/' + f2) + datafilename = datafile + fileformat = get_file_type('media/' + datafile) if len(fileformat) == 3 and fileformat[0] == 'zip': - f_to_be_deleted = f2 - with zipfile.ZipFile('media/' + f2) as z: - f2 = z.extract(z.namelist()[0], path='media/')[6:] + with zipfile.ZipFile('media/' + datafilename) as zip_file: + datafilename = zip_file.extract( + zip_file.namelist()[0], + path='media/')[6:] fileformat = fileformat[2] if fileformat == 'unknown': - fcopy = "copy_of_"+f2 - with open(f2, 'r') as f_in, open(fcopy, 'w') as f_out: - shutil.copyfileobj(f_in,f_out) +# extension = datafilename[-4:].lower() +# fcopy = "media/"+datafilename[:-4]+"_copy"+extension +# with open('media/'+datafilename, 'r') as f_in, open(fcopy, 'w') as f_out: +# shutil.copyfileobj(f_in,f_out) + fcopy = "media/"+datafilename if settings.DEBUG: res = handle_sendemail_unrecognized.delay(fcopy, - "roosendaalsander@gmail.com") + rower.user.email) else: res = queuehigh.enqueue(handle_sendemail_unrecognized, - fcopy, "roosendaalsander@gmail.com") + fcopy, + rower.user.email) return 0 summary = '' # handle non-Painsled if fileformat != 'csv': - f3, summary, oarlength, inboard = dataprep.handle_nonpainsled( - 'media/' + f2, fileformat, summary) + filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled( + 'media/' + datafilename, fileformat, summary) else: - f3 = 'media/' + f2 + filename_mediadir = 'media/' + datafilename inboard = 0.88 oarlength = 2.89 - row = rdata(f3) + row = rdata(filename_mediadir) if row == 0: return 0 # change filename - if f2[:5] != 'media': + if datafilename[:5] != 'media': timestr = time.strftime("%Y%m%d-%H%M%S") - f2 = 'media/' + timestr + str(cntr) + 'o.csv' + datafilename = 'media/' + timestr + str(cntr) + 'o.csv' try: avglat = row.df[' latitude'].mean() @@ -124,19 +129,21 @@ def make_new_workout_from_email(rr, f2, name, cntr=0): except KeyError: pass - row.write_csv(f2, gzip=True) + row.write_csv(datafilename, gzip=True) dosummary = (fileformat != 'fit') if name == '': name = 'imported through email' - id, message = dataprep.save_workout_database(f2, rr, - workouttype=workouttype, - dosummary=dosummary, - inboard=inboard, - oarlength=oarlength, - title=name, - workoutsource=fileformat, - notes='imported through email') + id, message = dataprep.save_workout_database( + datafilename, rower, + workouttype=workouttype, + dosummary=dosummary, + inboard=inboard, + oarlength=oarlength, + title=name, + workoutsource=fileformat, + notes='imported through email' + ) return id diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index bc2ff090..5bd2d6f1 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -11,6 +11,7 @@ from time import strftime from django.core.management.base import BaseCommand from django_mailbox.models import Message, MessageAttachment from django.core.urlresolvers import reverse +from django.conf import settings from rowers.models import Workout, Rower @@ -26,6 +27,9 @@ sys.path.append('$path_to_root_of_project$/$project_name$') os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' +if not getattr(__builtins__, "WindowsError", None): + class WindowsError(OSError): pass + def rdata(file_obj, rower=rrower()): """ Read rowing data file and return 0 if file doesn't exist""" try: @@ -35,15 +39,20 @@ def rdata(file_obj, rower=rrower()): return result -def processattachment(rower, filename, title, uploadoptions): +def processattachment(rower, fileobj, title, uploadoptions): + try: + filename = fileobj.name + except AttributeError: + filename = fileobj[6:] + workoutid = [ - make_new_workout_from_email(rower, filename[6:], title) + make_new_workout_from_email(rower, filename, title) ] - if workoutid: - link = 'https://rowsandall.com'+reverse( + if workoutid[0]: + link = settings.SITE_URL+reverse( rower.defaultlandingpage, kwargs = { - 'id':workoutid, + 'id':workoutid[0], } ) @@ -62,23 +71,23 @@ def processattachment(rower, filename, title, uploadoptions): plottype, title, imagename=imagename ) + try: + if workoutid: + email_sent = send_confirm( + rower.user, title, link, + uploadoptions + ) + time.sleep(10) + except: try: + time.sleep(10) if workoutid: email_sent = send_confirm( rower.user, title, link, uploadoptions ) - time.sleep(10) except: - try: - time.sleep(10) - if workoutid: - email_sent = send_confirm( - rower.user, title, link, - uploadoptions - ) - except: - pass + pass return workoutid @@ -119,6 +128,12 @@ class Command(BaseCommand): attachment.delete() except IOError: pass + except WindowsError: + time.sleep(2) + try: + attachment.delete() + except WindowsError: + pass if message.attachments.exists() is False: # no attachments, so can be deleted diff --git a/rowsandall_app/settings.py b/rowsandall_app/settings.py index a7f25d88..beadcf86 100644 --- a/rowsandall_app/settings.py +++ b/rowsandall_app/settings.py @@ -261,6 +261,9 @@ TP_CLIENT_SECRET = CFG["tp_client_secret"] TP_REDIRECT_URI = CFG["tp_redirect_uri"] TP_CLIENT_KEY = TP_CLIENT_ID +# Full Site URL +SITE_URL = "https://rowsandall.com" + # RQ stuff RQ_QUEUES = { diff --git a/rowsandall_app/settings_dev.py b/rowsandall_app/settings_dev.py index f851d022..b37aaed8 100644 --- a/rowsandall_app/settings_dev.py +++ b/rowsandall_app/settings_dev.py @@ -76,6 +76,8 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/' SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" +SITE_URL = "http://localhost:8000" + #EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' #EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'