sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

cleaned up processemail

Browse Source
This commit is contained in:
Sander Roosendaal
2017-10-22 11:49:47 +02:00
parent e541289066
commit a866b253a1
2 changed files with 202 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
rowers/mailprocessing.py
View File

@@ -2,21 +2,21 @@
import time import time
from django.conf import settings from django.conf import settings
from rowers.tasks import handle_sendemail_unrecognized from rowers.tasks import handle_sendemail_unrecognized
from django_mailbox.models import Mailbox,Message,MessageAttachment from django_mailbox.models import Mailbox, Message, MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
from django.core.mail import send_mail, BadHeaderError,EmailMessage from django.core.mail import send_mail, BadHeaderError, EmailMessage
from rowsandall_app.settings import BASE_DIR from rowsandall_app.settings import BASE_DIR
from rowingdata import rower as rrower from rowingdata import rower as rrower
from rowingdata import main as rmain from rowingdata import main as rmain
from rowingdata import rowingdata as rrdata from rowingdata import rowingdata as rrdata
from rowingdata import TCXParser,RowProParser,ErgDataParser from rowingdata import TCXParser, RowProParser, ErgDataParser
from rowingdata import MysteryParser,BoatCoachParser from rowingdata import MysteryParser, BoatCoachParser
from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser
from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata
from rowingdata import make_cumvalues from rowingdata import make_cumvalues
from rowingdata import summarydata,get_file_type from rowingdata import summarydata, get_file_type
from scipy.signal import savgol_filter from scipy.signal import savgol_filter
@@ -30,19 +30,20 @@ queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('default') queuehigh = django_rq.get_queue('default')
# Sends a confirmation with a link to the workout # Sends a confirmation with a link to the workout
def send_confirm(u,name,link,options):
def send_confirm(u, name, link, options):
fullemail = u.email fullemail = u.email
subject = 'Workout added: '+name subject = 'Workout added: ' + name
message = 'Dear '+u.first_name+',\n\n' message = 'Dear ' + u.first_name + ',\n\n'
message += "Your workout has been added to Rowsandall.com.\n" message += "Your workout has been added to Rowsandall.com.\n"
message += "Link to workout: "+link+"\n\n" message += "Link to workout: " + link + "\n\n"
message += "Best Regards, the Rowsandall Team" message += "Best Regards, the Rowsandall Team"
if options: if options:
message += "\n\n"+str(options) message += "\n\n" + str(options)
email = EmailMessage(subject, message,
email = EmailMessage(subject,message,
'Rowsandall <info@rowsandall.com>', 'Rowsandall <info@rowsandall.com>',
[fullemail]) [fullemail])
@@ -51,203 +52,204 @@ def send_confirm(u,name,link,options):
return 1 return 1
# Reads a "rowingdata" object, plus some error protections # Reads a "rowingdata" object, plus some error protections
def rdata(file,rower=rrower()):
def rdata(file, rower=rrower()):
try: try:
res = rrdata(file,rower=rower) res = rrdata(file, rower=rower)
except IOError: except IOError:
try: try:
res = rrdata(file+'.gz',rower=rower) res = rrdata(file + '.gz', rower=rower)
except IOError: except IOError:
res = 0 res = 0
return res return res
# Some error protection around process attachments # Some error protection around process attachments
def safeprocessattachments(): def safeprocessattachments():
try: try:
return processattachments() return processattachments()
except: except:
return [0] return [0]
# This is duplicated in management/commands/processemail # This is duplicated in management/commands/processemail
# Need to double check the code there, update here, and only # Need to double check the code there, update here, and only
# use the code here. # use the code here.
def processattachments(): def processattachments():
# in res, we store the ids of the new workouts # in res, we store the ids of the new workouts
res = [] res = []
attachments = MessageAttachment.objects.all() attachments = MessageAttachment.objects.all()
for a in attachments: for a in attachments:
donotdelete = 0 donotdelete = 0
m = Message.objects.get(id=a.message_id) m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0] from_address = m.from_address[0]
name = m.subject name = m.subject
# get a list of users # get a list of users
theusers = User.objects.filter(email=from_address) theusers = User.objects.filter(email=from_address)
for u in theusers: for u in theusers:
try: try:
rr = Rower.objects.get(user=u.id) rr = Rower.objects.get(user=u.id)
# move attachment and make workout # move attachment and make workout
try: try:
wid = [make_new_workout_from_email(rr,a.document,name)] wid = [make_new_workout_from_email(rr, a.document, name)]
res += wid res += wid
link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' link = 'https://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if wid != 1: if wid != 1:
dd = send_confirm(u,name,link) dd = send_confirm(u, name, link)
except: except:
# replace with code to process error # replace with code to process error
res += ['fail: '+name] res += ['fail: ' + name]
donotdelete = 1 donotdelete = 1
except Rower.DoesNotExist: except Rower.DoesNotExist:
pass pass
# remove attachment
if donotdelete == 0:
a.delete()
# remove attachment if m.attachments.exists() == False:
if donotdelete == 0: # no attachments, so can be deleted
a.delete() m.delete()
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
# Delete remaining messages (which should not have attachments) # Delete remaining messages (which should not have attachments)
mm = Message.objects.all() mm = Message.objects.all()
for m in mm: for m in mm:
if m.attachments.exists()==False: if m.attachments.exists() == False:
m.delete() m.delete()
return res return res
# As above, but with some print commands for debugging purposes # As above, but with some print commands for debugging purposes
def processattachments_debug(): def processattachments_debug():
res = [] res = []
attachments = MessageAttachment.objects.all() attachments = MessageAttachment.objects.all()
for a in attachments: for a in attachments:
donotdelete = 1 donotdelete = 1
m = Message.objects.get(id=a.message_id) m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0] from_address = m.from_address[0]
name = m.subject name = m.subject
# get a list of users # get a list of users
theusers = User.objects.filter(email=from_address) theusers = User.objects.filter(email=from_address)
print theusers print theusers
for u in theusers: for u in theusers:
try: try:
rr = Rower.objects.get(user=u.id) rr = Rower.objects.get(user=u.id)
doorgaan = 1 doorgaan = 1
except: except:
doorgaan = 0 doorgaan = 0
if doorgaan: if doorgaan:
# move attachment and make workout # move attachment and make workout
print a.document print a.document
print name print name
wid = [make_new_workout_from_email(rr,a.document,name)] wid = [make_new_workout_from_email(rr, a.document, name)]
res += wid res += wid
link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' link = 'https://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if wid != 1: if wid != 1:
dd = send_confirm(u,name,link) dd = send_confirm(u, name, link)
# remove attachment
if donotdelete == 0:
a.delete()
if m.attachments.exists() == False:
# remove attachment # no attachments, so can be deleted
if donotdelete == 0: m.delete()
a.delete()
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
mm = Message.objects.all() mm = Message.objects.all()
for m in mm: for m in mm:
if m.attachments.exists()==False: if m.attachments.exists() == False:
m.delete() m.delete()
return res return res
# Process the attachment file, create new workout # Process the attachment file, create new workout
# The code here is duplication of the code in views.py (workout_upload_view) # The code here is duplication of the code in views.py (workout_upload_view)
# Need to move the code to a subroutine used both in views.py and here # Need to move the code to a subroutine used both in views.py and here
def make_new_workout_from_email(rr,f2,name,cntr=0):
def make_new_workout_from_email(rr, f2, name, cntr=0):
workouttype = 'rower' workouttype = 'rower'
try: try:
f2 = f2.name f2 = f2.name
fileformat = get_file_type('media/'+f2) fileformat = get_file_type('media/' + f2)
except IOError: except IOError:
f2 = f2.name+'.gz' f2 = f2.name + '.gz'
fileformat = get_file_type('media/'+f2) fileformat = get_file_type('media/' + f2)
except AttributeError: except AttributeError:
fileformat = get_file_type('media/'+f2) fileformat = get_file_type('media/' + f2)
if len(fileformat)==3 and fileformat[0]=='zip': if len(fileformat) == 3 and fileformat[0] == 'zip':
f_to_be_deleted = f2 f_to_be_deleted = f2
with zipfile.ZipFile('media/'+f2) as z: with zipfile.ZipFile('media/' + f2) as z:
f2 = z.extract(z.namelist()[0],path='media/')[6:] f2 = z.extract(z.namelist()[0], path='media/')[6:]
fileformat = fileformat[2] fileformat = fileformat[2]
if fileformat == 'unknown': if fileformat == 'unknown':
if settings.DEBUG: if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2, res = handle_sendemail_unrecognized.delay(f2,
"roosendaalsander@gmail.com") "roosendaalsander@gmail.com")
else: else:
res = queuehigh.enqueue(handle_sendemail_unrecognized, res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2,"roosendaalsander@gmail.com") f2, "roosendaalsander@gmail.com")
return 1 return 1
summary = '' summary = ''
# handle non-Painsled # handle non-Painsled
if fileformat != 'csv': if fileformat != 'csv':
f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary) f3, summary, oarlength, inboard = dataprep.handle_nonpainsled(
'media/' + f2, fileformat, summary)
else: else:
f3 = 'media/'+f2 f3 = 'media/' + f2
inboard = 0.88 inboard = 0.88
oarlength = 2.89 oarlength = 2.89
# make workout and put in database # make workout and put in database
#r = rrower(hrmax=rr.max,hrut2=rr.ut2, # r = rrower(hrmax=rr.max,hrut2=rr.ut2,
# hrut1=rr.ut1,hrat=rr.at, # hrut1=rr.ut1,hrat=rr.at,
# hrtr=rr.tr,hran=rr.an,ftp=r.ftp) # hrtr=rr.tr,hran=rr.an,ftp=r.ftp)
row = rdata(f3) #,rower=r) row = rdata(f3) # ,rower=r)
if row == 0: if row == 0:
return 0 return 0
# change filename # change filename
if f2[:5] != 'media': if f2[:5] != 'media':
timestr = time.strftime("%Y%m%d-%H%M%S") timestr = time.strftime("%Y%m%d-%H%M%S")
f2 = 'media/'+timestr+str(cntr)+'o.csv' f2 = 'media/' + timestr + str(cntr) + 'o.csv'
try: try:
avglat = row.df[' latitude'].mean() avglat = row.df[' latitude'].mean()
avglon = row.df[' longitude'].mean() avglon = row.df[' longitude'].mean()
if avglat != 0 or avglon != 0: if avglat != 0 or avglon != 0:
workouttype = 'water' workouttype = 'water'
except KeyError: except KeyError:
pass pass
row.write_csv(f2,gzip=True) row.write_csv(f2, gzip=True)
dosummary = (fileformat != 'fit') dosummary = (fileformat != 'fit')
if name == '': if name == '':
name = 'imported through email' name = 'imported through email'
id,message = dataprep.save_workout_database(f2,rr,
workouttype=workouttype,
dosummary=dosummary,
inboard=inboard,
oarlength=oarlength,
title=name,
workoutsource=fileformat,
notes='imported through email')
id, message = dataprep.save_workout_database(f2, rr,
workouttype=workouttype,
dosummary=dosummary,
inboard=inboard,
oarlength=oarlength,
title=name,
workoutsource=fileformat,
notes='imported through email')
return id return id

153
rowers/management/commands/processemail.py
View File

@@ -1,139 +1,137 @@
#!/srv/venv/bin/python #!/srv/venv/bin/python
""" Process emails """
import sys import sys
import os import os
import zipfile
from django.core.management.base import BaseCommand
import time
from time import strftime
from django.conf import settings
from django_mailbox.models import Message, MessageAttachment
from rowers.models import Workout, Rower
from rowingdata import rower as rrower
from rowingdata import rowingdata as rrdata
from rowers.mailprocessing import make_new_workout_from_email, send_confirm
import rowers.uploads as uploads
# If you find a solution that does not need the two paths, please comment! # If you find a solution that does not need the two paths, please comment!
sys.path.append('$path_to_root_of_project$') sys.path.append('$path_to_root_of_project$')
sys.path.append('$path_to_root_of_project$/$project_name$') sys.path.append('$path_to_root_of_project$/$project_name$')
os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings' os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings'
import zipfile def rdata(file, rower=rrower()):
""" Read rowing data file and return 0 if file doesn't exist"""
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
#from rowers.mailprocessing import processattachments
import time
from time import strftime
from django.conf import settings
from rowers.tasks import handle_sendemail_unrecognized
from django_mailbox.models import Mailbox,Message,MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
from django.core.files.base import ContentFile
from rowsandall_app.settings import BASE_DIR
from rowingdata import rower as rrower
from rowingdata import main as rmain
from rowingdata import rowingdata as rrdata
from rowingdata import make_cumvalues
from rowingdata import summarydata,get_file_type
from scipy.signal import savgol_filter
from rowers.mailprocessing import make_new_workout_from_email,send_confirm
import rowers.uploads as uploads
def rdata(file,rower=rrower()):
try: try:
res = rrdata(file,rower=rower) res = rrdata(file, rower=rower)
except IOError: except IOError:
res = 0 res = 0
return res return res
class Command(BaseCommand): class Command(BaseCommand):
"""Run the Email processing command """
def handle(self, *args, **options): def handle(self, *args, **options):
res = [] res = []
attachments = MessageAttachment.objects.all() attachments = MessageAttachment.objects.all()
cntr = 0 cntr = 0
for a in attachments: for a in attachments:
extension = a.document.name[-3:].lower() extension = a.document.name[-3:].lower()
donotdelete = 0 donotdelete = 0
m = Message.objects.get(id=a.message_id) m = Message.objects.get(id=a.message_id)
body = "\n".join(m.text.splitlines()) body = "\n".join(m.text.splitlines())
uploadoptions = uploads.upload_options(body) uploadoptions = uploads.upload_options(body)
from_address = m.from_address[0].lower() from_address = m.from_address[0].lower()
name = m.subject name = m.subject
cntr += 1 cntr += 1
# get a list of users # get a list of users
# theusers = User.objects.filter(email=from_address) # theusers = User.objects.filter(email=from_address)
ther = [ ther = [
r for r in Rower.objects.all() if r.user.email.lower() == from_address r for r in Rower.objects.all() if r.user.email.lower() == from_address
] ]
for rr in ther: for rr in ther:
if extension == 'zip': if extension == 'zip':
z = zipfile.ZipFile(a.document) z = zipfile.ZipFile(a.document)
for f in z.namelist(): for f in z.namelist():
f2 = z.extract(f,path='media/') f2 = z.extract(f, path='media/')
title = os.path.basename(f2) title = os.path.basename(f2)
wid = [ wid = [
make_new_workout_from_email(rr,f2[6:],title) make_new_workout_from_email(rr, f2[6:], title)
] ]
res += wid res += wid
link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' link = 'http://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if uploadoptions and not 'error' in uploadoptions: if uploadoptions and not 'error' in uploadoptions:
w = Workout.objects.get(id=wid[0]) w = Workout.objects.get(id=wid[0])
r = w.user r = w.user
uploads.do_sync(w,uploadoptions) uploads.do_sync(w, uploadoptions)
uploads.make_private(w,uploadoptions) uploads.make_private(w, uploadoptions)
if 'make_plot' in uploadoptions: if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype'] plottype = uploadoptions['plottype']
f1 = w.csvfilename[6:-4] f1 = w.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S") timestr = strftime("%Y%m%d-%H%M%S")
imagename = f1+timestr+'.png' imagename = f1 + timestr + '.png'
resu = uploads.make_plot(r,w,f1, resu = uploads.make_plot(r, w, f1,
w.csvfilename, w.csvfilename,
plottype,name, plottype, name,
imagename=imagename) imagename=imagename)
try: try:
if wid != 1: if wid != 1:
dd = send_confirm(rr.user,title,link, dd = send_confirm(rr.user, title, link,
uploadoptions) uploadoptions)
time.sleep(10) time.sleep(10)
except: except:
try: try:
time.sleep(10) time.sleep(10)
if wid != 1: if wid != 1:
dd = send_confirm(rr.user,title,link, dd = send_confirm(rr.user, title, link,
uploadoptions) uploadoptions)
except: except:
pass pass
else: else:
# move attachment and make workout # move attachment and make workout
try: try:
wid = [ wid = [
make_new_workout_from_email(rr, make_new_workout_from_email(rr,
a.document, a.document,
name) name)
] ]
res += wid res += wid
link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit' link = 'http://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if uploadoptions: if uploadoptions:
w = Workout.objects.get(id=wid[0]) w = Workout.objects.get(id=wid[0])
r = w.user r = w.user
uploads.do_sync(w,uploadoptions) uploads.do_sync(w, uploadoptions)
uploads.make_private(w,uploadoptions) uploads.make_private(w, uploadoptions)
if 'make_plot' in uploadoptions: if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype'] plottype = uploadoptions['plottype']
f1 = w.csvfilename[6:-4] f1 = w.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S") timestr = strftime("%Y%m%d-%H%M%S")
imagename = f1+timestr+'.png' imagename = f1 + timestr + '.png'
resu = uploads.make_plot(r,w,f1, resu = uploads.make_plot(r, w, f1,
w.csvfilename, w.csvfilename,
plottype,name, plottype, name,
imagename=imagename) imagename=imagename)
except: except:
# replace with code to process error # replace with code to process error
res += ['fail: '+name] res += ['fail: ' + name]
donotdelete = 1 donotdelete = 1
wid = 1 wid = 1
try: try:
if wid != 1: if wid != 1:
dd = send_confirm(rr.user,name,link, dd = send_confirm(rr.user, name, link,
uploadoptions) uploadoptions)
time.sleep(10) time.sleep(10)
except: except:
@@ -144,19 +142,16 @@ class Command(BaseCommand):
except IOError: except IOError:
pass pass
# remove attachment # remove attachment
#if donotdelete == 0: # if donotdelete == 0:
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
self.stdout.write(self.style.SUCCESS('Successfully processed email attachments'))
if m.attachments.exists() is False:
# no attachments, so can be deleted
m.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists() is False:
m.delete()
self.stdout.write(self.style.SUCCESS(
'Successfully processed email attachments'))