sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

cleaned up processemail

Browse Source
This commit is contained in:
Sander Roosendaal
2017-10-22 11:49:47 +02:00
parent e541289066
commit a866b253a1
2 changed files with 202 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

250
rowers/mailprocessing.py
View File

@@ -2,21 +2,21 @@
import time
from django.conf import settings
from rowers.tasks import handle_sendemail_unrecognized
from django_mailbox.models import Mailbox,Message,MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
from django_mailbox.models import Mailbox, Message, MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm, RowerForm, GraphImage, AdvancedWorkoutForm
from django.core.files.base import ContentFile
from django.core.mail import send_mail, BadHeaderError,EmailMessage
from django.core.mail import send_mail, BadHeaderError, EmailMessage
from rowsandall_app.settings import BASE_DIR
from rowingdata import rower as rrower
from rowingdata import main as rmain
from rowingdata import rowingdata as rrdata
from rowingdata import TCXParser,RowProParser,ErgDataParser
from rowingdata import MysteryParser,BoatCoachParser
from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser
from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata
from rowingdata import TCXParser, RowProParser, ErgDataParser
from rowingdata import MysteryParser, BoatCoachParser
from rowingdata import painsledDesktopParser, speedcoachParser, ErgStickParser
from rowingdata import SpeedCoach2Parser, FITParser, fitsummarydata
from rowingdata import make_cumvalues
from rowingdata import summarydata,get_file_type
from rowingdata import summarydata, get_file_type
from scipy.signal import savgol_filter
@@ -30,19 +30,20 @@ queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('default')
# Sends a confirmation with a link to the workout
def send_confirm(u,name,link,options):
def send_confirm(u, name, link, options):
fullemail = u.email
subject = 'Workout added: '+name
message = 'Dear '+u.first_name+',\n\n'
subject = 'Workout added: ' + name
message = 'Dear ' + u.first_name + ',\n\n'
message += "Your workout has been added to Rowsandall.com.\n"
message += "Link to workout: "+link+"\n\n"
message += "Link to workout: " + link + "\n\n"
message += "Best Regards, the Rowsandall Team"
if options:
message += "\n\n"+str(options)
message += "\n\n" + str(options)
email = EmailMessage(subject,message,
email = EmailMessage(subject, message,
'Rowsandall <info@rowsandall.com>',
[fullemail])
@@ -51,203 +52,204 @@ def send_confirm(u,name,link,options):
return 1
# Reads a "rowingdata" object, plus some error protections
def rdata(file,rower=rrower()):
def rdata(file, rower=rrower()):
try:
res = rrdata(file,rower=rower)
res = rrdata(file, rower=rower)
except IOError:
try:
res = rrdata(file+'.gz',rower=rower)
res = rrdata(file + '.gz', rower=rower)
except IOError:
res = 0
res = 0
return res
# Some error protection around process attachments
def safeprocessattachments():
try:
return processattachments()
return processattachments()
except:
return [0]
return [0]
# This is duplicated in management/commands/processemail
# Need to double check the code there, update here, and only
# use the code here.
def processattachments():
# in res, we store the ids of the new workouts
res = []
attachments = MessageAttachment.objects.all()
for a in attachments:
donotdelete = 0
m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0]
name = m.subject
# get a list of users
theusers = User.objects.filter(email=from_address)
for u in theusers:
try:
rr = Rower.objects.get(user=u.id)
# move attachment and make workout
try:
wid = [make_new_workout_from_email(rr,a.document,name)]
res += wid
link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
if wid != 1:
dd = send_confirm(u,name,link)
except:
# replace with code to process error
res += ['fail: '+name]
donotdelete = 1
except Rower.DoesNotExist:
pass
# remove attachment
if donotdelete == 0:
a.delete()
donotdelete = 0
m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0]
name = m.subject
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
# get a list of users
theusers = User.objects.filter(email=from_address)
for u in theusers:
try:
rr = Rower.objects.get(user=u.id)
# move attachment and make workout
try:
wid = [make_new_workout_from_email(rr, a.document, name)]
res += wid
link = 'https://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if wid != 1:
dd = send_confirm(u, name, link)
except:
# replace with code to process error
res += ['fail: ' + name]
donotdelete = 1
except Rower.DoesNotExist:
pass
# remove attachment
if donotdelete == 0:
a.delete()
if m.attachments.exists() == False:
# no attachments, so can be deleted
m.delete()
# Delete remaining messages (which should not have attachments)
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
if m.attachments.exists() == False:
m.delete()
return res
# As above, but with some print commands for debugging purposes
def processattachments_debug():
res = []
attachments = MessageAttachment.objects.all()
for a in attachments:
donotdelete = 1
m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0]
name = m.subject
# get a list of users
theusers = User.objects.filter(email=from_address)
donotdelete = 1
m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0]
name = m.subject
# get a list of users
theusers = User.objects.filter(email=from_address)
print theusers
for u in theusers:
for u in theusers:
try:
rr = Rower.objects.get(user=u.id)
rr = Rower.objects.get(user=u.id)
doorgaan = 1
except:
doorgaan = 0
if doorgaan:
# move attachment and make workout
# move attachment and make workout
print a.document
print name
wid = [make_new_workout_from_email(rr,a.document,name)]
wid = [make_new_workout_from_email(rr, a.document, name)]
res += wid
link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
link = 'https://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if wid != 1:
dd = send_confirm(u,name,link)
# remove attachment
if donotdelete == 0:
a.delete()
dd = send_confirm(u, name, link)
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
# remove attachment
if donotdelete == 0:
a.delete()
if m.attachments.exists() == False:
# no attachments, so can be deleted
m.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
if m.attachments.exists() == False:
m.delete()
return res
# Process the attachment file, create new workout
# The code here is duplication of the code in views.py (workout_upload_view)
# Need to move the code to a subroutine used both in views.py and here
def make_new_workout_from_email(rr,f2,name,cntr=0):
def make_new_workout_from_email(rr, f2, name, cntr=0):
workouttype = 'rower'
try:
f2 = f2.name
fileformat = get_file_type('media/'+f2)
fileformat = get_file_type('media/' + f2)
except IOError:
f2 = f2.name+'.gz'
fileformat = get_file_type('media/'+f2)
f2 = f2.name + '.gz'
fileformat = get_file_type('media/' + f2)
except AttributeError:
fileformat = get_file_type('media/'+f2)
fileformat = get_file_type('media/' + f2)
if len(fileformat)==3 and fileformat[0]=='zip':
if len(fileformat) == 3 and fileformat[0] == 'zip':
f_to_be_deleted = f2
with zipfile.ZipFile('media/'+f2) as z:
f2 = z.extract(z.namelist()[0],path='media/')[6:]
with zipfile.ZipFile('media/' + f2) as z:
f2 = z.extract(z.namelist()[0], path='media/')[6:]
fileformat = fileformat[2]
if fileformat == 'unknown':
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2,
"roosendaalsander@gmail.com")
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2,
"roosendaalsander@gmail.com")
else:
res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2,"roosendaalsander@gmail.com")
else:
res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2, "roosendaalsander@gmail.com")
return 1
return 1
summary = ''
# handle non-Painsled
if fileformat != 'csv':
f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary)
f3, summary, oarlength, inboard = dataprep.handle_nonpainsled(
'media/' + f2, fileformat, summary)
else:
f3 = 'media/'+f2
f3 = 'media/' + f2
inboard = 0.88
oarlength = 2.89
# make workout and put in database
#r = rrower(hrmax=rr.max,hrut2=rr.ut2,
# hrut1=rr.ut1,hrat=rr.at,
# hrtr=rr.tr,hran=rr.an,ftp=r.ftp)
row = rdata(f3) #,rower=r)
# r = rrower(hrmax=rr.max,hrut2=rr.ut2,
# hrut1=rr.ut1,hrat=rr.at,
# hrtr=rr.tr,hran=rr.an,ftp=r.ftp)
row = rdata(f3) # ,rower=r)
if row == 0:
return 0
return 0
# change filename
if f2[:5] != 'media':
timestr = time.strftime("%Y%m%d-%H%M%S")
f2 = 'media/'+timestr+str(cntr)+'o.csv'
timestr = time.strftime("%Y%m%d-%H%M%S")
f2 = 'media/' + timestr + str(cntr) + 'o.csv'
try:
avglat = row.df[' latitude'].mean()
avglon = row.df[' longitude'].mean()
avglat = row.df[' latitude'].mean()
avglon = row.df[' longitude'].mean()
if avglat != 0 or avglon != 0:
workouttype = 'water'
except KeyError:
pass
row.write_csv(f2,gzip=True)
row.write_csv(f2, gzip=True)
dosummary = (fileformat != 'fit')
if name == '':
name = 'imported through email'
id,message = dataprep.save_workout_database(f2,rr,
workouttype=workouttype,
dosummary=dosummary,
inboard=inboard,
oarlength=oarlength,
title=name,
workoutsource=fileformat,
notes='imported through email')
name = 'imported through email'
id, message = dataprep.save_workout_database(f2, rr,
workouttype=workouttype,
dosummary=dosummary,
inboard=inboard,
oarlength=oarlength,
title=name,
workoutsource=fileformat,
notes='imported through email')
return id

157
rowers/management/commands/processemail.py
View File

@@ -1,139 +1,137 @@
#!/srv/venv/bin/python
""" Process emails """
import sys
import os
import zipfile
from django.core.management.base import BaseCommand
import time
from time import strftime
from django.conf import settings
from django_mailbox.models import Message, MessageAttachment
from rowers.models import Workout, Rower
from rowingdata import rower as rrower
from rowingdata import rowingdata as rrdata
from rowers.mailprocessing import make_new_workout_from_email, send_confirm
import rowers.uploads as uploads
# If you find a solution that does not need the two paths, please comment!
sys.path.append('$path_to_root_of_project$')
sys.path.append('$path_to_root_of_project$/$project_name$')
os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings'
import zipfile
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
#from rowers.mailprocessing import processattachments
import time
from time import strftime
from django.conf import settings
from rowers.tasks import handle_sendemail_unrecognized
from django_mailbox.models import Mailbox,Message,MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
from django.core.files.base import ContentFile
from rowsandall_app.settings import BASE_DIR
from rowingdata import rower as rrower
from rowingdata import main as rmain
from rowingdata import rowingdata as rrdata
from rowingdata import make_cumvalues
from rowingdata import summarydata,get_file_type
from scipy.signal import savgol_filter
from rowers.mailprocessing import make_new_workout_from_email,send_confirm
import rowers.uploads as uploads
def rdata(file,rower=rrower()):
def rdata(file, rower=rrower()):
""" Read rowing data file and return 0 if file doesn't exist"""
try:
res = rrdata(file,rower=rower)
res = rrdata(file, rower=rower)
except IOError:
res = 0
res = 0
return res
class Command(BaseCommand):
"""Run the Email processing command """
def handle(self, *args, **options):
res = []
attachments = MessageAttachment.objects.all()
cntr = 0
for a in attachments:
res = []
attachments = MessageAttachment.objects.all()
cntr = 0
for a in attachments:
extension = a.document.name[-3:].lower()
donotdelete = 0
m = Message.objects.get(id=a.message_id)
donotdelete = 0
m = Message.objects.get(id=a.message_id)
body = "\n".join(m.text.splitlines())
uploadoptions = uploads.upload_options(body)
from_address = m.from_address[0].lower()
name = m.subject
cntr += 1
# get a list of users
# theusers = User.objects.filter(email=from_address)
from_address = m.from_address[0].lower()
name = m.subject
cntr += 1
# get a list of users
# theusers = User.objects.filter(email=from_address)
ther = [
r for r in Rower.objects.all() if r.user.email.lower() == from_address
]
for rr in ther:
for rr in ther:
if extension == 'zip':
z = zipfile.ZipFile(a.document)
for f in z.namelist():
f2 = z.extract(f,path='media/')
f2 = z.extract(f, path='media/')
title = os.path.basename(f2)
wid = [
make_new_workout_from_email(rr,f2[6:],title)
make_new_workout_from_email(rr, f2[6:], title)
]
res += wid
link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
link = 'http://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if uploadoptions and not 'error' in uploadoptions:
w = Workout.objects.get(id=wid[0])
r = w.user
uploads.do_sync(w,uploadoptions)
uploads.make_private(w,uploadoptions)
uploads.do_sync(w, uploadoptions)
uploads.make_private(w, uploadoptions)
if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype']
f1 = w.csvfilename[6:-4]
f1 = w.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S")
imagename = f1+timestr+'.png'
resu = uploads.make_plot(r,w,f1,
imagename = f1 + timestr + '.png'
resu = uploads.make_plot(r, w, f1,
w.csvfilename,
plottype,name,
plottype, name,
imagename=imagename)
try:
if wid != 1:
dd = send_confirm(rr.user,title,link,
dd = send_confirm(rr.user, title, link,
uploadoptions)
time.sleep(10)
except:
try:
time.sleep(10)
if wid != 1:
dd = send_confirm(rr.user,title,link,
dd = send_confirm(rr.user, title, link,
uploadoptions)
except:
pass
else:
# move attachment and make workout
try:
wid = [
# move attachment and make workout
try:
wid = [
make_new_workout_from_email(rr,
a.document,
name)
]
res += wid
link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
link = 'http://rowsandall.com/rowers/workout/' + \
str(wid[0]) + '/edit'
if uploadoptions:
w = Workout.objects.get(id=wid[0])
r = w.user
uploads.do_sync(w,uploadoptions)
uploads.make_private(w,uploadoptions)
uploads.do_sync(w, uploadoptions)
uploads.make_private(w, uploadoptions)
if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype']
f1 = w.csvfilename[6:-4]
f1 = w.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S")
imagename = f1+timestr+'.png'
resu = uploads.make_plot(r,w,f1,
imagename = f1 + timestr + '.png'
resu = uploads.make_plot(r, w, f1,
w.csvfilename,
plottype,name,
plottype, name,
imagename=imagename)
except:
# replace with code to process error
res += ['fail: '+name]
donotdelete = 1
except:
# replace with code to process error
res += ['fail: ' + name]
donotdelete = 1
wid = 1
try:
if wid != 1:
dd = send_confirm(rr.user,name,link,
dd = send_confirm(rr.user, name, link,
uploadoptions)
time.sleep(10)
except:
@@ -144,19 +142,16 @@ class Command(BaseCommand):
except IOError:
pass
# remove attachment
#if donotdelete == 0:
# if donotdelete == 0:
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
if m.attachments.exists() is False:
# no attachments, so can be deleted
m.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists() is False:
m.delete()
self.stdout.write(self.style.SUCCESS('Successfully processed email attachments'))
self.stdout.write(self.style.SUCCESS(
'Successfully processed email attachments'))