Private
Public Access
1
0

Merge branch 'feature/emailprocessing' into develop

This commit is contained in:
Sander Roosendaal
2017-10-22 16:06:23 +02:00
4 changed files with 199 additions and 312 deletions

View File

@@ -1,24 +1,19 @@
# Processes emails sent to workouts@rowsandall.com
""" Processes emails sent to workouts@rowsandall.com """
import shutil
import time
from django.conf import settings
from rowers.tasks import handle_sendemail_unrecognized
from django_mailbox.models import Mailbox,Message,MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
from django.core.files.base import ContentFile
from django.core.mail import send_mail, BadHeaderError,EmailMessage
from rowsandall_app.settings import BASE_DIR
from django_mailbox.models import Message, MessageAttachment
from rowers.models import User, Rower, RowerForm
from django.core.mail import EmailMessage
from rowingdata import rower as rrower
from rowingdata import main as rmain
from rowingdata import rowingdata as rrdata
from rowingdata import TCXParser,RowProParser,ErgDataParser
from rowingdata import MysteryParser,BoatCoachParser
from rowingdata import painsledDesktopParser,speedcoachParser,ErgStickParser
from rowingdata import SpeedCoach2Parser,FITParser,fitsummarydata
from rowingdata import make_cumvalues
from rowingdata import summarydata,get_file_type
from scipy.signal import savgol_filter
from rowingdata import rowingdata as rrdata
from rowingdata import get_file_type
import zipfile
import os
@@ -30,19 +25,21 @@ queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('default')
# Sends a confirmation with a link to the workout
def send_confirm(u,name,link,options):
fullemail = u.email
subject = 'Workout added: '+name
message = 'Dear '+u.first_name+',\n\n'
def send_confirm(user, name, link, options):
""" Send confirmation email to user when email has been processed """
fullemail = user.email
subject = 'Workout added: ' + name
message = 'Dear ' + user.first_name + ',\n\n'
message += "Your workout has been added to Rowsandall.com.\n"
message += "Link to workout: "+link+"\n\n"
message += "Link to workout: " + link + "\n\n"
message += "Best Regards, the Rowsandall Team"
if options:
message += "\n\n"+str(options)
message += "\n\n" + str(options)
email = EmailMessage(subject,message,
email = EmailMessage(subject, message,
'Rowsandall <info@rowsandall.com>',
[fullemail])
@@ -51,203 +48,102 @@ def send_confirm(u,name,link,options):
return 1
# Reads a "rowingdata" object, plus some error protections
def rdata(file,rower=rrower()):
def rdata(file, rower=rrower()):
""" Reads rowingdata data or returns 0 on Error """
try:
res = rrdata(file,rower=rower)
result = rrdata(file, rower=rower)
except IOError:
try:
res = rrdata(file+'.gz',rower=rower)
result = rrdata(file + '.gz', rower=rower)
except IOError:
res = 0
result = 0
return res
return result
# Some error protection around process attachments
def safeprocessattachments():
try:
return processattachments()
except:
return [0]
# This is duplicated in management/commands/processemail
# Need to double check the code there, update here, and only
# use the code here.
def processattachments():
# in res, we store the ids of the new workouts
res = []
attachments = MessageAttachment.objects.all()
for a in attachments:
donotdelete = 0
m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0]
name = m.subject
# get a list of users
theusers = User.objects.filter(email=from_address)
for u in theusers:
try:
rr = Rower.objects.get(user=u.id)
# move attachment and make workout
try:
wid = [make_new_workout_from_email(rr,a.document,name)]
res += wid
link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
if wid != 1:
dd = send_confirm(u,name,link)
except:
# replace with code to process error
res += ['fail: '+name]
donotdelete = 1
except Rower.DoesNotExist:
pass
# remove attachment
if donotdelete == 0:
a.delete()
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
# Delete remaining messages (which should not have attachments)
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
return res
# As above, but with some print commands for debugging purposes
def processattachments_debug():
res = []
attachments = MessageAttachment.objects.all()
for a in attachments:
donotdelete = 1
m = Message.objects.get(id=a.message_id)
from_address = m.from_address[0]
name = m.subject
# get a list of users
theusers = User.objects.filter(email=from_address)
print theusers
for u in theusers:
try:
rr = Rower.objects.get(user=u.id)
doorgaan = 1
except:
doorgaan = 0
if doorgaan:
# move attachment and make workout
print a.document
print name
wid = [make_new_workout_from_email(rr,a.document,name)]
res += wid
link = 'https://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
if wid != 1:
dd = send_confirm(u,name,link)
# remove attachment
if donotdelete == 0:
a.delete()
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
return res
# Process the attachment file, create new workout
# The code here is duplication of the code in views.py (workout_upload_view)
# Need to move the code to a subroutine used both in views.py and here
def make_new_workout_from_email(rr,f2,name,cntr=0):
def make_new_workout_from_email(rower, datafile, name, cntr=0):
""" This one is used in processemail """
workouttype = 'rower'
try:
f2 = f2.name
fileformat = get_file_type('media/'+f2)
datafilename = datafile.name
fileformat = get_file_type('media/' + datafilename)
except IOError:
f2 = f2.name+'.gz'
fileformat = get_file_type('media/'+f2)
datafilename = datafile.name + '.gz'
fileformat = get_file_type('media/' + datafilename)
except AttributeError:
fileformat = get_file_type('media/'+f2)
datafilename = datafile
fileformat = get_file_type('media/' + datafile)
if len(fileformat)==3 and fileformat[0]=='zip':
f_to_be_deleted = f2
with zipfile.ZipFile('media/'+f2) as z:
f2 = z.extract(z.namelist()[0],path='media/')[6:]
if len(fileformat) == 3 and fileformat[0] == 'zip':
with zipfile.ZipFile('media/' + datafilename) as zip_file:
datafilename = zip_file.extract(
zip_file.namelist()[0],
path='media/')[6:]
fileformat = fileformat[2]
if fileformat == 'unknown':
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(f2,
"roosendaalsander@gmail.com")
# extension = datafilename[-4:].lower()
# fcopy = "media/"+datafilename[:-4]+"_copy"+extension
# with open('media/'+datafilename, 'r') as f_in, open(fcopy, 'w') as f_out:
# shutil.copyfileobj(f_in,f_out)
fcopy = "media/"+datafilename
if settings.DEBUG:
res = handle_sendemail_unrecognized.delay(fcopy,
rower.user.email)
else:
res = queuehigh.enqueue(handle_sendemail_unrecognized,
f2,"roosendaalsander@gmail.com")
else:
res = queuehigh.enqueue(handle_sendemail_unrecognized,
fcopy,
rower.user.email)
return 0
return 1
summary = ''
# handle non-Painsled
if fileformat != 'csv':
f3,summary,oarlength,inboard = dataprep.handle_nonpainsled('media/'+f2,fileformat,summary)
filename_mediadir, summary, oarlength, inboard = dataprep.handle_nonpainsled(
'media/' + datafilename, fileformat, summary)
else:
f3 = 'media/'+f2
filename_mediadir = 'media/' + datafilename
inboard = 0.88
oarlength = 2.89
# make workout and put in database
#r = rrower(hrmax=rr.max,hrut2=rr.ut2,
# hrut1=rr.ut1,hrat=rr.at,
# hrtr=rr.tr,hran=rr.an,ftp=r.ftp)
row = rdata(f3) #,rower=r)
row = rdata(filename_mediadir)
if row == 0:
return 0
return 0
# change filename
if f2[:5] != 'media':
timestr = time.strftime("%Y%m%d-%H%M%S")
f2 = 'media/'+timestr+str(cntr)+'o.csv'
if datafilename[:5] != 'media':
timestr = time.strftime("%Y%m%d-%H%M%S")
datafilename = 'media/' + timestr + str(cntr) + 'o.csv'
try:
avglat = row.df[' latitude'].mean()
avglon = row.df[' longitude'].mean()
avglat = row.df[' latitude'].mean()
avglon = row.df[' longitude'].mean()
if avglat != 0 or avglon != 0:
workouttype = 'water'
except KeyError:
pass
row.write_csv(f2,gzip=True)
row.write_csv(datafilename, gzip=True)
dosummary = (fileformat != 'fit')
if name == '':
name = 'imported through email'
id,message = dataprep.save_workout_database(f2,rr,
workouttype=workouttype,
dosummary=dosummary,
inboard=inboard,
oarlength=oarlength,
title=name,
workoutsource=fileformat,
notes='imported through email')
name = 'imported through email'
id, message = dataprep.save_workout_database(
datafilename, rower,
workouttype=workouttype,
dosummary=dosummary,
inboard=inboard,
oarlength=oarlength,
title=name,
workoutsource=fileformat,
notes='imported through email'
)
return id

View File

@@ -1,162 +1,148 @@
#!/srv/venv/bin/python
""" Process emails """
import sys
import os
import zipfile
import time
from time import strftime
from django.core.management.base import BaseCommand
from django_mailbox.models import Message, MessageAttachment
from django.core.urlresolvers import reverse
from django.conf import settings
from rowers.models import Workout, Rower
from rowingdata import rower as rrower
from rowingdata import rowingdata as rrdata
import rowers.uploads as uploads
from rowers.mailprocessing import make_new_workout_from_email, send_confirm
# If you find a solution that does not need the two paths, please comment!
sys.path.append('$path_to_root_of_project$')
sys.path.append('$path_to_root_of_project$/$project_name$')
os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings'
import zipfile
if not getattr(__builtins__, "WindowsError", None):
class WindowsError(OSError): pass
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
#from rowers.mailprocessing import processattachments
import time
from time import strftime
from django.conf import settings
from rowers.tasks import handle_sendemail_unrecognized
from django_mailbox.models import Mailbox,Message,MessageAttachment
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
from django.core.files.base import ContentFile
from rowsandall_app.settings import BASE_DIR
from rowingdata import rower as rrower
from rowingdata import main as rmain
from rowingdata import rowingdata as rrdata
from rowingdata import make_cumvalues
from rowingdata import summarydata,get_file_type
from scipy.signal import savgol_filter
from rowers.mailprocessing import make_new_workout_from_email,send_confirm
import rowers.uploads as uploads
def rdata(file,rower=rrower()):
def rdata(file_obj, rower=rrower()):
""" Read rowing data file and return 0 if file doesn't exist"""
try:
res = rrdata(file,rower=rower)
result = rrdata(file_obj, rower=rower)
except IOError:
res = 0
result = 0
return res
return result
def processattachment(rower, fileobj, title, uploadoptions):
try:
filename = fileobj.name
except AttributeError:
filename = fileobj[6:]
workoutid = [
make_new_workout_from_email(rower, filename, title)
]
if workoutid[0]:
link = settings.SITE_URL+reverse(
rower.defaultlandingpage,
kwargs = {
'id':workoutid[0],
}
)
if uploadoptions and not 'error' in uploadoptions:
workout = Workout.objects.get(id=workoutid[0])
uploads.do_sync(workout, uploadoptions)
uploads.make_private(workout, uploadoptions)
if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype']
workoutcsvfilename = workout.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S")
imagename = workoutcsvfilename + timestr + '.png'
result = uploads.make_plot(
workout.user, workout, workoutcsvfilename,
workout.csvfilename,
plottype, title,
imagename=imagename
)
try:
if workoutid:
email_sent = send_confirm(
rower.user, title, link,
uploadoptions
)
time.sleep(10)
except:
try:
time.sleep(10)
if workoutid:
email_sent = send_confirm(
rower.user, title, link,
uploadoptions
)
except:
pass
return workoutid
class Command(BaseCommand):
"""Run the Email processing command """
def handle(self, *args, **options):
res = []
attachments = MessageAttachment.objects.all()
cntr = 0
for a in attachments:
extension = a.document.name[-3:].lower()
donotdelete = 0
m = Message.objects.get(id=a.message_id)
body = "\n".join(m.text.splitlines())
attachments = MessageAttachment.objects.all()
cntr = 0
for attachment in attachments:
extension = attachment.document.name[-3:].lower()
message = Message.objects.get(id=attachment.message_id)
body = "\n".join(message.text.splitlines())
uploadoptions = uploads.upload_options(body)
from_address = m.from_address[0].lower()
name = m.subject
cntr += 1
# get a list of users
# theusers = User.objects.filter(email=from_address)
ther = [
from_address = message.from_address[0].lower()
name = message.subject
# get a list of users
# theusers = User.objects.filter(email=from_address)
rowers = [
r for r in Rower.objects.all() if r.user.email.lower() == from_address
]
for rr in ther:
for rower in rowers:
if extension == 'zip':
z = zipfile.ZipFile(a.document)
for f in z.namelist():
f2 = z.extract(f,path='media/')
title = os.path.basename(f2)
wid = [
make_new_workout_from_email(rr,f2[6:],title)
]
res += wid
link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
if uploadoptions and not 'error' in uploadoptions:
w = Workout.objects.get(id=wid[0])
r = w.user
uploads.do_sync(w,uploadoptions)
uploads.make_private(w,uploadoptions)
if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype']
f1 = w.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S")
imagename = f1+timestr+'.png'
resu = uploads.make_plot(r,w,f1,
w.csvfilename,
plottype,name,
imagename=imagename)
try:
if wid != 1:
dd = send_confirm(rr.user,title,link,
uploadoptions)
time.sleep(10)
except:
try:
time.sleep(10)
if wid != 1:
dd = send_confirm(rr.user,title,link,
uploadoptions)
except:
pass
zip_file = zipfile.ZipFile(attachment.document)
for filename in zip_file.namelist():
datafile = zip_file.extract(filename, path='media/')
title = os.path.basename(datafile)
workoutid = processattachment(
rower, datafile, title, uploadoptions
)
else:
# move attachment and make workout
try:
wid = [
make_new_workout_from_email(rr,
a.document,
name)
]
res += wid
link = 'http://rowsandall.com/rowers/workout/'+str(wid[0])+'/edit'
if uploadoptions:
w = Workout.objects.get(id=wid[0])
r = w.user
uploads.do_sync(w,uploadoptions)
uploads.make_private(w,uploadoptions)
if 'make_plot' in uploadoptions:
plottype = uploadoptions['plottype']
f1 = w.csvfilename[6:-4]
timestr = strftime("%Y%m%d-%H%M%S")
imagename = f1+timestr+'.png'
resu = uploads.make_plot(r,w,f1,
w.csvfilename,
plottype,name,
imagename=imagename)
except:
# replace with code to process error
res += ['fail: '+name]
donotdelete = 1
wid = 1
try:
if wid != 1:
dd = send_confirm(rr.user,name,link,
uploadoptions)
time.sleep(10)
except:
pass
# move attachment and make workout
workoutid = processattachment(
rower, attachment.document, name, uploadoptions
)
# We're done with the attachment. It can be deleted
try:
a.delete()
attachment.delete()
except IOError:
pass
# remove attachment
#if donotdelete == 0:
except WindowsError:
time.sleep(2)
try:
attachment.delete()
except WindowsError:
pass
if m.attachments.exists()==False:
# no attachments, so can be deleted
m.delete()
if message.attachments.exists() is False:
# no attachments, so can be deleted
message.delete()
mm = Message.objects.all()
for m in mm:
if m.attachments.exists()==False:
m.delete()
messages = Message.objects.all()
for message in messages:
if message.attachments.exists() is False:
message.delete()
self.stdout.write(self.style.SUCCESS('Successfully processed email attachments'))
self.stdout.write(self.style.SUCCESS(
'Successfully processed email attachments'))

View File

@@ -261,6 +261,9 @@ TP_CLIENT_SECRET = CFG["tp_client_secret"]
TP_REDIRECT_URI = CFG["tp_redirect_uri"]
TP_CLIENT_KEY = TP_CLIENT_ID
# Full Site URL
SITE_URL = "https://rowsandall.com"
# RQ stuff
RQ_QUEUES = {

View File

@@ -76,6 +76,8 @@ LOGIN_REDIRECT_URL = '/rowers/list-workouts/'
SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"
SITE_URL = "http://localhost:8000"
#EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
#EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend'