From 537b7392e62806f2d272fe2453fc55ac4135852d Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 30 Oct 2017 17:27:13 +0100 Subject: [PATCH 1/9] works on development - needs testing on production --- rowers/templates/async_tasks.html | 56 ++++++++ rowers/urls.py | 3 + rowers/views.py | 219 ++++++++++++++++++++++++++---- 3 files changed, 250 insertions(+), 28 deletions(-) create mode 100644 rowers/templates/async_tasks.html diff --git a/rowers/templates/async_tasks.html b/rowers/templates/async_tasks.html new file mode 100644 index 00000000..0828105d --- /dev/null +++ b/rowers/templates/async_tasks.html @@ -0,0 +1,56 @@ +{% extends "base.html" %} +{% load staticfiles %} +{% load rowerfilters %} + +{% block title %}Rowsandall - Tasks {% endblock %} + +{% block content %} + +

Your Tasks Status

+

Manage the asynchronous tasks running for you

+ + +
+ + + + + + + + + + + {% for task in taskstatus %} + + + + + {% if task|lookup:'failed' %} + + {% elif task|lookup:'finished' %} + + {% else %} + + {% endif %} + + {% endfor %} + +
IDTaskStatusAction
+ {{ task|lookup:'id' }} + + {{ task|lookup:'verbose' }} + + {{ task|lookup:'status' }} + + Remove + + Remove + +   +
+
+ + + +{% endblock %} diff --git a/rowers/urls.py b/rowers/urls.py index 9a4653f9..84a78222 100644 --- a/rowers/urls.py +++ b/rowers/urls.py @@ -138,6 +138,9 @@ urlpatterns = [ url(r'^user-multiflex-select/user/(?P\d+)/$',views.user_multiflex_select), url(r'^user-multiflex-select/(?P\w+.*)/(?P\w+.*)$',views.user_multiflex_select), url(r'^user-multiflex-select/$',views.user_multiflex_select), + url(r'^list-jobs/$',views.session_jobs_view), + url(r'^jobs-status/$',views.session_jobs_status), + url(r'^job-kill/(?P.*)$',views.kill_async_job), url(r'^list-graphs/$',views.graphs_view), url(r'^(?P\d+)/ote-bests/(?P\w+.*)/(?P\w+.*)$',views.rankings_view), url(r'^(?P\d+)/ote-bests/(?P\d+)$',views.rankings_view), diff --git a/rowers/views.py b/rowers/views.py index e44ba678..fdcb429e 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -130,14 +130,17 @@ import django_rq queue = django_rq.get_queue('default') queuelow = django_rq.get_queue('low') queuehigh = django_rq.get_queue('low') - -from redis import StrictRedis + +from redis import StrictRedis,Redis from rq.exceptions import NoSuchJobError from rq.registry import StartedJobRegistry -from rq import Queue +from rq import Queue,cancel_job +queuefailed = Queue("failed",connection=Redis()) redis_connection = StrictRedis() rq_registry = StartedJobRegistry(queue.name,connection=redis_connection) +rq_registryhigh = StartedJobRegistry(queuehigh.name,connection=redis_connection) +rq_registrylow = StartedJobRegistry(queuelow.name,connection=redis_connection) from rq.job import Job @@ -170,6 +173,20 @@ from rowers.celery import result as celery_result # Define the API documentation schema_view = get_swagger_view(title='Rowsandall API') +def remove_asynctask(request,id): + try: + oldtasks = request.session['async_tasks'] + except KeyError: + oldtasks = [] + + newtasks = [] + for task in oldtasks: + print task[0] + if id not in task[0]: + newtasks += [(task[0],task[1])] + + request.session['async_tasks'] = newtasks + def get_job_result(jobid): if settings.DEBUG: result = celery_result.AsyncResult(jobid).result @@ -188,6 +205,131 @@ def get_job_result(jobid): return result +verbose_job_status = { + 'updatecp': 'Critical Power Calculation for Ergometer Workouts', + 'updatecpwater': 'Critical Power Calculation for OTW Workouts', + 'otwsetpower': 'Rowing Physics OTW Power Calculation' + } + +def get_job_status(jobid): + if settings.DEBUG: + job = celery_result.AsyncResult(jobid) + jobresult = job.result + if 'fail' in job.status.lower(): + jobresult = '0' + summary = { + 'status': job.status, + 'result': jobresult, + } + else: + try: + job = Job.fetch(jobid,connection=redis_connection) + summary = { + 'status':job.status, + 'result':job.result + } + except NoSuchJobError: + summary = {} + + + if 'fail' in summary['status'].lower(): + summary['failed'] = True + else: + summary['failed'] = False + + if 'success' in summary['status'].lower(): + summary['finished'] = True + elif 'finished' in summary['status'].lower(): + summary['finished'] = True + else: + summary['finished'] = False + + return summary + +def kill_async_job(request,id='aap'): + if settings.DEBUG: + job = celery_result.AsyncResult(id) + job.revoke() + remove_asynctask(request,id) + else: + try: + cancel_job(id) + except NoSuchJobError: + pass + + url = reverse(session_jobs_status) + + return HttpResponseRedirect(url) + + +def get_all_queued_jobs(userid=0): + r = StrictRedis() + + jobs = [] + + celerykeys = r.keys('celery*') + for key in celerykeys: + id= key[17:] + job = celery_result.AsyncResult(id) + jobresult = job.result + if 'fail' in job.status.lower(): + jobresult = '0' + jobs.append( + (id,{ + 'status':job.status, + 'result':jobresult, + 'function':'', + 'meta':job.info, + })) + + ids = [j.id for j in queue.jobs] + ids += [j.id for j in queuehigh.jobs] + ids += [j.id for j in queuelow.jobs] + ids += [j.id for j in queuefailed.jobs] + + + for id in ids: + job = Job.fetch(id,connection=redis_connection) + jobs.append( + (id,{ + 'status':job.get_status(), + 'result':job.result, + 'function':job.func_name, + 'meta':job.meta, + })) + + return jobs + +def get_stored_tasks_status(request): + try: + taskids = request.session['async_tasks'] + except KeyError: + taskids = [] + + taskstatus = [{ + 'id':id, + 'status':get_job_status(id)['status'], + 'failed':get_job_status(id)['failed'], + 'finished':get_job_status(id)['finished'], + 'func_name':func_name, + 'verbose': verbose_job_status[func_name] + } for id,func_name in taskids] + + return taskstatus + +@login_required() +def session_jobs_view(request): + taskstatus = get_stored_tasks_status(request) + + return HttpResponse(json.dumps(taskstatus)) + +@login_required() +def session_jobs_status(request): + taskstatus = get_stored_tasks_status(request) + + return render(request, + 'async_tasks.html', + {'taskstatus':taskstatus}) # Test if row data include candidates def rowhascoordinates(row): @@ -3109,19 +3251,24 @@ def otwrankings_view(request,theuser=0, runningjob = 0 - try: - jobid = request.session['job_id'] - if jobid: - result = get_job_result(jobid) - if result: - messages.info(request,'Your calculation is ready') + taskstatus = get_stored_tasks_status(request) + for task in taskstatus: + if task['func_name'] == 'updatecpwater': + if 'success' in task['status'].lower() or 'finished' in task['status'].lower(): runningjob = 1 - request.session['job_id'] = 0 - else: + messages.info(request,'CP chart data have been updated') + remove_asynctask(request,task['id']) + elif 'fail' in task['status'].lower(): + runningjob = 0 + remove_asynctask(request,task[id]) + messages.error(request,'Oh, your task failed') + elif 'started' in task['status'].lower(): + messages.info(request,'Busy updating CP chart data') runningjob = 1 - messages.info(request,'Your job is still running') - except KeyError: - pass + elif 'queued' in task['status'].lower() or 'pending' in task['status'].lower(): + messages.info(request,'Getting ready to update CP chart data') + runningjob = 1 + if not runningjob: @@ -3131,6 +3278,10 @@ def otwrankings_view(request,theuser=0, enddate=enddate ) request.session['job_id'] = job.id + try: + request.session['async_tasks'] += [(job.id,'updatecpwater')] + except KeyError: + request.session['async_tasks'] = [(job.id,'updatecpwater')] messages.info(request,'New calculation queued. Refresh page or resubmit the date form to get the result') powerdf = pd.DataFrame({ @@ -3358,19 +3509,23 @@ def oterankings_view(request,theuser=0, runningjob = 0 - try: - jobid = request.session['job_id'] - if jobid: - result = get_job_result(jobid) - if result: - messages.info(request,'Your calculation is ready') + taskstatus = get_stored_tasks_status(request) + for task in taskstatus: + if task['func_name'] == 'updatecp': + if 'success' in task['status'].lower() or 'finished' in task['status'].lower(): runningjob = 1 - request.session['job_id'] = 0 - else: + messages.info(request,'CP chart data have been updated') + remove_asynctask(request,task['id']) + elif 'fail' in task['status'].lower(): + runningjob = 0 + remove_asynctask(request,task[id]) + messages.error(request,'Oh, your task failed') + elif 'started' in task['status'].lower(): + messages.info(request,'Busy updating CP chart data') + runningjob = 1 + elif 'queued' in task['status'].lower(): + messages.info(request,'Getting ready to update CP chart data') runningjob = 1 - messages.info(request,'Your job is still running') - except KeyError: - pass if not runningjob: @@ -3380,6 +3535,10 @@ def oterankings_view(request,theuser=0, enddate=enddate ) request.session['job_id'] = job.id + try: + request.session['async_tasks'] += [(job.id,'updatecp')] + except KeyError: + request.session['async_tasks'] = [(job.id,'updatecp')] messages.info(request,'New calculation queued. Refresh page or resubmit the date form to get the result') powerdf = pd.DataFrame({ @@ -5504,19 +5663,23 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): emailaddress = u.email if settings.DEBUG: - res = handle_otwsetpower.delay(f1,boattype,weightvalue, + job = handle_otwsetpower.delay(f1,boattype,weightvalue, first_name,last_name, emailaddress,id,debug=True, ps=[r.p0,r.p1,r.p2,r.p3], ratio=r.cpratio) else: - res = queuelow.enqueue(handle_otwsetpower,f1,boattype, + job = queuelow.enqueue(handle_otwsetpower,f1,boattype, weightvalue, first_name,last_name,emailaddress,id, ps=[r.p0,r.p1,r.p2,r.p3], ratio=r.cpratio) - + try: + request.session['async_tasks'] += [(job.id,'otwsetpower')] + except KeyError: + request.session['async_tasks'] = [(job.id,'otwsetpower')] + successmessage = "Your calculations have been submitted. You will receive an email when they are done." messages.info(request,successmessage) kwargs = { From 5fe6db02dcd6617811afa9d89f68278aec6f346e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 30 Oct 2017 17:33:58 +0100 Subject: [PATCH 2/9] bug fix --- rowers/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rowers/views.py b/rowers/views.py index fdcb429e..8b5d4f7a 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -250,13 +250,13 @@ def kill_async_job(request,id='aap'): if settings.DEBUG: job = celery_result.AsyncResult(id) job.revoke() - remove_asynctask(request,id) else: try: - cancel_job(id) + cancel_job(id,connect=redis_connection) except NoSuchJobError: pass + remove_asynctask(request,id) url = reverse(session_jobs_status) return HttpResponseRedirect(url) From 21be4d278b14cf4b4127d0655548775b7783aabe Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 30 Oct 2017 21:22:07 +0100 Subject: [PATCH 3/9] bux fix --- rowers/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rowers/views.py b/rowers/views.py index 8b5d4f7a..c0c6ab01 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -5680,7 +5680,7 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): except KeyError: request.session['async_tasks'] = [(job.id,'otwsetpower')] - successmessage = "Your calculations have been submitted. You will receive an email when they are done." + successmessage = 'Your calculations have been submitted. You will receive an email when they are done. You can check the status of your calculations here' messages.info(request,successmessage) kwargs = { 'id':int(id)} From 98abc68289ac606573e2e09ab94aaa95f7b16dd9 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Mon, 30 Oct 2017 22:32:34 +0100 Subject: [PATCH 4/9] with jquery to refresh when tasks are complete jquery lives in basebase.html. Perhaps should live in individual pages --- rowers/management/commands/processemail.py | 2 +- rowers/templates/async_tasks.html | 1 + rowers/uploads.py | 6 +++--- rowers/views.py | 16 +++++++++----- templates/basebase.html | 25 ++++++++++++++++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index a0413753..e4f88e28 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -65,7 +65,7 @@ def processattachment(rower, fileobj, title, uploadoptions): workoutcsvfilename = workout.csvfilename[6:-4] timestr = strftime("%Y%m%d-%H%M%S") imagename = workoutcsvfilename + timestr + '.png' - result = uploads.make_plot( + result,jobid = uploads.make_plot( workout.user, workout, workoutcsvfilename, workout.csvfilename, plottype, title, diff --git a/rowers/templates/async_tasks.html b/rowers/templates/async_tasks.html index 0828105d..1986acfb 100644 --- a/rowers/templates/async_tasks.html +++ b/rowers/templates/async_tasks.html @@ -6,6 +6,7 @@ {% block content %} +

Your Tasks Status

Manage the asynchronous tasks running for you

diff --git a/rowers/uploads.py b/rowers/uploads.py index 578d1807..a6fab4a1 100644 --- a/rowers/uploads.py +++ b/rowers/uploads.py @@ -175,11 +175,11 @@ def make_plot(r,w,f1,f2,plottype,title,imagename='',plotnr=0): if settings.DEBUG: - res = handle_makeplot.delay(f1,f2,title, + job = handle_makeplot.delay(f1,f2,title, hrpwrdata,plotnr, imagename) else: - res = queue.enqueue(handle_makeplot,f1,f2, + job = queue.enqueue(handle_makeplot,f1,f2, title,hrpwrdata, plotnr,imagename) @@ -195,7 +195,7 @@ def make_plot(r,w,f1,f2,plottype,title,imagename='',plotnr=0): width=width,height=height) i.save() - return i.id + return i.id,job.id import c2stuff,stravastuff,sporttracksstuff,runkeeperstuff import underarmourstuff,tpstuff diff --git a/rowers/views.py b/rowers/views.py index c0c6ab01..4cd08a09 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -208,7 +208,8 @@ def get_job_result(jobid): verbose_job_status = { 'updatecp': 'Critical Power Calculation for Ergometer Workouts', 'updatecpwater': 'Critical Power Calculation for OTW Workouts', - 'otwsetpower': 'Rowing Physics OTW Power Calculation' + 'otwsetpower': 'Rowing Physics OTW Power Calculation', + 'make_plot': 'Create static chart', } def get_job_status(jobid): @@ -3539,7 +3540,7 @@ def oterankings_view(request,theuser=0, request.session['async_tasks'] += [(job.id,'updatecp')] except KeyError: request.session['async_tasks'] = [(job.id,'updatecp')] - messages.info(request,'New calculation queued. Refresh page or resubmit the date form to get the result') + messages.info(request,'New calculation queued.') powerdf = pd.DataFrame({ 'Delta':delta, @@ -7468,9 +7469,14 @@ def workout_add_chart_view(request,id,plotnr=1): u = w.user.user r = getrower(u) title = w.name - res = uploads.make_plot(r,w,f1,w.csvfilename,'timeplot',title,plotnr=plotnr, - imagename=imagename) - + res,jobid = uploads.make_plot( + r,w,f1,w.csvfilename,'timeplot',title,plotnr=plotnr, + imagename=imagename + ) + try: + request.session['async_tasks'] += [(jobid,'make_plot')] + except KeyError: + request.session['async_tasks'] = [(jobid,'make_plot')] try: url = request.session['referer'] diff --git a/templates/basebase.html b/templates/basebase.html index 96cd74d4..1379b2c4 100644 --- a/templates/basebase.html +++ b/templates/basebase.html @@ -9,6 +9,31 @@ + + + {% analytical_head_top %} {% if GOOGLE_ANALYTICS_PROPERTY_ID %} From 092d24d9b24fde0066f9d459dade58bff5b79f9d Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 31 Oct 2017 08:37:47 +0100 Subject: [PATCH 5/9] reload uses timeout --- templates/basebase.html | 42 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/templates/basebase.html b/templates/basebase.html index 1379b2c4..33184670 100644 --- a/templates/basebase.html +++ b/templates/basebase.html @@ -9,29 +9,31 @@ - - + + - setInterval(function() { - $.getJSON('http://localhost:8000/rowers/list-jobs/', function(json) { + + From 8c31f5b7786c2690698b97a1fdf8e581d84d6485 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 31 Oct 2017 11:12:00 +0100 Subject: [PATCH 6/9] a wrapper around the task queue --- rowers/tasks.py | 21 ++++-- rowers/utils.py | 13 +++- rowers/views.py | 184 ++++++++++++++++-------------------------------- 3 files changed, 85 insertions(+), 133 deletions(-) diff --git a/rowers/tasks.py b/rowers/tasks.py index bbb3c88f..e1b87ae9 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -30,7 +30,7 @@ from rowers.dataprepnodjango import ( ) from django.core.mail import send_mail, EmailMessage - +from django.db.utils import OperationalError import datautils import utils @@ -376,9 +376,13 @@ def handle_otwsetpower(f1, boattype, weightvalue, res, btvalues, res2 = utils.isbreakthrough( delta, cpvalues, ps[0], ps[1], ps[2], ps[3], ratio) if res: - handle_sendemail_breakthrough(workoutid, email, - first_name, - last_name, btvalues=btvalues.to_json()) + try: + handle_sendemail_breakthrough( + workoutid, email, + first_name, + last_name, btvalues=btvalues.to_json()) + except OperationalError: + pass # send email fullemail = first_name + " " + last_name + " " + "<" + email + ">" @@ -394,9 +398,12 @@ def handle_otwsetpower(f1, boattype, weightvalue, message += "Please report any bugs/inconsistencies/unexpected results at rowsandall.slack.com or by reply to this email.\n\n" message += "Best Regards, The Rowsandall Physics Department." - send_mail(subject, message, - 'Rowsandall Physics Department ', - [fullemail]) + try: + send_mail(subject, message, + 'Rowsandall Physics Department ', + [fullemail]) + except OperationalError: + return 0 return 1 diff --git a/rowers/utils.py b/rowers/utils.py index 71f23547..f4081639 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -2,6 +2,8 @@ import math import numpy as np import pandas as pd import colorsys +from django.conf import settings + lbstoN = 4.44822 @@ -100,7 +102,7 @@ def range_to_color_hex(groupcols,palette='monochrome_blue'): try: plt = palettes[palette] - except KeyErro: + except KeyError: plt = palettes['monochrome_blue'] rgb = [colorsys.hsv_to_rgb((plt[0]+plt[1]*x)/360., @@ -214,3 +216,12 @@ def isbreakthrough(delta,cpvalues,p0,p1,p2,p3,ratio): return res>1,btdf,res2>1 + + +def myqueue(queue,function,*args,**kwargs): + if settings.DEBUG: + job = function.delay(*args,**kwargs) + else: + job = queue.enqueue(function,*args,**kwargs) + + return job diff --git a/rowers/views.py b/rowers/views.py index 4cd08a09..72ae1af9 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -188,7 +188,7 @@ def remove_asynctask(request,id): request.session['async_tasks'] = newtasks def get_job_result(jobid): - if settings.DEBUG: + if settings.EBUG: result = celery_result.AsyncResult(jobid).result else: running_job_ids = rq_registry.get_job_ids() @@ -476,7 +476,7 @@ def splitstdata(lijst): from utils import ( geo_distance,serialize_list,deserialize_list,uniqify, - str2bool,range_to_color_hex,absolute + str2bool,range_to_color_hex,absolute,myqueue ) import datautils @@ -1298,25 +1298,21 @@ def workout_tcxemail_view(request,id=0): except Workout.DoesNotExist: raise Http404("Workout doesn't exist") if (checkworkoutuser(request.user,w)): - try: - tcxfile,tcxmessg = stravastuff.createstravaworkoutdata(w,dozip=False) - if tcxfile == 0: - message = "Something went wrong (TCX export) "+tcxmessg - messages.error(request,message) - url = reverse(workout_export_view, - kwargs = { - 'id':str(w.id), - }) - return HttpResponseRedirect(url) - if settings.DEBUG and tcxfile: - res = handle_sendemailtcx.delay(r.user.first_name, - r.user.last_name, - r.user.email,tcxfile) - - elif tcxfile: - res = queuehigh.enqueue(handle_sendemailtcx,r.user.first_name, - r.user.last_name, - r.user.email,tcxfile) + tcxfile,tcxmessg = stravastuff.createstravaworkoutdata(w,dozip=False) + if tcxfile == 0: + message = "Something went wrong (TCX export) "+tcxmessg + messages.error(request,message) + url = reverse(workout_export_view, + kwargs = { + 'id':str(w.id), + }) + return HttpResponseRedirect(url) + if tcxfile: + res = myqueue(queuehigh,handle_sendemailtcx, + r.user.first_name, + r.user.last_name, + r.user.email, + tcxfile) successmessage = "The TCX file was sent to you per email" messages.info(request,successmessage) @@ -1324,19 +1320,6 @@ def workout_tcxemail_view(request,id=0): kwargs = { 'id':str(w.id), }) - except: - successmessage = "" - message = "Something went wrong (TCX export) "+str(sys.exc_info()[0]) - with open("media/c2errors.log","a") as errorlog: - errorstring = str(sys.exc_info()[0]) - timestr = strftime("%Y%m%d-%H%M%S") - errorlog.write(timestr+errorstring+"\r\n") - - messages.error(request,message) - url = reverse(workout_export_view, - kwargs = { - 'id':str(w.id), - }) response = HttpResponseRedirect(url) @@ -1366,15 +1349,11 @@ def workout_gpxemail_view(request,id=0): row = rdata(filename) gpxfilename = filename[:-4]+'.gpx' row.exporttogpx(gpxfilename) - if settings.DEBUG: - res = handle_sendemailtcx.delay(r.user.first_name, - r.user.last_name, - r.user.email,gpxfilename) - - else: - res = queuehigh.enqueue(handle_sendemailtcx,r.user.first_name, - r.user.last_name, - r.user.email,gpxfilename) + res = myqueue(queuehigh,handle_sendemailtcx, + r.user.first_name, + r.user.last_name, + r.user.email,gpxfilename) + successmessage = "The GPX file was sent to you per email" messages.info(request,successmessage) @@ -1407,15 +1386,9 @@ def workout_csvemail_view(request,id=0): raise Http404("Workout doesn't exist") if (checkworkoutuser(request.user,w)): csvfile = w.csvfilename - if settings.DEBUG: - res = handle_sendemailcsv.delay(r.user.first_name, - r.user.last_name, - r.user.email,csvfile) - - else: - res = queuehigh.enqueue(handle_sendemailcsv,r.user.first_name, - r.user.last_name, - r.user.email,csvfile) + res = myqueue(queuehigh,handle_sendemailcsv,r.user.first_name, + r.user.last_name, + r.user.email,csvfile) successmessage = "The CSV file was sent to you per email" messages.info(request,successmessage) @@ -1447,19 +1420,12 @@ def workout_csvtoadmin_view(request,id=0): raise Http404("Workout doesn't exist") csvfile = w.csvfilename - if settings.DEBUG: - res = handle_sendemailcsv.delay( - 'Sander', - 'Roosendaal', - 'roosendaalsander@gmail.com',csvfile) - - else: - res = queuehigh.enqueue( - handle_sendemailcsv, - 'Sander', - 'Roosendaal', - 'roosendaalsander@gmail.com', - csvfile) + res = myqueue(queuehigh, + handle_sendemailcsv, + 'Sander', + 'Roosendaal', + 'roosendaalsander@gmail.com', + csvfile) successmessage = "The CSV file was sent to the site admin per email" messages.info(request,successmessage) @@ -2579,11 +2545,7 @@ def workout_forcecurve_view(request,id=0,workstrokesonly=False): @login_required() def workout_test_task_view(request,id=0): row = Workout.objects.get(id=id) - if settings.DEBUG: - res = addcomment2.delay(request.user.id,row.id) - else: - #res = queuehigh.enqueue(addcomment2,request.user.id,row.id) - res = queuehigh.enqueue(addcomment2,request.user.id,row.id) + res = myqueue(queuehigh,addcomment2,request.user.id,row.id) url = reverse(workout_edit_view, @@ -5662,19 +5624,13 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): first_name = u.first_name last_name = u.last_name emailaddress = u.email - - if settings.DEBUG: - job = handle_otwsetpower.delay(f1,boattype,weightvalue, - first_name,last_name, - emailaddress,id,debug=True, - ps=[r.p0,r.p1,r.p2,r.p3], - ratio=r.cpratio) - else: - job = queuelow.enqueue(handle_otwsetpower,f1,boattype, - weightvalue, - first_name,last_name,emailaddress,id, - ps=[r.p0,r.p1,r.p2,r.p3], - ratio=r.cpratio) + + job = myqueue(queuelow, + handle_otwsetpower,f1,boattype, + weightvalue, + first_name,last_name,emailaddress,id, + ps=[r.p0,r.p1,r.p2,r.p3], + ratio=r.cpratio) try: request.session['async_tasks'] += [(job.id,'otwsetpower')] @@ -6963,49 +6919,30 @@ def workout_comment_view(request,id=0): ) if request.user != r.user: a_messages.info(r.user,message) - - if settings.DEBUG and request.user != r.user: - res = handle_sendemailnewcomment.delay(r.user.first_name, - r.user.last_name, - r.user.email, - request.user.first_name, - request.user.last_name, - comment,w.name, - w.id) - - elif request.user != r.user: - res = queuehigh.enqueue(handle_sendemailnewcomment,r.user.first_name, - r.user.last_name, - r.user.email, - request.user.first_name, - request.user.last_name, - comment,w.name,w.id) + res = myqueue(queuehigh, + handle_sendemailnewcomment,r.user.first_name, + r.user.last_name, + r.user.email, + request.user.first_name, + request.user.last_name, + comment,w.name,w.id) commenters = {oc.user for oc in comments if oc.notification} for u in commenters: a_messages.info(u,message) - if settings.DEBUG and u != request.user and u != r.user: - res = handle_sendemailnewresponse.delay(u.first_name, - u.last_name, - u.email, - request.user.first_name, - request.user.last_name, - comment, - w.name, - w.id, - c.id) - elif u != request.user and u != r.user: - res = queuelow.enqueue(handle_sendemailnewresponse, - u.first_name, - u.last_name, - u.email, - request.user.first_name, - request.user.last_name, - comment, - w.name, - w.id, - c.id) + if u != request.user and u != r.user: + res = myqueue(queuelow, + handle_sendemailnewresponse, + u.first_name, + u.last_name, + u.email, + request.user.first_name, + request.user.last_name, + comment, + w.name, + w.id, + c.id) url = reverse(workout_comment_view,kwargs = { 'id':id}) @@ -9654,10 +9591,7 @@ def rower_calcdps_view(request): r = getrower(request.user) ws = [(w.id,w.csvfilename) for w in Workout.objects.filter(user=r)] - if settings.DEBUG: - res = handle_updatedps.delay(r.user.email,ws,debug=True) - else: - queue.enqueue(handle_updatedps,r.user.email,ws,debug=False) + res = myqueue(queue,handle_updatedps,r.user.email,ws,debug=False) messages.info(request,"Your workouts are being updated in the background. You will receive email when this is done.") From 2939accbf22bb2442158cb8a04d56949d5512188 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 31 Oct 2017 11:18:14 +0100 Subject: [PATCH 7/9] added myqueue wrapper to teams.py --- rowers/tasks.py | 4 +-- rowers/teams.py | 68 ++++++++++++++++++------------------------------- 2 files changed, 27 insertions(+), 45 deletions(-) diff --git a/rowers/tasks.py b/rowers/tasks.py index e1b87ae9..1319e010 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -381,7 +381,7 @@ def handle_otwsetpower(f1, boattype, weightvalue, workoutid, email, first_name, last_name, btvalues=btvalues.to_json()) - except OperationalError: + except: pass # send email @@ -402,7 +402,7 @@ def handle_otwsetpower(f1, boattype, weightvalue, send_mail(subject, message, 'Rowsandall Physics Department ', [fullemail]) - except OperationalError: + except: return 0 return 1 diff --git a/rowers/teams.py b/rowers/teams.py index e2b45f5e..ba802fd2 100644 --- a/rowers/teams.py +++ b/rowers/teams.py @@ -9,6 +9,7 @@ from django.db import IntegrityError import uuid from django.conf import settings +from utils import myqueue import django_rq queue = django_rq.get_queue('default') @@ -289,11 +290,9 @@ def send_invite_email(id): teamname = invitation.team.name manager = invitation.team.manager.first_name+' '+invitation.team.manager.last_name - if settings.DEBUG: - res = handle_sendemail_invite.delay(email,name,code,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_invite, - email,name,code,teamname,manager) + res = myqueue(queuehigh, + handle_sendemail_invite, + email,name,code,teamname,manager) return (1,'Invitation email sent') @@ -303,11 +302,9 @@ def send_team_delete_mail(t,rower): email = u.email name = u.first_name+' '+u.last_name manager = t.manager.first_name+' '+t.manager.last_name - if settings.DEBUG: - res = handle_sendemail_team_removed.delay(email,name,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_team_removed, - email,name,teamname,manager) + res = myqueue(queuehigh, + handle_sendemail_team_removed, + email,name,teamname,manager) return (1,'Team delete email sent') @@ -319,11 +316,8 @@ def send_email_member_dropped(teamid,rower): name = u.first_name+' '+u.last_name manager = t.manager.first_name+' '+t.manager.last_name - if settings.DEBUG: - res = handle_sendemail_member_dropped.delay(email,name,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_member_dropped, - email,name,teamname,manager) + res = myqueue(queuehigh,handle_sendemail_member_dropped, + email,name,teamname,manager) return (1,'Member dropped email sent') @@ -334,12 +328,9 @@ def send_request_accept_email(rekwest): teamname = rekwest.team.name name = rekwest.user.first_name+' '+rekwest.user.last_name manager = rekwest.team.manager.first_name+' '+rekwest.team.manager.last_name - - if settings.DEBUG: - res = handle_sendemail_request_accept.delay(email,name,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_request_accept, - email,name,teamname,manager) + res = myqueue(queuehigh, + handle_sendemail_request_accept, + email,name,teamname,manager) return (1,'Invitation email sent') @@ -350,12 +341,10 @@ def send_request_reject_email(rekwest): email = rekwest.user.email name = rekwest.user.first_name+' '+rekwest.user.last_name manager = rekwest.team.manager.first_name+' '+rekwest.team.manager.last_name - - if settings.DEBUG: - res = handle_sendemail_request_reject.delay(email,name,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_request_reject, - email,name,teamname,manager) + + res = myqueue(queuehigh, + handle_sendemail_request_reject, + email,name,teamname,manager) return (1,'Invitation email sent') @@ -371,11 +360,9 @@ def send_invite_reject_email(invitation): teamname = invitation.team.name manager = invitation.team.manager.first_name+' '+invitation.team.manager.last_name - if settings.DEBUG: - res = handle_sendemail_invite_reject.delay(email,name,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_invite_reject, - email,name,teamname,manager) + res = myqueue(queuehigh, + handle_sendemail_invite_reject, + email,name,teamname,manager) return (1,'Invitation email sent') @@ -391,11 +378,9 @@ def send_invite_accept_email(invitation): teamname = invitation.team.name manager = invitation.team.manager.first_name+' '+invitation.team.manager.last_name - if settings.DEBUG: - res = handle_sendemail_invite_accept.delay(email,name,teamname,manager) - else: - res = queuehigh.enqueue(handle_sendemail_invite_accept, - email,name,teamname,manager) + res = myqueue(queuehigh, + handle_sendemail_invite_accept, + email,name,teamname,manager) return (1,'Invitation email sent') @@ -409,12 +394,9 @@ def send_request_email(rekwest): teamname = rekwest.team.name requestor = rekwest.user.first_name+' '+rekwest.user.last_name - if settings.DEBUG: - res = handle_sendemail_request.delay(email,name,code,teamname, - requestor,id) - else: - res = queuehigh.enqueue(handle_sendemail_request, - email,name,code,teamname,requestor,id) + res = myqueue(queuehigh, + handle_sendemail_request, + email,name,code,teamname,requestor,id) return (1,'Invitation email sent') From 0ecd8c588dd6efdc97e1db8027db78196a97354b Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 31 Oct 2017 14:52:52 +0100 Subject: [PATCH 8/9] debugged some odd behavior --- rowers/dataprepnodjango.py | 3 +- rowers/tasks.py | 59 ++++++++++++++++++++++++-------------- rowers/utils.py | 2 ++ 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/rowers/dataprepnodjango.py b/rowers/dataprepnodjango.py index 6873e7de..eb5ebae6 100644 --- a/rowers/dataprepnodjango.py +++ b/rowers/dataprepnodjango.py @@ -472,7 +472,7 @@ def delete_strokedata(id,debug=False): engine.dispose() def update_strokedata(id,df,debug=False): - delete_strokedata(id) + delete_strokedata(id,debug=debug) if debug: print "updating ",id rowdata = dataprep(df,id=id,bands=True,barchart=True,otwpower=True, @@ -935,7 +935,6 @@ def dataprep(rowdatadf,id=0,bands=True,barchart=True,otwpower=True, # write data if id given if id != 0: - data['workoutid'] = id if debug: engine = create_engine(database_url_debug, echo=False) else: diff --git a/rowers/tasks.py b/rowers/tasks.py index 1319e010..1427f0f9 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -50,7 +50,7 @@ def handle_new_workout_from_file(r, f2, workouttype='rower', title='Workout', makeprivate=False, - notes=''): + notes='',debug=False): return new_workout_from_file(r, f2, workouttype, title, makeprivate, notes) @@ -92,7 +92,8 @@ def handle_updatedps(useremail, workoutids, debug=False): @app.task def handle_sendemail_breakthrough(workoutid, useremail, userfirstname, userlastname, - btvalues=pd.DataFrame().to_json()): + btvalues=pd.DataFrame().to_json(), + debug=False): # send email with attachment subject = "A breakthrough workout on rowsandall.com" @@ -151,7 +152,8 @@ def handle_sendemail_breakthrough(workoutid, useremail, @app.task def handle_sendemail_hard(workoutid, useremail, userfirstname, userlastname, - btvalues=pd.DataFrame().to_json()): + btvalues=pd.DataFrame().to_json(), + debug=False): # send email with attachment subject = "That was a pretty hard workout on rowsandall.com" @@ -183,7 +185,8 @@ def handle_sendemail_hard(workoutid, useremail, # send email to me when an unrecognized file is uploaded @app.task -def handle_sendemail_unrecognized(unrecognizedfile, useremail): +def handle_sendemail_unrecognized(unrecognizedfile, useremail, + debug=False): # send email with attachment fullemail = 'roosendaalsander@gmail.com' @@ -215,7 +218,8 @@ def handle_sendemail_unrecognized(unrecognizedfile, useremail): # send email to owner when an unrecognized file is uploaded @app.task -def handle_sendemail_unrecognizedowner(useremail, userfirstname): +def handle_sendemail_unrecognizedowner(useremail, userfirstname, + debug=False): # send email with attachment fullemail = useremail @@ -246,7 +250,7 @@ def handle_sendemail_unrecognizedowner(useremail, userfirstname): # Send email with TCX attachment @app.task -def handle_sendemailtcx(first_name, last_name, email, tcxfile): +def handle_sendemailtcx(first_name, last_name, email, tcxfile,debug=False): # send email with attachment fullemail = first_name + " " + last_name + " " + "<" + email + ">" @@ -269,7 +273,7 @@ def handle_sendemailtcx(first_name, last_name, email, tcxfile): @app.task -def handle_zip_file(emailfrom, subject, file): +def handle_zip_file(emailfrom, subject, file,debug=False): message = "... zip processing ... " email = EmailMessage(subject, message, emailfrom, @@ -283,7 +287,7 @@ def handle_zip_file(emailfrom, subject, file): @app.task -def handle_sendemailcsv(first_name, last_name, email, csvfile): +def handle_sendemailcsv(first_name, last_name, email, csvfile,debug=False): # send email with attachment fullemail = first_name + " " + last_name + " " + "<" + email + ">" @@ -365,7 +369,9 @@ def handle_otwsetpower(f1, boattype, weightvalue, totaltime = totaltime + rowdata.df.ix[0, ' ElapsedTime (sec)'] except KeyError: pass - df = getsmallrowdata_db(['power', 'workoutid', 'time'], ids=[workoutid]) + df = getsmallrowdata_db( + ['power', 'workoutid', 'time'], ids=[workoutid], + debug=debug) thesecs = totaltime maxt = 1.05 * thesecs logarr = datautils.getlogarr(maxt) @@ -447,7 +453,8 @@ def handle_updatecp(rower_id,workoutids,debug=False,table='cpdata'): return 1 @app.task -def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename): +def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename, + debug=False): hrmax = hrdata['hrmax'] hrut2 = hrdata['hrut2'] @@ -519,7 +526,8 @@ def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename): @app.task -def handle_sendemail_invite(email, name, code, teamname, manager): +def handle_sendemail_invite(email, name, code, teamname, manager, + debug=False): fullemail = name + ' <' + email + '>' subject = 'Invitation to join team ' + teamname message = 'Dear ' + name + ',\n\n' @@ -560,7 +568,8 @@ def handle_sendemailnewresponse(first_name, last_name, commenter_first_name, commenter_last_name, comment, - workoutname, workoutid, commentid): + workoutname, workoutid, commentid, + debug=False): fullemail = first_name + ' ' + last_name + ' <' + email + '>' subject = 'New comment on workout ' + workoutname message = 'Dear ' + first_name + ',\n\n' @@ -594,7 +603,8 @@ def handle_sendemailnewcomment(first_name, commenter_first_name, commenter_last_name, comment, workoutname, - workoutid): + workoutid, + debug=False): fullemail = first_name + ' ' + last_name + ' <' + email + '>' subject = 'New comment on workout ' + workoutname message = 'Dear ' + first_name + ',\n\n' @@ -617,7 +627,8 @@ def handle_sendemailnewcomment(first_name, @app.task -def handle_sendemail_request(email, name, code, teamname, requestor, id): +def handle_sendemail_request(email, name, code, teamname, requestor, id, + debug=False): fullemail = name + ' <' + email + '>' subject = 'Request to join team ' + teamname message = 'Dear ' + name + ',\n\n' @@ -641,7 +652,8 @@ def handle_sendemail_request(email, name, code, teamname, requestor, id): @app.task -def handle_sendemail_request_accept(email, name, teamname, managername): +def handle_sendemail_request_accept(email, name, teamname, managername, + debug=False): fullemail = name + ' <' + email + '>' subject = 'Welcome to ' + teamname message = 'Dear ' + name + ',\n\n' @@ -661,7 +673,8 @@ def handle_sendemail_request_accept(email, name, teamname, managername): @app.task -def handle_sendemail_request_reject(email, name, teamname, managername): +def handle_sendemail_request_reject(email, name, teamname, managername, + debug=False): fullemail = name + ' <' + email + '>' subject = 'Your application to ' + teamname + ' was rejected' message = 'Dear ' + name + ',\n\n' @@ -682,7 +695,8 @@ def handle_sendemail_request_reject(email, name, teamname, managername): @app.task -def handle_sendemail_member_dropped(email, name, teamname, managername): +def handle_sendemail_member_dropped(email, name, teamname, managername, + debug=False): fullemail = name + ' <' + email + '>' subject = 'You were removed from ' + teamname message = 'Dear ' + name + ',\n\n' @@ -703,7 +717,8 @@ def handle_sendemail_member_dropped(email, name, teamname, managername): @app.task -def handle_sendemail_team_removed(email, name, teamname, managername): +def handle_sendemail_team_removed(email, name, teamname, managername, + debug=False): fullemail = name + ' <' + email + '>' subject = 'Team ' + teamname + ' was deleted' message = 'Dear ' + name + ',\n\n' @@ -725,7 +740,8 @@ def handle_sendemail_team_removed(email, name, teamname, managername): @app.task -def handle_sendemail_invite_reject(email, name, teamname, managername): +def handle_sendemail_invite_reject(email, name, teamname, managername, + debug=False): fullemail = managername + ' <' + email + '>' subject = 'Your invitation to ' + name + ' was rejected' message = 'Dear ' + managername + ',\n\n' @@ -746,7 +762,8 @@ def handle_sendemail_invite_reject(email, name, teamname, managername): @app.task -def handle_sendemail_invite_accept(email, name, teamname, managername): +def handle_sendemail_invite_accept(email, name, teamname, managername, + debug=False): fullemail = managername + ' <' + email + '>' subject = 'Your invitation to ' + name + ' was accepted' message = 'Dear ' + managername + ',\n\n' @@ -763,5 +780,5 @@ def handle_sendemail_invite_accept(email, name, teamname, managername): # Another simple task for debugging purposes -def add2(x, y): +def add2(x, y,debug=False): return x + y diff --git a/rowers/utils.py b/rowers/utils.py index f4081639..b8553ca9 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -220,6 +220,8 @@ def isbreakthrough(delta,cpvalues,p0,p1,p2,p3,ratio): def myqueue(queue,function,*args,**kwargs): if settings.DEBUG: + kwargs['debug'] = True + print kwargs job = function.delay(*args,**kwargs) else: job = queue.enqueue(function,*args,**kwargs) From 24482cca8a071420185ca2fe8da2686b6aac1e5b Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Tue, 31 Oct 2017 14:57:41 +0100 Subject: [PATCH 9/9] removed some debug print statements --- rowers/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rowers/utils.py b/rowers/utils.py index b8553ca9..b6b81eec 100644 --- a/rowers/utils.py +++ b/rowers/utils.py @@ -221,7 +221,6 @@ def isbreakthrough(delta,cpvalues,p0,p1,p2,p3,ratio): def myqueue(queue,function,*args,**kwargs): if settings.DEBUG: kwargs['debug'] = True - print kwargs job = function.delay(*args,**kwargs) else: job = queue.enqueue(function,*args,**kwargs)