sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

Merge branch 'feature/progress' into develop

Browse Source
This commit is contained in:
Sander Roosendaal
2017-11-01 17:39:09 +01:00
parent 5525dad1f6 e1a2c509e7
commit 255f4df8d1
8 changed files with 245 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rowers/longtask.py
+58
View File
@@ -0,0 +1,58 @@
from __future__ import absolute_import
import numpy as np
import time
from redis import StrictRedis,Redis
from celery import result as celery_result
import json
redis_connection = StrictRedis()
import redis
import threading
def getvalue(data):
perc = 0
total = 1
done = 0
id = 0
session_key = 'noot'
for i in data.iteritems():
if i[0] == 'total':
total = float(i[1])
if i[0] == 'done':
done = float(i[1])
if i[0] == 'id':
id = i[1]
if i[0] == 'session_key':
session_key = i[1]
return total,done,id,session_key
def longtask(aantal,jobid=None,debug=False,
session_key=None):
counter = 0
channel = 'tasks'
for i in range(aantal):
time.sleep(1)
counter += 1
if counter > 10:
counter = 0
if debug:
progress = 100.*i/aantal
if jobid != None:
redis_connection.publish(channel,json.dumps(
{
'done':i,
'total':aantal,
'id':jobid,
'session_key':session_key,
}
))
return 1
rowers/tasks.py
+10 -2
View File
@@ -35,6 +35,8 @@ from django.db.utils import OperationalError
import datautils
import utils
import longtask
# testing task
@@ -42,9 +44,15 @@ import utils
def add(x, y):
return x + y
@app.task(bind=True)
def long_test_task(self,aantal,debug=False,job=None,session_key=None):
job = self.request
return longtask.longtask(aantal,jobid=job.id,debug=debug,
session_key=session_key)
# create workout
@app.task
def handle_new_workout_from_file(r, f2,
workouttype='rower',
rowers/templates/async_tasks.html
+55 -1
View File
@@ -4,6 +4,53 @@
{% block title %}Rowsandall - Tasks {% endblock %}
{% block meta %}
<link rel="stylesheet" href="//code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css">
<style type="text/css">
.progressBar div {
border-radius:5px;
height: 100%;
/* padding: 4px 10px; */
font-size: 15px;
color: #fff;
text-align: right;
line-height: 22px;
width: 0;
vertical-align: middle;
background-color: #0099ff;
}
</style>
<script type='text/javascript'
src='https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js'>
</script>
<script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
<script>
$(document).ready(function(){
$('.progressBar').each(function() {
var bar = $(this);
var maxvalue = $(this).attr('data');
maxvalue = 0;
var text = $(this).children('div').data('show');
progress1(maxvalue, bar, text);
});
$('.progressBar').each(function() {
var bar = $(this);
var maxvalue = $(this).attr('data');
maxvalue = maxvalue.substring(3);
var text = $(this).children('div').data('show');
progress(maxvalue, bar, text);
});
});
function progress1(percent, element, text) {
element.find('div').animate({ width: percent+'%' }, 1).html(text +"&nbsp;"+ percent + "%&nbsp;");
}
function progress(percent, element, text) {
element.find('div').animate({ width: percent+'%' }, 'slow').html(text +"&nbsp;"+ percent + "%&nbsp;");
}
</script>
{% endblock %}
{% block content %}
@@ -16,7 +63,8 @@
<thead>
<tr>
<th>ID</th>
<th style="width:180">Task</th>
<th>Task</th>
<th>Progress</th>
<th>Status</th>
<th>Action</th>
</tr>
@@ -31,6 +79,12 @@
{{ task|lookup:'verbose' }}
</td>
<td>
<div class="progressBar" data="max{{ task|lookup:'progress' }}" style="width: 100%;">
<div data-show=""> {{ task|lookup:'progress' }}</div>
</div>
</td>
<td>
{{ task|lookup:'status' }}
</td>
{% if task|lookup:'failed' %}
rowers/templatetags/rowerfilters.py
+7
View File
@@ -1,6 +1,8 @@
from django import template
from django.utils.safestring import mark_safe
from time import strftime
import dateutil.parser
import json
register = template.Library()
@@ -65,6 +67,11 @@ def deltatimeprint(d):
return strfdeltah(d)
@register.filter(is_safe=True)
def jsdict(dict,key):
s = dict.get(key)
return mark_safe(json.dumps(s))
@register.filter
def lookup(dict, key):
s = dict.get(key)
rowers/urls.py
+1
View File
@@ -141,6 +141,7 @@ urlpatterns = [
url(r'^list-jobs/$',views.session_jobs_view),
url(r'^jobs-status/$',views.session_jobs_status),
url(r'^job-kill/(?P<id>.*)$',views.kill_async_job),
url(r'^test-job/(?P<aantal>\d+)$',views.test_job_view),
url(r'^list-graphs/$',views.graphs_view),
url(r'^(?P<theuser>\d+)/ote-bests/(?P<startdatestring>\w+.*)/(?P<enddatestring>\w+.*)$',views.rankings_view),
url(r'^(?P<theuser>\d+)/ote-bests/(?P<deltadays>\d+)$',views.rankings_view),
rowers/utils.py
+3
View File
@@ -5,6 +5,7 @@ import colorsys
from django.conf import settings
lbstoN = 4.44822
landingpages = (
@@ -226,3 +227,5 @@ def myqueue(queue,function,*args,**kwargs):
job = queue.enqueue(function,*args,**kwargs)
return job
rowers/views.py
+108 -14
View File
@@ -14,6 +14,7 @@ from django.views.generic.base import TemplateView
from django.db.models import Q
from django import template
from django.db import IntegrityError, transaction
from django.shortcuts import render
from django.http import (
HttpResponse, HttpResponseRedirect,
@@ -99,7 +100,7 @@ from rowers.tasks import handle_makeplot,handle_otwsetpower,handle_sendemailtcx,
from rowers.tasks import (
handle_sendemail_unrecognized,handle_sendemailnewcomment,
handle_sendemailnewresponse, handle_updatedps,
handle_updatecp
handle_updatecp,long_test_task
)
from scipy.signal import savgol_filter
@@ -131,13 +132,69 @@ queue = django_rq.get_queue('default')
queuelow = django_rq.get_queue('low')
queuehigh = django_rq.get_queue('low')
import redis
import threading
from redis import StrictRedis,Redis
from rq.exceptions import NoSuchJobError
from rq.registry import StartedJobRegistry
from rq import Queue,cancel_job
from django.core.cache import cache
def getvalue(data):
perc = 0
total = 1
done = 0
id = 0
session_key = 'noot'
for i in data.iteritems():
if i[0] == 'total':
total = float(i[1])
if i[0] == 'done':
done = float(i[1])
if i[0] == 'id':
id = i[1]
if i[0] == 'session_key':
session_key = i[1]
return total,done,id,session_key
class SessionTaskListener(threading.Thread):
def __init__(self, r, channels):
threading.Thread.__init__(self)
self.redis = r
self.pubsub = self.redis.pubsub()
self.pubsub.subscribe(channels)
def work(self, item):
try:
data = json.loads(item['data'])
total,done,id,session_key = getvalue(data)
perc = int(100.*done/total)
cache.set(id,perc,3600)
except TypeError:
pass
def run(self):
for item in self.pubsub.listen():
if item['data'] == "KILL":
self.pubsub.unsubscribe()
print self, "unsubscribed and finished"
break
else:
self.work(item)
queuefailed = Queue("failed",connection=Redis())
redis_connection = StrictRedis()
r = Redis()
client = SessionTaskListener(r,['tasks'])
client.start()
rq_registry = StartedJobRegistry(queue.name,connection=redis_connection)
rq_registryhigh = StartedJobRegistry(queuehigh.name,connection=redis_connection)
rq_registrylow = StartedJobRegistry(queuelow.name,connection=redis_connection)
@@ -181,14 +238,13 @@ def remove_asynctask(request,id):
newtasks = []
for task in oldtasks:
print task[0]
if id not in task[0]:
newtasks += [(task[0],task[1])]
request.session['async_tasks'] = newtasks
def get_job_result(jobid):
if settings.EBUG:
if settings.DEBUG:
result = celery_result.AsyncResult(jobid).result
else:
running_job_ids = rq_registry.get_job_ids()
@@ -210,12 +266,14 @@ verbose_job_status = {
'updatecpwater': 'Critical Power Calculation for OTW Workouts',
'otwsetpower': 'Rowing Physics OTW Power Calculation',
'make_plot': 'Create static chart',
'long_test_task': 'Long Test Task',
}
def get_job_status(jobid):
if settings.DEBUG:
job = celery_result.AsyncResult(jobid)
jobresult = job.result
if 'fail' in job.status.lower():
jobresult = '0'
summary = {
@@ -261,6 +319,25 @@ def kill_async_job(request,id='aap'):
pass
remove_asynctask(request,id)
cache.delete(id)
url = reverse(session_jobs_status)
return HttpResponseRedirect(url)
@login_required()
def test_job_view(request,aantal=100):
session_key = request.session._session_key
job = myqueue(queuehigh,long_test_task,int(aantal),
session_key=session_key)
try:
request.session['async_tasks'] += [(job.id,'long_test_task')]
except KeyError:
request.session['async_tasks'] = [(job.id,'long_test_task')]
url = reverse(session_jobs_status)
return HttpResponseRedirect(url)
@@ -310,14 +387,31 @@ def get_stored_tasks_status(request):
except KeyError:
taskids = []
taskstatus = [{
'id':id,
'status':get_job_status(id)['status'],
'failed':get_job_status(id)['failed'],
'finished':get_job_status(id)['finished'],
'func_name':func_name,
'verbose': verbose_job_status[func_name]
} for id,func_name in taskids]
taskstatus = []
for id,func_name in taskids:
progress = 0
cached_progress = cache.get(id)
finished = get_job_status(id)['finished']
if finished:
cache.set(id,100)
progress = 100
elif cached_progress>0:
progress = cached_progress
else:
progress = 0
this_task_status = {
'id':id,
'status':get_job_status(id)['status'],
'failed':get_job_status(id)['failed'],
'finished':get_job_status(id)['finished'],
'func_name':func_name,
'verbose': verbose_job_status[func_name],
'progress': progress,
}
taskstatus.append(this_task_status)
return taskstatus
@@ -6528,7 +6622,6 @@ def workout_flexchart3_view(request,*args,**kwargs):
if request.user == row.user.user:
mayedit=1
workouttype = 'ote'
if row.workouttype in ('water','coastal'):
workouttype = 'otw'
@@ -6578,8 +6671,9 @@ def workout_flexchart3_view(request,*args,**kwargs):
else:
yparam2 = 'hr'
if favoritenr>=0 and r.showfavoritechartnotes:
favoritechartnotes = favorites[favoritenr].notes
if not request.user.is_anonymous():
if favoritenr>=0 and r.showfavoritechartnotes:
favoritechartnotes = favorites[favoritenr].notes
else:
favoritechartnotes = ''
favoritenr = 0
rowsandall_app/settings.py
+1
View File
@@ -287,6 +287,7 @@ RQ_QUEUES = {
#SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"
#SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_SAVE_EVERY_REQUEST = True
# admin stuff for error reporting
SERVER_EMAIL='admin@rowsandall.com'