From b6c67f7e2b3297dc6d1215b6e700a8c7e36d633f Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 17 May 2018 16:59:19 +0200 Subject: [PATCH 1/2] submit race result now starts async process --- rowers/courses.py | 90 +------------------ rowers/courseutils.py | 96 ++++++++++++++++++++ rowers/models.py | 4 +- rowers/plannedsessions.py | 35 +++----- rowers/tasks.py | 139 ++++++++++++++++++++++++++++- rowers/templates/virtualevent.html | 4 + rowers/views.py | 12 ++- 7 files changed, 261 insertions(+), 119 deletions(-) create mode 100644 rowers/courseutils.py diff --git a/rowers/courses.py b/rowers/courses.py index d63ec28e..7a3066a2 100644 --- a/rowers/courses.py +++ b/rowers/courses.py @@ -33,14 +33,8 @@ from rowers.models import ( ) from utils import geo_distance +from rowers.courseutils import coursetime_paths, coursetime_first -# low level methods -class InvalidTrajectoryError(Exception): - def __init__(self,value): - self.value=value - - def __str__(self): - return repr(self.value) def get_course_timezone(course): @@ -65,31 +59,6 @@ def get_course_timezone(course): -def time_in_path(df,p,maxmin='max'): - - if df.empty: - return 0 - - latitude = df.latitude - longitude = df.longitude - - f = lambda x: coordinate_in_path(x['latitude'],x['longitude'],p) - - df['inpolygon'] = df.apply(f,axis=1) - - if maxmin=='max': - b = (~df['inpolygon']).shift(-1)+df['inpolygon'] - else: - b = (~df['inpolygon']).shift(1)+df['inpolygon'] - - - if len(df[b==2]): - return df[b==2]['time'].min(),df[b==2]['cum_dist'].min() - - raise InvalidTrajectoryError("Trajectory doesn't go through path") - - - return 0 def crewnerdcourse(doc): courses = [] @@ -198,63 +167,6 @@ def createcourse( return c -def coursetime_first(data,paths): - - entrytime = data['time'].max() - entrydistance = data['cum_dist'].max() - coursecompleted = False - - try: - entrytime,entrydistance = time_in_path(data,paths[0],maxmin='max') - coursecompleted = True - except InvalidTrajectoryError: - entrytime = data['time'].max() - entrydistance = data['cum_dist'].max() - coursecompleted = False - return entrytime, entrydistance, coursecompleted - -def coursetime_paths(data,paths,finalmaxmin='min'): - - entrytime = data['time'].max() - entrydistance = data['cum_dist'].max() - coursecompleted = False - - # corner case - empty list of paths - if len(paths) == 0: - return 0,True - - # end - just the Finish polygon - if len(paths) == 1: - try: - ( - entrytime, - entrydistance - ) = time_in_path(data,paths[0],maxmin=finalmaxmin) - coursecompleted = True - except InvalidTrajectoryError: - entrytime = data['time'].max() - entrydistance = data['cum_dist'].max() - coursecompleted = False - return entrytime,entrydistance,coursecompleted - - if len(paths) > 1: - try: - time,dist = time_in_path(data, paths[0]) - data = data[data['time']>time] - data['time'] = data['time']-time - data['cum_dist'] = data['cum_dist']-dist - ( - timenext, - distnext, - coursecompleted - ) = coursetime_paths(data,paths[1:]) - return time+timenext, dist+distnext,coursecompleted - except InvalidTrajectoryError: - entrytime = data['time'].max() - entrydistance = data['cum_dist'].max() - coursecompleted = False - - return entrytime, entrydistance, coursecompleted def get_time_course(ws,course): coursetimeseconds = 0.0 diff --git a/rowers/courseutils.py b/rowers/courseutils.py new file mode 100644 index 00000000..d3b8d636 --- /dev/null +++ b/rowers/courseutils.py @@ -0,0 +1,96 @@ +# low level methods +def coordinate_in_path(latitude,longitude, p): + + return p.contains_points([(latitude,longitude)])[0] + +class InvalidTrajectoryError(Exception): + def __init__(self,value): + self.value=value + + def __str__(self): + return repr(self.value) + +def time_in_path(df,p,maxmin='max'): + + if df.empty: + return 0 + + latitude = df.latitude + longitude = df.longitude + + f = lambda x: coordinate_in_path(x['latitude'],x['longitude'],p) + + df['inpolygon'] = df.apply(f,axis=1) + + if maxmin=='max': + b = (~df['inpolygon']).shift(-1)+df['inpolygon'] + else: + b = (~df['inpolygon']).shift(1)+df['inpolygon'] + + + if len(df[b==2]): + return df[b==2]['time'].min(),df[b==2]['cum_dist'].min() + + raise InvalidTrajectoryError("Trajectory doesn't go through path") + + + return 0 + + +def coursetime_first(data,paths): + + entrytime = data['time'].max() + entrydistance = data['cum_dist'].max() + coursecompleted = False + + try: + entrytime,entrydistance = time_in_path(data,paths[0],maxmin='max') + coursecompleted = True + except InvalidTrajectoryError: + entrytime = data['time'].max() + entrydistance = data['cum_dist'].max() + coursecompleted = False + return entrytime, entrydistance, coursecompleted + +def coursetime_paths(data,paths,finalmaxmin='min'): + + entrytime = data['time'].max() + entrydistance = data['cum_dist'].max() + coursecompleted = False + + # corner case - empty list of paths + if len(paths) == 0: + return 0,True + + # end - just the Finish polygon + if len(paths) == 1: + try: + ( + entrytime, + entrydistance + ) = time_in_path(data,paths[0],maxmin=finalmaxmin) + coursecompleted = True + except InvalidTrajectoryError: + entrytime = data['time'].max() + entrydistance = data['cum_dist'].max() + coursecompleted = False + return entrytime,entrydistance,coursecompleted + + if len(paths) > 1: + try: + time,dist = time_in_path(data, paths[0]) + data = data[data['time']>time] + data['time'] = data['time']-time + data['cum_dist'] = data['cum_dist']-dist + ( + timenext, + distnext, + coursecompleted + ) = coursetime_paths(data,paths[1:]) + return time+timenext, dist+distnext,coursecompleted + except InvalidTrajectoryError: + entrytime = data['time'].max() + entrydistance = data['cum_dist'].max() + coursecompleted = False + + return entrytime, entrydistance, coursecompleted diff --git a/rowers/models.py b/rowers/models.py index 23455021..374339ff 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -376,9 +376,7 @@ def polygon_to_path(polygon): return p -def coordinate_in_path(latitude,longitude, p): - - return p.contains_points([(latitude,longitude)])[0] +from rowers.courseutils import coordinate_in_path def course_spline(coordinates): latitudes = coordinates['latitude'].values diff --git a/rowers/plannedsessions.py b/rowers/plannedsessions.py index 10a11d8f..3911458f 100644 --- a/rowers/plannedsessions.py +++ b/rowers/plannedsessions.py @@ -26,6 +26,8 @@ import numpy as np import dataprep import courses +from rowers.tasks import handle_check_race_course + # Low Level functions - to be called by higher level methods def add_workouts_plannedsession(ws,ps,r): result = 0 @@ -660,11 +662,11 @@ def add_workout_race(ws,race,r): dates = [w.date for w in ws] if (not all(d == dates[0] for d in dates)) and race.sessiontype not in ['challenge','cycletarget']: errors.append('For tests and training sessions, selected workouts must all be done on the same date') - return result,comments,errors + return result,comments,errors,0 if len(ws)>1 and race.sessiontype == 'test': errors.append('For tests, you can only attach one workout') - return result,comments,errors + return result,comments,errors,0 @@ -673,7 +675,7 @@ def add_workout_race(ws,race,r): if len(ids)>1 and race.sessiontype in ['test','coursetest','race']: errors.append('For tests, you can only attach one workout') - return result,comments,errors + return result,comments,errors,0 # start adding sessions for w in ws: @@ -682,10 +684,9 @@ def add_workout_race(ws,race,r): w.save() result += 1 - comments.append('Your result has been submitted') else: errors.append('Workout %i did not match the race window' % w.id) - return result,comments,errors + return result,comments,errors,0 if result>0: username = r.user.first_name+' '+r.user.last_name @@ -693,16 +694,6 @@ def add_workout_race(ws,race,r): age = calculate_age(r.birthdate) else: age = None - ( - coursetime, - coursemeters, - coursecompleted - ) = courses.get_time_course(ws,race.course) - if not coursecompleted: - errors.append('Your trajectory did not match the race course') - return result,comments,errors - - duration = totaltime_sec_to_string(coursetime) records = VirtualRaceResult.objects.filter( userid=r.id, @@ -713,22 +704,20 @@ def add_workout_race(ws,race,r): if ws[0].boattype != record.boattype: errors.append('Your workout boat type did not match the boat type you registered') - return result,comments,errors + return result,comments,errors,0 if ws[0].weightcategory != record.weightcategory: errors.append('Your workout weight category did not match the weight category you registered') - return result,comments, errors + return result,comments, errors,0 - record.coursecompleted=coursecompleted - record.distance = int(coursemeters) - record.workoutid=ws[0].id - record.duration = duration - record.save() + + job = myqueue(queue,handle_check_race_course,ws[0].csvfilename, + ws[0].id,race.course.id,record.id) add_workouts_plannedsession(ws,race,r) - return result,comments,errors + return result,comments,errors,job.id def delete_race_result(workout,race): results = VirtualRaceResult.objects.filter(workoutid=workout.id,race=race) diff --git a/rowers/tasks.py b/rowers/tasks.py index d9abc222..67e1d34c 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -12,6 +12,8 @@ from scipy import optimize import rowingdata from rowingdata import rowingdata as rdata +from datetime import timedelta +from sqlalchemy import create_engine from celery import app import datetime @@ -21,6 +23,7 @@ import iso8601 from matplotlib.backends.backend_agg import FigureCanvas #from matplotlib.backends.backend_cairo import FigureCanvasCairo as FigureCanvas import matplotlib.pyplot as plt +from matplotlib import path from rowsandall_app.settings import SITE_URL from rowsandall_app.settings_dev import SITE_URL as SITE_URL_DEV @@ -41,7 +44,8 @@ from rowers.dataprepnodjango import ( getsmallrowdata_db, updatecpdata_sql, update_agegroup_db,fitnessmetric_to_sql, add_c2_stroke_data_db,totaltime_sec_to_string, - create_c2_stroke_data_db,update_empower + create_c2_stroke_data_db,update_empower, + database_url_debug,database_url, ) @@ -66,6 +70,7 @@ siteurl = SITE_URL # testing task from rowers.emails import send_template_email +from rowers.courseutils import coursetime_paths, coursetime_first @app.task def add(x, y): @@ -171,6 +176,136 @@ def getagegrouprecord(age,sex='male',weightcategory='hwt', return power +def polygon_to_path(polygon,debug=True): + pid = polygon[0] + query = 'SELECT "rowers_geopoint"."id", "rowers_geopoint"."latitude", "rowers_geopoint"."longitude" FROM "rowers_geopoint" WHERE "rowers_geopoint"."polygon_id" = {pid} ORDER BY "rowers_geopoint"."order_in_poly" ASC'.format( + pid=pid + ) + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + with engine.connect() as conn, conn.begin(): + result = conn.execute(query) + points = result.fetchall() + + conn.close() + engine.dispose() + s = [] + + for point in points: + s.append([point[1],point[2]]) + + p = path.Path(s[:-1]) + + return p + +@app.task(bind=True) +def handle_check_race_course(self, + f1,workoutid,courseid, + recordid,**kwargs): + + if 'debug' in kwargs: + debug = kwargs['debug'] + else: + debug = False + + columns = ['time',' latitude',' longitude','cum_dist'] + + try: + row = rdata(csvfile=f1) + except IOError: + try: + row = rdata(f1 + '.csv') + except IOError: + try: + row = rdata(f1 + '.gz') + except IOError: + return 0 + + + rowdata = row.df + + rowdata.rename(columns = { + ' latitude':'latitude', + ' longitude':'longitude', + ' ElapsedTime (sec)': 'time', + }, inplace=True) + + + rowdata.fillna(method='backfill',inplace=True) + + rowdata['time'] = rowdata['time']-rowdata.ix[0,'time'] + # we may want to expand the time (interpolate) + rowdata['dt'] = rowdata['time'].apply( + lambda x: timedelta(seconds=x) + ) + rowdata = rowdata.resample('100ms',on='dt').mean() + rowdata = rowdata.interpolate() + + # initiate database engine + + if debug: + engine = create_engine(database_url_debug, echo=False) + else: + engine = create_engine(database_url, echo=False) + + # get polygons + query = 'SELECT "rowers_geopolygon"."id" FROM "rowers_geopolygon" WHERE "rowers_geopolygon"."course_id" = {courseid} ORDER BY "rowers_geopolygon"."order_in_course" ASC'.format( + courseid=courseid + ) + + with engine.connect() as conn, conn.begin(): + try: + result = conn.execute(query) + polygons = result.fetchall() + except: + print "Database locked" + conn.close() + engine.dispose() + + paths = [] + for polygon in polygons: + path = polygon_to_path(polygon,debug=debug) + paths.append(path) + + ( + coursetimeseconds, + coursemeters, + coursecompleted, + + ) = coursetime_paths(rowdata,paths) + ( + coursetimefirst, + coursemetersfirst, + firstcompleted + ) = coursetime_first( + rowdata,paths) + + coursetimeseconds = coursetimeseconds-coursetimefirst + coursemeters = coursemeters-coursemetersfirst + + if coursecompleted: + query = 'UPDATE "rowers_virtualraceresult" SET "coursecompleted" = 1, "duration" = "{duration}", "distance" = {distance}, "workoutid" = {workoutid} WHERE "id"="{recordid}"'.format( + recordid=recordid, + duration=totaltime_sec_to_string(coursetimeseconds), + distance=int(coursemeters), + workoutid=workoutid, + ) + + with engine.connect() as conn, conn.begin(): + result = conn.execute(query) + + conn.close() + engine.dispose() + + return 1 + + else: + return 2 + + return 0 + @app.task(bind=True) def handle_getagegrouprecords(self, @@ -287,8 +422,6 @@ def handle_update_empower(self, boattype = workoutdict['boattype'] f1 = workoutdict['filename'] - print wid - # oarlength consistency checks will be done in view havedata = 1 diff --git a/rowers/templates/virtualevent.html b/rowers/templates/virtualevent.html index 36d75779..0765650b 100644 --- a/rowers/templates/virtualevent.html +++ b/rowers/templates/virtualevent.html @@ -4,6 +4,10 @@ {% block title %}Rowsandall Virtual Race{% endblock %} +{% block scripts %} +{% include "monitorjobs.html" %} +{% endblock %} + {% block content %}
diff --git a/rowers/views.py b/rowers/views.py index 0598d3d4..57c45db9 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -1,3 +1,4 @@ + import time import colorsys import timestring @@ -371,6 +372,7 @@ verbose_job_status = { 'long_test_task': 'Long Test Task', 'long_test_task2': 'Long Test Task 2', 'update_empower': 'Correct Empower Inflated Power Bug', + 'submit_race': 'Checking Race Course Result', } def get_job_status(jobid): @@ -13788,13 +13790,21 @@ def virtualevent_submit_result_view(request,id=0): workouts = Workout.objects.filter(id=selectedworkout) - result,comments,errors = add_workout_race(workouts,race,r) + result,comments,errors,jobid = add_workout_race(workouts,race,r) for c in comments: messages.info(request,c) for er in errors: messages.error(request,er) + if jobid: + try: + request.session['async_tasks'] += [(jobid,'submit_race')] + except KeyError: + request.session['async_tasks'] = [(jobid,'submit_race')] + + messages.info(request,"We are evaluating your result. The page will reload when we're done. Your result will show up if you adhered to the course") + # redirect to race page url = reverse(virtualevent_view, kwargs = { From 4f8bdf15437c9457d877344ce040fe40c2cfc70e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 17 May 2018 18:19:42 +0200 Subject: [PATCH 2/2] made everything work with async --- rowers/models.py | 8 +++ rowers/plannedsessions.py | 77 ++++++++++++++++++++++++++- rowers/tasks.py | 13 +++++ rowers/templates/plannedsessions.html | 10 ++++ rowers/views.py | 44 ++++++++++----- 5 files changed, 137 insertions(+), 15 deletions(-) diff --git a/rowers/models.py b/rowers/models.py index 374339ff..c5dd1270 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -1575,6 +1575,14 @@ class VirtualRaceResult(models.Model): age = models.IntegerField(null=True) +class CourseTestResult(models.Model): + userid = models.IntegerField(default=0) + workoutid = models.IntegerField(null=True) + plannedsession = models.ForeignKey(PlannedSession) + duration = models.TimeField(default=datetime.time(1,0)) + distance = models.IntegerField(default=0) + coursecompleted = models.BooleanField(default=False) + class VirtualRaceResultForm(ModelForm): class Meta: model = VirtualRaceResult diff --git a/rowers/plannedsessions.py b/rowers/plannedsessions.py index 3911458f..7f01d08c 100644 --- a/rowers/plannedsessions.py +++ b/rowers/plannedsessions.py @@ -1,6 +1,7 @@ # Python from django.utils import timezone from datetime import datetime +import datetime as dt from datetime import timedelta from datetime import date import time @@ -18,7 +19,7 @@ queuehigh = django_rq.get_queue('low') from rowers.models import ( Rower, Workout,Team, GeoCourse, TrainingMicroCycle,TrainingMesoCycle,TrainingMacroCycle, - TrainingPlan,PlannedSession,VirtualRaceResult + TrainingPlan,PlannedSession,VirtualRaceResult,CourseTestResult ) import metrics @@ -61,6 +62,17 @@ def add_workouts_plannedsession(ws,ps,r): w.save() result += 1 comments.append('Attached workout %i to session' % w.id) + if ps.sessiontype == 'coursetest': + record = CourseTestResult( + userid=w.user.id, + plannedsession=ps, + duration=dt.time(0,0), + coursecompleted=False, + ) + record.save() + job = myqueue(queue,handle_check_race_course,w.csvfilename, + w.id,ps.course.id,record.id, + mode='coursetest') else: errors.append('Workout %i did not match session dates' % w.id) @@ -273,7 +285,68 @@ def is_session_complete_ws(ws,ps): if not completiondate: completiondate = ws.reverse()[0].date return ratio,'partial',completiondate - elif ps.sessiontype in ['coursetest','race']: + elif ps.sessiontype == 'race': + vs = VirtualRaceResult.objects.filter(race=ps) + wids = [w.id for w in ws] + for record in vs: + if record.workoutid in wids: + if record.coursecompleted: + ratio = record.distance/ps.sessionvalue + return ratio,'completed',completiondate + + if ps.course: + ( + coursetime, + coursemeters, + coursecompleted + ) = courses.get_time_course(ws,ps.course) + if coursecompleted: + return 1.0,'completed',completiondate + else: + return ratio,'partial',completiondate + else: + if ps.criterium == 'exact': + if ratio == 1.0: + return ratio,'completed',completiondate + else: + if not completiondate: + completiondate = ws.reverse()[0].date + return ratio,'partial',completiondate + elif ps.criterium == 'minimum': + if ratio >= 1.0: + return ratio,'completed',completiondate + else: + if not completiondate: + completiondate = ws.reverse()[0].date + + return ratio,'partial',completiondate + else: + if ratio>cratiomin and ratio {{ ps.startdate|date:"Y-m-d" }} {{ ps.enddate|date:"Y-m-d" }} + {% if ps.sessiontype != 'race' %} {% if ps.name != '' %} {{ ps.name }} @@ -122,6 +123,15 @@ Unnamed Session {% endif %} + {% else %} + {% if ps.name != '' %} + {{ ps.name }} + {% else %} + Unnamed Race + {% endif %} + {% endif %} {{ ps.get_sessiontype_display }} {{ ps.get_sessionmode_display }} diff --git a/rowers/views.py b/rowers/views.py index 57c45db9..bba37d17 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -12857,6 +12857,9 @@ def plannedsessions_view(request,timeperiod='thisweek',rowerid=0): sps = get_sessions(r,startdate=startdate,enddate=enddate) + for ps in sps: + print ps.name,ps.sessiontype + completeness = {} actualvalue = {} completiondate = {} @@ -13187,7 +13190,7 @@ def plannedsession_view(request,id=0,rowerid=0, ws = get_workouts_session(r,ps) ratio,status,completiondate = is_session_complete(r,ps) - + ratio = int(100.*ratio) # ranking for test @@ -13214,19 +13217,34 @@ def plannedsession_view(request,id=0,rowerid=0, 'type': w.workouttype, } if ps.sessiontype == 'coursetest': - ( - coursetimeseconds, - coursemeters, - coursecompleted - ) = courses.get_time_course([w],ps.course) - intsecs = int(coursetimeseconds) - microsecs = int(1.e6*(coursetimeseconds-intsecs)) + vs = CourseTestResult.objects.filter(plannedsession=ps) + if vs: + for record in vs: + if record.workoutid == w.id: + coursemeters = record.distance + coursecompleted = record.coursecompleted + t = record.duration + wdict['time'] = datetime.timedelta( + hours=t.hour, + seconds=t.second, + minutes=t.minute, + microseconds=t.microsecond + ) + wdict['distance'] = int(round(coursemeters)) + else: + ( + coursetimeseconds, + coursemeters, + coursecompleted + ) = courses.get_time_course([w],ps.course) + intsecs = int(coursetimeseconds) + microsecs = int(1.e6*(coursetimeseconds-intsecs)) - wdict['time'] = datetime.timedelta( - seconds=intsecs, - microseconds=microsecs + wdict['time'] = datetime.timedelta( + seconds=intsecs, + microseconds=microsecs ) - wdict['distance'] = int(round(coursemeters)) + wdict['distance'] = int(round(coursemeters)) ranking.append(wdict) @@ -13631,7 +13649,7 @@ def virtualevent_create_view(request): end_time=end_time, course=geocourse, comment=comment, - sessiontype = 'coursetest', + sessiontype = 'race', timezone=timezone_str, evaluation_closure=evaluation_closure, registration_closure=registration_closure,