From 11cec956b7e98ea58de303c88144fc0cc471305a Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 06:30:24 +0100 Subject: [PATCH 1/8] fix --- rowers/views/analysisviews.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/rowers/views/analysisviews.py b/rowers/views/analysisviews.py index 0ad27124..07e5e71c 100644 --- a/rowers/views/analysisviews.py +++ b/rowers/views/analysisviews.py @@ -209,13 +209,16 @@ def analysis_new(request,userid=0,function='boxplot',teamid=0,id=''): query = request.POST.get('q') if query: query_list = query.split() - workouts = workouts.filter( - reduce(operator.and_, - (Q(name__icontains=q) for q in query_list)) | - reduce(operator.and_, + try: + workouts = workouts.filter( + reduce(operator.and_, + (Q(name__icontains=q) for q in query_list)) | + reduce(operator.and_, (Q(notes__icontains=q) for q in query_list)) - ) - searchform = SearchForm(initial={'q':query}) + ) + searchform = SearchForm(initial={'q':query}) + except TypeError: + searchform = SearchForm() else: searchform = SearchForm() From e55682edd0428052c1ca193ad9de86e6152488e3 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 06:56:50 +0100 Subject: [PATCH 2/8] refactoring and adding email confirmation / trimp calc to api v2 --- apilog.log | 22 ++++++++++ rowers/dataprep.py | 90 +++++++++++++++++++++------------------- rowers/views/apiviews.py | 19 ++++++++- 3 files changed, 88 insertions(+), 43 deletions(-) create mode 100644 apilog.log diff --git a/apilog.log b/apilog.log new file mode 100644 index 00000000..06d4998a --- /dev/null +++ b/apilog.log @@ -0,0 +1,22 @@ +2020-12-31 05:38:33.044672+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:39:20.796493+00:00: sander (strokedatajson_v2 POST) +No data in request +2020-12-31 05:39:35.245274+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:40:51.057825+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:40:52.842430+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 05:50:10.246269+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:50:54.083948+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:51:50.273905+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:51:52.185284+00:00: sander (strokedatajson_v2 POST completed successfully) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 3d7ae8fb..d72c1517 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1430,6 +1430,53 @@ def create_row_df(r,distance,duration,startdatetime,workouttype='rower', from rowers.utils import totaltime_sec_to_string +def checkbreakthrough(w, r): + isbreakthrough = False + ishard = False + workouttype = w.workouttype + if workouttype in rowtypes: + cpdf,delta,cpvalues = setcp(w) + if not cpdf.empty: + if workouttype in otwtypes: + res, btvalues, res2 = utils.isbreakthrough( + delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) + success = update_rolling_cp(r,otwtypes,'water') + + elif workouttype in otetypes: + res, btvalues, res2 = utils.isbreakthrough( + delta, cpvalues, r.ep0, r.ep1, r.ep2, r.ep3, r.ecpratio) + success = update_rolling_cp(r,otetypes,'erg') + else: + res = 0 + res2 = 0 + if res: + isbreakthrough = True + if res2 and not isbreakthrough: + ishard = True + + # submit email task to send email about breakthrough workout + if isbreakthrough: + if r.getemailnotifications and not r.emailbounced: + job = myqueue(queuehigh,handle_sendemail_breakthrough, + w.id, + r.user.email, + r.user.first_name, + r.user.last_name, + btvalues=btvalues.to_json()) + + # submit email task to send email about breakthrough workout + if ishard: + if r.getemailnotifications and not r.emailbounced: + job = myqueue(queuehigh,handle_sendemail_hard, + w.id, + r.user.email, + r.user.first_name, + r.user.last_name, + btvalues=btvalues.to_json()) + + return isbreakthrough, ishard + + # Processes painsled CSV file to database def save_workout_database(f2, r, dosmooth=True, workouttype='rower', boattype='1x', @@ -1701,48 +1748,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', job = myqueue(queuehigh,handle_calctrimp,w.id,f2,r.ftp,r.sex,r.hrftp,r.max,r.rest) - - isbreakthrough = False - ishard = False - if workouttype in rowtypes: - cpdf,delta,cpvalues = setcp(w) - if not cpdf.empty: - if workouttype in otwtypes: - res, btvalues, res2 = utils.isbreakthrough( - delta, cpvalues, r.p0, r.p1, r.p2, r.p3, r.cpratio) - success = update_rolling_cp(r,otwtypes,'water') - - elif workouttype in otetypes: - res, btvalues, res2 = utils.isbreakthrough( - delta, cpvalues, r.ep0, r.ep1, r.ep2, r.ep3, r.ecpratio) - success = update_rolling_cp(r,otetypes,'erg') - else: - res = 0 - res2 = 0 - if res: - isbreakthrough = True - if res2 and not isbreakthrough: - ishard = True - - # submit email task to send email about breakthrough workout - if isbreakthrough: - if r.getemailnotifications and not r.emailbounced: - job = myqueue(queuehigh,handle_sendemail_breakthrough, - w.id, - r.user.email, - r.user.first_name, - r.user.last_name, - btvalues=btvalues.to_json()) - - # submit email task to send email about breakthrough workout - if ishard: - if r.getemailnotifications and not r.emailbounced: - job = myqueue(queuehigh,handle_sendemail_hard, - w.id, - r.user.email, - r.user.first_name, - r.user.last_name, - btvalues=btvalues.to_json()) + isbreakthrough, ishard = checkbreakthrough(w, r) return (w.id, message) diff --git a/rowers/views/apiviews.py b/rowers/views/apiviews.py index 8e446f2b..b8ceaf67 100644 --- a/rowers/views/apiviews.py +++ b/rowers/views/apiviews.py @@ -4,7 +4,8 @@ from __future__ import print_function from __future__ import unicode_literals from rowers.views.statements import * - +from rowers.tasks import handle_calctrimp +from rowers.mailprocessing import send_confirm # Stroke data form to test API upload @login_required() @@ -310,6 +311,22 @@ def strokedatajson_v2(request,id): datadf = dataprep.dataprep(rowdata,id=row.id,bands=True,barchart=True,otwpower=True,empower=True) + job = myqueue(queuehigh, handle_calctrimp, row.id, row.csvfilename, r.ftp,r.sex,r.hrftp, r.max, r.rest) + + isbreakthrough, ishard = dataprep.checkbreakthrough(row, r) + + if r.getemailnotifications and not r.emailbounced: + link = settings.SITE_URL+reverse( + r.defaultlandingpage, + kwargs = { + 'id':encoder.encode_hex(row.id), + } + ) + email_sent = send_confirm(r.user, row.name, link, '') + + result = uploads.do_sync(row,{},quick=True) + + with open('apilog.log','a') as logfile: logfile.write(str(timezone.now())+": ") logfile.write(request.user.username+" (strokedatajson_v2 POST completed successfully) \n") From a85d19c008264cabd79a839e2fe351e9d4ab296e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 07:13:12 +0100 Subject: [PATCH 3/8] improved api logging --- apilog.log | 56 ++++++++++++++++++++++++++++++++++++++++ rowers/uploads.py | 1 - rowers/views/apiviews.py | 10 +++++-- 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/apilog.log b/apilog.log index 06d4998a..e19acd55 100644 --- a/apilog.log +++ b/apilog.log @@ -20,3 +20,59 @@ No data in request No data in request 1595667900.0: sander(strokedatajson_v2 POST - data parsed) 2020-12-31 05:51:52.185284+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 05:59:52.889322+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 05:59:54.598443+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:01:15.553477+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:01:17.410378+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:02:02.098587+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:02:03.864872+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:03:31.462548+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:03:33.169393+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:05:44.285370+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:05:46.002410+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:07:49.175386+00:00: sander (strokedatajson_v2 POST) +No data in request +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:07:50.792027+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:09:10.741327+00:00: sander (strokedatajson_v2 POST) +{"distance": 5, "power": 112, "hr": 132, "pace": 145800, "time": 0, "spm": 11, "latitude": 52.2264097, "longitude": 6.8493638} +{"distance": 12, "power": 221, "hr": 131, "pace": 116400, "time": 2200, "spm": 41, "latitude": 52.2263474, "longitude": 6.8495814} +{"distance": 19, "power": 511, "hr": 131, "pace": 88100, "time": 4599, "spm": 56, "latitude": 52.2262715, "longitude": 6.8496975} +{"distance": 27, "power": 673, "hr": 132, "pace": 80400, "time": 7000, "spm": 59, "latitude": 52.2262003, "longitude": 6.8498095} +{"distance": 35, "power": 744, "hr": 133, "pace": 77700, "time": 9599, "spm": 55, "latitude": 52.2261312, "longitude": 6.8499267} +{"distance": 43, "power": 754, "hr": 136, "pace": 77400, "time": 12000, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} +{"distance": 51, "power": 754, "hr": 139, "pace": 77400, "time": 14400, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} +{"distance": 59, "power": 749, "hr": 142, "pace": 77600, "time": 16799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} +{"distance": 67, "power": 729, "hr": 145, "pace": 78300, "time": 19400, "spm": 48, "latitude": 52.2257749, "longitude": 6.8503918} +{"distance": 75, "power": 729, "hr": 147, "pace": 78300, "time": 21799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} +{"distance": 82, "power": 726, "hr": 150, "pace": 78400, "time": 24200, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} +{"distance": 90, "power": 709, "hr": 152, "pace": 79000, "time": 26599, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} +{"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:09:12.451036+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:10:46.757769+00:00: sander (strokedatajson_v2 POST) +{"distance": 5, "power": 112, "hr": 132, "pace": 145800, "time": 0, "spm": 11, "latitude": 52.2264097, "longitude": 6.8493638} +{"distance": 12, "power": 221, "hr": 131, "pace": 116400, "time": 2200, "spm": 41, "latitude": 52.2263474, "longitude": 6.8495814} +{"distance": 19, "power": 511, "hr": 131, "pace": 88100, "time": 4599, "spm": 56, "latitude": 52.2262715, "longitude": 6.8496975} +{"distance": 27, "power": 673, "hr": 132, "pace": 80400, "time": 7000, "spm": 59, "latitude": 52.2262003, "longitude": 6.8498095} +{"distance": 35, "power": 744, "hr": 133, "pace": 77700, "time": 9599, "spm": 55, "latitude": 52.2261312, "longitude": 6.8499267} +{"distance": 43, "power": 754, "hr": 136, "pace": 77400, "time": 12000, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} +{"distance": 51, "power": 754, "hr": 139, "pace": 77400, "time": 14400, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} +{"distance": 59, "power": 749, "hr": 142, "pace": 77600, "time": 16799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} +{"distance": 67, "power": 729, "hr": 145, "pace": 78300, "time": 19400, "spm": 48, "latitude": 52.2257749, "longitude": 6.8503918} +{"distance": 75, "power": 729, "hr": 147, "pace": 78300, "time": 21799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} +{"distance": 82, "power": 726, "hr": 150, "pace": 78400, "time": 24200, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} +{"distance": 90, "power": 709, "hr": 152, "pace": 79000, "time": 26599, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} +{"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} +1595667900.0: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:10:48.555945+00:00: sander (strokedatajson_v2 POST completed successfully) diff --git a/rowers/uploads.py b/rowers/uploads.py index b122e3df..0104a2fd 100644 --- a/rowers/uploads.py +++ b/rowers/uploads.py @@ -542,7 +542,6 @@ def do_sync(w,options, quick=False): w.uploadedtogarmin = options['garminid'] w.save() except KeyError: - print('keyerror') pass diff --git a/rowers/views/apiviews.py b/rowers/views/apiviews.py index b8ceaf67..333a281c 100644 --- a/rowers/views/apiviews.py +++ b/rowers/views/apiviews.py @@ -7,6 +7,8 @@ from rowers.views.statements import * from rowers.tasks import handle_calctrimp from rowers.mailprocessing import send_confirm +import sys + # Stroke data form to test API upload @login_required() def strokedataform(request,id=0): @@ -118,10 +120,14 @@ def strokedatajson_v2(request,id): logfile.write(str(timezone.now())+": ") logfile.write(request.user.username+" (strokedatajson_v2 POST) \n") try: - logfile.write(request.data['data']+"\n") + for d in request.data['data']: + logfile.write(json.dumps(d)) + logfile.write("\n") except KeyError: try: - logfile.write(request.data['strokedata']+"\n") + for d in request.data['strokedata']: + logfile.write(json.dumps(d)) + logfile.write("\n") except KeyError: logfile.write("No data in request.data\n") except (AttributeError,TypeError): From f8c44b3c3e18e9b1b409097200bc8a77a25005ba Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 11:22:06 +0100 Subject: [PATCH 4/8] adding apilog.log to gitignore --- .gitignore | 1 + apilog.log | 16 ++++++++++++++++ rowers/views/apiviews.py | 3 ++- rowers/views/importviews.py | 2 +- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f808e834..c2fcce79 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /django_cache/ *.crt *.key +*.log garminlog.log strava_webhooks.log diff --git a/apilog.log b/apilog.log index e19acd55..261622bc 100644 --- a/apilog.log +++ b/apilog.log @@ -76,3 +76,19 @@ No data in request {"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} 1595667900.0: sander(strokedatajson_v2 POST - data parsed) 2020-12-31 06:10:48.555945+00:00: sander (strokedatajson_v2 POST completed successfully) +2020-12-31 06:14:28.182452+00:00: sander (strokedatajson_v2 POST) +{"distance": 5, "power": 112, "hr": 132, "pace": 145800, "time": 0, "spm": 11, "latitude": 52.2264097, "longitude": 6.8493638} +{"distance": 12, "power": 221, "hr": 131, "pace": 116400, "time": 2200, "spm": 41, "latitude": 52.2263474, "longitude": 6.8495814} +{"distance": 19, "power": 511, "hr": 131, "pace": 88100, "time": 4599, "spm": 56, "latitude": 52.2262715, "longitude": 6.8496975} +{"distance": 27, "power": 673, "hr": 132, "pace": 80400, "time": 7000, "spm": 59, "latitude": 52.2262003, "longitude": 6.8498095} +{"distance": 35, "power": 744, "hr": 133, "pace": 77700, "time": 9599, "spm": 55, "latitude": 52.2261312, "longitude": 6.8499267} +{"distance": 43, "power": 754, "hr": 136, "pace": 77400, "time": 12000, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} +{"distance": 51, "power": 754, "hr": 139, "pace": 77400, "time": 14400, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} +{"distance": 59, "power": 749, "hr": 142, "pace": 77600, "time": 16799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} +{"distance": 67, "power": 729, "hr": 145, "pace": 78300, "time": 19400, "spm": 48, "latitude": 52.2257749, "longitude": 6.8503918} +{"distance": 75, "power": 729, "hr": 147, "pace": 78300, "time": 21799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} +{"distance": 82, "power": 726, "hr": 150, "pace": 78400, "time": 24200, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} +{"distance": 90, "power": 709, "hr": 152, "pace": 79000, "time": 26599, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} +{"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} +2020-07-25 09:05:00+00:00: sander(strokedatajson_v2 POST - data parsed) +2020-12-31 06:14:29.859936+00:00: sander (strokedatajson_v2 POST completed successfully) diff --git a/rowers/views/apiviews.py b/rowers/views/apiviews.py index 333a281c..19f60590 100644 --- a/rowers/views/apiviews.py +++ b/rowers/views/apiviews.py @@ -8,6 +8,7 @@ from rowers.tasks import handle_calctrimp from rowers.mailprocessing import send_confirm import sys +import arrow # Stroke data form to test API upload @login_required() @@ -262,7 +263,7 @@ def strokedatajson_v2(request,id): unixtime = starttime+time with open('apilog.log','a') as logfile: - logfile.write(str(starttime)+": ") + logfile.write(str(arrow.get(starttime).datetime)+": ") logfile.write(request.user.username+"(strokedatajson_v2 POST - data parsed) \r\n") diff --git a/rowers/views/importviews.py b/rowers/views/importviews.py index 215edc07..5056926d 100644 --- a/rowers/views/importviews.py +++ b/rowers/views/importviews.py @@ -1717,7 +1717,7 @@ def workout_getimportview(request,externalid,source = 'c2'): try: notes = data['comments'] name = notes[:40] - except KeyError: + except (KeyError,TypeError): comments = 'C2 Import Workout from {startdatetime}'.format(startdatetime=startdatetime) name = notes From b07de9e9672035c5acebd5cefc1e0a496254bcb6 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 11:22:21 +0100 Subject: [PATCH 5/8] removing apilog --- apilog.log | 94 ------------------------------------------------------ 1 file changed, 94 deletions(-) delete mode 100644 apilog.log diff --git a/apilog.log b/apilog.log deleted file mode 100644 index 261622bc..00000000 --- a/apilog.log +++ /dev/null @@ -1,94 +0,0 @@ -2020-12-31 05:38:33.044672+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:39:20.796493+00:00: sander (strokedatajson_v2 POST) -No data in request -2020-12-31 05:39:35.245274+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:40:51.057825+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:40:52.842430+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 05:50:10.246269+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:50:54.083948+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:51:50.273905+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:51:52.185284+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 05:59:52.889322+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 05:59:54.598443+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:01:15.553477+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:01:17.410378+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:02:02.098587+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:02:03.864872+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:03:31.462548+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:03:33.169393+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:05:44.285370+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:05:46.002410+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:07:49.175386+00:00: sander (strokedatajson_v2 POST) -No data in request -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:07:50.792027+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:09:10.741327+00:00: sander (strokedatajson_v2 POST) -{"distance": 5, "power": 112, "hr": 132, "pace": 145800, "time": 0, "spm": 11, "latitude": 52.2264097, "longitude": 6.8493638} -{"distance": 12, "power": 221, "hr": 131, "pace": 116400, "time": 2200, "spm": 41, "latitude": 52.2263474, "longitude": 6.8495814} -{"distance": 19, "power": 511, "hr": 131, "pace": 88100, "time": 4599, "spm": 56, "latitude": 52.2262715, "longitude": 6.8496975} -{"distance": 27, "power": 673, "hr": 132, "pace": 80400, "time": 7000, "spm": 59, "latitude": 52.2262003, "longitude": 6.8498095} -{"distance": 35, "power": 744, "hr": 133, "pace": 77700, "time": 9599, "spm": 55, "latitude": 52.2261312, "longitude": 6.8499267} -{"distance": 43, "power": 754, "hr": 136, "pace": 77400, "time": 12000, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} -{"distance": 51, "power": 754, "hr": 139, "pace": 77400, "time": 14400, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} -{"distance": 59, "power": 749, "hr": 142, "pace": 77600, "time": 16799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} -{"distance": 67, "power": 729, "hr": 145, "pace": 78300, "time": 19400, "spm": 48, "latitude": 52.2257749, "longitude": 6.8503918} -{"distance": 75, "power": 729, "hr": 147, "pace": 78300, "time": 21799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} -{"distance": 82, "power": 726, "hr": 150, "pace": 78400, "time": 24200, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} -{"distance": 90, "power": 709, "hr": 152, "pace": 79000, "time": 26599, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} -{"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:09:12.451036+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:10:46.757769+00:00: sander (strokedatajson_v2 POST) -{"distance": 5, "power": 112, "hr": 132, "pace": 145800, "time": 0, "spm": 11, "latitude": 52.2264097, "longitude": 6.8493638} -{"distance": 12, "power": 221, "hr": 131, "pace": 116400, "time": 2200, "spm": 41, "latitude": 52.2263474, "longitude": 6.8495814} -{"distance": 19, "power": 511, "hr": 131, "pace": 88100, "time": 4599, "spm": 56, "latitude": 52.2262715, "longitude": 6.8496975} -{"distance": 27, "power": 673, "hr": 132, "pace": 80400, "time": 7000, "spm": 59, "latitude": 52.2262003, "longitude": 6.8498095} -{"distance": 35, "power": 744, "hr": 133, "pace": 77700, "time": 9599, "spm": 55, "latitude": 52.2261312, "longitude": 6.8499267} -{"distance": 43, "power": 754, "hr": 136, "pace": 77400, "time": 12000, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} -{"distance": 51, "power": 754, "hr": 139, "pace": 77400, "time": 14400, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} -{"distance": 59, "power": 749, "hr": 142, "pace": 77600, "time": 16799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} -{"distance": 67, "power": 729, "hr": 145, "pace": 78300, "time": 19400, "spm": 48, "latitude": 52.2257749, "longitude": 6.8503918} -{"distance": 75, "power": 729, "hr": 147, "pace": 78300, "time": 21799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} -{"distance": 82, "power": 726, "hr": 150, "pace": 78400, "time": 24200, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} -{"distance": 90, "power": 709, "hr": 152, "pace": 79000, "time": 26599, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} -{"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} -1595667900.0: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:10:48.555945+00:00: sander (strokedatajson_v2 POST completed successfully) -2020-12-31 06:14:28.182452+00:00: sander (strokedatajson_v2 POST) -{"distance": 5, "power": 112, "hr": 132, "pace": 145800, "time": 0, "spm": 11, "latitude": 52.2264097, "longitude": 6.8493638} -{"distance": 12, "power": 221, "hr": 131, "pace": 116400, "time": 2200, "spm": 41, "latitude": 52.2263474, "longitude": 6.8495814} -{"distance": 19, "power": 511, "hr": 131, "pace": 88100, "time": 4599, "spm": 56, "latitude": 52.2262715, "longitude": 6.8496975} -{"distance": 27, "power": 673, "hr": 132, "pace": 80400, "time": 7000, "spm": 59, "latitude": 52.2262003, "longitude": 6.8498095} -{"distance": 35, "power": 744, "hr": 133, "pace": 77700, "time": 9599, "spm": 55, "latitude": 52.2261312, "longitude": 6.8499267} -{"distance": 43, "power": 754, "hr": 136, "pace": 77400, "time": 12000, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} -{"distance": 51, "power": 754, "hr": 139, "pace": 77400, "time": 14400, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} -{"distance": 59, "power": 749, "hr": 142, "pace": 77600, "time": 16799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} -{"distance": 67, "power": 729, "hr": 145, "pace": 78300, "time": 19400, "spm": 48, "latitude": 52.2257749, "longitude": 6.8503918} -{"distance": 75, "power": 729, "hr": 147, "pace": 78300, "time": 21799, "spm": 48, "latitude": 52.2259154, "longitude": 6.8502769} -{"distance": 82, "power": 726, "hr": 150, "pace": 78400, "time": 24200, "spm": 48, "latitude": 52.2259872, "longitude": 6.8501561} -{"distance": 90, "power": 709, "hr": 152, "pace": 79000, "time": 26599, "spm": 48, "latitude": 52.2260576, "longitude": 6.8500366} -{"distance": 100, "power": 707, "hr": 153, "pace": 79100, "time": 29000, "spm": 49, "latitude": 52.2261312, "longitude": 6.8499267} -2020-07-25 09:05:00+00:00: sander(strokedatajson_v2 POST - data parsed) -2020-12-31 06:14:29.859936+00:00: sander (strokedatajson_v2 POST completed successfully) From 65e580dd9237dec8565572f664fe3670002c8856 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 14:08:58 +0100 Subject: [PATCH 6/8] new python 3.8.6 install - new requirements.txt --- .gitignore | 1 + requirements.txt | 50 ++++++++++++++++-------------------------------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index c2fcce79..05b750b7 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ config.yaml # virtualenv /venv/ /venv38/ +/venv39/ /py27/ /py2/ /django2/ diff --git a/requirements.txt b/requirements.txt index f882b9d3..34437479 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ apipkg==1.5 appdirs==1.4.3 arcgis==1.6.0 arrow==0.13.1 -asgiref==3.2.10 asn1crypto==0.24.0 atomicwrites==1.3.0 attrs==19.1.0 @@ -13,8 +12,7 @@ billiard==3.6.0.0 bleach==3.1.0 bokeh==2.2.3 boto==2.49.0 -boto3==1.14.7 -botocore==1.17.7 +bottle==0.12.18 braintree==3.55.0 cairocffi==1.0.2 celery==4.3.0 @@ -23,13 +21,13 @@ cffi==1.14.0 chardet==3.0.4 Click==7.0 cloudpickle==1.2.2 -colorama==0.4.4 +colorama==0.4.1 colorclass==2.2.0 cookies==2.2.1 coreapi==2.3.3 coreschema==0.0.4 coverage==4.5.3 -cryptography==2.9.2 +cryptography==2.6.1 cycler==0.10.0 Cython==0.29.21 dask==2.20.0 @@ -57,11 +55,10 @@ django-rest-framework==0.1.0 django-rest-swagger==2.2.0 django-rq==1.3.1 django-rq-dashboard==0.3.3 -django-ses==1.0.0 +django-ses==0.8.10 django-shell-plus==1.1.7 django-social-share==1.3.2 -django-sslserver==0.22 -django-suit==0.2.26 +django-suit==0.2.28 django-suit-rq==1.0.1 django-tz-detect==0.2.9 djangorestframework==3.9.2 @@ -71,17 +68,17 @@ entrypoints==0.3 execnet==1.5.0 factory-boy==2.11.1 Faker==1.0.4 -fastparquet==0.4.1 +fastparquet==0.3.2 fitparse==1.1.0 Flask==1.0.2 fsspec==0.5.2 future==0.17.1 -garminconnect==0.1.14 geocoder==1.38.1 geoip2==3.0.0 geos==0.2.1 grpcio==1.26.0 grpcio-tools==1.26.0 +gunicorn==20.0.4 holoviews==1.13.5 html5lib==1.0.1 htmlmin==0.1.12 @@ -103,15 +100,14 @@ itypes==1.1.0 jedi==0.13.3 jeepney==0.4 Jinja2==2.10 -jmespath==0.10.0 json5==0.8.5 jsonschema==3.0.1 jupyter==1.0.0 -jupyter-client==5.2.4 -jupyter-console==6.0.0 -jupyter-core==4.4.0 -jupyterlab==0.35.4 -jupyterlab-server==0.3.0 +jupyter-client==6.1.7 +jupyter-console==6.2.0 +jupyter-core==4.7.0 +jupyterlab==0.35.6 +jupyterlab-server==0.2.0 keyring==18.0.0 kiwisolver==1.0.1 kombu==4.5.0 @@ -121,7 +117,6 @@ Markdown==3.0.1 MarkupSafe==1.1.1 matplotlib==3.0.3 maxminddb==1.5.4 -minify==0.1.4 MiniMockTest==0.5 mistune==0.8.4 mock==2.0.0 @@ -151,7 +146,6 @@ pexpect==4.6.0 pickleshare==0.7.5 Pillow==8.0.1 pip-upgrader==1.4.6 -pkginfo==1.6.0 pluggy==0.9.0 prometheus-client==0.6.0 prompt-toolkit==2.0.9 @@ -159,13 +153,11 @@ protobuf==3.11.1 psycopg2==2.8.1 ptyprocess==0.6.0 py==1.8.0 -pyarrow==0.17.1 -pycairo==1.20.0 +pyarrow==2.0.0 +pycairo==1.19.0 pycparser==2.19 pyct==0.4.8 -pygeoip==0.3.2 -Pygments==2.7.1 -pyOpenSSL==19.1.0 +Pygments==2.3.1 pyparsing==2.3.1 pyrsistent==0.14.11 pyshp==2.1.0 @@ -174,7 +166,6 @@ pytest-django==3.4.8 pytest-forked==1.0.2 pytest-runner==4.4 pytest-sugar==0.9.2 -pytest-timeout==1.4.2 pytest-xdist==1.27.0 python-dateutil==2.8.0 python-memcached==1.59 @@ -182,23 +173,18 @@ python-twitter==3.5 pytz==2020.1 pyviz-comms==0.7.6 pywin32-ctypes==0.2.0 -pywinpty==0.5.5 PyYAML==5.1 pyzmq==18.0.1 qtconsole==4.4.3 ratelim==0.1.6 -readme-renderer==28.0 -redis==3.2.1 +redis==3.5.3 requests==2.23.0 requests-oauthlib==1.2.0 -requests-toolbelt==0.9.1 -rfc3986==1.4.0 rowingdata==3.0.6 rowingphysics==0.5.0 rq==0.13.0 rules==2.1 -s3transfer==0.3.3 -scipy==1.5.0 +scipy==1.5.4 SecretStorage==3.1.1 Send2Trash==1.5.0 shell==1.0.1 @@ -221,7 +207,6 @@ toolz==0.10.0 tornado==6.0.1 tqdm==4.31.1 traitlets==4.3.2 -twine==3.2.0 typing-extensions==3.7.4.3 units==0.7 uritemplate==3.0.0 @@ -236,4 +221,3 @@ xlrd==1.2.0 xmltodict==0.12.0 yamjam==0.1.7 yamllint==1.15.0 -yuicompressor==2.4.8 From 96b4b641b65d95a06ff857ca37afd73f680f6c7e Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 31 Dec 2020 16:33:55 +0100 Subject: [PATCH 7/8] better api processing (sets duplicate) --- rowers/dataprep.py | 47 +++++++++++++++++++++++++--------------- rowers/views/apiviews.py | 9 ++++++++ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index d72c1517..705e398c 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1477,6 +1477,35 @@ def checkbreakthrough(w, r): return isbreakthrough, ishard +def checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime): + print(workoutdate,workoutstartdatetime,workoutenddatetime) + duplicate = False + ws = Workout.objects.filter(user=r,date=workoutdate,duplicate=False).exclude( + startdatetime__gt=workoutenddatetime + ) + + ws2 = [] + + for ww in ws: + print(ww) + t = ww.duration + delta = datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) + enddatetime = ww.startdatetime+delta + print(enddatetime,workoutstartdatetime) + if enddatetime > workoutstartdatetime: + print('ja') + ws2.append(ww) + + + if (len(ws2) != 0): + print('len ws2 ',len(ws2)) + message = "Warning: This workout overlaps with an existing one and was marked as a duplicate" + duplicate = True + return duplicate + + return duplicate + + # Processes painsled CSV file to database def save_workout_database(f2, r, dosmooth=True, workouttype='rower', boattype='1x', @@ -1685,23 +1714,7 @@ def save_workout_database(f2, r, dosmooth=True, workouttype='rower', workoutenddatetime = workoutstartdatetime+delta # check for duplicate start times and duration - ws = Workout.objects.filter(user=r,date=workoutdate,duplicate=False).exclude( - startdatetime__gt=workoutenddatetime - ) - - ws2 = [] - - for ww in ws: - t = ww.duration - delta = datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) - enddatetime = ww.startdatetime+delta - if enddatetime > workoutstartdatetime: - ws2.append(ww) - - - if (len(ws2) != 0): - message = "Warning: This workout overlaps with an existing one and was marked as a duplicate" - duplicate = True + duplicate = checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime) # test title length if title is not None and len(title)>140: diff --git a/rowers/views/apiviews.py b/rowers/views/apiviews.py index 19f60590..4f532bc3 100644 --- a/rowers/views/apiviews.py +++ b/rowers/views/apiviews.py @@ -296,6 +296,15 @@ def strokedatajson_v2(request,id): timestr = row.startdatetime.strftime("%Y%m%d-%H%M%S") csvfilename ='media/Import_'+timestr+'.csv' + workoutdate = row.date + workoutstartdatetime = row.startdatetime + workoutenddatetime = workoutstartdatetime+datetime.timedelta(seconds=data[' ElapsedTime (sec)'].max()) + + duplicate = dataprep.checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime) + if duplicate: + row.duplicate = True + row.save() + res = data.to_csv(csvfilename+'.gz',index_label='index', compression='gzip') row.csvfilename = csvfilename From 3acfa9d4a061a2e0fa0a373967c4652ed48be483 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 1 Jan 2021 11:02:47 +0100 Subject: [PATCH 8/8] removing print statements --- rowers/dataprep.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rowers/dataprep.py b/rowers/dataprep.py index 705e398c..9aa856ac 100644 --- a/rowers/dataprep.py +++ b/rowers/dataprep.py @@ -1478,7 +1478,6 @@ def checkbreakthrough(w, r): def checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime): - print(workoutdate,workoutstartdatetime,workoutenddatetime) duplicate = False ws = Workout.objects.filter(user=r,date=workoutdate,duplicate=False).exclude( startdatetime__gt=workoutenddatetime @@ -1487,18 +1486,15 @@ def checkduplicates(r,workoutdate,workoutstartdatetime,workoutenddatetime): ws2 = [] for ww in ws: - print(ww) t = ww.duration delta = datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second) enddatetime = ww.startdatetime+delta print(enddatetime,workoutstartdatetime) if enddatetime > workoutstartdatetime: - print('ja') ws2.append(ww) if (len(ws2) != 0): - print('len ws2 ',len(ws2)) message = "Warning: This workout overlaps with an existing one and was marked as a duplicate" duplicate = True return duplicate