From 921e58eef761ebd0eaa7954365ab554054d46655 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 19 Dec 2019 12:44:07 +0100 Subject: [PATCH 1/9] not working --- rowers/models.py | 1 + rowers/otw_power_calculator_pb2.py | 175 ++++++++++++++++++++++++ rowers/otw_power_calculator_pb2_grpc.py | 46 +++++++ rowers/tasks.py | 87 +++++++++--- rowers/views/workoutviews.py | 2 + 5 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 rowers/otw_power_calculator_pb2.py create mode 100644 rowers/otw_power_calculator_pb2_grpc.py diff --git a/rowers/models.py b/rowers/models.py index dcd7df70..d73bb683 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -3203,6 +3203,7 @@ class WorkoutForm(ModelForm): # Used for the rowing physics calculations class AdvancedWorkoutForm(ModelForm): quick_calc = forms.BooleanField(initial=True,required=False) + go_service = forms.BooleanField(initial=False,required=False,label='Experimental') class Meta: model = Workout diff --git a/rowers/otw_power_calculator_pb2.py b/rowers/otw_power_calculator_pb2.py new file mode 100644 index 00000000..79887198 --- /dev/null +++ b/rowers/otw_power_calculator_pb2.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: otw-power-calculator.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='otw-power-calculator.proto', + package='otw_power_calculator', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x1aotw-power-calculator.proto\x12\x14otw_power_calculator\"\x97\x01\n\x13WorkoutPowerRequest\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x10\n\x08\x62oattype\x18\x02 \x01(\t\x12\x10\n\x08\x63rewmass\x18\x03 \x01(\x01\x12\x15\n\rpowermeasured\x18\x04 \x01(\x08\x12\x13\n\x0bprogressurl\x18\x05 \x01(\t\x12\x0e\n\x06secret\x18\x06 \x01(\t\x12\x0e\n\x06silent\x18\x07 \x01(\x08\"#\n\x11\x43\x61lculationResult\x12\x0e\n\x06result\x18\x01 \x01(\x05\x32j\n\x05Power\x12\x61\n\tCalcPower\x12).otw_power_calculator.WorkoutPowerRequest\x1a\'.otw_power_calculator.CalculationResult\"\x00\x62\x06proto3') +) + + + + +_WORKOUTPOWERREQUEST = _descriptor.Descriptor( + name='WorkoutPowerRequest', + full_name='otw_power_calculator.WorkoutPowerRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='filename', full_name='otw_power_calculator.WorkoutPowerRequest.filename', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='boattype', full_name='otw_power_calculator.WorkoutPowerRequest.boattype', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='crewmass', full_name='otw_power_calculator.WorkoutPowerRequest.crewmass', index=2, + number=3, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='powermeasured', full_name='otw_power_calculator.WorkoutPowerRequest.powermeasured', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='progressurl', full_name='otw_power_calculator.WorkoutPowerRequest.progressurl', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='secret', full_name='otw_power_calculator.WorkoutPowerRequest.secret', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='silent', full_name='otw_power_calculator.WorkoutPowerRequest.silent', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=53, + serialized_end=204, +) + + +_CALCULATIONRESULT = _descriptor.Descriptor( + name='CalculationResult', + full_name='otw_power_calculator.CalculationResult', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='result', full_name='otw_power_calculator.CalculationResult.result', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=206, + serialized_end=241, +) + +DESCRIPTOR.message_types_by_name['WorkoutPowerRequest'] = _WORKOUTPOWERREQUEST +DESCRIPTOR.message_types_by_name['CalculationResult'] = _CALCULATIONRESULT +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +WorkoutPowerRequest = _reflection.GeneratedProtocolMessageType('WorkoutPowerRequest', (_message.Message,), { + 'DESCRIPTOR' : _WORKOUTPOWERREQUEST, + '__module__' : 'otw_power_calculator_pb2' + # @@protoc_insertion_point(class_scope:otw_power_calculator.WorkoutPowerRequest) + }) +_sym_db.RegisterMessage(WorkoutPowerRequest) + +CalculationResult = _reflection.GeneratedProtocolMessageType('CalculationResult', (_message.Message,), { + 'DESCRIPTOR' : _CALCULATIONRESULT, + '__module__' : 'otw_power_calculator_pb2' + # @@protoc_insertion_point(class_scope:otw_power_calculator.CalculationResult) + }) +_sym_db.RegisterMessage(CalculationResult) + + + +_POWER = _descriptor.ServiceDescriptor( + name='Power', + full_name='otw_power_calculator.Power', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=243, + serialized_end=349, + methods=[ + _descriptor.MethodDescriptor( + name='CalcPower', + full_name='otw_power_calculator.Power.CalcPower', + index=0, + containing_service=None, + input_type=_WORKOUTPOWERREQUEST, + output_type=_CALCULATIONRESULT, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_POWER) + +DESCRIPTOR.services_by_name['Power'] = _POWER + +# @@protoc_insertion_point(module_scope) diff --git a/rowers/otw_power_calculator_pb2_grpc.py b/rowers/otw_power_calculator_pb2_grpc.py new file mode 100644 index 00000000..12e20178 --- /dev/null +++ b/rowers/otw_power_calculator_pb2_grpc.py @@ -0,0 +1,46 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import rowers.otw_power_calculator_pb2 as otw__power__calculator__pb2 + + +class PowerStub(object): + """Power service definition + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CalcPower = channel.unary_unary( + '/otw_power_calculator.Power/CalcPower', + request_serializer=otw__power__calculator__pb2.WorkoutPowerRequest.SerializeToString, + response_deserializer=otw__power__calculator__pb2.CalculationResult.FromString, + ) + + +class PowerServicer(object): + """Power service definition + """ + + def CalcPower(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_PowerServicer_to_server(servicer, server): + rpc_method_handlers = { + 'CalcPower': grpc.unary_unary_rpc_method_handler( + servicer.CalcPower, + request_deserializer=otw__power__calculator__pb2.WorkoutPowerRequest.FromString, + response_serializer=otw__power__calculator__pb2.CalculationResult.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'otw_power_calculator.Power', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/rowers/tasks.py b/rowers/tasks.py index ab2c9261..cbd62a7c 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -35,6 +35,10 @@ from matplotlib.backends.backend_agg import FigureCanvas import matplotlib.pyplot as plt from matplotlib import path +import grpc +import rowers.otw_power_calculator_pb2 as calculator_pb2 +import rowers.otw_power_calculator_pb2_grpc as calculator_pb2_grpc + from rowsandall_app.settings import SITE_URL from rowsandall_app.settings_dev import SITE_URL as SITE_URL_DEV from rowsandall_app.settings import PROGRESS_CACHE_SECRET @@ -1517,32 +1521,36 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, else: usetable = False + if 'go_service' in kwargs: + goservice = kwargs['go_service'] + print('Setting goservice to ', goservice) + else: + goservice = False + kwargs['jobid'] = job_id - try: - rowdata = rdata(csvfile=f1) - except IOError: - try: - rowdata = rdata(csvfile=f1 + '.csv') - except IOError: - rowdata = rdata(csvfile=f1 + '.gz') - weightvalue = float(weightvalue) + # check what the real file name is + if os.path.exists(f1): + csvfile = f1 + elif os.path.exists(f1+'.csv'): + csvfile = f1+'.csv' + elif os.path.exists(f1+'.gz'): + csvfile = f1+'.gz' + + csvfile = os.path.abspath(csvfile) + print('csvfile ',csvfile) + # do something with boat type - boatfile = { - '1x': 'static/rigging/1x.txt', - '2x': 'static/rigging/2x.txt', - '2-': 'static/rigging/2-.txt', - '4x': 'static/rigging/4x.txt', - '4-': 'static/rigging/4-.txt', - '8+': 'static/rigging/8+.txt', - } try: - rg = rowingdata.getrigging(boatfile[boattype]) - except KeyError: - rg = rowingdata.getrigging('static/rigging/1x.txt') + rowdata = rdata(csvfile) + except IOError: + try: + rowdata = rdata(csvfile) + except IOError: + rowdata = rdata(csvfile) # do calculation, but do not overwrite NK Empower Power data powermeasured = False @@ -1563,10 +1571,51 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, progressurl += "/rowers/record-progress/" progressurl += job_id+'/' + if goservice: + # do something (this should return from go service) + with grpc.insecure_channel( + target='localhost:50051', + options=[('grpc.lb_policy_name', 'pick_first'), + ('grpc.enable_retries', 0), ('grpc.keepalive_timeout_ms', + 10000)] + ) as channel: + try: + grpc.channel_ready_future(channel).result(timeout=10) + except grpc.FutureTimeoutError: + return 0 + else: + stub = calculator_pb2_grpc.PowerStub(channel) + response = stub.CalcPower(calculator_pb2.WorkoutPowerRequest( + filename = csvfile, + boattype = boattype, + crewmass = weightvalue, + powermeasured = powermeasured, + progressurl = progressurl, + secret = secret, + silent = False, + ),timeout=600) + return response.result + + + + boatfile = { + '1x': 'static/rigging/1x.txt', + '2x': 'static/rigging/2x.txt', + '2-': 'static/rigging/2-.txt', + '4x': 'static/rigging/4x.txt', + '4-': 'static/rigging/4-.txt', + '8+': 'static/rigging/8+.txt', + } + try: + rg = rowingdata.getrigging(boatfile[boattype]) + except KeyError: + rg = rowingdata.getrigging('static/rigging/1x.txt') + # determine cache file name physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue)) + rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg, powermeasured=powermeasured, progressurl=progressurl, diff --git a/rowers/views/workoutviews.py b/rowers/views/workoutviews.py index e113f19d..e50c618e 100644 --- a/rowers/views/workoutviews.py +++ b/rowers/views/workoutviews.py @@ -2624,6 +2624,7 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): if form.is_valid(): quick_calc = form.cleaned_data['quick_calc'] + go_service = form.cleaned_data['go_service'] boattype = form.cleaned_data['boattype'] weightvalue = form.cleaned_data['weightvalue'] w.boattype = boattype @@ -2669,6 +2670,7 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): ps=[r.p0,r.p1,r.p2,r.p3], ratio=r.cpratio, quick_calc = quick_calc, + go_service=go_service, emailbounced = r.emailbounced ) From a06792553a5bc05bfaf235b94450bc51f5234101 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 19 Dec 2019 20:53:06 +0100 Subject: [PATCH 2/9] working with go --- requirements.txt | 6 ++++- rowers/tasks.py | 60 +++++++++++++++++++++++++----------------------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/requirements.txt b/requirements.txt index 86e57035..58eb6402 100644 --- a/requirements.txt +++ b/requirements.txt @@ -72,6 +72,8 @@ fsspec==0.5.2 future==0.17.1 geocoder==1.38.1 geos==0.2.1 +grpcio==1.26.0 +grpcio-tools==1.26.0 holoviews==1.11.3 html5lib==1.0.1 htmlmin==0.1.12 @@ -93,6 +95,7 @@ itypes==1.1.0 jedi==0.13.3 jeepney==0.4 Jinja2==2.10 +json5==0.8.5 jsonschema==3.0.1 jupyter==1.0.0 jupyter-client==5.2.4 @@ -140,6 +143,7 @@ pip-upgrader==1.4.6 pluggy==0.9.0 prometheus-client==0.6.0 prompt-toolkit==2.0.9 +protobuf==3.11.1 psycopg2==2.8.1 ptyprocess==0.6.0 py==1.8.0 @@ -169,7 +173,7 @@ ratelim==0.1.6 redis==3.2.1 requests==2.21.0 requests-oauthlib==1.2.0 -rowingdata==2.5.5 +rowingdata==2.5.7 rowingphysics==0.5.0 rq==0.13.0 scipy==1.2.1 diff --git a/rowers/tasks.py b/rowers/tasks.py index cbd62a7c..7edad3ed 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -1523,7 +1523,6 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, if 'go_service' in kwargs: goservice = kwargs['go_service'] - print('Setting goservice to ', goservice) else: goservice = False @@ -1541,7 +1540,6 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, csvfile = f1+'.gz' csvfile = os.path.abspath(csvfile) - print('csvfile ',csvfile) # do something with boat type try: @@ -1594,38 +1592,42 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, secret = secret, silent = False, ),timeout=600) - return response.result + result = response.result + if result == 0: + # send failure email + return 0 + + else: + boatfile = { + '1x': 'static/rigging/1x.txt', + '2x': 'static/rigging/2x.txt', + '2-': 'static/rigging/2-.txt', + '4x': 'static/rigging/4x.txt', + '4-': 'static/rigging/4-.txt', + '8+': 'static/rigging/8+.txt', + } + try: + rg = rowingdata.getrigging(boatfile[boattype]) + except KeyError: + rg = rowingdata.getrigging('static/rigging/1x.txt') + + # determine cache file name + physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue)) - boatfile = { - '1x': 'static/rigging/1x.txt', - '2x': 'static/rigging/2x.txt', - '2-': 'static/rigging/2-.txt', - '4x': 'static/rigging/4x.txt', - '4-': 'static/rigging/4-.txt', - '8+': 'static/rigging/8+.txt', - } - try: - rg = rowingdata.getrigging(boatfile[boattype]) - except KeyError: - rg = rowingdata.getrigging('static/rigging/1x.txt') + rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg, + powermeasured=powermeasured, + progressurl=progressurl, + secret=secret, + silent=True, + usetable=usetable,storetable=physics_cache, + ) - # determine cache file name - physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue)) + # save data + rowdata.write_csv(f1, gzip=True) - - - rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg, - powermeasured=powermeasured, - progressurl=progressurl, - secret=secret, - silent=True, - usetable=usetable,storetable=physics_cache, - ) - - # save data - rowdata.write_csv(f1, gzip=True) + # continuing for both update_strokedata(workoutid, rowdata.df, debug=debug) totaltime = rowdata.df['TimeStamp (sec)'].max( From 0b22c9744831f308f656f9c3f0f9c8697ee14571 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Dec 2019 12:12:13 +0100 Subject: [PATCH 3/9] extended timeout for otw_setpower --- rowers/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rowers/tasks.py b/rowers/tasks.py index 7edad3ed..aec4f4d6 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -1591,7 +1591,7 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, progressurl = progressurl, secret = secret, silent = False, - ),timeout=600) + ),timeout=1200) result = response.result if result == 0: # send failure email From 77f7cf4844ac54b98436ca4d5a0db0878452fdb9 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 20 Dec 2019 16:41:54 +0100 Subject: [PATCH 4/9] returning 0 is makeplot gives no image --- rowers/tasks.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/rowers/tasks.py b/rowers/tasks.py index aec4f4d6..d39ca012 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -1887,13 +1887,16 @@ def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename, t += ' - Power Distribution' fig1 = row.get_power_piechart(t) - if fig1: - canvas = FigureCanvas(fig1) + if fig1 is None: + return 0 - canvas.print_figure('static/plots/' + imagename) - plt.close(fig1) - fig1.clf() - gc.collect() + + canvas = FigureCanvas(fig1) + + canvas.print_figure('static/plots/' + imagename) + plt.close(fig1) + fig1.clf() + gc.collect() return imagename From 2bd9d962733b64b5e0b1256cf54dece0245ef075 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Dec 2019 15:14:39 +0100 Subject: [PATCH 5/9] removed unnecessary else --- rowers/tasks.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/rowers/tasks.py b/rowers/tasks.py index d39ca012..6e25ac0d 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -1581,21 +1581,29 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, grpc.channel_ready_future(channel).result(timeout=10) except grpc.FutureTimeoutError: return 0 - else: - stub = calculator_pb2_grpc.PowerStub(channel) - response = stub.CalcPower(calculator_pb2.WorkoutPowerRequest( - filename = csvfile, - boattype = boattype, - crewmass = weightvalue, - powermeasured = powermeasured, - progressurl = progressurl, - secret = secret, - silent = False, - ),timeout=1200) - result = response.result - if result == 0: - # send failure email - return 0 + + stub = calculator_pb2_grpc.PowerStub(channel) + response = stub.CalcPower(calculator_pb2.WorkoutPowerRequest( + filename = csvfile, + boattype = boattype, + crewmass = weightvalue, + powermeasured = powermeasured, + progressurl = progressurl, + secret = secret, + silent = False, + ),timeout=1200) + result = response.result + if result == 0: + # send failure email + return 0 + # do something with boat type + try: + rowdata = rdata(csvfile) + except IOError: + try: + rowdata = rdata(csvfile) + except IOError: + rowdata = rdata(csvfile) else: boatfile = { From b38fb91c03009044a76a0e4c719acc47b41a9927 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Sun, 22 Dec 2019 18:53:57 +0100 Subject: [PATCH 6/9] better progress bars --- rowers/templates/async_tasks.html | 60 +++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/rowers/templates/async_tasks.html b/rowers/templates/async_tasks.html index 812221fa..83c916f0 100644 --- a/rowers/templates/async_tasks.html +++ b/rowers/templates/async_tasks.html @@ -9,7 +9,7 @@