diff --git a/requirements.txt b/requirements.txt index 86e57035..58eb6402 100644 --- a/requirements.txt +++ b/requirements.txt @@ -72,6 +72,8 @@ fsspec==0.5.2 future==0.17.1 geocoder==1.38.1 geos==0.2.1 +grpcio==1.26.0 +grpcio-tools==1.26.0 holoviews==1.11.3 html5lib==1.0.1 htmlmin==0.1.12 @@ -93,6 +95,7 @@ itypes==1.1.0 jedi==0.13.3 jeepney==0.4 Jinja2==2.10 +json5==0.8.5 jsonschema==3.0.1 jupyter==1.0.0 jupyter-client==5.2.4 @@ -140,6 +143,7 @@ pip-upgrader==1.4.6 pluggy==0.9.0 prometheus-client==0.6.0 prompt-toolkit==2.0.9 +protobuf==3.11.1 psycopg2==2.8.1 ptyprocess==0.6.0 py==1.8.0 @@ -169,7 +173,7 @@ ratelim==0.1.6 redis==3.2.1 requests==2.21.0 requests-oauthlib==1.2.0 -rowingdata==2.5.5 +rowingdata==2.5.7 rowingphysics==0.5.0 rq==0.13.0 scipy==1.2.1 diff --git a/rowers/c2stuff.py b/rowers/c2stuff.py index 5d184895..87895291 100644 --- a/rowers/c2stuff.py +++ b/rowers/c2stuff.py @@ -821,7 +821,10 @@ def get_userid(access_token): return 0 - me_json = response.json() + try: + me_json = response.json() + except JSONDecodeError: + return 0 try: res = me_json['data']['id'] except KeyError: diff --git a/rowers/management/commands/processemail.py b/rowers/management/commands/processemail.py index e4171355..8720b2cb 100644 --- a/rowers/management/commands/processemail.py +++ b/rowers/management/commands/processemail.py @@ -61,7 +61,7 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False): if testing: print('Attribute Error', filename) - + # test if file exists and is not empty try: with io.open('media/'+filename,'rb') as fop: @@ -83,13 +83,13 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False): return 0 else: therower = rower - - + + workoutid = [ make_new_workout_from_email(therower, filename, title,testing=testing) ] - + if 'raceid' in uploadoptions and workoutid[0] and rower.user.is_staff: if testing and workoutid[0]: w = Workout.objects.get(id = workoutid[0]) @@ -142,7 +142,7 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False): try: time.sleep(10) if workoutid: - if therower.getemailnotifications and not therower.emailbounced: + if therower.getemailnotifications and not therower.emailbounced: email_sent = send_confirm( therower.user, title, link, uploadoptions @@ -155,7 +155,7 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False): def get_from_address(message): from_address = message.from_address[0].lower() - + if message.encoded: body = message.text.splitlines() else: @@ -194,7 +194,7 @@ class Command(BaseCommand): default='workouts', help="Changing mailbox name", ) - + """Run the Email processing command """ def handle(self, *args, **options): if 'testing' in options: @@ -221,7 +221,7 @@ class Command(BaseCommand): for r in rowers: c2stuff.get_c2_workouts(r) - + messages = Message.objects.filter(mailbox_id = workoutmailbox.id) message_ids = [m.id for m in messages] attachments = MessageAttachment.objects.filter( @@ -239,11 +239,11 @@ class Command(BaseCommand): body = "\n".join(message.text.splitlines()) else: body = message.get_body() - + uploadoptions = uploads.upload_options(body) from_address = get_from_address(message) - + name = message.subject # get a list of users # theusers = User.objects.filter(email=from_address) @@ -265,7 +265,7 @@ class Command(BaseCommand): title = name+' ('+str(id+1)+')' else: title = name - + workoutid = processattachment( rower, datafile, title, uploadoptions, testing=testing @@ -284,7 +284,7 @@ class Command(BaseCommand): print(attachment.document) except UnicodeEncodeError: pass - + workoutid = processattachment( rower, attachment.document, name, uploadoptions, testing=testing diff --git a/rowers/models.py b/rowers/models.py index dcd7df70..d73bb683 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -3203,6 +3203,7 @@ class WorkoutForm(ModelForm): # Used for the rowing physics calculations class AdvancedWorkoutForm(ModelForm): quick_calc = forms.BooleanField(initial=True,required=False) + go_service = forms.BooleanField(initial=False,required=False,label='Experimental') class Meta: model = Workout diff --git a/rowers/otw_power_calculator_pb2.py b/rowers/otw_power_calculator_pb2.py new file mode 100644 index 00000000..79887198 --- /dev/null +++ b/rowers/otw_power_calculator_pb2.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: otw-power-calculator.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='otw-power-calculator.proto', + package='otw_power_calculator', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x1aotw-power-calculator.proto\x12\x14otw_power_calculator\"\x97\x01\n\x13WorkoutPowerRequest\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x10\n\x08\x62oattype\x18\x02 \x01(\t\x12\x10\n\x08\x63rewmass\x18\x03 \x01(\x01\x12\x15\n\rpowermeasured\x18\x04 \x01(\x08\x12\x13\n\x0bprogressurl\x18\x05 \x01(\t\x12\x0e\n\x06secret\x18\x06 \x01(\t\x12\x0e\n\x06silent\x18\x07 \x01(\x08\"#\n\x11\x43\x61lculationResult\x12\x0e\n\x06result\x18\x01 \x01(\x05\x32j\n\x05Power\x12\x61\n\tCalcPower\x12).otw_power_calculator.WorkoutPowerRequest\x1a\'.otw_power_calculator.CalculationResult\"\x00\x62\x06proto3') +) + + + + +_WORKOUTPOWERREQUEST = _descriptor.Descriptor( + name='WorkoutPowerRequest', + full_name='otw_power_calculator.WorkoutPowerRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='filename', full_name='otw_power_calculator.WorkoutPowerRequest.filename', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='boattype', full_name='otw_power_calculator.WorkoutPowerRequest.boattype', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='crewmass', full_name='otw_power_calculator.WorkoutPowerRequest.crewmass', index=2, + number=3, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='powermeasured', full_name='otw_power_calculator.WorkoutPowerRequest.powermeasured', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='progressurl', full_name='otw_power_calculator.WorkoutPowerRequest.progressurl', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='secret', full_name='otw_power_calculator.WorkoutPowerRequest.secret', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='silent', full_name='otw_power_calculator.WorkoutPowerRequest.silent', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=53, + serialized_end=204, +) + + +_CALCULATIONRESULT = _descriptor.Descriptor( + name='CalculationResult', + full_name='otw_power_calculator.CalculationResult', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='result', full_name='otw_power_calculator.CalculationResult.result', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=206, + serialized_end=241, +) + +DESCRIPTOR.message_types_by_name['WorkoutPowerRequest'] = _WORKOUTPOWERREQUEST +DESCRIPTOR.message_types_by_name['CalculationResult'] = _CALCULATIONRESULT +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +WorkoutPowerRequest = _reflection.GeneratedProtocolMessageType('WorkoutPowerRequest', (_message.Message,), { + 'DESCRIPTOR' : _WORKOUTPOWERREQUEST, + '__module__' : 'otw_power_calculator_pb2' + # @@protoc_insertion_point(class_scope:otw_power_calculator.WorkoutPowerRequest) + }) +_sym_db.RegisterMessage(WorkoutPowerRequest) + +CalculationResult = _reflection.GeneratedProtocolMessageType('CalculationResult', (_message.Message,), { + 'DESCRIPTOR' : _CALCULATIONRESULT, + '__module__' : 'otw_power_calculator_pb2' + # @@protoc_insertion_point(class_scope:otw_power_calculator.CalculationResult) + }) +_sym_db.RegisterMessage(CalculationResult) + + + +_POWER = _descriptor.ServiceDescriptor( + name='Power', + full_name='otw_power_calculator.Power', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=243, + serialized_end=349, + methods=[ + _descriptor.MethodDescriptor( + name='CalcPower', + full_name='otw_power_calculator.Power.CalcPower', + index=0, + containing_service=None, + input_type=_WORKOUTPOWERREQUEST, + output_type=_CALCULATIONRESULT, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_POWER) + +DESCRIPTOR.services_by_name['Power'] = _POWER + +# @@protoc_insertion_point(module_scope) diff --git a/rowers/otw_power_calculator_pb2_grpc.py b/rowers/otw_power_calculator_pb2_grpc.py new file mode 100644 index 00000000..12e20178 --- /dev/null +++ b/rowers/otw_power_calculator_pb2_grpc.py @@ -0,0 +1,46 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +import rowers.otw_power_calculator_pb2 as otw__power__calculator__pb2 + + +class PowerStub(object): + """Power service definition + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CalcPower = channel.unary_unary( + '/otw_power_calculator.Power/CalcPower', + request_serializer=otw__power__calculator__pb2.WorkoutPowerRequest.SerializeToString, + response_deserializer=otw__power__calculator__pb2.CalculationResult.FromString, + ) + + +class PowerServicer(object): + """Power service definition + """ + + def CalcPower(self, request, context): + # missing associated documentation comment in .proto file + pass + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_PowerServicer_to_server(servicer, server): + rpc_method_handlers = { + 'CalcPower': grpc.unary_unary_rpc_method_handler( + servicer.CalcPower, + request_deserializer=otw__power__calculator__pb2.WorkoutPowerRequest.FromString, + response_serializer=otw__power__calculator__pb2.CalculationResult.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'otw_power_calculator.Power', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/rowers/tasks.py b/rowers/tasks.py index e812a363..9f31af9b 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -35,6 +35,10 @@ from matplotlib.backends.backend_agg import FigureCanvas import matplotlib.pyplot as plt from matplotlib import path +import grpc +import rowers.otw_power_calculator_pb2 as calculator_pb2 +import rowers.otw_power_calculator_pb2_grpc as calculator_pb2_grpc + from rowsandall_app.settings import SITE_URL from rowsandall_app.settings_dev import SITE_URL as SITE_URL_DEV from rowsandall_app.settings import PROGRESS_CACHE_SECRET @@ -1517,32 +1521,34 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, else: usetable = False + if 'go_service' in kwargs: + goservice = kwargs['go_service'] + else: + goservice = False + kwargs['jobid'] = job_id - try: - rowdata = rdata(csvfile=f1) - except IOError: - try: - rowdata = rdata(csvfile=f1 + '.csv') - except IOError: - rowdata = rdata(csvfile=f1 + '.gz') - weightvalue = float(weightvalue) + # check what the real file name is + if os.path.exists(f1): + csvfile = f1 + elif os.path.exists(f1+'.csv'): + csvfile = f1+'.csv' + elif os.path.exists(f1+'.gz'): + csvfile = f1+'.gz' + + csvfile = os.path.abspath(csvfile) + # do something with boat type - boatfile = { - '1x': 'static/rigging/1x.txt', - '2x': 'static/rigging/2x.txt', - '2-': 'static/rigging/2-.txt', - '4x': 'static/rigging/4x.txt', - '4-': 'static/rigging/4-.txt', - '8+': 'static/rigging/8+.txt', - } try: - rg = rowingdata.getrigging(boatfile[boattype]) - except KeyError: - rg = rowingdata.getrigging('static/rigging/1x.txt') + rowdata = rdata(csvfile) + except IOError: + try: + rowdata = rdata(csvfile) + except IOError: + rowdata = rdata(csvfile) # do calculation, but do not overwrite NK Empower Power data powermeasured = False @@ -1563,20 +1569,73 @@ def handle_otwsetpower(self,f1, boattype, weightvalue, progressurl += "/rowers/record-progress/" progressurl += job_id+'/' - # determine cache file name - physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue)) + if goservice: + # do something (this should return from go service) + with grpc.insecure_channel( + target='localhost:50051', + options=[('grpc.lb_policy_name', 'pick_first'), + ('grpc.enable_retries', 0), ('grpc.keepalive_timeout_ms', + 10000)] + ) as channel: + try: + grpc.channel_ready_future(channel).result(timeout=10) + except grpc.FutureTimeoutError: + return 0 + + stub = calculator_pb2_grpc.PowerStub(channel) + response = stub.CalcPower(calculator_pb2.WorkoutPowerRequest( + filename = csvfile, + boattype = boattype, + crewmass = weightvalue, + powermeasured = powermeasured, + progressurl = progressurl, + secret = secret, + silent = False, + ),timeout=1200) + result = response.result + if result == 0: + # send failure email + return 0 + # do something with boat type + try: + rowdata = rdata(csvfile) + except IOError: + try: + rowdata = rdata(csvfile) + except IOError: + rowdata = rdata(csvfile) + + else: + boatfile = { + '1x': 'static/rigging/1x.txt', + '2x': 'static/rigging/2x.txt', + '2-': 'static/rigging/2-.txt', + '4x': 'static/rigging/4x.txt', + '4-': 'static/rigging/4-.txt', + '8+': 'static/rigging/8+.txt', + } + try: + rg = rowingdata.getrigging(boatfile[boattype]) + except KeyError: + rg = rowingdata.getrigging('static/rigging/1x.txt') + + # determine cache file name + physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue)) - rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg, - powermeasured=powermeasured, - progressurl=progressurl, - secret=secret, - silent=True, - usetable=usetable,storetable=physics_cache, - ) - # save data - rowdata.write_csv(f1, gzip=True) + rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg, + powermeasured=powermeasured, + progressurl=progressurl, + secret=secret, + silent=True, + usetable=usetable,storetable=physics_cache, + ) + + # save data + rowdata.write_csv(f1, gzip=True) + + # continuing for both update_strokedata(workoutid, rowdata.df, debug=debug) totaltime = rowdata.df['TimeStamp (sec)'].max( @@ -1836,13 +1895,16 @@ def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename, t += ' - Power Distribution' fig1 = row.get_power_piechart(t) - if fig1: - canvas = FigureCanvas(fig1) + if fig1 is None: + return 0 - canvas.print_figure('static/plots/' + imagename) - plt.close(fig1) - fig1.clf() - gc.collect() + + canvas = FigureCanvas(fig1) + + canvas.print_figure('static/plots/' + imagename) + plt.close(fig1) + fig1.clf() + gc.collect() return imagename diff --git a/rowers/templates/async_tasks.html b/rowers/templates/async_tasks.html index 812221fa..83c916f0 100644 --- a/rowers/templates/async_tasks.html +++ b/rowers/templates/async_tasks.html @@ -9,7 +9,7 @@