sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

Merge branch 'develop' into feature/gauges

Browse Source
This commit is contained in:
Sander Roosendaal
2020-01-01 15:23:48 +01:00
parent 1ce39e6421 0ac7e5ee95
commit 5c1988461c
10 changed files with 404 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
requirements.txt
View File

@@ -72,6 +72,8 @@ fsspec==0.5.2
future==0.17.1
geocoder==1.38.1
geos==0.2.1
grpcio==1.26.0
grpcio-tools==1.26.0
holoviews==1.11.3
html5lib==1.0.1
htmlmin==0.1.12
@@ -93,6 +95,7 @@ itypes==1.1.0
jedi==0.13.3
jeepney==0.4
Jinja2==2.10
json5==0.8.5
jsonschema==3.0.1
jupyter==1.0.0
jupyter-client==5.2.4
@@ -140,6 +143,7 @@ pip-upgrader==1.4.6
pluggy==0.9.0
prometheus-client==0.6.0
prompt-toolkit==2.0.9
protobuf==3.11.1
psycopg2==2.8.1
ptyprocess==0.6.0
py==1.8.0
@@ -169,7 +173,7 @@ ratelim==0.1.6
redis==3.2.1
requests==2.21.0
requests-oauthlib==1.2.0
rowingdata==2.5.5
rowingdata==2.5.7
rowingphysics==0.5.0
rq==0.13.0
scipy==1.2.1

5
rowers/c2stuff.py
View File

@@ -821,7 +821,10 @@ def get_userid(access_token):
return 0
me_json = response.json()
try:
me_json = response.json()
except JSONDecodeError:
return 0
try:
res = me_json['data']['id']
except KeyError:

24
rowers/management/commands/processemail.py
View File

@@ -61,7 +61,7 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
if testing:
print('Attribute Error', filename)
# test if file exists and is not empty
try:
with io.open('media/'+filename,'rb') as fop:
@@ -83,13 +83,13 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
return 0
else:
therower = rower
workoutid = [
make_new_workout_from_email(therower, filename, title,testing=testing)
]
if 'raceid' in uploadoptions and workoutid[0] and rower.user.is_staff:
if testing and workoutid[0]:
w = Workout.objects.get(id = workoutid[0])
@@ -142,7 +142,7 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
try:
time.sleep(10)
if workoutid:
if therower.getemailnotifications and not therower.emailbounced:
if therower.getemailnotifications and not therower.emailbounced:
email_sent = send_confirm(
therower.user, title, link,
uploadoptions
@@ -155,7 +155,7 @@ def processattachment(rower, fileobj, title, uploadoptions,testing=False):
def get_from_address(message):
from_address = message.from_address[0].lower()
if message.encoded:
body = message.text.splitlines()
else:
@@ -194,7 +194,7 @@ class Command(BaseCommand):
default='workouts',
help="Changing mailbox name",
)
"""Run the Email processing command """
def handle(self, *args, **options):
if 'testing' in options:
@@ -221,7 +221,7 @@ class Command(BaseCommand):
for r in rowers:
c2stuff.get_c2_workouts(r)
messages = Message.objects.filter(mailbox_id = workoutmailbox.id)
message_ids = [m.id for m in messages]
attachments = MessageAttachment.objects.filter(
@@ -239,11 +239,11 @@ class Command(BaseCommand):
body = "\n".join(message.text.splitlines())
else:
body = message.get_body()
uploadoptions = uploads.upload_options(body)
from_address = get_from_address(message)
name = message.subject
# get a list of users
# theusers = User.objects.filter(email=from_address)
@@ -265,7 +265,7 @@ class Command(BaseCommand):
title = name+' ('+str(id+1)+')'
else:
title = name
workoutid = processattachment(
rower, datafile, title, uploadoptions,
testing=testing
@@ -284,7 +284,7 @@ class Command(BaseCommand):
print(attachment.document)
except UnicodeEncodeError:
pass
workoutid = processattachment(
rower, attachment.document, name, uploadoptions,
testing=testing

1
rowers/models.py
View File

@@ -3203,6 +3203,7 @@ class WorkoutForm(ModelForm):
# Used for the rowing physics calculations
class AdvancedWorkoutForm(ModelForm):
quick_calc = forms.BooleanField(initial=True,required=False)
go_service = forms.BooleanField(initial=False,required=False,label='Experimental')
class Meta:
model = Workout

175
rowers/otw_power_calculator_pb2.py Normal file
View File

@@ -0,0 +1,175 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: otw-power-calculator.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='otw-power-calculator.proto',
package='otw_power_calculator',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x1aotw-power-calculator.proto\x12\x14otw_power_calculator\"\x97\x01\n\x13WorkoutPowerRequest\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x10\n\x08\x62oattype\x18\x02 \x01(\t\x12\x10\n\x08\x63rewmass\x18\x03 \x01(\x01\x12\x15\n\rpowermeasured\x18\x04 \x01(\x08\x12\x13\n\x0bprogressurl\x18\x05 \x01(\t\x12\x0e\n\x06secret\x18\x06 \x01(\t\x12\x0e\n\x06silent\x18\x07 \x01(\x08\"#\n\x11\x43\x61lculationResult\x12\x0e\n\x06result\x18\x01 \x01(\x05\x32j\n\x05Power\x12\x61\n\tCalcPower\x12).otw_power_calculator.WorkoutPowerRequest\x1a\'.otw_power_calculator.CalculationResult\"\x00\x62\x06proto3')
)
_WORKOUTPOWERREQUEST = _descriptor.Descriptor(
name='WorkoutPowerRequest',
full_name='otw_power_calculator.WorkoutPowerRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='filename', full_name='otw_power_calculator.WorkoutPowerRequest.filename', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='boattype', full_name='otw_power_calculator.WorkoutPowerRequest.boattype', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='crewmass', full_name='otw_power_calculator.WorkoutPowerRequest.crewmass', index=2,
number=3, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='powermeasured', full_name='otw_power_calculator.WorkoutPowerRequest.powermeasured', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='progressurl', full_name='otw_power_calculator.WorkoutPowerRequest.progressurl', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='secret', full_name='otw_power_calculator.WorkoutPowerRequest.secret', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='silent', full_name='otw_power_calculator.WorkoutPowerRequest.silent', index=6,
number=7, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=53,
serialized_end=204,
)
_CALCULATIONRESULT = _descriptor.Descriptor(
name='CalculationResult',
full_name='otw_power_calculator.CalculationResult',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='result', full_name='otw_power_calculator.CalculationResult.result', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=206,
serialized_end=241,
)
DESCRIPTOR.message_types_by_name['WorkoutPowerRequest'] = _WORKOUTPOWERREQUEST
DESCRIPTOR.message_types_by_name['CalculationResult'] = _CALCULATIONRESULT
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
WorkoutPowerRequest = _reflection.GeneratedProtocolMessageType('WorkoutPowerRequest', (_message.Message,), {
'DESCRIPTOR' : _WORKOUTPOWERREQUEST,
'__module__' : 'otw_power_calculator_pb2'
# @@protoc_insertion_point(class_scope:otw_power_calculator.WorkoutPowerRequest)
})
_sym_db.RegisterMessage(WorkoutPowerRequest)
CalculationResult = _reflection.GeneratedProtocolMessageType('CalculationResult', (_message.Message,), {
'DESCRIPTOR' : _CALCULATIONRESULT,
'__module__' : 'otw_power_calculator_pb2'
# @@protoc_insertion_point(class_scope:otw_power_calculator.CalculationResult)
})
_sym_db.RegisterMessage(CalculationResult)
_POWER = _descriptor.ServiceDescriptor(
name='Power',
full_name='otw_power_calculator.Power',
file=DESCRIPTOR,
index=0,
serialized_options=None,
serialized_start=243,
serialized_end=349,
methods=[
_descriptor.MethodDescriptor(
name='CalcPower',
full_name='otw_power_calculator.Power.CalcPower',
index=0,
containing_service=None,
input_type=_WORKOUTPOWERREQUEST,
output_type=_CALCULATIONRESULT,
serialized_options=None,
),
])
_sym_db.RegisterServiceDescriptor(_POWER)
DESCRIPTOR.services_by_name['Power'] = _POWER
# @@protoc_insertion_point(module_scope)

46
rowers/otw_power_calculator_pb2_grpc.py Normal file
View File

@@ -0,0 +1,46 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
import grpc
import rowers.otw_power_calculator_pb2 as otw__power__calculator__pb2
class PowerStub(object):
"""Power service definition
"""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.CalcPower = channel.unary_unary(
'/otw_power_calculator.Power/CalcPower',
request_serializer=otw__power__calculator__pb2.WorkoutPowerRequest.SerializeToString,
response_deserializer=otw__power__calculator__pb2.CalculationResult.FromString,
)
class PowerServicer(object):
"""Power service definition
"""
def CalcPower(self, request, context):
# missing associated documentation comment in .proto file
pass
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_PowerServicer_to_server(servicer, server):
rpc_method_handlers = {
'CalcPower': grpc.unary_unary_rpc_method_handler(
servicer.CalcPower,
request_deserializer=otw__power__calculator__pb2.WorkoutPowerRequest.FromString,
response_serializer=otw__power__calculator__pb2.CalculationResult.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'otw_power_calculator.Power', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))

134
rowers/tasks.py
View File

@@ -35,6 +35,10 @@ from matplotlib.backends.backend_agg import FigureCanvas
import matplotlib.pyplot as plt
from matplotlib import path
import grpc
import rowers.otw_power_calculator_pb2 as calculator_pb2
import rowers.otw_power_calculator_pb2_grpc as calculator_pb2_grpc
from rowsandall_app.settings import SITE_URL
from rowsandall_app.settings_dev import SITE_URL as SITE_URL_DEV
from rowsandall_app.settings import PROGRESS_CACHE_SECRET
@@ -1517,32 +1521,34 @@ def handle_otwsetpower(self,f1, boattype, weightvalue,
else:
usetable = False
if 'go_service' in kwargs:
goservice = kwargs['go_service']
else:
goservice = False
kwargs['jobid'] = job_id
try:
rowdata = rdata(csvfile=f1)
except IOError:
try:
rowdata = rdata(csvfile=f1 + '.csv')
except IOError:
rowdata = rdata(csvfile=f1 + '.gz')
weightvalue = float(weightvalue)
# check what the real file name is
if os.path.exists(f1):
csvfile = f1
elif os.path.exists(f1+'.csv'):
csvfile = f1+'.csv'
elif os.path.exists(f1+'.gz'):
csvfile = f1+'.gz'
csvfile = os.path.abspath(csvfile)
# do something with boat type
boatfile = {
'1x': 'static/rigging/1x.txt',
'2x': 'static/rigging/2x.txt',
'2-': 'static/rigging/2-.txt',
'4x': 'static/rigging/4x.txt',
'4-': 'static/rigging/4-.txt',
'8+': 'static/rigging/8+.txt',
}
try:
rg = rowingdata.getrigging(boatfile[boattype])
except KeyError:
rg = rowingdata.getrigging('static/rigging/1x.txt')
rowdata = rdata(csvfile)
except IOError:
try:
rowdata = rdata(csvfile)
except IOError:
rowdata = rdata(csvfile)
# do calculation, but do not overwrite NK Empower Power data
powermeasured = False
@@ -1563,20 +1569,73 @@ def handle_otwsetpower(self,f1, boattype, weightvalue,
progressurl += "/rowers/record-progress/"
progressurl += job_id+'/'
# determine cache file name
physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue))
if goservice:
# do something (this should return from go service)
with grpc.insecure_channel(
target='localhost:50051',
options=[('grpc.lb_policy_name', 'pick_first'),
('grpc.enable_retries', 0), ('grpc.keepalive_timeout_ms',
10000)]
) as channel:
try:
grpc.channel_ready_future(channel).result(timeout=10)
except grpc.FutureTimeoutError:
return 0
stub = calculator_pb2_grpc.PowerStub(channel)
response = stub.CalcPower(calculator_pb2.WorkoutPowerRequest(
filename = csvfile,
boattype = boattype,
crewmass = weightvalue,
powermeasured = powermeasured,
progressurl = progressurl,
secret = secret,
silent = False,
),timeout=1200)
result = response.result
if result == 0:
# send failure email
return 0
# do something with boat type
try:
rowdata = rdata(csvfile)
except IOError:
try:
rowdata = rdata(csvfile)
except IOError:
rowdata = rdata(csvfile)
else:
boatfile = {
'1x': 'static/rigging/1x.txt',
'2x': 'static/rigging/2x.txt',
'2-': 'static/rigging/2-.txt',
'4x': 'static/rigging/4x.txt',
'4-': 'static/rigging/4-.txt',
'8+': 'static/rigging/8+.txt',
}
try:
rg = rowingdata.getrigging(boatfile[boattype])
except KeyError:
rg = rowingdata.getrigging('static/rigging/1x.txt')
# determine cache file name
physics_cache = 'media/'+str(boattype)+'_'+str(int(weightvalue))
rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg,
powermeasured=powermeasured,
progressurl=progressurl,
secret=secret,
silent=True,
usetable=usetable,storetable=physics_cache,
)
# save data
rowdata.write_csv(f1, gzip=True)
rowdata.otw_setpower(skiprows=5, mc=weightvalue, rg=rg,
powermeasured=powermeasured,
progressurl=progressurl,
secret=secret,
silent=True,
usetable=usetable,storetable=physics_cache,
)
# save data
rowdata.write_csv(f1, gzip=True)
# continuing for both
update_strokedata(workoutid, rowdata.df, debug=debug)
totaltime = rowdata.df['TimeStamp (sec)'].max(
@@ -1836,13 +1895,16 @@ def handle_makeplot(f1, f2, t, hrdata, plotnr, imagename,
t += ' - Power Distribution'
fig1 = row.get_power_piechart(t)
if fig1:
canvas = FigureCanvas(fig1)
if fig1 is None:
return 0
canvas.print_figure('static/plots/' + imagename)
plt.close(fig1)
fig1.clf()
gc.collect()
canvas = FigureCanvas(fig1)
canvas.print_figure('static/plots/' + imagename)
plt.close(fig1)
fig1.clf()
gc.collect()
return imagename

60
rowers/templates/async_tasks.html
View File

@@ -9,7 +9,7 @@
<style type="text/css">
.progressBar div {
border-radius:5px;
height: 100%;
height: 100%;
/* padding: 4px 10px; */
font-size: 15px;
color: #fff;
@@ -38,10 +38,10 @@
var maxvalue = $(this).attr('data');
maxvalue = maxvalue.substring(3);
var text = $(this).children('div').data('show');
progress(maxvalue, bar, text);
progress(maxvalue, bar, text);
});
});
function progress1(percent, element, text) {
element.find('div').animate({ width: percent+'%' }, 1).html(text +"&nbsp;"+ percent + "%&nbsp;");
}
@@ -52,13 +52,59 @@
{% endblock %}
{% block scripts %}
{% include "monitorjobs.html" %}
<script type='text/javascript'
src='https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js'>
</script>
<script>
var previous = null;
var current = null;
var formNotChanged = true;
(function checkJobs() {
$.getJSON(window.location.protocol + '//'+window.location.host + '/rowers/list-jobs/', function(json) {
current = JSON.stringify(json);
$('form').each(function() {
$(this).change(function() {
formNotChanged = false;
});
});
if (previous && current && previous !== current && formNotChanged) {
if (previous.length != current.length) {
console.log('refresh');
location.reload();
}
$('.progressBar').each(function() {
var bar = $(this);
var maxvalue = $(this).attr('data').substring(3);
var barid = $(this).attr('id').substring(0,10)
$.each(json, function(index,record) {
var recordid = record['id'].substring(0,10)
if (recordid == barid) {
if (record['progrss']<100 || parseInt(maxvalue)<100) {
maxvalue = record['progress'].toString();
}
}
});
var text = $(this).children('div').data('show');
progress(maxvalue, bar, text);
});
};
previous = current;
});
setTimeout(checkJobs, 1000);
}());
</script>
{% endblock %}
{% block main %}
<h1>Your Tasks Status</h1>
<p>Manage the asynchronous tasks running for you</p>
@@ -88,7 +134,7 @@
{{ task|lookup:'started_at' }}
</td>
<td>
<div class="progressBar" data="max{{ task|lookup:'progress' }}" style="width: 100%;">
<div class="progressBar" id="{{ task|lookup:'id' }}" data="max{{ task|lookup:'progress' }}" style="width: 100%;">
<div data-show=""> {{ task|lookup:'progress' }}</div>
</div>

16
rowers/uploads.py
View File

@@ -506,6 +506,7 @@ def do_sync(w,options):
try:
if options['stravaid'] != 0:
w.uploadedtostrava = options['stravaid']
options['upload_to_Strava'] = False
w.save()
except KeyError:
pass
@@ -524,14 +525,13 @@ def do_sync(w,options):
if ('upload_to_Strava' in options and not options['upload_to_Strava']):
pass
else:
if options['stravaid'] != 0:
try:
message,id = stravastuff.workout_strava_upload(
w.user.user,w
)
except NoTokenError:
id = 0
message = "Please connect to Strava first"
try:
message,id = stravastuff.workout_strava_upload(
w.user.user,w
)
except NoTokenError:
id = 0
message = "Please connect to Strava first"
if ('upload_to_SportTracks' in options and options['upload_to_SportTracks']) or (w.user.sporttracks_auto_export and isprorower(w.user)):

2
rowers/views/workoutviews.py
View File

@@ -2624,6 +2624,7 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""):
if form.is_valid():
quick_calc = form.cleaned_data['quick_calc']
go_service = form.cleaned_data['go_service']
boattype = form.cleaned_data['boattype']
weightvalue = form.cleaned_data['weightvalue']
w.boattype = boattype
@@ -2669,6 +2670,7 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""):
ps=[r.p0,r.p1,r.p2,r.p3],
ratio=r.cpratio,
quick_calc = quick_calc,
go_service=go_service,
emailbounced = r.emailbounced
)