sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

Merge branch 'develop' into feature/restapi

Browse Source
This commit is contained in:
Sander Roosendaal
2016-12-09 08:35:13 +01:00
parent 5708aebf1e 14e67a07eb
commit 52dcd6b24c
506 changed files with 207 additions and 3059962 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
rowers/dataprep.py
View File

@@ -13,6 +13,7 @@ import itertools
from django.conf import settings
from sqlalchemy import create_engine
import sqlalchemy as sa
import sys
user = settings.DATABASES['default']['USER']
password = settings.DATABASES['default']['PASSWORD']
@@ -79,11 +80,48 @@ def timedeltaconv(x):
return dt
def compare_data(id):
row = Workout.objects.get(id=id)
f1 = row.csvfilename
rowdata = rdata(f1)
try:
l1 = len(rowdata.df)
except AttributeError:
l1 = 0
engine = create_engine(database_url, echo=False)
query = sa.text('SELECT COUNT(*) FROM strokedata WHERE workoutid={id};'.format(
id=id,
))
with engine.connect() as conn, conn.begin():
try:
res = conn.execute(query)
l2 = res.fetchall()[0][0]
except:
print "Database Locked"
conn.close()
engine.dispose()
return l1==l2
def repair_data(verbose=False):
ws = Workout.objects.all()
for w in ws:
if verbose:
sys.stdout.write(".")
if not compare_data(w.id):
if verbose:
print w.id
rowdata = rdata(w.csvfilename)
update_strokedata(w.id,rowdata.df)
def rdata(file,rower=rrower()):
try:
res = rrdata(file,rower=rower)
except IOError:
res = 0
try:
res = rrdata(file+'.gz',rower=rower)
except IOError:
res = 0
return res
@@ -245,7 +283,18 @@ def smalldataprep(therows,xparam,yparam1,yparam2):
)
df = pd.concat([df,rowdata],ignore_index=True)
except IOError:
pass
try:
rowdata = dataprep(rrdata(f1+'.gz').df)
rowdata = pd.DataFrame({xparam: rowdata[xparam],
yparam1: rowdata[yparam1],
yparam2: rowdata[yparam2],
'distance': rowdata['distance'],
'spm': rowdata['spm'],
}
)
df = pd.concat([df,rowdata],ignore_index=True)
except IOError:
pass
return df

18
rowers/dataprepnodjango.py
View File

@@ -82,7 +82,10 @@ def rdata(file,rower=rrower()):
try:
res = rrdata(file,rower=rower)
except IOError:
res = 0
try:
res = rrdata(file+'.gz',rower=rower)
except IOError:
res = 0
return res
@@ -220,7 +223,18 @@ def smalldataprep(therows,xparam,yparam1,yparam2):
)
df = pd.concat([df,rowdata],ignore_index=True)
except IOError:
pass
try:
rowdata = dataprep(rrdata(f1+'.gz').df)
rowdata = pd.DataFrame({xparam: rowdata[xparam],
yparam1: rowdata[yparam1],
yparam2: rowdata[yparam2],
'distance': rowdata['distance'],
'spm': rowdata['spm'],
}
)
df = pd.concat([df,rowdata],ignore_index=True)
except IOError:
pass
return df

6
rowers/interactiveplots.py
View File

@@ -437,7 +437,8 @@ def interactive_windchart(id=0,promember=0):
vwind = rowdata.df.ix[:,'vwind']
winddirection = rowdata.df.ix[:,'winddirection']
bearing = rowdata.df.ix[:,'winddirection']
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
dataprep.update_strokedata(id,rowdata.df)
winddirection = winddirection % 360
winddirection = (winddirection + 360) % 360
@@ -520,7 +521,8 @@ def interactive_streamchart(id=0,promember=0):
except KeyError:
rowdata.add_stream(0)
vstream = rowdata.df.ix[:,'vstream']
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
dataprep.update_strokedata(id,rowdata.df)
# plot tools
if (promember==1):

11
rowers/mailprocessing.py
View File

@@ -39,7 +39,10 @@ def rdata(file,rower=rrower()):
try:
res = rrdata(file,rower=rower)
except IOError:
res = 0
try:
res = rrdata(file+'.gz',rower=rower)
except IOError:
res = 0
return res
@@ -160,7 +163,7 @@ def make_new_workout_from_email(rr,f2,name,cntr=0):
timestr = time.strftime("%Y%m%d-%H%M%S")
filename = timestr+str(cntr)+'o.csv'
row.write_csv('media/'+filename)
row.write_csv('media/'+filename,gzip=True)
f2 = filename
# make workout and put in database
@@ -196,14 +199,14 @@ def make_new_workout_from_email(rr,f2,name,cntr=0):
row.df = row.df.fillna(0)
row.write_csv(f2)
row.write_csv(f2,gzip=True)
# recalculate power data
if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
try:
row.erg_recalculatepower()
# row.spm_fromtimestamps()
row.write_csv(f2)
row.write_csv(f2,gzip=True)
except:
pass

25
rowers/management/commands/repairstrokedata.py Normal file
View File

@@ -0,0 +1,25 @@
#!/srv/venv/bin/python
import sys
import os
# If you find a solution that does not need the two paths, please comment!
sys.path.append('$path_to_root_of_project$')
sys.path.append('$path_to_root_of_project$/$project_name$')
os.environ['DJANGO_SETTINGS_MODULE'] = '$project_name$.settings'
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
#from rowers.mailprocessing import processattachments
import time
from django.conf import settings
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage,AdvancedWorkoutForm
from django.core.files.base import ContentFile
from rowsandall_app.settings import BASE_DIR
from rowers.dataprep import *
class Command(BaseCommand):
def handle(self, *args, **options):
repair_data(verbose=True)
self.stdout.write(self.style.SUCCESS('Successfully cleaned data'))

4
rowers/models.py
View File

@@ -255,6 +255,10 @@ def auto_delete_file_on_delete(sender, instance, **kwargs):
if instance.csvfilename:
if os.path.isfile(instance.csvfilename):
os.remove(instance.csvfilename)
if instance.csvfilename+'.gz':
if os.path.isfile(instance.csvfilename+'.gz'):
os.remove(instance.csvfilename+'.gz')
@receiver(models.signals.post_delete,sender=Workout)
def auto_delete_strokedata_on_delete(sender, instance, **kwargs):

26
rowers/tasks.py
View File

@@ -2,6 +2,8 @@ from celery import Celery,app
import os
import time
import gc
import gzip
import shutil
import rowingdata
from rowingdata import main as rmain
@@ -88,8 +90,16 @@ def handle_sendemailcsv(first_name,last_name,email,csvfile):
[fullemail])
email.attach_file(csvfile)
if os.path.isfile(csvfile):
email.attach_file(csvfile)
else:
csvfile2 = csvfile
with gzip.open(csvfile+'.gz','rb') as f_in, open(csvfile2,'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
email.attach_file(csvfile2)
os.remove(csvfile2)
res = email.send()
return 1
@@ -98,7 +108,11 @@ def handle_sendemailcsv(first_name,last_name,email,csvfile):
def handle_otwsetpower(f1,boattype,weightvalue,
first_name,last_name,email,workoutid,
debug=False):
rowdata = rdata(f1)
try:
rowdata = rdata(f1)
except IOError:
rowdata = rdata(f1+'.csv')
weightvalue = float(weightvalue)
# do something with boat type
@@ -165,7 +179,11 @@ def handle_makeplot(f1,f2,t,hrdata,plotnr,imagename):
rr = rowingdata.rower(hrmax=hrmax,hrut2=hrut2,
hrut1=hrut1,hrat=hrat,
hrtr=hrtr,hran=hran)
row = rdata(f2,rower=rr)
try:
row = rdata(f2,rower=rr)
except IOError:
row = rdata(f2+'.gz',rower=rr)
nr_rows = len(row.df)
if (plotnr in [1,2,4,5,8,11,9,12]) and (nr_rows > 1200):
bin = int(nr_rows/1200.)

38
rowers/tests.py
View File

@@ -453,7 +453,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_notloggedin(self):
@@ -501,7 +501,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
@@ -535,7 +535,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
@@ -580,7 +580,7 @@ class ViewTest(TestCase):
self.assertEqual(response.status_code, 200)
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_TCX_SpeedCoach2(self):
self.c.login(username='john',password='koeinsloot')
@@ -610,7 +610,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_TCX_SpeedCoach2(self):
self.c.login(username='john',password='koeinsloot')
@@ -640,7 +640,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
@@ -672,7 +672,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_SpeedCoach2v127(self):
self.c.login(username='john',password='koeinsloot')
@@ -702,7 +702,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_SpeedCoach2v127intervals(self):
@@ -733,7 +733,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_TCX_NoHR(self):
@@ -764,7 +764,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_TCX_CN(self):
self.c.login(username='john',password='koeinsloot')
@@ -791,7 +791,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_RP(self):
self.c.login(username='john',password='koeinsloot')
@@ -818,7 +818,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_Mystery(self):
self.c.login(username='john',password='koeinsloot')
@@ -845,7 +845,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_RP_interval(self):
self.c.login(username='john',password='koeinsloot')
@@ -872,7 +872,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_sled_desktop(self):
self.c.login(username='john',password='koeinsloot')
@@ -899,7 +899,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_sled_ergdata(self):
self.c.login(username='john',password='koeinsloot')
@@ -926,7 +926,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_sled_boatcoach(self):
self.c.login(username='john',password='koeinsloot')
@@ -953,7 +953,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
def test_upload_view_sled_ergstick(self):
self.c.login(username='john',password='koeinsloot')
@@ -980,7 +980,7 @@ class ViewTest(TestCase):
w = Workout.objects.get(id=1)
f_to_be_deleted = w.csvfilename
os.remove(f_to_be_deleted)
os.remove(f_to_be_deleted+'.gz')
# def test_upload_view_FIT_SpeedCoach2a(self):
# self.c.login(username='john',password='koeinsloot')
@@ -1013,7 +1013,7 @@ class ViewTest(TestCase):
# w = Workout.objects.get(id=1)
# f_to_be_deleted = w.csvfilename
# os.remove(f_to_be_deleted)
# os.remove(f_to_be_deleted+'.gz')
class subroutinetests(TestCase):
def setUp(self):

72
rowers/views.py
View File

@@ -135,7 +135,10 @@ def rdata(file,rower=rrower()):
try:
res = rrdata(file,rower=rower)
except IOError:
res = 0
try:
res = rrdata(file+'.gz',rower=rower)
except IOError:
res = 0
return res
@@ -445,7 +448,8 @@ def add_workout_from_strokedata(user,importid,data,strokedata,source='c2'):
csvfilename ='media/Import_'+str(importid)+'.csv'
res = df.to_csv(csvfilename,index_label='index')
res = df.to_csv(csvfilename+'.gz',index_label='index',
compression='gzip')
averagehr = df[' HRCur (bpm)'].mean()
maxhr = df[' HRCur (bpm)'].max()
@@ -677,7 +681,8 @@ def add_workout_from_stdata(user,importid,data):
csvfilename ='media/Import_'+str(importid)+'.csv'
res = df.to_csv(csvfilename,index_label='index')
res = df.to_csv(csvfilename+'.gz',index_label='index',
compression='gzip')
averagehr = df[' HRCur (bpm)'].mean()
maxhr = df[' HRCur (bpm)'].max()
@@ -2048,7 +2053,7 @@ def workout_undo_smoothenpace_view(request,id=0,message="",successmessage=""):
velo = row.df['originalvelo'].values
row.df[' Stroke500mPace (sec/500m)'] = 500./velo
row.write_csv(filename)
row.write_csv(filename,gzip=True)
dataprep.update_strokedata(id,row.df)
url = "/rowers/workout/"+str(id)+"/advanced"
@@ -2085,7 +2090,7 @@ def workout_smoothenpace_view(request,id=0,message="",successmessage=""):
row.df = row.df.fillna(0)
row.write_csv(filename)
row.write_csv(filename,gzip=True)
dataprep.update_strokedata(id,row.df)
url = "/rowers/workout/"+str(id)+"/advanced"
@@ -2155,7 +2160,7 @@ def workout_downloadwind_view(request,id=0,message="",successmessage=""):
bearing = rowdata.df.ix[:,'bearing'].values
except KeyError:
rowdata.add_bearing()
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
# get wind
try:
@@ -2174,7 +2179,7 @@ def workout_downloadwind_view(request,id=0,message="",successmessage=""):
row.notes += "\n"+message
row.save()
rowdata.add_wind(windspeed,windbearing)
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
kwargs = {'successmessage':str(message),
'id':str(id)}
@@ -2224,7 +2229,7 @@ def workout_wind_view(request,id=0,message="",successmessage=""):
bearing = rowdata.df.ix[:,'bearing'].values
except KeyError:
rowdata.add_bearing()
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
@@ -2249,7 +2254,7 @@ def workout_wind_view(request,id=0,message="",successmessage=""):
dist1,dist2,
units=windunit)
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
else:
@@ -2323,7 +2328,7 @@ def workout_stream_view(request,id=0,message="",successmessage=""):
rowdata.update_stream(stream1,stream2,dist1,dist2,
units=streamunit)
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
else:
@@ -2381,19 +2386,19 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""):
vstream = rowdata.df['vstream']
except KeyError:
rowdata.add_stream(0)
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
try:
bearing = rowdata.df['bearing']
except KeyError:
rowdata.add_bearing()
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
try:
vwind = rowdata.df['vwind']
except KeyError:
rowdata.add_wind(0,0)
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
# do power calculation (asynchronous)
u = request.user
@@ -2959,7 +2964,7 @@ def workout_edit_view(request,id=0,message="",successmessage=""):
if r == 0:
return HttpResponse("Error: CSV Data File Not Found")
r.rowdatetime = startdatetime
r.write_csv(row.csvfilename)
r.write_csv(row.csvfilename,gzip=True)
dataprep.update_strokedata(id,r.df)
successmessage = "Changes saved"
url = "/rowers/workout/"+str(row.id)+"/edit"
@@ -3706,8 +3711,13 @@ def workout_upload_view(request,message=""):
f_to_be_deleted = f2
# should delete file
f2 = f2[:-4]+'o.csv'
row.write_csv(f2)
os.remove(f_to_be_deleted)
row.write_csv(f2,gzip=True)
#os.remove(f2)
try:
os.remove(f_to_be_deleted)
except:
os.remove(f_to_be_deleted+'.gz')
# make workout and put in database
r = Rower.objects.get(user=request.user)
@@ -3721,7 +3731,7 @@ def workout_upload_view(request,message=""):
# auto smoothing
pace = row.df[' Stroke500mPace (sec/500m)'].values
velo = 500./pace
f = row.df['TimeStamp (sec)'].diff().mean()
windowsize = 2*(int(10./(f)))+1
@@ -3743,14 +3753,19 @@ def workout_upload_view(request,message=""):
row.df = row.df.fillna(0)
row.write_csv(f2)
row.write_csv(f2,gzip=True)
try:
os.remove(f2)
except:
pass
# recalculate power data
if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides':
try:
row.erg_recalculatepower()
# row.spm_fromtimestamps()
row.write_csv(f2)
row.write_csv(f2,gzip=True)
except:
pass
@@ -3987,8 +4002,12 @@ def workout_upload_view_debug(request,message=""):
f_to_be_deleted = f2
# should delete file
f2 = f2[:-4]+'o.csv'
row.write_csv(f2)
os.remove(f_to_be_deleted)
row.write_csv(f2,gzip=True)
try:
os.remove(f_to_be_deleted)
except:
os.remove(f_to_be_deleted+'.gz')
# remove uncompressed
# make workout and put in database
r = Rower.objects.get(user=request.user)
@@ -3996,8 +4015,10 @@ def workout_upload_view_debug(request,message=""):
hrut1=r.ut1,hrat=r.at,
hrtr=r.tr,hran=r.an,ftp=r.ftp)
row = rdata(f2,rower=rr)
if row == 0:
return HttpResponse("Error: CSV Data File Not Found")
row.write_csv(f2,gzip=True)
averagehr = row.df[' HRCur (bpm)'].mean()
maxhr = row.df[' HRCur (bpm)'].max()
totaldist = row.df['cum_dist'].max()
@@ -4302,7 +4323,7 @@ def workout_summary_restore_view(request,id,message="",successmessage=""):
if rowdata == 0:
return HttpResponse("Error: CSV Data File Not Found")
rowdata.restoreintervaldata()
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
dataprep.update_strokedata(id,rowdata.df)
intervalstats = rowdata.allstats()
row.summary = intervalstats
@@ -4387,7 +4408,7 @@ def workout_summary_edit_view(request,id,message="",successmessage=""
#intervalstats = rowdata.allstats()
row.notes += "\n"+s
row.save()
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
dataprep.update_strokedata(id,rowdata.df)
data = {'intervalstring':s}
form = SummaryStringForm(initial=data)
@@ -4439,7 +4460,7 @@ def workout_summary_edit_view(request,id,message="",successmessage=""
row.summary = intervalstats
row.notes += "\n"+s
row.save()
rowdata.write_csv(f1)
rowdata.write_csv(f1,gzip=True)
dataprep.update_strokedata(id,rowdata.df)
@@ -4791,7 +4812,8 @@ def strokedatajson(request,id):
timestr = row.startdatetime.strftime("%Y%m%d-%H%M%S")
csvfilename ='media/Import_'+timestr+'.csv'
res = data.to_csv(csvfilename,index_label='index')
res = data.to_csv(csvfilename+'.gz',index_label='index',
compression='gzip')
row.csvfilename = csvfilename
row.save()