From 8895f61e730f6ae1ae0e469d6dbb0fdbd0bc8676 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Thu, 8 Dec 2016 23:07:57 +0100 Subject: [PATCH] gzip --- rowers/interactiveplots.py | 6 ++-- rowers/mailprocessing.py | 6 ++-- rowers/views.py | 57 +++++++++++++++++++++++--------------- rowsandall_app/rows.py | 5 +++- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 4560905c..e50b9e93 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -437,7 +437,8 @@ def interactive_windchart(id=0,promember=0): vwind = rowdata.df.ix[:,'vwind'] winddirection = rowdata.df.ix[:,'winddirection'] bearing = rowdata.df.ix[:,'winddirection'] - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) + dataprep.update_strokedata(id,rowdata.df) winddirection = winddirection % 360 winddirection = (winddirection + 360) % 360 @@ -520,7 +521,8 @@ def interactive_streamchart(id=0,promember=0): except KeyError: rowdata.add_stream(0) vstream = rowdata.df.ix[:,'vstream'] - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) + dataprep.update_strokedata(id,rowdata.df) # plot tools if (promember==1): diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index 071612e9..424241d3 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -163,7 +163,7 @@ def make_new_workout_from_email(rr,f2,name,cntr=0): timestr = time.strftime("%Y%m%d-%H%M%S") filename = timestr+str(cntr)+'o.csv' - row.write_csv('media/'+filename) + row.write_csv('media/'+filename,gzip=True) f2 = filename # make workout and put in database @@ -199,14 +199,14 @@ def make_new_workout_from_email(rr,f2,name,cntr=0): row.df = row.df.fillna(0) - row.write_csv(f2) + row.write_csv(f2,gzip=True) # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: row.erg_recalculatepower() # row.spm_fromtimestamps() - row.write_csv(f2) + row.write_csv(f2,gzip=True) except: pass diff --git a/rowers/views.py b/rowers/views.py index 75fd14f5..107ae277 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -444,7 +444,8 @@ def add_workout_from_strokedata(user,importid,data,strokedata,source='c2'): csvfilename ='media/Import_'+str(importid)+'.csv' - res = df.to_csv(csvfilename,index_label='index') + res = df.to_csv(csvfilename+'.gz',index_label='index', + compression='gzip') averagehr = df[' HRCur (bpm)'].mean() maxhr = df[' HRCur (bpm)'].max() @@ -676,7 +677,8 @@ def add_workout_from_stdata(user,importid,data): csvfilename ='media/Import_'+str(importid)+'.csv' - res = df.to_csv(csvfilename,index_label='index') + res = df.to_csv(csvfilename+'.gz',index_label='index', + compression='gzip') averagehr = df[' HRCur (bpm)'].mean() maxhr = df[' HRCur (bpm)'].max() @@ -2047,7 +2049,7 @@ def workout_undo_smoothenpace_view(request,id=0,message="",successmessage=""): velo = row.df['originalvelo'].values row.df[' Stroke500mPace (sec/500m)'] = 500./velo - row.write_csv(filename) + row.write_csv(filename,gzip=True) dataprep.update_strokedata(id,row.df) url = "/rowers/workout/"+str(id)+"/advanced" @@ -2084,7 +2086,7 @@ def workout_smoothenpace_view(request,id=0,message="",successmessage=""): row.df = row.df.fillna(0) - row.write_csv(filename) + row.write_csv(filename,gzip=True) dataprep.update_strokedata(id,row.df) url = "/rowers/workout/"+str(id)+"/advanced" @@ -2154,7 +2156,7 @@ def workout_downloadwind_view(request,id=0,message="",successmessage=""): bearing = rowdata.df.ix[:,'bearing'].values except KeyError: rowdata.add_bearing() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) # get wind try: @@ -2173,7 +2175,7 @@ def workout_downloadwind_view(request,id=0,message="",successmessage=""): row.notes += "\n"+message row.save() rowdata.add_wind(windspeed,windbearing) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) kwargs = {'successmessage':str(message), 'id':str(id)} @@ -2223,7 +2225,7 @@ def workout_wind_view(request,id=0,message="",successmessage=""): bearing = rowdata.df.ix[:,'bearing'].values except KeyError: rowdata.add_bearing() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) @@ -2248,7 +2250,7 @@ def workout_wind_view(request,id=0,message="",successmessage=""): dist1,dist2, units=windunit) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) else: @@ -2322,7 +2324,7 @@ def workout_stream_view(request,id=0,message="",successmessage=""): rowdata.update_stream(stream1,stream2,dist1,dist2, units=streamunit) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) else: @@ -2380,19 +2382,19 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): vstream = rowdata.df['vstream'] except KeyError: rowdata.add_stream(0) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) try: bearing = rowdata.df['bearing'] except KeyError: rowdata.add_bearing() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) try: vwind = rowdata.df['vwind'] except KeyError: rowdata.add_wind(0,0) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) # do power calculation (asynchronous) u = request.user @@ -2958,7 +2960,7 @@ def workout_edit_view(request,id=0,message="",successmessage=""): if r == 0: return HttpResponse("Error: CSV Data File Not Found") r.rowdatetime = startdatetime - r.write_csv(row.csvfilename) + r.write_csv(row.csvfilename,gzip=True) dataprep.update_strokedata(id,r.df) successmessage = "Changes saved" url = "/rowers/workout/"+str(row.id)+"/edit" @@ -3705,7 +3707,9 @@ def workout_upload_view(request,message=""): f_to_be_deleted = f2 # should delete file f2 = f2[:-4]+'o.csv' - row.write_csv(f2) + row.write_csv(f2,gzip=True) + + #os.remove(f2) os.remove(f_to_be_deleted) # make workout and put in database @@ -3720,7 +3724,7 @@ def workout_upload_view(request,message=""): # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values velo = 500./pace - + f = row.df['TimeStamp (sec)'].diff().mean() windowsize = 2*(int(10./(f)))+1 @@ -3742,14 +3746,19 @@ def workout_upload_view(request,message=""): row.df = row.df.fillna(0) - row.write_csv(f2) + row.write_csv(f2,gzip=True) + try: + os.remove(f2) + except: + pass + # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: row.erg_recalculatepower() # row.spm_fromtimestamps() - row.write_csv(f2) + row.write_csv(f2,gzip=True) except: pass @@ -3986,8 +3995,9 @@ def workout_upload_view_debug(request,message=""): f_to_be_deleted = f2 # should delete file f2 = f2[:-4]+'o.csv' - row.write_csv(f2) + row.write_csv(f2,gzip=True) os.remove(f_to_be_deleted) + # remove uncompressed # make workout and put in database r = Rower.objects.get(user=request.user) @@ -3995,8 +4005,10 @@ def workout_upload_view_debug(request,message=""): hrut1=r.ut1,hrat=r.at, hrtr=r.tr,hran=r.an,ftp=r.ftp) row = rdata(f2,rower=rr) + if row == 0: return HttpResponse("Error: CSV Data File Not Found") + row.write_csv(f2,gzip=True) averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() totaldist = row.df['cum_dist'].max() @@ -4301,7 +4313,7 @@ def workout_summary_restore_view(request,id,message="",successmessage=""): if rowdata == 0: return HttpResponse("Error: CSV Data File Not Found") rowdata.restoreintervaldata() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) dataprep.update_strokedata(id,rowdata.df) intervalstats = rowdata.allstats() row.summary = intervalstats @@ -4386,7 +4398,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" #intervalstats = rowdata.allstats() row.notes += "\n"+s row.save() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) dataprep.update_strokedata(id,rowdata.df) data = {'intervalstring':s} form = SummaryStringForm(initial=data) @@ -4438,7 +4450,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" row.summary = intervalstats row.notes += "\n"+s row.save() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) dataprep.update_strokedata(id,rowdata.df) @@ -4790,7 +4802,8 @@ def strokedatajson(request,id): timestr = row.startdatetime.strftime("%Y%m%d-%H%M%S") csvfilename ='media/Import_'+timestr+'.csv' - res = data.to_csv(csvfilename,index_label='index') + res = data.to_csv(csvfilename+'.gz',index_label='index', + compression='gzip') row.csvfilename = csvfilename row.save() diff --git a/rowsandall_app/rows.py b/rowsandall_app/rows.py index bad5eac3..1d9848c9 100644 --- a/rowsandall_app/rows.py +++ b/rowsandall_app/rows.py @@ -1,4 +1,7 @@ import time +import gzip +import shutil + from django.core.exceptions import ValidationError def format_pace_tick(x,pos=None): @@ -65,7 +68,7 @@ def handle_uploaded_file(f): with open(fname2,'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) - + return fname,fname2 # this might work on windows