diff --git a/rowers/interactiveplots.py b/rowers/interactiveplots.py index 4560905c..e50b9e93 100644 --- a/rowers/interactiveplots.py +++ b/rowers/interactiveplots.py @@ -437,7 +437,8 @@ def interactive_windchart(id=0,promember=0): vwind = rowdata.df.ix[:,'vwind'] winddirection = rowdata.df.ix[:,'winddirection'] bearing = rowdata.df.ix[:,'winddirection'] - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) + dataprep.update_strokedata(id,rowdata.df) winddirection = winddirection % 360 winddirection = (winddirection + 360) % 360 @@ -520,7 +521,8 @@ def interactive_streamchart(id=0,promember=0): except KeyError: rowdata.add_stream(0) vstream = rowdata.df.ix[:,'vstream'] - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) + dataprep.update_strokedata(id,rowdata.df) # plot tools if (promember==1): diff --git a/rowers/mailprocessing.py b/rowers/mailprocessing.py index 071612e9..424241d3 100644 --- a/rowers/mailprocessing.py +++ b/rowers/mailprocessing.py @@ -163,7 +163,7 @@ def make_new_workout_from_email(rr,f2,name,cntr=0): timestr = time.strftime("%Y%m%d-%H%M%S") filename = timestr+str(cntr)+'o.csv' - row.write_csv('media/'+filename) + row.write_csv('media/'+filename,gzip=True) f2 = filename # make workout and put in database @@ -199,14 +199,14 @@ def make_new_workout_from_email(rr,f2,name,cntr=0): row.df = row.df.fillna(0) - row.write_csv(f2) + row.write_csv(f2,gzip=True) # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: row.erg_recalculatepower() # row.spm_fromtimestamps() - row.write_csv(f2) + row.write_csv(f2,gzip=True) except: pass diff --git a/rowers/tests.py b/rowers/tests.py index 8a5a4b7d..aa4666df 100644 --- a/rowers/tests.py +++ b/rowers/tests.py @@ -453,7 +453,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_notloggedin(self): @@ -501,7 +501,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') @@ -535,7 +535,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') @@ -580,7 +580,7 @@ class ViewTest(TestCase): self.assertEqual(response.status_code, 200) w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_TCX_SpeedCoach2(self): self.c.login(username='john',password='koeinsloot') @@ -610,7 +610,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_TCX_SpeedCoach2(self): self.c.login(username='john',password='koeinsloot') @@ -640,7 +640,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') @@ -672,7 +672,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_SpeedCoach2v127(self): self.c.login(username='john',password='koeinsloot') @@ -702,7 +702,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_SpeedCoach2v127intervals(self): @@ -733,7 +733,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_TCX_NoHR(self): @@ -764,7 +764,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_TCX_CN(self): self.c.login(username='john',password='koeinsloot') @@ -791,7 +791,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_RP(self): self.c.login(username='john',password='koeinsloot') @@ -818,7 +818,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_Mystery(self): self.c.login(username='john',password='koeinsloot') @@ -845,7 +845,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_RP_interval(self): self.c.login(username='john',password='koeinsloot') @@ -872,7 +872,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_sled_desktop(self): self.c.login(username='john',password='koeinsloot') @@ -899,7 +899,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_sled_ergdata(self): self.c.login(username='john',password='koeinsloot') @@ -926,7 +926,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_sled_boatcoach(self): self.c.login(username='john',password='koeinsloot') @@ -953,7 +953,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') def test_upload_view_sled_ergstick(self): self.c.login(username='john',password='koeinsloot') @@ -980,7 +980,7 @@ class ViewTest(TestCase): w = Workout.objects.get(id=1) f_to_be_deleted = w.csvfilename - os.remove(f_to_be_deleted) + os.remove(f_to_be_deleted+'.gz') # def test_upload_view_FIT_SpeedCoach2a(self): # self.c.login(username='john',password='koeinsloot') @@ -1013,7 +1013,7 @@ class ViewTest(TestCase): # w = Workout.objects.get(id=1) # f_to_be_deleted = w.csvfilename -# os.remove(f_to_be_deleted) +# os.remove(f_to_be_deleted+'.gz') class subroutinetests(TestCase): def setUp(self): diff --git a/rowers/views.py b/rowers/views.py index 75fd14f5..2c8af572 100644 --- a/rowers/views.py +++ b/rowers/views.py @@ -444,7 +444,8 @@ def add_workout_from_strokedata(user,importid,data,strokedata,source='c2'): csvfilename ='media/Import_'+str(importid)+'.csv' - res = df.to_csv(csvfilename,index_label='index') + res = df.to_csv(csvfilename+'.gz',index_label='index', + compression='gzip') averagehr = df[' HRCur (bpm)'].mean() maxhr = df[' HRCur (bpm)'].max() @@ -676,7 +677,8 @@ def add_workout_from_stdata(user,importid,data): csvfilename ='media/Import_'+str(importid)+'.csv' - res = df.to_csv(csvfilename,index_label='index') + res = df.to_csv(csvfilename+'.gz',index_label='index', + compression='gzip') averagehr = df[' HRCur (bpm)'].mean() maxhr = df[' HRCur (bpm)'].max() @@ -2047,7 +2049,7 @@ def workout_undo_smoothenpace_view(request,id=0,message="",successmessage=""): velo = row.df['originalvelo'].values row.df[' Stroke500mPace (sec/500m)'] = 500./velo - row.write_csv(filename) + row.write_csv(filename,gzip=True) dataprep.update_strokedata(id,row.df) url = "/rowers/workout/"+str(id)+"/advanced" @@ -2084,7 +2086,7 @@ def workout_smoothenpace_view(request,id=0,message="",successmessage=""): row.df = row.df.fillna(0) - row.write_csv(filename) + row.write_csv(filename,gzip=True) dataprep.update_strokedata(id,row.df) url = "/rowers/workout/"+str(id)+"/advanced" @@ -2154,7 +2156,7 @@ def workout_downloadwind_view(request,id=0,message="",successmessage=""): bearing = rowdata.df.ix[:,'bearing'].values except KeyError: rowdata.add_bearing() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) # get wind try: @@ -2173,7 +2175,7 @@ def workout_downloadwind_view(request,id=0,message="",successmessage=""): row.notes += "\n"+message row.save() rowdata.add_wind(windspeed,windbearing) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) kwargs = {'successmessage':str(message), 'id':str(id)} @@ -2223,7 +2225,7 @@ def workout_wind_view(request,id=0,message="",successmessage=""): bearing = rowdata.df.ix[:,'bearing'].values except KeyError: rowdata.add_bearing() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) @@ -2248,7 +2250,7 @@ def workout_wind_view(request,id=0,message="",successmessage=""): dist1,dist2, units=windunit) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) else: @@ -2322,7 +2324,7 @@ def workout_stream_view(request,id=0,message="",successmessage=""): rowdata.update_stream(stream1,stream2,dist1,dist2, units=streamunit) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) else: @@ -2380,19 +2382,19 @@ def workout_otwsetpower_view(request,id=0,message="",successmessage=""): vstream = rowdata.df['vstream'] except KeyError: rowdata.add_stream(0) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) try: bearing = rowdata.df['bearing'] except KeyError: rowdata.add_bearing() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) try: vwind = rowdata.df['vwind'] except KeyError: rowdata.add_wind(0,0) - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) # do power calculation (asynchronous) u = request.user @@ -2958,7 +2960,7 @@ def workout_edit_view(request,id=0,message="",successmessage=""): if r == 0: return HttpResponse("Error: CSV Data File Not Found") r.rowdatetime = startdatetime - r.write_csv(row.csvfilename) + r.write_csv(row.csvfilename,gzip=True) dataprep.update_strokedata(id,r.df) successmessage = "Changes saved" url = "/rowers/workout/"+str(row.id)+"/edit" @@ -3705,8 +3707,13 @@ def workout_upload_view(request,message=""): f_to_be_deleted = f2 # should delete file f2 = f2[:-4]+'o.csv' - row.write_csv(f2) - os.remove(f_to_be_deleted) + row.write_csv(f2,gzip=True) + + #os.remove(f2) + try: + os.remove(f_to_be_deleted) + except: + os.remove(f_to_be_deleted+'.gz') # make workout and put in database r = Rower.objects.get(user=request.user) @@ -3720,7 +3727,7 @@ def workout_upload_view(request,message=""): # auto smoothing pace = row.df[' Stroke500mPace (sec/500m)'].values velo = 500./pace - + f = row.df['TimeStamp (sec)'].diff().mean() windowsize = 2*(int(10./(f)))+1 @@ -3742,14 +3749,19 @@ def workout_upload_view(request,message=""): row.df = row.df.fillna(0) - row.write_csv(f2) + row.write_csv(f2,gzip=True) + try: + os.remove(f2) + except: + pass + # recalculate power data if workouttype == 'rower' or workouttype == 'dynamic' or workouttype == 'slides': try: row.erg_recalculatepower() # row.spm_fromtimestamps() - row.write_csv(f2) + row.write_csv(f2,gzip=True) except: pass @@ -3986,8 +3998,12 @@ def workout_upload_view_debug(request,message=""): f_to_be_deleted = f2 # should delete file f2 = f2[:-4]+'o.csv' - row.write_csv(f2) - os.remove(f_to_be_deleted) + row.write_csv(f2,gzip=True) + try: + os.remove(f_to_be_deleted) + except: + os.remove(f_to_be_deleted+'.gz') + # remove uncompressed # make workout and put in database r = Rower.objects.get(user=request.user) @@ -3995,8 +4011,10 @@ def workout_upload_view_debug(request,message=""): hrut1=r.ut1,hrat=r.at, hrtr=r.tr,hran=r.an,ftp=r.ftp) row = rdata(f2,rower=rr) + if row == 0: return HttpResponse("Error: CSV Data File Not Found") + row.write_csv(f2,gzip=True) averagehr = row.df[' HRCur (bpm)'].mean() maxhr = row.df[' HRCur (bpm)'].max() totaldist = row.df['cum_dist'].max() @@ -4301,7 +4319,7 @@ def workout_summary_restore_view(request,id,message="",successmessage=""): if rowdata == 0: return HttpResponse("Error: CSV Data File Not Found") rowdata.restoreintervaldata() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) dataprep.update_strokedata(id,rowdata.df) intervalstats = rowdata.allstats() row.summary = intervalstats @@ -4386,7 +4404,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" #intervalstats = rowdata.allstats() row.notes += "\n"+s row.save() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) dataprep.update_strokedata(id,rowdata.df) data = {'intervalstring':s} form = SummaryStringForm(initial=data) @@ -4438,7 +4456,7 @@ def workout_summary_edit_view(request,id,message="",successmessage="" row.summary = intervalstats row.notes += "\n"+s row.save() - rowdata.write_csv(f1) + rowdata.write_csv(f1,gzip=True) dataprep.update_strokedata(id,rowdata.df) @@ -4790,7 +4808,8 @@ def strokedatajson(request,id): timestr = row.startdatetime.strftime("%Y%m%d-%H%M%S") csvfilename ='media/Import_'+timestr+'.csv' - res = data.to_csv(csvfilename,index_label='index') + res = data.to_csv(csvfilename+'.gz',index_label='index', + compression='gzip') row.csvfilename = csvfilename row.save() diff --git a/rowsandall_app/rows.py b/rowsandall_app/rows.py index bad5eac3..1d9848c9 100644 --- a/rowsandall_app/rows.py +++ b/rowsandall_app/rows.py @@ -1,4 +1,7 @@ import time +import gzip +import shutil + from django.core.exceptions import ValidationError def format_pace_tick(x,pos=None): @@ -65,7 +68,7 @@ def handle_uploaded_file(f): with open(fname2,'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) - + return fname,fname2 # this might work on windows