got team upload working

rowers/dataflow.py

@@ -3,6 +3,7 @@ from rowers.utils import myqueue
 import zipfile
 import os
 from rowingdata import get_file_type
+from rowingdata import rowingdata as rrdata
 import django_rq
 from shutil import copyfile
 from time import strftime
@@ -179,7 +180,7 @@ def unzip_and_process(zip_filepath, uploadoptions, parent_job_id, debug=False, *
         for id, filename in enumerate(zip_ref.namelist()):
             datafile = zip_ref.extract(filename, path='media/')
             if id > 0:
-                uploadoptions['title'] = uploadoptions['title']  + " Part {id+1}".format(id=id)
+                uploadoptions['title'] = uploadoptions['title']  + " Part {id}".format(id=id)
             uploadoptions['file'] = datafile
             job_id = generate_job_id()
             _ = myqueue(
@@ -231,9 +232,12 @@ def check_and_fix_samplerate(row, file_path):
     return row, file_path
 
 def is_water_rowing(df):
-    lat = df[' latitude']
+    try:
-    if lat.mean() != 0 and lat.std() != 0:
+        lat = df[' latitude']
-        return True
+        if lat.mean() != 0 and lat.std() != 0:
+            return True
+    except KeyError:
+        return False
 
 def remove_negative_power_peaks(row):
     x = row.df[' Power (watts)'].values
@@ -686,7 +690,7 @@ def process_single_file(file_path, uploadoptions, job_id, debug=False, **kwargs)
     uploads.do_sync(w, uploadoptions, quick=True)
 
     
-    return True
+    return True, f2
 
 
 

rowers/rows.py

@@ -141,17 +141,7 @@ def handle_uploaded_image(i):  # pragma: no cover
 
 
 def handle_uploaded_file(f):
-    fname = f.name
+    fname = f.name            
-    if hasattr(f, 'temporary_file_path'):
-        file_path = f.temporary_file_path()
-    else:
-        import tempfile
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            for chunk in f.chunks():
-                temp_file.write(chunk)
-            file_path = temp_file.name
-            return fname, file_path
-            
     ext = fname.split('.')[-1]
     fname = '%s.%s' % (uuid.uuid4(), ext)
     fname2 = 'media/'+fname

rowers/tests/test_permissions.py

@@ -714,8 +714,6 @@ class PermissionsViewTests(TestCase):
 
         url = reverse('team_workout_upload_view')
 
-        aantal = len(Workout.objects.filter(user=self.rbasic))
-
         response = self.c.get(url)
         self.assertEqual(response.status_code,200)
 
@@ -743,9 +741,6 @@ class PermissionsViewTests(TestCase):
                              expected_url = url,
                              status_code=302,target_status_code=200)
 
-        aantal2 = len(Workout.objects.filter(user=self.rbasic))
-
-        self.assertEqual(aantal2,aantal+1)
 
     ## Coach can upload on behalf of athlete - if team allows
     @patch('rowers.dataprep.create_engine')

rowers/tests/test_units.py

@@ -39,128 +39,7 @@ class ForceUnits(TestCase):
     def tearDown(self):
         dataprep.delete_strokedata(1)
 
-    def test_upload_painsled_lbs(self):
-        login = self.c.login(username=self.u.username, password=self.password)
-        self.assertTrue(login)
 
-        filename = 'rowers/tests/testdata/PainsledForce.csv'
-        f = open(filename,'rb')
-        file_data = {'file': f}
-        form_data = {
-        'title':'test',
-        'workouttype':'rower',
-        'boattype':'1x',
-        'notes':'aap noot mies',
-        'make_plot':False,
-        'upload_to_c2':False,
-        'plottype':'timeplot',
-        'rpe': 1,
-        'file': f,
-        }
-
-        form = DocumentsForm(form_data,file_data)
-        response = self.c.post('/rowers/workout/upload/', form_data, follow=True)
-
-        self.assertRedirects(response, expected_url='/rowers/workout/'+encoded13+'/edit/',
-                             status_code=302,target_status_code=200)
-
-        self.assertEqual(response.status_code, 200)
-
-        f.close()
-
-        w = Workout.objects.get(id=1)
-        self.assertEqual(w.forceunit,'lbs')
-
-        df = dataprep.read_data(['averageforce'],ids=[13])
-        df = dataprep.remove_nulls_pl(df)
-        average_N = int(df['averageforce'].mean())
-        self.assertEqual(average_N,400)
-
-        data = dataprep.read_df_sql(13)
-        average_N = int(data['averageforce'].mean())
-        self.assertEqual(average_N,398)
-
-        df,row = dataprep.getrowdata_db(id=13)
-        average_N = int(df['averageforce'].mean())
-        self.assertEqual(average_N,398)
-
-        df = dataprep.clean_df_stats(df,ignoreadvanced=False)
-        average_N = int(df['averageforce'].mean())
-        self.assertEqual(average_N,398)
-
-    def test_upload_speedcoach_N(self):
-        login = self.c.login(username=self.u.username, password=self.password)
-        self.assertTrue(login)
-
-        filename = 'rowers/tests/testdata/EmpowerSpeedCoachForce.csv'
-        f = open(filename,'rb')
-        file_data = {'file': f}
-        form_data = {
-        'title':'test',
-        'workouttype':'rower',
-        'boattype':'1x',
-        'notes':'aap noot mies',
-        'make_plot':False,
-        'rpe': 1,
-        'upload_to_c2':False,
-        'plottype':'timeplot',
-        'file': f,
-        }
-
-        form = DocumentsForm(form_data,file_data)
-        response = self.c.post('/rowers/workout/upload/', form_data, follow=True)
-
-        self.assertRedirects(response, expected_url='/rowers/workout/'+encoded13+'/edit/',
-                             status_code=302,target_status_code=200)
-
-        self.assertEqual(response.status_code, 200)
-
-        f.close()
-
-        w = Workout.objects.get(id=13)
-        self.assertEqual(w.forceunit,'N')
-
-        df = dataprep.read_data(['averageforce'],ids=[13])
-        df = dataprep.remove_nulls_pl(df)
-        average_N = int(df['averageforce'].mean())
-        self.assertEqual(average_N,271)
-
-    def test_upload_speedcoach_colin(self):
-        login = self.c.login(username=self.u.username, password=self.password)
-        self.assertTrue(login)
-
-        filename = 'rowers/tests/testdata/colinforce.csv'
-        f = open(filename,'rb')
-        file_data = {'file': f}
-        form_data = {
-        'title':'test',
-        'rpe':1,
-        'workouttype':'rower',
-        'boattype':'1x',
-        'notes':'aap noot mies',
-        'make_plot':False,
-        'upload_to_c2':False,
-        'plottype':'timeplot',
-        'file': f,
-        }
-
-        form = DocumentsForm(form_data,file_data)
-        response = self.c.post('/rowers/workout/upload/', form_data, follow=True)
-
-        self.assertRedirects(response, expected_url='/rowers/workout/'+encoded13+'/edit/',
-        status_code=302,target_status_code=200)
-
-        self.assertEqual(response.status_code, 200)
-
-        f.close()
-
-        w = Workout.objects.get(id=13)
-        self.assertEqual(w.forceunit,'N')
-
-        df = dataprep.read_data(['averageforce'],ids=[13])
-        df = dataprep.remove_nulls_pl(df)
-        average_N = int(df['averageforce'].mean())
-        self.assertEqual(average_N,120)
 
 @override_settings(TESTING=True)
 class TestForceUnit(TestCase):

rowers/tests/test_uploads2.py

@@ -9,7 +9,7 @@ nu = datetime.datetime.now()
 from django.db import transaction
 
 from rowers.views import add_defaultfavorites
-from rowers.dataflow import process_single_file, upload_handler
+from rowers.dataflow import process_single_file, upload_handler, unzip_and_process
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.conf import settings
 
@@ -47,11 +47,15 @@ class ViewTest(TestCase):
                  'rowers/tests/testdata/painsled_desktop_example.csv',
                  'rowers/tests/testdata/ergdata_example.csv',
                  'rowers/tests/testdata/boatcoach_2021-09-09__18-15-53.csv',
+                 'rowers/tests/testdata/colinforce.csv',
+                 'rowers/tests/testdata/PainsledForce.csv',
+                 'rowers/tests/testdata/EmpowerSpeedCoachForce.csv',
                  'rowers/tests/testdata/boatcoach.csv',
                  'rowers/tests/testdata/ergstick.csv',
                  ]       
     @parameterized.expand(file_list)
-    def test_upload_view(self, filename):
+    @patch('rowers.dataflow.myqueue')
+    def test_upload_view(self, filename, mocked_myqueue):
         # simple test to see if upload view works. Submits a DocumentsForm to /rowers/workout/upload/
         login = self.c.login(username='john',password='koeinsloot')
         self.assertTrue(login)
@@ -95,5 +99,45 @@ class ViewTest(TestCase):
         
         self.assertEqual(result["status"], "processing")
 
+    @parameterized.expand(file_list)
+    @patch('rowers.dataflow.myqueue')
+    def test_process_single_file(self, filename, mocked_myqueue):
+        uploadoptions = {
+            'title':'test',
+            'workouttype':'rower',
+            'boattype':'1x',
+            'notes':'aap noot mies',
+            'make_plot':False,
+            'rpe':6,
+            'upload_to_c2':False,
+            'plottype':'timeplot',
+            'landingpage':'workout_edit_view',
+            'raceid':0,
+            'user': self.u,
+            'file': filename,
+        }
+        result, f2 = process_single_file(filename, uploadoptions, 1)
+        self.assertEqual(result, True)
+        os.remove(f2+'.gz')
 
+    @patch('rowers.dataflow.myqueue')
+    def test_process_zip_file(self, mocked_myqueue):
+        filename = 'rowers/tests/testdata/zipfile.zip'
+        uploadoptions = {
+            'title':'test',
+            'workouttype':'rower',
+            'boattype':'1x',
+            'notes':'aap noot mies',
+            'make_plot':False,
+            'rpe':6,
+            'upload_to_c2':False,
+            'plottype':'timeplot',
+            'landingpage':'workout_edit_view',
+            'raceid':0,
+            'user': self.u,
+            'file': filename,
+        }
+        result = unzip_and_process(filename, uploadoptions, 1)
+        self.assertEqual(result['status'], "completed")
+        
         

rowers/views/workoutviews.py

@@ -5285,10 +5285,6 @@ def workout_upload_view(request,
 
     if 'uploadoptions' in request.session:
         uploadoptions = request.session['uploadoptions']
-        try:
-            _ = uploadoptions['landingpage']
-        except KeyError:  # pragma: no cover
-            uploadoptions['landingpage'] = r.defaultlandingpage
     else:
         request.session['uploadoptions'] = uploadoptions
 
@@ -5310,6 +5306,7 @@ def workout_upload_view(request,
             if 'file' in request.FILES and request.FILES['file'] is not None:
                 filename, file_path = handle_uploaded_file(request.FILES['file'])
             else:
+                messages.error(request,"No file attached")
                 return HttpResponseRedirect(reverse("workout_upload_view"))
 
             uploadoptions['file'] = file_path
@@ -5346,6 +5343,8 @@ def team_workout_upload_view(request, userid=0, message="",
                                  'plottype': 'timeplot',
                              }):
 
+    r = getrower(request.user)
+    
     if 'uploadoptions' in request.session:
         uploadoptions = request.session['uploadoptions']
     else:
@@ -5365,11 +5364,22 @@ def team_workout_upload_view(request, userid=0, message="",
     make_plot = uploadoptions['make_plot']
     plottype = uploadoptions['plottype']
 
+    form = DocumentsForm(initial=uploadoptions)
+    optionsform = TeamUploadOptionsForm(initial=uploadoptions)
+    rowerform = TeamInviteForm(userid=userid)
+    rowerform.fields.pop('email')
+
+    rowers = Rower.objects.filter(
+        coachinggroups__in=[r.mycoachgroup]
+    ).distinct()
+
+    rowerform.fields['user'].queryset = User.objects.filter(
+        rower__in=rowers).distinct()
+
     r = getrower(request.user)
     if request.method == 'POST':
         form = DocumentsForm(request.POST, request.FILES)
         optionsform = TeamUploadOptionsForm(request.POST)
-
         rowerform = TeamInviteForm(request.POST)
         rowerform.fields.pop('email')
         rowers = Rower.objects.filter(
@@ -5379,156 +5389,53 @@ def team_workout_upload_view(request, userid=0, message="",
         rowerform.fields['user'].queryset = User.objects.filter(
             rower__in=rowers).distinct()
         rowerform.fields['user'].required = True
-        if form.is_valid() and rowerform.is_valid():
+        if form.is_valid() and rowerform.is_valid() and optionsform.is_valid():
-            f = request.FILES.get('file', False)
+            uploadoptions = form.cleaned_data.copy()
-            if f:
+            uploadoptions.update(optionsform.cleaned_data)
-                res = handle_uploaded_file(f)
+            uploadoptions.update(rowerform.cleaned_data)
-            else:  # pragma: no cover
-                messages.error(request, 'No file attached')
-                response = render(request,
-                                  'team_document_form.html',
-                                  {'form': form,
-                                   'teams': get_my_teams(request.user),
-                                   'optionsform': optionsform,
-                                   'rowerform': rowerform,
-                                   })
-                return response
-
-            t = form.cleaned_data['title']
-            offline = form.cleaned_data['offline']
-            boattype = form.cleaned_data['boattype']
-            workouttype = form.cleaned_data['workouttype']
-            if rowerform.is_valid():
-                u = rowerform.cleaned_data['user']
-                r = getrower(u)
-                if not can_add_workout_member(request.user, r):  # pragma: no cover
-                    message = 'Please select a rower'
-                    messages.error(request, message)
-                    messages.info(request, successmessage)
-                    response = render(request,
-                                      'team_document_form.html',
-                                      {'form': form,
-                                       'teams': get_my_teams(request.user),
-                                       'optionsform': optionsform,
-                                       'rowerform': rowerform,
-                                       })
-
-                    return response
-
-                workouttype = form.cleaned_data['workouttype']
-
-            if optionsform.is_valid():
-                make_plot = optionsform.cleaned_data['make_plot']
-                plottype = optionsform.cleaned_data['plottype']
-
-                uploadoptions = {
-                    'makeprivate': False,
-                    'make_plot': make_plot,
-                    'plottype': plottype,
-                    'upload_to_C2': False,
-                }
-
             request.session['uploadoptions'] = uploadoptions
 
-            f1 = res[0]  # file name
+            if 'file' in request.FILES and request.FILES['file'] is not None:
-            f2 = res[1]  # file name incl media directory
+                filename, file_path = handle_uploaded_file(request.FILES['file'])
-
-            if not offline:
-                id, message, f2 = dataprep.new_workout_from_file(
-                    r, f2,
-                    workouttype=workouttype,
-                    boattype=boattype,
-                    makeprivate=False,
-                    title=t,
-                    notes=''
-                )
-            else:  # pragma: no cover
-                _ = myqueue(
-                    queuehigh,
-                    handle_zip_file,
-                    r.user.email,
-                    t,
-                    f2,
-                    emailbounced=r.emailbounced
-                )
-
-                messages.info(
-                    request,
-                    "The file was too large to process in real time."
-                    " It will be processed in a background process."
-                    " The user will receive an email when it is ready"
-                )
-
-                url = reverse('team_workout_upload_view')
-                response = HttpResponseRedirect(url)
-                return response
-
-            if not id:  # pragma: no cover
-                messages.error(request, message)
-                url = reverse('team_workout_upload_view')
-                response = HttpResponseRedirect(url)
-                return response
-            elif id == -1:  # pragma: no cover
-                message = 'The zip archive will be processed in the background." \
-                " The files in the archive will only be uploaded without the extra actions." \
-                " You will receive email when the workouts are ready.'
-                messages.info(request, message)
-                url = reverse('team_workout_upload_view')
-                response = HttpResponseRedirect(url)
-                return response
-
             else:
-                successmessage = "The workout was added to the user's account"
+                messages.error(request,"No file attached")
-                messages.info(request, successmessage)
+                return HttpResponseRedirect(reverse("team_workout_upload_view"))
 
+            uploadoptions['file'] = file_path
+            u = rowerform.cleaned_data['user']
+            r = getrower(u)
+            if not can_add_workout_member(request.user, r):  # pragma: no cover
+                message = 'Please select a rower'
+                messages.error(request, message)
+
+            uploadoptions['user'] = u.id
+
+            response = upload_handler(uploadoptions, file_path)
+            if response["status"] not in ["processing"]:
+                messages.error(request, response["message"])
                 url = reverse('team_workout_upload_view')
+                return HttpResponseRedirect(url)
+            else:
+                messages.info(request, response["message"])
 
-                response = HttpResponseRedirect(url)
+            # redirect to workouts_view
-                w = Workout.objects.get(id=id)
+            url = reverse('team_workout_upload_view')
-
+            return HttpResponseRedirect(url)
-            r = getrower(request.user)
-            if (make_plot):  # pragma: no cover
-                id, jobid = uploads.make_plot(r, w, f1, f2, plottype, t)
-            elif r.staticchartonupload:
-                plottype = r.staticchartonupload
-                id, jobid = uploads.make_plot(r, w, f1, f2, plottype, t)
-
         else:
-            response = render(request,
+            messages.error(request, "error")
-                              'team_document_form.html',
-                              {'form': form,
-                               'teams': get_my_teams(request.user),
-                               'active': 'nav-workouts',
-                               'breadcrumbs': breadcrumbs,
-                               'optionsform': optionsform,
-                               'rowerform': rowerform,
-                               })
 
-        return response
+    response = render(request,
-    else:
+                      'team_document_form.html',
-        form = DocumentsForm()
-        optionsform = TeamUploadOptionsForm(initial=uploadoptions,
-                                            request=request,raceid=raceid)
-        rowerform = TeamInviteForm(userid=userid)
-        rowerform.fields.pop('email')
-
-        rowers = Rower.objects.filter(
-            coachinggroups__in=[r.mycoachgroup]
-        ).distinct()
-
-        rowerform.fields['user'].queryset = User.objects.filter(
-            rower__in=rowers).distinct()
-
-        return render(request, 'team_document_form.html',
                       {'form': form,
-                       #                       'teams':get_my_teams(request.user),
+                       'teams': get_my_teams(request.user),
-                       'optionsform': optionsform,
                        'active': 'nav-workouts',
                        'breadcrumbs': breadcrumbs,
-                       #                       'rower':r,
+                       'optionsform': optionsform,
                        'rowerform': rowerform,
                        })
 
+    return response
+
 
 # A page with all the recent graphs (searchable on workout name)
 @login_required()