flow flush manual upload - still saving as private

rowers/dataflow.py

@@ -4,16 +4,66 @@ import zipfile
 import os
 from rowingdata import get_file_type
 import django_rq
+from shutil import copyfile
+from time import strftime
+import numpy as np
+from scipy.signal import find_peaks, savgol_filter
+import pandas as pd
+import datetime
+import math
+
+os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+from YamJam import yamjam
+CFG = yamjam()['rowsandallapp']
+
+try:
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE",CFG['settings_name'])
+except KeyError: # pragma: no cover
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE","rowsandall_app.settings")
+
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
 
 queue = django_rq.get_queue('default')
 queuelow = django_rq.get_queue('low')
 queuehigh = django_rq.get_queue('default')
 
 from django.conf import settings
+from django.urls import reverse
+from django.utils import timezone as tz
+
 from rowers.forms import DocumentsForm, TeamUploadOptionsForm
-from rowers.models import TeamInviteForm, Workout
+from rowers.models import TeamInviteForm, Workout, User, Rower, Team
+from rowers.opaque import encoder
+from rowers import uploads
+
+from rowingdata import rower as rrower
+
+from rowers.dataroutines import rdata, get_startdate_time_zone, df_resample, checkduplicates, dataplep
+from rowers.mytypes import otetypes, otwtypes
+from rowers.utils import totaltime_sec_to_string
+from rowers.dataprep import check_marker, checkbreakthrough, update_wps 
+from rowers.emails import send_confirm
+from rowers.tasks import handle_sendemail_unrecognized, handle_sendemail_breakthrough, handle_sendemail_hard, handle_calctrimp
+
 from uuid import uuid4
 
+def getrower(user):
+    try:
+        if user is None or user.is_anonymous:
+            return None
+    except AttributeError:  # pragma: no cover
+        if User.objects.get(id=user).is_anonymous:
+            return None
+    try:
+        r = Rower.objects.get(user=user)
+    except Rower.DoesNotExist:  # pragma: no cover:
+        r = Rower(user=user)
+        r.save()
+
+    return r
+
+
 def generate_job_id():
     return str(uuid4())
 
@@ -75,7 +125,7 @@ def is_invalid_file(file_path):
     return True, ""
         
 
-def upload_handler(filename, uploadoptions):
+def upload_handler(uploadoptions, filename):
     if not valid_uploadoptions(uploadoptions):
         return {
             "status": "error",
@@ -84,14 +134,15 @@ def upload_handler(filename, uploadoptions):
         }
     is_valid, message = is_invalid_file(filename)
     if not is_valid:
+        os.remove(filename)
         return {
             "status": "error",
             "job_id": None,
             "message": message
         }
-    if is_zipfile(file):
+    if is_zipfile(filename):
         parent_job_id = generate_job_id()
-        _ = myqueue.enqueue(
+        _ = myqueue(
             queuehigh,
             unzip_and_process,
             filename,
@@ -103,7 +154,7 @@ def upload_handler(filename, uploadoptions):
             "message": "Your zip file is being processed. You will be notified when it is complete."
         }
     job_id = generate_job_id()
-    _ = myqueue.enqueue(
+    _ = myqueue(
         queuehigh,
         process_single_file,
         filename,
@@ -116,7 +167,7 @@ def upload_handler(filename, uploadoptions):
     }
 
 @app.task
-def unzip_and_process(zip_filepath, uploadoptions, parent_job_id):
+def unzip_and_process(zip_filepath, uploadoptions, parent_job_id, debug=False, **kwargs):
     with zipfile.ZipFile(zip_filepath, 'r') as zip_ref:
         for id, filename in enumerate(zip_ref.namelist()):
             datafile = zip_ref.extract(filename, path='media/')
@@ -124,7 +175,7 @@ def unzip_and_process(zip_filepath, uploadoptions, parent_job_id):
                 uploadoptions['title'] = uploadoptions['title']  + " Part {id+1}".format(id=id)
             uploadoptions['file'] = datafile
             job_id = generate_job_id()
-            _ = myqueue.enqueue(
+            _ = myqueue(
                 queuehigh,
                 process_single_file,
                 datafile,
@@ -137,13 +188,452 @@ def unzip_and_process(zip_filepath, uploadoptions, parent_job_id):
         "message": "All files from the zip have been processed."
     }
 
+def get_rower_from_uploadoptions(uploadoptions):
+    rowerform = TeamInviteForm(uploadoptions)
+    if not rowerform.is_valid():
+        return None
+    try:
+        u = rowerform.cleaned_data['user']
+        r = getrower(u)
+    except KeyError:
+        if 'useremail' in uploadoptions:
+            us = User.objects.filter(email=uploadoptions['useremail'])
+            if len(us):
+                u = us[0]
+                r = getrower(u)
+            else:
+                r = None
+                for rwr in Rower.objects.all():
+                    if rwr.emailalternatives is not None:
+                        if uploadoptions['useremail'] in rwr.emailalternatives:
+                            r = rwr
+                            break
+    return r 
 
+def check_and_fix_samplerate(row, file_path):
+    # implement sample rate check and fix here
+    dtavg = row.df['TimeStamp (sec)'].diff().mean()
+    if dtavg < 1:
+        newdf = df_resample(row.df)
+        try:
+            os.remove(file_path)
+        except Exception:
+            pass
+        row = rrdata(df=newdf)
+        row.write_csv(file_path, gzip=True)
+    return row, file_path
+
+def is_water_rowing(df):
+    lat = df[' latitude']
+    if lat.mean() != 0 and lat.std() != 0:
+        return True
+
+def remove_negative_power_peaks(row):
+    x = row.df[' Power (watts)'].values
+    x = x * - 1    
+    neg_peaks, _ = find_peaks(x, height=0) # hieght is the threshold value
+
+    row.df[' Power (watts)'][neg_peaks] = row.df[' Power (watts)'][neg_peaks-1]
+    x = row.df[' Power (watts)'].values
+    x = x * - 1    
+    neg_peaks, _ = find_peaks(x, height=0) # hieght is the threshold value
+
+    row.df[' Power (watts)'][neg_peaks] = row.df[' Power (watts)'][neg_peaks-1]
+
+    return row
+
+def do_smooth(row, f2):
+    # implement smoothing here if needed
+    pace = row.df[' Stroke500mPace (sec/500m)'].values
+    velo = 500. / pace
+    
+    f = row.df['TimeStamp (sec)'].diff().mean()
+    if f != 0 and not np.isnan(f):
+        windowsize = 2 * (int(10. / (f))) + 1
+    else:  # pragma: no cover
+        windowsize = 1
+    if 'originalvelo' not in row.df:
+        row.df['originalvelo'] = velo
+
+    if windowsize > 3 and windowsize < len(velo):
+        velo2 = savgol_filter(velo, windowsize, 3)
+    else:  # pragma: no cover
+        velo2 = velo
+
+    velo3 = pd.Series(velo2, dtype='float')
+    velo3 = velo3.replace([-np.inf, np.inf], np.nan)
+    velo3 = velo3.fillna(method='ffill')
+
+    pace2 = 500. / abs(velo3)
+
+    row.df[' Stroke500mPace (sec/500m)'] = pace2
+
+    row.df = row.df.fillna(0)
+
+    row.write_csv(f2, gzip=True)
+    try:
+        os.remove(f2)
+    except:
+        pass
+    
+    return row
+
+def update_workout_attributes(w, row, file_path, uploadoptions, 
+                              startdatetime='',
+                              timezone='', forceunit='lbs'):
+    
+    # calculate
+    startdatetime, startdate, starttime, timezone_str, partofday = get_startdate_time_zone(
+        w.user, row, startdatetime=startdatetime, timezone=timezone
+    )
+
+    boattype = uploadoptions.get('boattype', '1x')
+    workoutsource = 'unknown'
+    stravaid = uploadoptions.get('stravaid', 0)
+    rpe = uploadoptions.get('rpe', 0)
+    notes = uploadoptions.get('notes', '')
+    inboard = uploadoptions.get('inboard', 0.88)
+    oarlength = uploadoptions.get('oarlength', 2.89)
+    useImpeller = uploadoptions.get('useImpeller', False)
+    seatnumber = uploadoptions.get('seatNumber', 1)
+    boatname = uploadoptions.get('boatName','')
+    portStarboard = uploadoptions.get('portStarboard', 1)
+    empowerside = 'port'
+
+    if portStarboard == 1:
+        empowerside = 'starboard'
+    
+    stravaid = uploadoptions.get('stravaid','')
+    if stravaid != 0:
+        workoutsource = 'strava'
+
+    workouttype = uploadoptions.get('workouttype', 'rower')
+    title = uploadoptions.get('title', '')
+    if title is None or title == '':
+        title = 'Workout'
+        if partofday is not None:
+            title = '{partofday} {workouttype}'.format(
+                partofday=partofday,
+                workouttype=workouttype,
+            )
+    averagehr = row.df[' HRCur (bpm)'].mean()
+    maxhr = row.df[' HRCur (bpm)'].max()
+
+    totaldist = uploadoptions.get('distance', 0)
+    if totaldist == 0:
+        totaldist = row.df['cum_dist'].max()
+
+    totaltime = uploadoptions.get('duration', 0)
+    if totaltime == 0:
+        totaltime = row.df['TimeStamp (sec)'].max() - row.df['TimeStamp (sec)'].min()
+        try:
+            totaltime = totaltime + row.df.loc[:, ' ElapsedTime (sec)'].iloc[0]
+        except KeyError:
+            pass
+
+    if np.isnan(totaltime):
+        totaltime = 0
+
+    if uploadoptions.get('summary', '') == '':
+        summary = row.allstats()
+
+    if uploadoptions.get('makeprivate', False):
+        privacy = 'hidden'
+    elif workoutsource != 'strava':
+        privacy = 'visible'
+    else:
+        privacy = 'hidden'
+
+    # checking for in values
+    totaldist = np.nan_to_num(totaldist)
+    maxhr = np.nan_to_num(maxhr)
+    averagehr = np.nan_to_num(averagehr)
+
+    dragfactor = 0
+    if workouttype in otetypes:
+        dragfactor = row.dragfactor
+
+    delta = datetime.timedelta(seconds=totaltime)
+
+    try:
+        workoutenddatetime = startdatetime+delta
+    except AttributeError as e:
+        workoutstartdatetime = pendulum.parse(str(startdatetime))
+        workoutenddatetime = startdatetime+delta
+        
+
+    # check for duplicate start times and duration
+    duplicate = checkduplicates(
+        w.user, startdate, startdatetime, workoutenddatetime)
+    if duplicate:
+        rankingpiece = False
+
+    # test title length
+    if title is not None and len(title) > 140:  # pragma: no cover
+        title = title[0:140]
+
+    timezone_str = str(startdatetime.tzinfo)
+    
+        
+    duration = totaltime_sec_to_string(totaltime)
+    
+    # implement workout attribute updates here
+    w.name = title
+    w.date = startdate
+    w.workouttype = workouttype
+    w.boattype = boattype
+    w.dragfactor = dragfactor
+    w.duration = duration
+    w.distance = totaldist
+    w.weightcategory = w.user.weightcategory
+    w.adaptiveclass = w.user.adaptiveclass
+    w.starttime = starttime
+    w.duplicate = duplicate
+    w.workoutsource = workoutsource
+    w.rankingpiece = False
+    w.forceunit = forceunit
+    w.rpe = rpe
+    w.csvfilename = file_path
+    w.notes = notes
+    w.summary = summary
+    w.maxhr = maxhr
+    w.averagehr = averagehr
+    w.startdatetime = startdatetime
+    w.inboard = inboard
+    w.oarlength = oarlength
+    w.seatnumber = seatnumber
+    w.boatname = boatname
+    w.empowerside = empowerside
+    w.timezone = timezone_str
+    w.privacy = privacy
+    w.impeller = useImpeller
+    w.save()
+    
+    return w
+
+def send_upload_confirmation_email(rower, workout):
+    # implement email sending here
+    if rower.getemailnotifications and not rower.emailbounced:  # pragma: no cover
+        link = settings.SITE_URL+reverse(
+            rower.defaultlandingpage,
+            kwargs={
+                'id': encoder.encode_hex(workout.id),
+            }
+        )
+        _ = send_confirm(rower.user, workout.name, link, '')
+
+
+def update_running_wps(r, w, row):
+    # implement wps update here
+    if not w.duplicate and w.workouttype in otetypes:
+        cntr = Workout.objects.filter(user=r, workouttype__in=otetypes,
+                                      startdatetime__gt=tz.now()-tz.timedelta(days=42),
+                                      duplicate=False).count()
+        new_value = (cntr*r.running_wps_erg + row.df['driveenergy'].mean())/(cntr+1.0)
+        # if new_value is not zero or infinite or -inf, r.running_wps can be set to value
+        if not (math.isnan(new_value) or math.isinf(new_value) or new_value == 0):
+            r.running_wps_erg = new_value
+        elif not (math.isnan(r.running_wps_erg) or math.isinf(r.running_wps_erg) or r.running_wps_erg == 0):
+            pass
+        else:
+            r.running_wps_erg = 600.
+        r.save()
+
+    if not w.duplicate and w.workouttype in otwtypes:
+        cntr = Workout.objects.filter(user=r, workouttype__in=otwtypes,
+                                      startdatetime__gt=tz.now()-tz.timedelta(days=42),
+                                      duplicate=False).count()
+        try:
+            new_value = (cntr*r.running_wps_erg + row.df['driveenergy'].mean())/(cntr+1.0)
+        except TypeError:
+            new_value = r.running_wps
+        if not (math.isnan(new_value) or math.isinf(new_value) or new_value == 0):
+            r.running_wps = new_value
+        elif not (math.isnan(r.running_wps) or math.isinf(r.running_wps) or r.running_wps == 0):
+            pass
+        else:
+            r.running_wps = 400.
+        r.save()
+        
 @app.task
-def process_single_file(file_path, uploadoptions, job_id):
+def process_single_file(file_path, uploadoptions, job_id, debug=False, **kwargs):
-    # placeholder
+    # copy file to a unique name in media folder
+    f2 = file_path
+    try:
+        nn, ext = os.path.splitext(f2)
+        if ext == '.gz':
+            nn, ext2 = os.path.splitext(nn)
+            ext = ext2 + ext
+        f1  = uuid4().hex[:10]+'-'+strftime('%Y%m%d-%H%M%S')+ext
+        f2 = 'media/'+f1
+        copyfile(file_path, f2)
+    except FileNotFoundError:
+        return {
+            "status": "error",
+            "job_id": job_id,
+            "message": "File not found during processing."
+        }
+
+    # determine the user
+    r = get_rower_from_uploadoptions(uploadoptions)
+    if r is None:
+        os.remove(f2)
+        return {
+            "status": "error",
+            "job_id": job_id,
+            "message": "Rower not found for the provided upload options."
+        }
+
+    try:
+        fileformat = get_file_type(f2)
+    except Exception as e:
+        os.remove(f2)
+        return {
+            "status": "error",
+            "job_id": job_id,
+            "message": "Error determining file format: {error}".format(error=str(e))
+        }
+
+    # Get fileformat from fit & tcx
+    if fileformat == 'fit':
+        workouttype = get_workouttype_from_fit(f2)
+        uploadoptions['workouttype'] = workouttype
+        new_title = get_title_from_fit(f2)
+        if new_title:
+            uploadoptions['title'] = new_title
+        new_notes = get_notes_from_fit(f2)
+        if new_notes:
+            uploadoptions['notes'] = new_notes
+
+    # handle non-Painsled
+    if fileformat != 'csv':
+        f2, summary, oarlength, inboard, fileformat, impeller = handle_nonpainsled(
+            f2,
+            fileformat,
+        )
+        if not f2:
+            return {
+                "status": "error",
+                "job_id": job_id,
+                "message": "Error processing non-Painsled file."
+            }
+
+    # create raw row data object
+    powerperc = 100 * np.array([r.pw_ut2,
+                                r.pw_ut1,
+                                r.pw_at,
+                                r.pw_tr, r.pw_an]) / r.ftp
+
+    rr = rrower(hrmax=r.max, hrut2=r.ut2,
+                hrut1=r.ut1, hrat=r.at,
+                hrtr=r.tr, hran=r.an, ftp=r.ftp,
+                powerperc=powerperc, powerzones=r.powerzones)
+    row = rdata(f2, rower=rr)
+
+    if row.df.empty:
+        os.remove(f2)
+        return {
+            "status": "error",
+            "job_id": job_id,
+            "message": "No valid data found in the uploaded file."
+        }
+
+    if row == 0:
+        os.remove(f2)
+        return {
+            "status": "error",
+            "job_id": job_id,
+            "message": "Error creating row data from the file."
+        }
+
+    # check and fix sample rate
+    row, f2 = check_and_fix_samplerate(row, f2)
+
+    # change rower type to water if GPS data is present
+    if is_water_rowing(row.df):
+        uploadoptions['workouttype'] = 'water'
+
+    # remove negative power peaks
+    row = remove_negative_power_peaks(row)
+
+    # optional auto smoothing
+    row = do_smooth(row, f2)
+
+    # recalculate power data
+    if uploadoptions['workouttype'] in otetypes:
+        try:
+            if r.erg_recalculatepower:
+                row.erg_recalculatepower()
+                row.write_csv(f2, gzip=True)
+        except Exception as e:
+            pass
+
+    workoutid = uploadoptions.get('workoutid', None)
+    if workoutid is not None:
+        try:
+            w = Workout.objects.get(id=workoutid)
+        except Workout.DoesNotExist:
+            w = Workout(user=r, duration='00:00:00')
+            w.save()
+    else:
+        w = Workout(user=r, duration='00:00:00')
+        w.save()
+
+    # set workout attributes from uploadoptions and calculated values
+    w = update_workout_attributes(w, row, f2, uploadoptions)
+
+
+    # add teams
+    if w.privacy == 'visible':
+        ts = Team.objects.filter(rower=r
+                                 )
+        for t in ts:
+            w.team.add(t)
+
+    # put stroke data in file store through "dataplep"
+    try:
+        row = rrdata_pl(df=pl.form_pandas(row.df))
+    except:
+        pass
+
+    _ = dataplep(row.df, id=w.id, bands=True,
+                 barchart=True, otwpower=True, empower=True, inboard=w.inboard)
+
+    # send confirmation email
+    send_upload_confirmation_email(r, w)
+
+    # check for breakthroughs
+    isbreakthrough, ishard = checkbreakthrough(w, r)
+    _ = check_marker(w)
+    _ = update_wps(r, otwtypes)
+    _ = update_wps(r, otetypes)
+
+    # update running_wps
+    update_running_wps(r, w, row)
+
+    # calculate TRIMP
+    if w.workouttype in otwtypes:
+        wps_avg = r.median_wps
+    elif w.workouttype in otetypes:
+        wps_avg = r.median_wps_erg
+    else:
+        wps_avg = 0
+
+    _ = myqueue(queuehigh, handle_calctrimp, w.id, f2,
+                r.ftp, r.sex, r.hrftp, r.max, r.rest, wps_avg)
+        
+    # make plots
+    if uploadoptions['make_plot']:
+        res, jobid = uploads.make_plot(r, w, f1, f2, plottype, w.name)
+    elif r.staticchartonupload != 'None':  # pragma: no cover
+        plottype = r.staticchartonupload
+        res, jobid = uploads.make_plot(r, w, f1, f2, plottype, w.name)
+
+    # sync workouts to connected services
+    uploads.do_sync(w, uploadoptions, quick=True)
+
+    
     return True
 
-    # process data to create df
 
-    # 
 

rowers/forms.py

@@ -425,8 +425,6 @@ class DocumentsForm(forms.Form):
     notes = forms.CharField(required=False,
                             widget=forms.Textarea)
 
-    offline = forms.BooleanField(initial=False, required=False,
-                                 label='Process in Background')
 
     class Meta:
         fields = ['title', 'file', 'workouttype',
@@ -580,9 +578,6 @@ class UploadOptionsForm(forms.Form):
         label='Submit as challenge Result',
         required=False)
 
-    landingpage = forms.ChoiceField(choices=nextpages,
-                                    initial='workout_edit_view',
-                                    label='After Upload, go to')
 
     raceid = forms.IntegerField(initial=0, widget=HiddenInput())
 

rowers/templates/file_upload.html

@@ -0,0 +1,343 @@
+{% extends "newbase.html" %}
+{% load static %}
+{% load rowerfilters %}
+
+{% block title %}File loading{% endblock %}
+
+{% block meta %}
+<script type='text/javascript'
+        src='https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js'>
+</script>
+<script type='text/javascript'
+        src='https://ajax.aspnetcdn.com/ajax/jquery.validate/1.14.0/jquery.validate.min.js'>
+</script>
+{% endblock %}
+
+{% block main %}
+<div id="id_main">
+<ul class="main-content">
+  <li class="grid_2">
+    <div id="id_dropregion" class="watermark invisible">
+      <p>Drag and drop files here </p>
+    </div>
+    <div id="id_drop-files" class="drop-files">
+      <form id="file_form" enctype="multipart/form-data" action="{{ formloc }}" method="post">
+        <h1>Upload Workout File</h1>
+        {% if user.is_authenticated and user|coach_rowers %}
+        <p>Looking for <a href="/rowers/workout/upload/team/">Team Manager
+            Upload?</a></p>
+        {% endif %}
+        {% if form.errors %}
+        <p style="color: red;">
+          Please correct the error{{ form.errors|pluralize }} below.
+        </p>
+        {% endif %}
+
+        <table>
+          {{ form.as_table }}
+        </table>
+        {% csrf_token %}
+        <p>
+          &nbsp;<input type="submit" value="Submit">
+        </p>
+    </div>
+
+  </li>
+  <li class="grid_2">
+    <h1>Optional extra actions</h1>
+    <p>
+      <table>
+	{{ optionsform.as_table }}
+
+      </table>
+    </p>
+    <p>
+      You can select one static plot to be generated immediately for
+      this workout. You can select to export to major fitness
+      platforms automatically.
+      If you check "make private", this workout will not be visible to your followers and will not show up in your teams' workouts list. With the Landing Page option, you can select to which (workout related) page you will be
+      taken after a successfull upload.
+    </p>
+
+    <p>
+      If you don't have a workout file but have written down the splits,
+      you can create a workout file yourself from <a href="/static/dummy_workout_template.xls">this template</a>
+    </p>
+
+
+    <p><b>Select Files with the File button or drag them on the marked area</b></p>
+
+  </li>
+
+
+
+  </form>
+</ul>
+</div>
+{% endblock %}
+
+    {% block scripts %}
+
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script>
+<script>
+  $( document ).ready(function() {
+      var boattypes = {
+	  '1x': '1x (single)',
+	  '2x': '2x (double)',
+	  '2x+': '2x+ (coxed double)',
+	  '2-': '2- (pair)',
+	  '2+': '2+ (coxed pair)',
+	  '3x+': '3x+ (coxed triple)',
+	  '3x-': '3x- (triple)',
+	  '4x': '4x (quad)',
+	  '4x+': '4x+ (coxed quad)',
+	  '4-': '4- (four)',
+	  '4+': '4+ (coxed four)',
+	  '8+': '8+ (eight)',
+	  '8x+': '8x+ (octuple scull)',
+      }
+      
+      var ergtypes = {
+	  'static': 'Concept2 static',
+	  'dynamic': 'Concept2 dynamic',
+	  'slides': 'Concept2 slides',
+	  'rp3': 'RP3',
+	  'waterrower': 'Water Rower',
+	  'other': 'Other Indoor Rower',	  
+      }
+
+      $('#id_workouttype').on('change', function(){
+          if (
+              $(this).val() == 'water'
+		  || $(this).val() == 'coastal'
+		  || $(this).val() == 'c-boat'
+		  || $(this).val() == 'churchboat'
+          ) {
+              var $el = $('#id_boattype');
+	      $el.empty();
+	      $.each(boattypes, function(key,value) {
+		  if ( key == '{{ workout.boattype }}') {
+		      $el.append($("<option></option").attr("value", key).attr("selected", "selected").text(value));
+		  } else {
+		      $el.append($("<option></option").attr("value", key).text(value));
+		  }
+	      });
+	      $el.toggle(true);
+	      
+          }
+	  else if (
+	      $(this).val() == 'rower'
+	  ) {
+              var $el = $('#id_boattype');
+	      $el.empty();
+	      $.each(ergtypes, function(key,value) {
+		  if ( key == '{{ workout.boattype }}') {
+		      $el.append($("<option></option").attr("value", key).attr("selected", "selected").text(value));
+		  } else {
+		      $el.append($("<option></option").attr("value", key).text(value));
+		  }
+	      });
+	      $el.toggle(true);	    
+	  }
+	  else {
+              $('#id_boattype').toggle(false);
+              $('#id_boattype').val('1x');
+	  }
+	  if (
+              $(this).val() == 'rower'
+		  || $(this).val() == 'dynamic'
+		  || $(this).val() == 'slides'
+	  ) {
+              $('#id_dragfactor').toggle(true);
+	  } else {
+              $('#id_dragfactor').toggle(false);
+              $('#id_dragfactor').val('0');
+	  }
+	  
+      });
+      $('#id_workouttype').change();
+  });
+  </script>
+
+<script>
+  var td = new FormData();
+  var formdatasetok = false;
+  try {
+      td.set('aap','noot');
+      formdatasetok = true;
+      console.log('FormData.set OK');
+  }
+  catch(err) {
+      console.log('FormData.set not OK');
+      formdatasetok = false;
+  }
+  
+  if (!formdatasetok) {
+      $("#id_dropregion").remove();
+  }
+  
+  if (formdatasetok) {
+      
+      $(document).ready(function() {
+	  var csrftoken = jQuery("[name=csrfmiddlewaretoken]").val();
+	  console.log("CSRF token",csrftoken);
+	  
+	  function csrfSafeMethod(method) {
+	      // these HTTP methods do not require CSRF protection
+	      return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
+	  }
+	  $.ajaxSetup({
+	      beforeSend: function(xhr, settings) {
+		  if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
+		      xhr.setRequestHeader("X-CSRFToken", csrftoken);
+		  }
+	      }
+	  });
+	  console.log("Loading dropper");
+	  jQuery.event.props.push('dataTransfer');
+	  
+	  $(window).on('dragenter', function() {
+	      $("#id_drop-files").css("background-color","#E9E9E4");
+	      $("#id_dropregion").addClass("watermark").removeClass("invisible");})
+	  
+	  $(window).on('dragleave', function() {
+	      $("#id_drop-files").css("background-color","#FFFFFF");
+	      $("#id_dropregion").removeClass("watermark").addClass("invisible");})
+	  
+	  var frm = $("#file_form");
+	  
+	  if( window.FormData === undefined ) {
+	      console.log('no formdata');
+	      alert("No FormData");
+	  } else {
+	      console.log('we have formdata');
+	  }
+	  
+	  var data = new FormData(frm[0]);
+	  
+	  $('#id_file').on('change', function(evt) {
+	      var f = this.files[0];
+	      console.log(f);
+	      var istcx = false;
+	      var isgzip = false;
+	      var size1 = 10485760;
+	      var size2 = 2097152;
+	      if ((/\.(tcx|TCX)/i).test(f.name)) {
+		  istcx = true;
+		  console.log('tcx');
+		  if ((/\.(gz|GZ)/i).test(f.name)) {
+		      isgzip = true;
+		      console.log('gzip');
+		      size1 /= 5;
+		      size2 /= 5;
+		  }
+	      }
+	      console.log(size1)
+	      console.log(size2)
+	      if (f.size > size1) {
+		  alert("File Size must be smaller than 10 MB");
+		  this.value = null;
+	      } else {
+		  
+		  if (f.size > size2) {
+		      $('#id_offline').val('True');
+		      $('#id_offline').prop('checked','True');
+		      data.set($('#id_offline').attr('name'),$('#id_offline').prop('checked'));
+		      console.log("Set offline to True");
+		      $('#extra_message').text('Because of the large size, we recommend to use  background processing. You will receive email when it is done.');
+		      $('#extra_message').addClass('message');
+		  }
+	      }
+	      
+	  });
+	  
+	  $('input').each(function( i ) {
+	      $(this).change(function() {
+		  if ($(this).attr('type') == 'checkbox') {
+		      data.set($(this).attr('name'),$(this).prop('checked'));
+		      console.log($(this).attr('id'),$(this).attr('name'),$(this).prop('checked'));
+		  } else {
+		      data.set($(this).attr('name'),$(this).val());
+		      if ($(this).attr('id') == 'id_file') {
+			  data.set("file",this.files[0]);
+		      }
+		      console.log($(this).attr('id'),$(this).attr('name'),$(this).val());
+		  };
+	      });});
+	  
+	  $('textarea').each(function( i ) {
+	      $(this).change(function() {
+		  data.set($(this).attr('name'),$(this).val());
+		  console.log($(this).attr('id'),$(this).attr('name'),$(this).val());
+	      });});
+	  
+	  $('select').each(function( i ) {
+	      console.log($(this).attr('name'),$(this).val());
+	      $(this).change(function() {
+		  data.set($(this).attr('name'),$(this).val());
+		  console.log($(this).attr('id'),$(this).attr('name'),$(this).val());
+	      });
+	  });
+	  
+	  $('#id_drop-files').bind({
+	      drop: function(e) {
+		  e.preventDefault();
+		  console.log("you dropped something");
+		  var files = e.dataTransfer.files;
+		  console.log(files[0]);
+		  
+		  var f = files[0];
+		  var istcx = false;
+		  var isgzip = false;
+		  var size1 = 10485760;
+		  var size2 = 1048576;
+		  if ((/\.(tcx|TCX)/i).test(f.name)) {
+		      istcx = true;
+		      console.log('tcx');
+		      if ((/\.(gz|GZ)/i).test(f.name)) {
+			  isgzip = true;
+			  console.log('gzip');
+			  size1 /= 5;
+			  size2 /= 5;
+		      }
+		  }
+		  console.log(f);
+		  console.log(size1)
+		  console.log(size2)
+		  if (f.size > size1) {
+		      alert("File Size must be smaller than 10 MB");
+		      $("#id_file").value = 0;
+		      return false;
+		  }
+		  if (f.size > size2) {
+    $('#id_offline').val('True');
+		      $('#id_offline').prop('checked','True');
+		      data.set($('#id_offline').attr('name'),$('#id_offline').prop('checked'));
+		      console.log("Set offline to True");
+		      $('#extra_message').text('Because of the large size, we recommend to use  background processing. You will receive email when it is done.');
+		      $('#extra_message').addClass('message');
+		  }
+		  data.set("file",f);
+		  //    data.append("file",f);
+		  
+		  $("#id_file").replaceWith('<div id="id_file">'+files[0].name+'&nbsp; <a class="remove" href="javascript:void(0);"><b><font color="red">X</font></b></a></div>');
+	      },
+	      mouseenter:function(){$("#id_drop-files").css("background-color","#E9E9E4");},
+	      mouseleave:function(){$("#id_drop-files").css("background-color","#FFFFFF");},
+	      dragover:function(e){
+		  e.preventDefault();
+		  $("#id_drop-files").css("background-color","#E9E9E4");},
+	      dragleave:function(e){ e.preventDefault();},
+	  });
+	  $(document).on("click", "a.remove", function() {
+	      $(this).parent().replaceWith('<td><input id="id_file" name="file" type="file" /></td>');
+	  });
+      });
+  };
+  </script>
+{% endblock %}
+
+
+{% block sidebar %}
+{% include 'menu_workouts.html' %}
+{% endblock %}

rowers/views/workoutviews.py

@@ -5264,16 +5264,10 @@ def workout_upload_view(request,
                             'upload_to_C2': False,
                             'plottype': 'timeplot',
                             'landingpage': 'workout_edit_view',
-                        },
-                        docformoptions={
                             'workouttype': 'rower',
                         },
                         raceid=0):
 
-    is_ajax = request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest'
-    if settings.TESTING:
-        is_ajax = False
-
     r = getrower(request.user)
     if r.imports_are_private:
         uploadoptions['makeprivate'] = True
@@ -5289,420 +5283,44 @@ def workout_upload_view(request,
         }
     ]
 
-    if 'uploadoptions' in request.session:
+    form = DocumentsForm(initial=uploadoptions)
-        uploadoptions = request.session['uploadoptions']
+    optionsform = UploadOptionsForm(initial=uploadoptions,
-        try:
+                                    request=request, raceid=raceid)
-            _ = uploadoptions['landingpage']
+    
-        except KeyError:  # pragma: no cover
-            uploadoptions['landingpage'] = r.defaultlandingpage
-    else:
-        request.session['uploadoptions'] = uploadoptions
 
-    if 'docformoptions' in request.session:
-        docformoptions = request.session['docformoptions']
-    else:
-        request.session['docformoptions'] = docformoptions
-
-    makeprivate = uploadoptions.get('makeprivate', False)
-    make_plot = uploadoptions.get('make_plot', False)
-    workouttype = uploadoptions.get('WorkoutType', 'rower')
-    boattype = docformoptions.get('boattype', '1x')
-
-    try:
-        rpe = docformoptions['rpe']
-        try:  # pragma: no cover
-            rpe = int(rpe)
-        except ValueError:  # pragma: no cover
-            rpe = 0
-        if not rpe:  # pragma: no cover
-            rpe = -1
-    except KeyError:
-        rpe = -1
-
-    notes = docformoptions.get('notes', '')
-    workoutsource = uploadoptions.get('workoutsource', None)
-    plottype = uploadoptions.get('plottype', 'timeplot')
-    landingpage = uploadoptions.get('landingpage', r.defaultlandingpage)
-    upload_to_c2 = uploadoptions.get('upload_to_C2', False)
-    upload_to_strava = uploadoptions.get('upload_to_Strava', False)
-    upload_to_st = uploadoptions.get('upload_to_SportTracks', False)
-    upload_to_tp = uploadoptions.get('upload_to_TrainingPeaks', False)
-    upload_to_intervals = uploadoptions.get('upload_to_Intervals', False)
-
-    response = {}
     if request.method == 'POST':
         form = DocumentsForm(request.POST, request.FILES)
         optionsform = UploadOptionsForm(request.POST, request=request)
 
-        if form.is_valid():
+        if form.is_valid() and optionsform.is_valid():
-            #           f = request.FILES['file']
+            uploadoptions = form.cleaned_data.copy()
-            f = form.cleaned_data['file']
+            uploadoptions.update(optionsform.cleaned_data)
+        
+            uploadoptions['secret'] = settings.UPLOAD_SERVICE_SECRET
+            uploadoptions['user'] = r.user.id
+            if request.FILES['file'] is not None:
+                filename, file_path = handle_uploaded_file(request.FILES['file'])
+            uploadoptions['file'] = file_path
 
-            if f is not None:
+            response = upload_handler(uploadoptions, file_path)
-                res = handle_uploaded_file(f)
+            if response["status"] not in ["processing"]:
-            else:  # pragma: no cover
+                messages.error(request, response["message"])
-                messages.error(request,
-                               "Something went wrong - no file attached")
-                url = reverse('workout_upload_view')
-                if is_ajax:
-                    return JSONResponse({'result': 0, 'url': 0})
-                else:
-                    return HttpResponseRedirect(url)
-
-            t = form.cleaned_data['title']
-            workouttype = form.cleaned_data['workouttype']
-            boattype = form.cleaned_data['boattype']
-            try:
-                rpe = form.cleaned_data['rpe']
-                try:
-                    rpe = int(rpe)
-                except ValueError:
-                    rpe = 0
-            except KeyError:  # pragma: no cover
-                rpe = -1
-
-            request.session['docformoptions'] = {
-                'workouttype': workouttype,
-                'boattype': boattype,
-            }
-
-            notes = form.cleaned_data['notes']
-            offline = form.cleaned_data['offline']
-
-            registrationid = 0
-            if optionsform.is_valid():
-                make_plot = optionsform.cleaned_data['make_plot']
-                plottype = optionsform.cleaned_data['plottype']
-                upload_to_c2 = optionsform.cleaned_data['upload_to_C2']
-                upload_to_strava = optionsform.cleaned_data['upload_to_Strava']
-                upload_to_st = optionsform.cleaned_data['upload_to_SportTracks']
-                upload_to_tp = optionsform.cleaned_data['upload_to_TrainingPeaks']
-                upload_to_intervals = optionsform.cleaned_data['upload_to_Intervals']
-                makeprivate = optionsform.cleaned_data['makeprivate']
-                landingpage = optionsform.cleaned_data['landingpage']
-                raceid = optionsform.cleaned_data['raceid']
-
-                try:
-                    registrationid = optionsform.cleaned_data['submitrace']
-                except KeyError:
-                    registrationid = 0
-
-                uploadoptions = {
-                    'makeprivate': makeprivate,
-                    'make_plot': make_plot,
-                    'plottype': plottype,
-                    'upload_to_C2': upload_to_c2,
-                    'upload_to_Strava': upload_to_strava,
-                    'upload_to_SportTracks': upload_to_st,
-                    'upload_to_TrainingPeaks': upload_to_tp,
-                    'upload_to_Intervals': upload_to_intervals,
-                    'landingpage': landingpage,
-                    'boattype': boattype,
-                    'rpe': rpe,
-                    'workouttype': workouttype,
-                }
-
-            request.session['uploadoptions'] = uploadoptions
-
-            f1 = res[0]  # file name
-            f2 = res[1]  # file name incl media directory
-
-            if not offline:
-                id, message, f2 = dataprep.new_workout_from_file(
-                    r, f2,
-                    workouttype=workouttype,
-                    workoutsource=workoutsource,
-                    boattype=boattype,
-                    rpe=rpe,
-                    makeprivate=makeprivate,
-                    title=t,
-                    notes=notes,
-                )
             else:
-                uploadoptions['secret'] = settings.UPLOAD_SERVICE_SECRET
+                messages.info(request, response["message"])
-                uploadoptions['user'] = r.user.id
-                uploadoptions['title'] = t
-                uploadoptions['file'] = f2
 
-                url = settings.UPLOAD_SERVICE_URL
+            # redirect to workouts_view
-
+            url = reverse('workouts_view')
-                _ = myqueue(queuehigh,
+            return HttpResponseRedirect(url)
-                            handle_request_post,
-                            url,
-                            uploadoptions
-                            )
-
-                messages.info(
-                    request,
-                    "The file was too large to process in real time."
-                    " It will be processed in a background process."
-                    " You will receive an email when it is ready")
-                url = reverse('workout_upload_view')
-                if is_ajax:  # pragma: no cover
-                    return JSONResponse({'result': 1, 'url': url})
-                else:
-                    response = HttpResponseRedirect(url)
-                    return response
-
-            if not id:  # pragma: no cover
-                messages.error(request, message)
-                url = reverse('workout_upload_view')
-                if is_ajax:  # pragma: no cover
-                    return JSONResponse({'result': 0, 'url': url})
-                else:
-                    response = HttpResponseRedirect(url)
-                    return response
-            elif id == -1:  # pragma: no cover
-                message = 'The zip archive will be processed in the background." \
-                " The files in the archive will only be uploaded without the extra actions." \
-                " You will receive email when the workouts are ready.'
-                messages.info(request, message)
-                url = reverse('workout_upload_view')
-                if is_ajax:
-                    return JSONResponse({'result': 1, 'url': url})
-                else:
-                    response = HttpResponseRedirect(url)
-                    return response
-            else:
-                if message:  # pragma: no cover
-                    messages.error(request, message)
-
-                w = Workout.objects.get(id=id)
-
-                url = reverse('workout_edit_view',
-                              kwargs={
-                                  'id': encoder.encode_hex(w.id),
-                              })
-
-                if is_ajax:  # pragma: no cover
-                    response = {'result': 1, 'url': url}
-                else:
-                    response = HttpResponseRedirect(url)
-
-            r = getrower(request.user)
-            if (make_plot):  # pragma: no cover
-                res, jobid = uploads.make_plot(r, w, f1, f2, plottype, t)
-                if res == 0:
-                    messages.error(request, jobid)
-                else:
-                    try:
-                        request.session['async_tasks'] += [
-                            (jobid, 'make_plot')]
-                    except KeyError:
-                        request.session['async_tasks'] = [(jobid, 'make_plot')]
-            elif r.staticchartonupload is not None:
-                plottype = r.staticchartonupload
-                res, jobid = uploads.make_plot(r, w, f1, f2, plottype, t)
-
-            # upload to C2
-            if (upload_to_c2):  # pragma: no cover
-                try:
-                    c2integration = C2Integration(request.user)
-                    id = c2integration.workout_export(w)
-                except NoTokenError:
-                    id = 0
-                    message = "Something went wrong with the Concept2 sync"
-                    messages.error(request, message)
-
-            if (upload_to_strava):  # pragma: no cover
-                strava_integration = StravaIntegration(request.user)
-                try:
-                    id = strava_integration.workout_export(w)
-                except NoTokenError:
-                    id = 0
-                    message = "Please connect to Strava first"
-                    messages.error(request, message)
-
-            if (upload_to_st):  # pragma: no cover
-                st_integration = SportTracksIntegration(request.user)
-                try:
-                    id = st_integration.workout_export(w)
-                except NoTokenError:
-                    message = "Please connect to SportTracks first"
-                    id = 0
-                    messages.error(request, message)
-
-            if (upload_to_tp):  # pragma: no cover
-                tp_integration = TPIntegration(request.user)
-                try:
-                    id = tp_integration.workout_export(w)
-                except NoTokenError:
-                    message = "Please connect to TrainingPeaks first"
-                    messages.error(request, message)
-
-            if (upload_to_intervals):
-                intervals_integration = IntervalsIntegration(request.user)
-                try:
-                    id = intervals_integration.workout_export(w)
-                except NoTokenError:
-                    message = "Please connect to Intervals.icu first"
-                    messages.error(request, message)
-
-            if int(registrationid) < 0:  # pragma: no cover
-                race = VirtualRace.objects.get(id=-int(registrationid))
-                if race.sessiontype == 'race':
-                    result, comments, errors, jobid = add_workout_race(
-                        [w], race, r, doregister=True,
-                    )
-                    if result:
-                        messages.info(
-                            request,
-                            "We have submitted your workout to the race")
-
-                    for c in comments:
-                        messages.info(request, c)
-                    for er in errors:
-                        messages.error(request, er)
-                elif race.sessiontype == 'indoorrace':
-                    result, comments, errors, jobid = add_workout_indoorrace(
-                        [w], race, r, doregister=True,
-                    )
-
-                    if result:
-                        messages.info(
-                            request,
-                            "We have submitted your workout to the race")
-
-                    for c in comments:
-                        messages.info(request, c)
-                    for er in errors:
-                        messages.error(request, er)
-                elif race.sessiontype in ['fastest_time', 'fastest_distance']:
-                    result, comments, errors, jobid = add_workout_fastestrace(
-                        [w], race, r, doregister=True,
-                    )
-                    if result:
-                        messages.info(
-                            request, "We have submitted your workout to the race")
-                    for c in comments:
-                        messages.info(request, c)
-                    for er in errors:
-                        messages.error(request, er)
-
-            if int(registrationid) > 0:  # pragma: no cover
-                races = VirtualRace.objects.filter(
-                    registration_closure__gt=timezone.now()
-                )
-                if raceid != 0:
-                    races = VirtualRace.objects.filter(
-                        registration_closure__gt=timezone.now(),
-                        id=raceid,
-                    )
-
-                registrations = IndoorVirtualRaceResult.objects.filter(
-                    race__in=races,
-                    id=registrationid,
-                    userid=r.id,
-                )
-                registrations2 = VirtualRaceResult.objects.filter(
-                    race__in=races,
-                    id=registrationid,
-                    userid=r.id,
-                )
-
-                if int(registrationid) in [r.id for r in registrations]:  # pragma: no cover
-                    # indoor race
-                    registrations = registrations.filter(id=registrationid)
-                    if registrations:
-                        race = registrations[0].race
-                        if race.sessiontype == 'indoorrace':
-                            result, comments, errors, jobid = add_workout_indoorrace(
-                                [w], race, r, recordid=registrations[0].id
-                            )
-                        elif race.sessiontype in ['fastest_time', 'fastest_distance']:
-                            result, comments, errors, jobid = add_workout_fastestrace(
-                                [w], race, r, recordid=registrations[0].id
-                            )
-
-                        if result:
-                            messages.info(
-                                request,
-                                "We have submitted your workout to the race")
-
-                        for c in comments:
-                            messages.info(request, c)
-                        for er in errors:
-                            messages.error(request, er)
-
-                if int(registrationid) in [r.id for r in registrations2]:  # pragma: no cover
-                    # race
-                    registrations = registrations2.filter(id=registrationid)
-                    if registrations:
-                        race = registrations[0].race
-                        if race.sessiontype == 'race':
-                            result, comments, errors, jobid = add_workout_race(
-                                [w], race, r, recordid=registrations[0].id
-                            )
-                        elif race.sessiontype in ['fastest_time', 'fastest_distance']:
-                            result, comments, errors, jobid = add_workout_fastestrace(
-                                [w], race, r, recordid=registrations[0].id
-                            )
-                        if result:
-                            messages.info(
-                                request,
-                                "We have submitted your workout to the race")
-
-                        for c in comments:
-                            messages.info(request, c)
-                        for er in errors:
-                            messages.error(request, er)
-
-            if registrationid != 0:  # pragma: no cover
-                try:
-                    url = reverse('virtualevent_view',
-                                  kwargs={
-                                      'id': race.id,
-                                  })
-                except UnboundLocalError:
-                    if landingpage != 'workout_upload_view':
-                        url = reverse(landingpage,
-                                      kwargs={
-                                          'id': encoder.encode_hex(w.id),
-                                      })
-                    else:  # pragma: no cover
-                        url = reverse(landingpage)
-            elif landingpage != 'workout_upload_view':  # pragma: no cover
-                url = reverse(landingpage,
-                              kwargs={
-                                  'id': encoder.encode_hex(w.id),
-                              })
-
-            else:  # pragma: no cover
-                url = reverse(landingpage)
-
-            if is_ajax:  # pragma: no cover
-                response = {'result': 1, 'url': url}
-            else:
-                response = HttpResponseRedirect(url)
         else:
-            if not is_ajax:  # pragma: no cover
+            messages.error(request, "error")
-                response = render(request,
-                                  'document_form.html',
-                                  {'form': form,
-                                   'teams': get_my_teams(request.user),
-                                   'optionsform': optionsform,
-                                   })
-
-        if is_ajax:  # pragma: no cover
-            return JSONResponse(response)
-        else:
-            return response
-    else:
-        if not is_ajax:
-
-            form = DocumentsForm(initial=docformoptions)
-            optionsform = UploadOptionsForm(initial=uploadoptions,
-                                            request=request, raceid=raceid)
-            return render(request, 'document_form.html',
-                          {'form': form,
-                           'active': 'nav-workouts',
-                           'breadcrumbs': breadcrumbs,
-                           'teams': get_my_teams(request.user),
-                           'optionsform': optionsform,
-                           })
-        else:  # pragma: no cover
-            return {'result': 0}
 
+    return render(request, 'file_upload.html',
+                  {'form': form,
+                   'active': 'nav-workouts',
+                   'breadcrumbs': breadcrumbs,
+                   'teams': get_my_teams(request.user),
+                   'optionsform': optionsform,
+                   })
 
 # This is the main view for processing uploaded files
 @user_passes_test(ispromember, login_url="/rowers/paidplans", redirect_field_name=None,