added test to travel all links

rowers/dataprep.py

@@ -309,6 +309,8 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
     # clean data remove zeros and negative values
 
     # bring metrics which have negative values to positive domain
+    if datadf.empty:
+        return datadf
     try:
         datadf['catch'] = -datadf['catch']
     except KeyError:
@@ -1670,7 +1672,7 @@ def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True):
 
         try:
             f = row.df['TimeStamp (sec)'].diff().mean()
-        except AttributeError:
+        except (AttributeError,KeyError) as e:
             f = 0
             
         if f != 0 and not np.isnan(f):

rowers/templates/about_us.html

@@ -75,7 +75,7 @@
 <p>Rowsandall.com tries to be compatible with the most important tools
   that rowers use to capture the data (both indoor and OTW). For a full
   list of currently supported devides/apps, click
-  <a href="compatibility">here</a>.
+  <a href="/rowers/compatibility">here</a>.
 
 
 </div>

rowers/templates/compatibility.html

@@ -7,14 +7,15 @@
 <h2>Import Compatibility</h2>
 
 <p>Rowsandall.com tries to be compatible with the most important tools
-  that rowers use to capture the data (both indoor and OTW). For a full
+  that rowers use to capture the data (both indoor and OTW). 
-  list of currently supported devides/apps, click
-  <a href="compatibility.html">here</a>.
 
 <p>On The Water
 <ul>
 <li> CrewNerd (TCX)</li>
 <li> Rowing In Motion (TCX)</li>
+<li> BoatCoach </li>
+<li> RowingCoach </li>
+<li> Quiske RowP</li>
 <li> Speedcoach XL (CSV)</li>
 <li> Speedcoach GPS (FIT and CSV)</li></ul></p>
 

rowers/templates/workout_form.html

@@ -235,7 +235,7 @@
       </p>
       <div class="grid_2 alpha">
         <p>
-          <a class="button green small" href="recalcsummary">Update Summary</a>
+          <a class="button green small" href="/rowers/workout/{{ workout.id }}/recalcsummary">Update Summary</a>
         </p>
       </div>
     </div>

rowers/tests.py

@@ -1,3 +1,6 @@
+from __future__ import print_function
+from bs4 import BeautifulSoup
+import re
 from django.test import TestCase, Client,override_settings
 from django.core.management import call_command
 from django.utils.six import StringIO
@@ -34,8 +37,7 @@ from dataprep import delete_strokedata
 from redis import StrictRedis
 redis_connection = StrictRedis()
 
-
+        
-
 class DjangoTestCase(TestCase, MockTestCase):
     def _pre_setup(self):
 	MockTestCase.setUp(self)

rowers/traverselinktest.py

@@ -0,0 +1,186 @@
+from __future__ import print_function
+from bs4 import BeautifulSoup
+import re
+from django.test import TestCase, Client,override_settings
+from django.core.management import call_command
+from django.utils.six import StringIO
+from django.test.client import RequestFactory
+from .views import checkworkoutuser,c2_open
+from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage
+from rowers.forms import DocumentsForm,CNsummaryForm,RegistrationFormUniqueEmail
+import rowers.plots as plots
+import rowers.interactiveplots as iplots
+import datetime
+from rowingdata import rowingdata as rdata
+from rowingdata import rower as rrower
+from django.utils import timezone
+from rowers.rows import handle_uploaded_file
+from django.core.files.uploadedfile import SimpleUploadedFile
+from time import strftime,strptime,mktime,time,daylight
+import os
+from rowers.tasks import handle_makeplot
+from rowers.utils import serialize_list,deserialize_list
+from rowers.c2stuff import C2NoTokenError
+from shutil import copyfile
+
+from minimocktest import MockTestCase
+import pandas as pd
+
+import json
+import numpy as np
+
+from rowers import urls
+from rowers.views import error500_view,error404_view,error400_view,error403_view
+
+from dataprep import delete_strokedata
+
+from redis import StrictRedis
+redis_connection = StrictRedis()
+
+VERBOSE = True
+
+class TraverseLinksTest(TestCase):
+    def setUp(self):
+        self.u = User.objects.create_superuser(
+            'superuser1',
+            'superuser1@example.com','pwd')
+        self.r = Rower.objects.create(user=self.u,gdproptin=True,gdproptindate=timezone.now())
+        nu = datetime.datetime.now()
+
+	self.w = Workout.objects.create(
+            name='testworkout',workouttype='On-water',
+	    user=self.r,date=nu.strftime('%Y-%m-%d'),
+	    starttime=nu.strftime('%H:%M:%S'),
+	    duration="0:55:00",distance=8000)        
+	self.w2 = Workout.objects.create(
+            name='testworkout 2',workouttype='On-water',
+	    user=self.r,date=nu.strftime('%Y-%m-%d'),
+	    starttime=nu.strftime('%H:%M:%S'),
+	    duration="0:55:00",distance=8000)        
+        if self.client.login(
+                username="superuser1", password="pwd"):
+            if VERBOSE:
+                print('\nLogin as superuser OK')
+        else:
+            raise BaseException('Login failed')
+
+    @classmethod
+    def setUpTestData(cls):
+        # Initialise your database here as needed
+        pass
+
+
+    def test_traverse_urls(self):
+        # Fill these lists as needed with your site specific URLs to check and to avoid
+        to_traverse_list = ['/rowers/list-workouts']
+        to_avoid_list = ['^/$', '^$', 'javascript:history\.back()',
+                         'javascript:history\.go\(-1\)', '^mailto:.*',
+                         '.*github\.io.*', 'javascript:.*',
+                         '.*biorow\.com.*','.*facebook.*',
+                         '.*wordpress.*','.*analytics.*','.*freenet.*',
+                         '.*twitter.*','^blog.*',
+                         '.*\d+-\d+-\d+.*',
+                         '.*flexchart/.*',
+                         '.*heroku.*',
+                         '.*oauth.*',
+                         '.*rowingdata.*',
+                         '.*thisisant.*',
+                         '.*garmin.*',
+                         '.*sub7.*',
+                         '.*bitbucket.*',
+                         '.*rathburn.*',
+                         '.*team.*',
+                         '.*concept2.*',
+                         '.*static.*',
+                         '.*authorize.*',
+                         '.*youtu.*',
+                         '.*earth.*',
+                         '.*underarmour.*',
+                         '.*runkeeper.*',
+                         '.*c2list.*',
+                         '.*stravaimport.*',
+                         '.*performancephones.*',
+                         '.*sporttracks.*',
+                         '.*join-select.*',
+        ]
+
+        done_list = []
+        error_list = []
+        source_of_link = dict()
+        for link in to_traverse_list:
+            source_of_link[link] = 'initial'
+
+        (to_traverse_list, to_avoid_list, done_list, error_list, source_of_link) = \
+            self.recurse_into_path(to_traverse_list, to_avoid_list, done_list, error_list, source_of_link)
+
+        print('END REACHED\nStats:')
+        if VERBOSE: print('\nto_traverse_list = ' + str(to_traverse_list))
+        if VERBOSE: print('\nto_avoid_list = ' + str(to_avoid_list))
+        if VERBOSE: print('\nsource_of_link = ' + str(source_of_link))
+        if VERBOSE: print('\ndone_list = ' + str(done_list))
+        print('Followed ' + str(len(done_list)) + ' links successfully')
+        print('Avoided ' + str(len(to_avoid_list)) + ' links')
+
+        if error_list:
+            print('!! ' + str(len(error_list)) + ' error(s) : ')
+            for error in error_list:
+                print(str(error) + ' found in page ' + source_of_link[error[0]])
+
+            print('Errors found traversing links')
+            assert False
+        else:
+            print('No errors')
+
+    def recurse_into_path(self, to_traverse_list, to_avoid_list, done_list, error_list, source_of_link):
+        """ Dives into first item of to_traverse_list
+            Returns: (to_traverse_list, to_avoid_list, done_list, source_of_link)
+        """
+
+        if to_traverse_list:
+            url = to_traverse_list.pop()
+
+            if not match_any(url, to_avoid_list):
+                print('Surfing to ' + str(url) + ', discovered in ' + str(source_of_link[url]))
+                response = self.client.get(url, follow=True)
+
+                if response.status_code == 200:
+                    soup = BeautifulSoup(response.content, 'html.parser')
+
+                    text = soup.get_text()
+
+                    for link in soup.find_all('a'):
+                        new_link = link.get('href')
+                        if VERBOSE: print('  Found link: ' + str(new_link))
+                        if match_any(new_link, to_avoid_list):
+                            if VERBOSE: print('    Avoiding it')
+                        elif new_link in done_list:
+                            if VERBOSE: print('    Already done, ignoring')
+                        elif new_link in to_traverse_list:
+                            if VERBOSE: print('    Already in to traverse list, ignoring')
+                        else:
+                            if VERBOSE: print('    New, unknown link: Storing it to traverse later')
+                            source_of_link[new_link] = url
+                            to_traverse_list.append(new_link)
+
+                    done_list.append(url)
+                    if VERBOSE: print('Done')
+                else:
+                    error_list.append((url, response.status_code))
+                    to_avoid_list.append(url)
+
+            if VERBOSE: print('Diving into next level')
+            return self.recurse_into_path(to_traverse_list, to_avoid_list, done_list, error_list, source_of_link)
+
+        else:
+            # Nothing to traverse
+            if VERBOSE: print('Returning to upper level')
+            return to_traverse_list, to_avoid_list, done_list, error_list, source_of_link
+
+
+def match_any(my_string, regexp_list):
+    if my_string:
+        combined = "(" + ")|(".join(regexp_list) + ")"
+        return re.match(combined, my_string)
+    else:
+        # 'None' as string always matches
+        return True

rowers/views.py

@@ -2712,7 +2712,7 @@ def rower_process_testcallback(request):
 @login_required()
 def histo_all(request,theuser=0,
 	      startdate=timezone.now()-datetime.timedelta(days=10),
-	      enddate=timezone.now()+datetime.timedelta(days=1),
+	      enddate=timezone.now(),
 	      deltadays=-1,
 	      startdatestring="",
 	      enddatestring="",
@@ -2894,7 +2894,7 @@ def cum_flex(request,theuser=0,
 	     yparam1='power',
 	     yparam2='None',
 	     startdate=timezone.now()-datetime.timedelta(days=10),
-	     enddate=timezone.now()+datetime.timedelta(days=1),
+	     enddate=timezone.now(),
 	     deltadays=-1,
 	     startdatestring="",
 	     enddatestring="",
@@ -3623,7 +3623,7 @@ def rankings_view(request,theuser=0,
     # test to fix bug
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
     thedistances = []
     theworkouts = []
@@ -4029,7 +4029,7 @@ def rankings_view2(request,theuser=0,
     # test to fix bug
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
 
     thedistances = []
@@ -4407,7 +4407,7 @@ def otwrankings_view(request,theuser=0,
     # test to fix bug
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
  
 
@@ -4683,7 +4683,7 @@ def oterankings_view(request,theuser=0,
     # test to fix bug
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
  
     
@@ -5080,7 +5080,7 @@ def workouts_join_select(request,
                          message='',
                          successmessage='',
                          startdate=timezone.now()-datetime.timedelta(days=30),
-                         enddate=timezone.now()+datetime.timedelta(days=1),
+                         enddate=timezone.now(),
                          teamid=0):
 
     try:
@@ -5152,7 +5152,7 @@ def workouts_join_select(request,
             
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
     if startdatestring:
         startdate = iso8601.parse_date(startdatestring)
@@ -5239,7 +5239,7 @@ def team_comparison_select(request,
                            message='',
                            successmessage='',
                            startdate=timezone.now()-datetime.timedelta(days=30),
-                           enddate=timezone.now()+datetime.timedelta(days=1),
+                           enddate=timezone.now(),
                            teamid=0):
 
     try:
@@ -5311,7 +5311,7 @@ def team_comparison_select(request,
             
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
     if startdatestring:
         startdate = iso8601.parse_date(startdatestring)
@@ -5478,13 +5478,13 @@ def multi_compare_view(request):
 # Multi Flex Chart with Grouping
 @user_passes_test(ispromember,login_url="/",redirect_field_name=None)
 def user_multiflex_select(request,
-                        startdatestring="",
+                          startdatestring="",
-                        enddatestring="",
+                          enddatestring="",
-                        message='',
+                          message='',
-                        successmessage='',
+                          successmessage='',
-                        startdate=timezone.now()-datetime.timedelta(days=30),
+                          startdate=timezone.now()-datetime.timedelta(days=30),
-                        enddate=timezone.now()+datetime.timedelta(days=1),
+                          enddate=timezone.now(),
-                        userid=0):
+                          userid=0):
 
     if userid == 0:
         user = request.user
@@ -5575,7 +5575,7 @@ def user_multiflex_select(request,
             
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
     if startdatestring:
         startdate = iso8601.parse_date(startdatestring)
@@ -6017,7 +6017,7 @@ def user_boxplot_select(request,
                         message='',
                         successmessage='',
                         startdate=timezone.now()-datetime.timedelta(days=30),
-                        enddate=timezone.now()+datetime.timedelta(days=1),
+                        enddate=timezone.now(),
                         options={
                             'includereststrokes':False,
                             'workouttypes':['rower','dynamic','slides'],
@@ -6113,7 +6113,7 @@ def user_boxplot_select(request,
 
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
     if startdatestring:
         startdate = iso8601.parse_date(startdatestring)
@@ -6413,7 +6413,7 @@ def courses_view(request):
 def workouts_view(request,message='',successmessage='',
                   startdatestring="",enddatestring="",
                   startdate=timezone.now()-datetime.timedelta(days=365),
-                  enddate=timezone.now()+datetime.timedelta(days=1),
+                  enddate=timezone.now(),
                   teamid=0,rankingonly=False,rowerid=0,userid=0):
 
     request.session['referer'] = absolute(request)['PATH']
@@ -6448,7 +6448,7 @@ def workouts_view(request,message='',successmessage='',
 
     startdate = datetime.datetime.combine(startdate,datetime.time())
     enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-    enddate = enddate+datetime.timedelta(days=1)
+    #enddate = enddate+datetime.timedelta(days=1)
 
     if startdatestring:
         startdate = iso8601.parse_date(startdatestring)
@@ -6590,7 +6590,7 @@ def workout_comparison_list(request,id=0,message='',successmessage='',
 
         startdate = datetime.datetime.combine(startdate,datetime.time())
         enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-        enddate = enddate+datetime.timedelta(days=1)
+        #enddate = enddate+datetime.timedelta(days=1)
                 
         if enddate < startdate:
 	    s = enddate
@@ -6668,7 +6668,7 @@ def workout_fusion_list(request,id=0,message='',successmessage='',
 
         startdate = datetime.datetime.combine(startdate,datetime.time())
         enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
-        enddate = enddate+datetime.timedelta(days=1)
+        #enddate = enddate+datetime.timedelta(days=1)
                 
         if enddate < startdate:
 	    s = enddate
@@ -10988,7 +10988,10 @@ def workout_summary_edit_view(request,id,message="",successmessage=""
     if rowdata == 0:
 	return HttpResponse("Error: CSV Data File Not Found")
     intervalstats = rowdata.allstats()
-    itime,idist,itype = rowdata.intervalstats_values()
+    try:
+        itime,idist,itype = rowdata.intervalstats_values()
+    except TypeError:
+        return HttpResponse("Error: CSV Data File Not Found")
     nrintervals = len(idist)
 
     # create interactive plot
@@ -12209,7 +12212,7 @@ def plannedsession_multiclone_view(
         request,timeperiod='none',
         rowerid=0,
         startdate=timezone.now()-datetime.timedelta(days=30),
-        enddate=timezone.now()+datetime.timedelta(days=1)):
+        enddate=timezone.now()):
     
     if rowerid==0:
         r = getrower(request.user)