sander/rowsandall
Private
Public Access
1
0
Fork 0
Code Activity

added test to travel all links

Browse Source
This commit is contained in:
Sander Roosendaal
2018-04-10 15:42:38 +02:00
parent 7edc020747
commit c3448ebf2c
7 changed files with 228 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
rowers/dataprep.py
View File

@@ -309,6 +309,8 @@ def clean_df_stats(datadf, workstrokesonly=True, ignorehr=True,
# clean data remove zeros and negative values
# bring metrics which have negative values to positive domain
if datadf.empty:
return datadf
try:
datadf['catch'] = -datadf['catch']
except KeyError:
@@ -1670,7 +1672,7 @@ def getsmallrowdata_db(columns, ids=[], doclean=True, workstrokesonly=True):
try:
f = row.df['TimeStamp (sec)'].diff().mean()
except AttributeError:
except (AttributeError,KeyError) as e:
f = 0
if f != 0 and not np.isnan(f):

2
rowers/templates/about_us.html
View File

@@ -75,7 +75,7 @@
<p>Rowsandall.com tries to be compatible with the most important tools
that rowers use to capture the data (both indoor and OTW). For a full
list of currently supported devides/apps, click
<a href="compatibility">here</a>.
<a href="/rowers/compatibility">here</a>.
</div>

7
rowers/templates/compatibility.html
View File

@@ -7,14 +7,15 @@
<h2>Import Compatibility</h2>
<p>Rowsandall.com tries to be compatible with the most important tools
that rowers use to capture the data (both indoor and OTW). For a full
list of currently supported devides/apps, click
<a href="compatibility.html">here</a>.
that rowers use to capture the data (both indoor and OTW).
<p>On The Water
<ul>
<li> CrewNerd (TCX)</li>
<li> Rowing In Motion (TCX)</li>
<li> BoatCoach </li>
<li> RowingCoach </li>
<li> Quiske RowP</li>
<li> Speedcoach XL (CSV)</li>
<li> Speedcoach GPS (FIT and CSV)</li></ul></p>

2
rowers/templates/workout_form.html
View File

@@ -235,7 +235,7 @@
</p>
<div class="grid_2 alpha">
<p>
<a class="button green small" href="recalcsummary">Update Summary</a>
<a class="button green small" href="/rowers/workout/{{ workout.id }}/recalcsummary">Update Summary</a>
</p>
</div>
</div>

6
rowers/tests.py
View File

@@ -1,3 +1,6 @@
from __future__ import print_function
from bs4 import BeautifulSoup
import re
from django.test import TestCase, Client,override_settings
from django.core.management import call_command
from django.utils.six import StringIO
@@ -34,8 +37,7 @@ from dataprep import delete_strokedata
from redis import StrictRedis
redis_connection = StrictRedis()
class DjangoTestCase(TestCase, MockTestCase):
def _pre_setup(self):
MockTestCase.setUp(self)

186
rowers/traverselinktest.py Normal file
View File

@@ -0,0 +1,186 @@
from __future__ import print_function
from bs4 import BeautifulSoup
import re
from django.test import TestCase, Client,override_settings
from django.core.management import call_command
from django.utils.six import StringIO
from django.test.client import RequestFactory
from .views import checkworkoutuser,c2_open
from rowers.models import Workout, User, Rower, WorkoutForm,RowerForm,GraphImage
from rowers.forms import DocumentsForm,CNsummaryForm,RegistrationFormUniqueEmail
import rowers.plots as plots
import rowers.interactiveplots as iplots
import datetime
from rowingdata import rowingdata as rdata
from rowingdata import rower as rrower
from django.utils import timezone
from rowers.rows import handle_uploaded_file
from django.core.files.uploadedfile import SimpleUploadedFile
from time import strftime,strptime,mktime,time,daylight
import os
from rowers.tasks import handle_makeplot
from rowers.utils import serialize_list,deserialize_list
from rowers.c2stuff import C2NoTokenError
from shutil import copyfile
from minimocktest import MockTestCase
import pandas as pd
import json
import numpy as np
from rowers import urls
from rowers.views import error500_view,error404_view,error400_view,error403_view
from dataprep import delete_strokedata
from redis import StrictRedis
redis_connection = StrictRedis()
VERBOSE = True
class TraverseLinksTest(TestCase):
def setUp(self):
self.u = User.objects.create_superuser(
'superuser1',
'superuser1@example.com','pwd')
self.r = Rower.objects.create(user=self.u,gdproptin=True,gdproptindate=timezone.now())
nu = datetime.datetime.now()
self.w = Workout.objects.create(
name='testworkout',workouttype='On-water',
user=self.r,date=nu.strftime('%Y-%m-%d'),
starttime=nu.strftime('%H:%M:%S'),
duration="0:55:00",distance=8000)
self.w2 = Workout.objects.create(
name='testworkout 2',workouttype='On-water',
user=self.r,date=nu.strftime('%Y-%m-%d'),
starttime=nu.strftime('%H:%M:%S'),
duration="0:55:00",distance=8000)
if self.client.login(
username="superuser1", password="pwd"):
if VERBOSE:
print('\nLogin as superuser OK')
else:
raise BaseException('Login failed')
@classmethod
def setUpTestData(cls):
# Initialise your database here as needed
pass
def test_traverse_urls(self):
# Fill these lists as needed with your site specific URLs to check and to avoid
to_traverse_list = ['/rowers/list-workouts']
to_avoid_list = ['^/$', '^$', 'javascript:history\.back()',
'javascript:history\.go\(-1\)', '^mailto:.*',
'.*github\.io.*', 'javascript:.*',
'.*biorow\.com.*','.*facebook.*',
'.*wordpress.*','.*analytics.*','.*freenet.*',
'.*twitter.*','^blog.*',
'.*\d+-\d+-\d+.*',
'.*flexchart/.*',
'.*heroku.*',
'.*oauth.*',
'.*rowingdata.*',
'.*thisisant.*',
'.*garmin.*',
'.*sub7.*',
'.*bitbucket.*',
'.*rathburn.*',
'.*team.*',
'.*concept2.*',
'.*static.*',
'.*authorize.*',
'.*youtu.*',
'.*earth.*',
'.*underarmour.*',
'.*runkeeper.*',
'.*c2list.*',
'.*stravaimport.*',
'.*performancephones.*',
'.*sporttracks.*',
'.*join-select.*',
]
done_list = []
error_list = []
source_of_link = dict()
for link in to_traverse_list:
source_of_link[link] = 'initial'
(to_traverse_list, to_avoid_list, done_list, error_list, source_of_link) = \
self.recurse_into_path(to_traverse_list, to_avoid_list, done_list, error_list, source_of_link)
print('END REACHED\nStats:')
if VERBOSE: print('\nto_traverse_list = ' + str(to_traverse_list))
if VERBOSE: print('\nto_avoid_list = ' + str(to_avoid_list))
if VERBOSE: print('\nsource_of_link = ' + str(source_of_link))
if VERBOSE: print('\ndone_list = ' + str(done_list))
print('Followed ' + str(len(done_list)) + ' links successfully')
print('Avoided ' + str(len(to_avoid_list)) + ' links')
if error_list:
print('!! ' + str(len(error_list)) + ' error(s) : ')
for error in error_list:
print(str(error) + ' found in page ' + source_of_link[error[0]])
print('Errors found traversing links')
assert False
else:
print('No errors')
def recurse_into_path(self, to_traverse_list, to_avoid_list, done_list, error_list, source_of_link):
""" Dives into first item of to_traverse_list
Returns: (to_traverse_list, to_avoid_list, done_list, source_of_link)
"""
if to_traverse_list:
url = to_traverse_list.pop()
if not match_any(url, to_avoid_list):
print('Surfing to ' + str(url) + ', discovered in ' + str(source_of_link[url]))
response = self.client.get(url, follow=True)
if response.status_code == 200:
soup = BeautifulSoup(response.content, 'html.parser')
text = soup.get_text()
for link in soup.find_all('a'):
new_link = link.get('href')
if VERBOSE: print(' Found link: ' + str(new_link))
if match_any(new_link, to_avoid_list):
if VERBOSE: print(' Avoiding it')
elif new_link in done_list:
if VERBOSE: print(' Already done, ignoring')
elif new_link in to_traverse_list:
if VERBOSE: print(' Already in to traverse list, ignoring')
else:
if VERBOSE: print(' New, unknown link: Storing it to traverse later')
source_of_link[new_link] = url
to_traverse_list.append(new_link)
done_list.append(url)
if VERBOSE: print('Done')
else:
error_list.append((url, response.status_code))
to_avoid_list.append(url)
if VERBOSE: print('Diving into next level')
return self.recurse_into_path(to_traverse_list, to_avoid_list, done_list, error_list, source_of_link)
else:
# Nothing to traverse
if VERBOSE: print('Returning to upper level')
return to_traverse_list, to_avoid_list, done_list, error_list, source_of_link
def match_any(my_string, regexp_list):
if my_string:
combined = "(" + ")|(".join(regexp_list) + ")"
return re.match(combined, my_string)
else:
# 'None' as string always matches
return True

55
rowers/views.py
View File

@@ -2712,7 +2712,7 @@ def rower_process_testcallback(request):
@login_required()
def histo_all(request,theuser=0,
startdate=timezone.now()-datetime.timedelta(days=10),
enddate=timezone.now()+datetime.timedelta(days=1),
enddate=timezone.now(),
deltadays=-1,
startdatestring="",
enddatestring="",
@@ -2894,7 +2894,7 @@ def cum_flex(request,theuser=0,
yparam1='power',
yparam2='None',
startdate=timezone.now()-datetime.timedelta(days=10),
enddate=timezone.now()+datetime.timedelta(days=1),
enddate=timezone.now(),
deltadays=-1,
startdatestring="",
enddatestring="",
@@ -3623,7 +3623,7 @@ def rankings_view(request,theuser=0,
# test to fix bug
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
thedistances = []
theworkouts = []
@@ -4029,7 +4029,7 @@ def rankings_view2(request,theuser=0,
# test to fix bug
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
thedistances = []
@@ -4407,7 +4407,7 @@ def otwrankings_view(request,theuser=0,
# test to fix bug
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
@@ -4683,7 +4683,7 @@ def oterankings_view(request,theuser=0,
# test to fix bug
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
@@ -5080,7 +5080,7 @@ def workouts_join_select(request,
message='',
successmessage='',
startdate=timezone.now()-datetime.timedelta(days=30),
enddate=timezone.now()+datetime.timedelta(days=1),
enddate=timezone.now(),
teamid=0):
try:
@@ -5152,7 +5152,7 @@ def workouts_join_select(request,
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if startdatestring:
startdate = iso8601.parse_date(startdatestring)
@@ -5239,7 +5239,7 @@ def team_comparison_select(request,
message='',
successmessage='',
startdate=timezone.now()-datetime.timedelta(days=30),
enddate=timezone.now()+datetime.timedelta(days=1),
enddate=timezone.now(),
teamid=0):
try:
@@ -5311,7 +5311,7 @@ def team_comparison_select(request,
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if startdatestring:
startdate = iso8601.parse_date(startdatestring)
@@ -5478,13 +5478,13 @@ def multi_compare_view(request):
# Multi Flex Chart with Grouping
@user_passes_test(ispromember,login_url="/",redirect_field_name=None)
def user_multiflex_select(request,
startdatestring="",
enddatestring="",
message='',
successmessage='',
startdate=timezone.now()-datetime.timedelta(days=30),
enddate=timezone.now()+datetime.timedelta(days=1),
userid=0):
startdatestring="",
enddatestring="",
message='',
successmessage='',
startdate=timezone.now()-datetime.timedelta(days=30),
enddate=timezone.now(),
userid=0):
if userid == 0:
user = request.user
@@ -5575,7 +5575,7 @@ def user_multiflex_select(request,
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if startdatestring:
startdate = iso8601.parse_date(startdatestring)
@@ -6017,7 +6017,7 @@ def user_boxplot_select(request,
message='',
successmessage='',
startdate=timezone.now()-datetime.timedelta(days=30),
enddate=timezone.now()+datetime.timedelta(days=1),
enddate=timezone.now(),
options={
'includereststrokes':False,
'workouttypes':['rower','dynamic','slides'],
@@ -6113,7 +6113,7 @@ def user_boxplot_select(request,
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if startdatestring:
startdate = iso8601.parse_date(startdatestring)
@@ -6413,7 +6413,7 @@ def courses_view(request):
def workouts_view(request,message='',successmessage='',
startdatestring="",enddatestring="",
startdate=timezone.now()-datetime.timedelta(days=365),
enddate=timezone.now()+datetime.timedelta(days=1),
enddate=timezone.now(),
teamid=0,rankingonly=False,rowerid=0,userid=0):
request.session['referer'] = absolute(request)['PATH']
@@ -6448,7 +6448,7 @@ def workouts_view(request,message='',successmessage='',
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if startdatestring:
startdate = iso8601.parse_date(startdatestring)
@@ -6590,7 +6590,7 @@ def workout_comparison_list(request,id=0,message='',successmessage='',
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if enddate < startdate:
s = enddate
@@ -6668,7 +6668,7 @@ def workout_fusion_list(request,id=0,message='',successmessage='',
startdate = datetime.datetime.combine(startdate,datetime.time())
enddate = datetime.datetime.combine(enddate,datetime.time(23,59,59))
enddate = enddate+datetime.timedelta(days=1)
#enddate = enddate+datetime.timedelta(days=1)
if enddate < startdate:
s = enddate
@@ -10988,7 +10988,10 @@ def workout_summary_edit_view(request,id,message="",successmessage=""
if rowdata == 0:
return HttpResponse("Error: CSV Data File Not Found")
intervalstats = rowdata.allstats()
itime,idist,itype = rowdata.intervalstats_values()
try:
itime,idist,itype = rowdata.intervalstats_values()
except TypeError:
return HttpResponse("Error: CSV Data File Not Found")
nrintervals = len(idist)
# create interactive plot
@@ -12209,7 +12212,7 @@ def plannedsession_multiclone_view(
request,timeperiod='none',
rowerid=0,
startdate=timezone.now()-datetime.timedelta(days=30),
enddate=timezone.now()+datetime.timedelta(days=1)):
enddate=timezone.now()):
if rowerid==0:
r = getrower(request.user)