Allow Pa11ycrawler to run against a single url

Utilize scrapy parse

specify custom callback and add port

Getting rid of scrapy parse. A modified crawl will work

remove log

Remove unnecessary input options

Remove extra empty line

version bump v1.5.9

Reduce duplicate code

add in port and simplify further

clean command further

add port to basic command

Bump Pa11yversion to the newest version

pavelib/bok_choy.py

@@ -9,6 +9,7 @@ from pavelib.utils.test.bokchoy_options import (
     PA11Y_HTML,
     PA11Y_COURSE_KEY,
     PA11Y_FETCH_COURSE,
+    PA11Y_SINGLE_URL
 )
 from pavelib.utils.envs import Env
 from pavelib.utils.test.utils import check_firefox_version
@@ -100,7 +101,7 @@ def perf_report_bokchoy(options, passthrough_options):
 
 @needs('pavelib.prereqs.install_prereqs', 'get_test_course')
 @cmdopts(
-    BOKCHOY_OPTS + [PA11Y_HTML, PA11Y_COURSE_KEY, PA11Y_FETCH_COURSE],
+    BOKCHOY_OPTS + [PA11Y_SINGLE_URL, PA11Y_HTML, PA11Y_COURSE_KEY, PA11Y_FETCH_COURSE],
     share_with=['get_test_course', 'prepare_bokchoy_run', 'load_courses']
 )
 @PassthroughTask

pavelib/utils/test/bokchoy_options.py

@@ -81,6 +81,7 @@ BOKCHOY_OPTS = [
     ),
 ]
 
+PA11Y_SINGLE_URL = make_option('--single-url', help='Crawl only the specified url')
 PA11Y_HTML = ('with-html', 'w', 'Include html reports')
 PA11Y_COURSE_KEY = make_option('--course-key', help='Course key for test course')
 PA11Y_FETCH_COURSE = make_option(

pavelib/utils/test/suites/bokchoy_suite.py

@@ -350,6 +350,7 @@ class Pa11yCrawler(BokChoyTestSuite):
     def __init__(self, *args, **kwargs):
         super(Pa11yCrawler, self).__init__(*args, **kwargs)
         self.course_key = kwargs.get('course_key')
+        self.single_url = kwargs.get('single_url', False)
         self.ensure_scrapy_cfg()
 
     def ensure_scrapy_cfg(self):
@@ -396,7 +397,8 @@ class Pa11yCrawler(BokChoyTestSuite):
         """
         data_dir = os.path.join(self.report_dir, 'data')
         url = "https://raw.githubusercontent.com/edx/pa11ycrawler-ignore/master/ignore.yaml"
-        return [
+
+        command = [
             "scrapy",
             "crawl",
             "edx",
@@ -407,5 +409,13 @@ class Pa11yCrawler(BokChoyTestSuite):
             "-a",
             "pa11y_ignore_rules_url={url}".format(url=url),
             "-a",
-            "data_dir={dir}".format(dir=data_dir)
+            "data_dir={dir}".format(dir=data_dir),
         ]
+
+        if self.single_url:
+            command = command + [
+                "-a",
+                "single_url={url}".format(url=self.single_url),
+            ]
+
+        return command

requirements/edx/base.txt

@@ -170,7 +170,7 @@ nose-exclude
 nose-ignore-docstring
 nose-randomly==1.2.0
 nosexcover==1.0.7
-pa11ycrawler==1.5.8
+pa11ycrawler==1.6.0
 pep8==1.5.7
 PyContracts==1.7.1
 python-subunit==0.0.16