Merge pull request #19672 from edx/youngstrom/balance-shards

Bump bokchoy shards and rebalance to decrease test times
This commit is contained in:
Michael Youngstrom
2019-01-30 15:44:32 -05:00
committed by GitHub
29 changed files with 132 additions and 54 deletions

View File

@@ -5,7 +5,6 @@ End-to-end tests for the Account Settings page.
from datetime import datetime
from unittest import skip
import pytest
from bok_choy.page_object import XSS_INJECTION
from pytz import timezone, utc
@@ -24,6 +23,8 @@ class AccountSettingsTestMixin(EventsTestMixin, AcceptanceTest):
USER_SETTINGS_CHANGED_EVENT_NAME = 'edx.user.settings.changed'
ACCOUNT_SETTINGS_REFERER = u"/account/settings"
shard = 23
def visit_account_settings_page(self, gdpr=False):
"""
Visit the account settings page for the current user, and store the page instance
@@ -553,11 +554,11 @@ class AccountSettingsDeleteAccountTest(AccountSettingsTestMixin, AcceptanceTest)
)
@pytest.mark.a11y
class AccountSettingsA11yTest(AccountSettingsTestMixin, AcceptanceTest):
"""
Class to test account settings accessibility.
"""
a11y = True
def test_account_settings_a11y(self):
"""

View File

@@ -5,7 +5,6 @@ End-to-end tests for the courseware unit bookmarks.
import json
from unittest import skip
import pytest
import requests
from common.test.acceptance.fixtures.course import CourseFixture, XBlockFixtureDesc
@@ -586,11 +585,12 @@ class BookmarksTest(BookmarksTestMixin):
)
@pytest.mark.a11y
class BookmarksA11yTests(BookmarksTestMixin):
"""
Tests for checking the a11y of the bookmarks page.
"""
a11y = True
def test_view_a11y(self):
"""
Verify the basic accessibility of the bookmarks page while paginated.

View File

@@ -15,6 +15,7 @@ class ConditionalTest(UniqueCourseTest):
"""
Test the conditional module in the lms.
"""
shard = 23
def setUp(self):
super(ConditionalTest, self).setUp()

View File

@@ -6,8 +6,6 @@ from contextlib import contextmanager
from datetime import datetime
from unittest import skip
import pytest
from common.test.acceptance.pages.common.auto_auth import AutoAuthPage
from common.test.acceptance.pages.common.logout import LogoutPage
from common.test.acceptance.pages.lms.account_settings import AccountSettingsPage
@@ -679,11 +677,11 @@ class DifferentUserLearnerProfilePageTest(LearnerProfileTestMixin, AcceptanceTes
badge.close_modal()
@pytest.mark.a11y
class LearnerProfileA11yTest(LearnerProfileTestMixin, AcceptanceTest):
"""
Class to test learner profile accessibility.
"""
a11y = True
def test_editable_learner_profile_a11y(self):
"""

View File

@@ -47,6 +47,7 @@ class XBlockAcidNoChildTest(XBlockAcidBase):
"""
Tests of an AcidBlock with no children
"""
shard = 20
__test__ = True
def setup_fixtures(self):
@@ -80,6 +81,7 @@ class XBlockAcidChildTest(XBlockAcidBase):
"""
Tests of an AcidBlock with children
"""
shard = 20
__test__ = True
def setup_fixtures(self):
@@ -125,6 +127,7 @@ class XBlockAcidAsideTest(XBlockAcidBase):
"""
Tests of an AcidBlock with children
"""
shard = 20
__test__ = True
def setup_fixtures(self):

View File

@@ -16,6 +16,7 @@ class CourseDiscoveryTest(AcceptanceTest):
"""
Test searching for courses.
"""
shard = 20
STAFF_USERNAME = "STAFF_TESTER"
STAFF_EMAIL = "staff101@example.com"

View File

@@ -62,6 +62,8 @@ class CourseHomeTest(CourseHomeBaseTest):
"""
Tests the course home page with course outline.
"""
shard = 20
def test_course_home(self):
"""
Smoke test of course goals, course outline, breadcrumbs to and from course outline, and bookmarks.

View File

@@ -4,8 +4,6 @@ End-to-end tests for the main LMS Dashboard (aka, Student Dashboard).
"""
import datetime
import pytest
from common.test.acceptance.fixtures.course import CourseFixture
from common.test.acceptance.pages.common.auto_auth import AutoAuthPage
from common.test.acceptance.pages.lms.dashboard import DashboardPage
@@ -359,6 +357,7 @@ class LmsDashboardCourseUnEnrollDialogMessageTest(BaseLmsDashboardTestMultiple):
"""
Class to test lms student dashboard unenroll dialog messages.
"""
shard = 23
def test_audit_course_run_unenroll_dialog_msg(self):
"""
@@ -399,11 +398,11 @@ class LmsDashboardCourseUnEnrollDialogMessageTest(BaseLmsDashboardTestMultiple):
self.assertEqual(dialog_message['refund-info'][0], expected_refund_message)
@pytest.mark.a11y
class LmsDashboardA11yTest(BaseLmsDashboardTestMultiple):
"""
Class to test lms student dashboard accessibility.
"""
a11y = True
def test_dashboard_course_listings_a11y(self):
"""

View File

@@ -24,6 +24,8 @@ class GatingTest(UniqueCourseTest):
STUDENT_USERNAME = "STUDENT_TESTER"
STUDENT_EMAIL = "student101@example.com"
shard = 23
def setUp(self):
super(GatingTest, self).setUp()

View File

@@ -18,6 +18,8 @@ class TestCohortHelp(ContainerBase, CohortTestMixin):
"""
Tests help links in Cohort page
"""
shard = 2
def setUp(self, is_staff=True):
super(TestCohortHelp, self).setUp(is_staff=is_staff)
self.enable_cohorting(self.course_fixture)
@@ -82,6 +84,7 @@ class InstructorDashboardHelp(BaseInstructorDashboardTest):
"""
Tests opening help from the general Help button in the instructor dashboard.
"""
shard = 2
def setUp(self):
super(InstructorDashboardHelp, self).setUp()

View File

@@ -28,6 +28,7 @@ class BaseLmsIndexTest(AcceptanceTest):
class LmsIndexPageTest(BaseLmsIndexTest):
""" Test suite for the LMS Index (Home) page """
shard = 2
def setUp(self):
super(LmsIndexPageTest, self).setUp()

View File

@@ -90,6 +90,8 @@ class BulkEmailTest(BaseInstructorDashboardTest):
"""
End-to-end tests for bulk emailing from instructor dash.
"""
shard = 23
def setUp(self):
super(BulkEmailTest, self).setUp()
self.course_fixture = CourseFixture(**self.course_info).install()
@@ -239,6 +241,7 @@ class BatchBetaTestersTest(BaseInstructorDashboardTest):
"""
End-to-end tests for Batch beta testers functionality.
"""
shard = 23
def setUp(self):
super(BatchBetaTestersTest, self).setUp()
@@ -697,6 +700,8 @@ class DataDownloadsWithMultipleRoleTests(BaseInstructorDashboardTest):
"""
Bok Choy tests for the "Data Downloads" tab with multiple user roles.
"""
shard = 23
def setUp(self):
super(DataDownloadsWithMultipleRoleTests, self).setUp()
self.course_fixture = CourseFixture(**self.course_info).install()
@@ -1450,6 +1455,8 @@ class StudentAdminTest(BaseInstructorDashboardTest):
UNIT_NAME = 'Test Unit 1'
PROBLEM_NAME = 'Test Problem 1'
shard = 23
def setUp(self):
super(StudentAdminTest, self).setUp()
self.course_fix = CourseFixture(

View File

@@ -15,6 +15,8 @@ class MatlabProblemTest(ProblemsTest):
"""
Tests that verify matlab problem "Run Code".
"""
shard = 11
def get_problem(self):
"""
Create a matlab problem for the test.

View File

@@ -881,6 +881,8 @@ class ProblemMetaGradedTest(ProblemsTest):
"""
TestCase Class to verify that the graded variable is passed
"""
shard = 23
def get_problem(self):
"""
Problem structure
@@ -911,6 +913,8 @@ class ProblemMetaUngradedTest(ProblemsTest):
"""
TestCase Class to verify that the ungraded variable is passed
"""
shard = 23
def get_problem(self):
"""
Problem structure
@@ -941,6 +945,8 @@ class FormulaProblemTest(ProblemsTest):
"""
Test Class to verify the formula problem on LMS.
"""
shard = 23
def setUp(self):
"""
Setup the test suite to verify various behaviors involving formula problem type.
@@ -1037,6 +1043,7 @@ class FormulaProblemRandomizeTest(ProblemsTest):
"""
Test Class to verify the formula problem on LMS with Randomization enabled.
"""
shard = 23
def setUp(self):
"""

View File

@@ -12,6 +12,7 @@ class OAuth2PermissionDelegationTests(AcceptanceTest):
"""
Tests for acceptance/denial of permission delegation requests.
"""
shard = 16
def setUp(self):
super(OAuth2PermissionDelegationTests, self).setUp()

View File

@@ -643,6 +643,7 @@ class AnnotationProblemTypeTest(AnnotationProblemTypeBase, ProblemTypeTestMixin)
"""
Standard tests for the Annotation Problem Type
"""
shard = 24
pass
@@ -691,6 +692,8 @@ class CheckboxProblemTypeTest(CheckboxProblemTypeBase, ProblemTypeTestMixin, Cha
"""
Standard tests for the Checkbox Problem Type
"""
shard = 24
def test_can_show_answer(self):
"""
Scenario: Verifies that show answer button is working as expected.
@@ -788,6 +791,8 @@ class MultipleChoiceProblemTypeTest(MultipleChoiceProblemTypeBase, ProblemTypeTe
"""
Standard tests for the Multiple Choice Problem Type
"""
shard = 24
def test_can_show_answer(self):
"""
Scenario: Verifies that show answer button is working as expected.
@@ -822,6 +827,7 @@ class MultipleChoiceProblemResetCorrectnessAfterChangingAnswerTest(MultipleChoic
"""
Tests for Multiple choice problem with changing answers
"""
shard = 24
@ddt.data(['correct', '1/1 point (ungraded)'], ['incorrect', '0/1 point (ungraded)'])
@ddt.unpack
@@ -870,6 +876,8 @@ class MultipleChoiceProblemTypeTestNonRandomized(MultipleChoiceProblemTypeBase,
"""
Tests for non-randomized multiple choice problem
"""
shard = 24
def get_problem(self):
"""
Creates a {problem_type} problem
@@ -1064,6 +1072,7 @@ class RadioProblemTypeTest(RadioProblemTypeBase, ProblemTypeTestMixin):
"""
Standard tests for the Multiple Radio Problem Type
"""
shard = 24
pass
@@ -1072,6 +1081,7 @@ class RadioProblemResetCorrectnessAfterChangingAnswerTest(RadioProblemTypeBase):
"""
Tests for Radio problem with changing answers
"""
shard = 24
@ddt.data(['correct', '1/1 point (ungraded)'], ['incorrect', '0/1 point (ungraded)'])
@ddt.unpack
@@ -1119,6 +1129,7 @@ class RadioProblemTypeTestNonRandomized(RadioProblemTypeBase, NonRandomizedProbl
"""
Tests for non-randomized radio problem
"""
shard = 24
def get_problem(self):
"""
@@ -1172,6 +1183,7 @@ class DropdownProblemTypeTest(DropDownProblemTypeBase, ProblemTypeTestMixin, Cha
"""
Standard tests for the Dropdown Problem Type
"""
shard = 24
pass
@@ -1180,6 +1192,7 @@ class DropDownProblemTypeTestNonRandomized(DropDownProblemTypeBase, NonRandomize
"""
Tests for non-randomized Dropdown problem
"""
shard = 24
def get_problem(self):
"""
@@ -1254,6 +1267,7 @@ class StringProblemTypeTest(StringProblemTypeBase, ProblemTypeTestMixin):
"""
Standard tests for the String Problem Type
"""
shard = 24
pass
@@ -1325,7 +1339,8 @@ class NumericalProblemTypeTest(NumericalProblemTypeBase, ProblemTypeTestMixin, C
"""
Standard tests for the Numerical Problem Type
"""
@attr(shard=12)
shard = 12
def test_error_input_gentle_alert(self):
"""
Scenario: I can answer a problem with erroneous input and will see a gentle alert
@@ -1356,6 +1371,7 @@ class NumericalProblemTypeTestNonRandomized(NumericalProblemTypeBase, NonRandomi
"""
Tests for non-randomized Numerical problem
"""
shard = 12
def get_problem(self):
"""
@@ -1463,6 +1479,7 @@ class FormulaProblemTypeTest(FormulaProblemTypeBase, ProblemTypeTestMixin, Chang
"""
Standard tests for the Formula Problem Type
"""
shard = 24
pass
@@ -1470,6 +1487,7 @@ class FormulaProblemTypeTestNonRandomized(FormulaProblemTypeBase, NonRandomizedP
"""
Tests for non-randomized Formula problem
"""
shard = 24
def get_problem(self):
"""
@@ -1565,6 +1583,7 @@ class ScriptProblemTypeTest(ScriptProblemTypeBase, ProblemTypeTestMixin):
"""
Standard tests for the Script Problem Type
"""
shard = 24
pass
@@ -1573,6 +1592,7 @@ class ScriptProblemResetAfterAnswerTest(ScriptProblemTypeBase):
"""
Test Script problem by resetting answers
"""
shard = 24
@ddt.data(['correct', 'incorrect'], ['incorrect', 'correct'])
@ddt.unpack
@@ -1618,6 +1638,7 @@ class ScriptProblemTypeTestNonRandomized(ScriptProblemTypeBase, NonRandomizedPro
"""
Tests for non-randomized Script problem
"""
shard = 24
def get_problem(self):
"""
@@ -1704,7 +1725,8 @@ class CodeProblemTypeTest(CodeProblemTypeBase, ProblemTypeTestMixin):
"""
Standard tests for the Code Problem Type
"""
@attr(shard=12)
shard = 12
def test_answer_incorrectly(self):
"""
Overridden for script test because the testing grader always responds
@@ -1712,7 +1734,6 @@ class CodeProblemTypeTest(CodeProblemTypeBase, ProblemTypeTestMixin):
"""
pass
@attr(shard=12)
def test_submit_blank_answer(self):
"""
Overridden for script test because the testing grader always responds
@@ -1720,7 +1741,6 @@ class CodeProblemTypeTest(CodeProblemTypeBase, ProblemTypeTestMixin):
"""
pass
@attr(shard=12)
def test_cant_submit_blank_answer(self):
"""
Overridden for script test because the testing grader always responds
@@ -1728,7 +1748,6 @@ class CodeProblemTypeTest(CodeProblemTypeBase, ProblemTypeTestMixin):
"""
pass
@attr(shard=12)
def wait_for_status(self, status):
"""
Overridden for script test because the testing grader always responds
@@ -1846,6 +1865,7 @@ class RadioTextProblemTypeTest(RadioTextProblemTypeBase, ProblemTypeTestMixin):
"""
Standard tests for the Radio Text Problem Type
"""
shard = 24
pass
@@ -1854,6 +1874,7 @@ class RadioTextProblemResetCorrectnessAfterChangingAnswerTest(RadioTextProblemTy
"""
Tests for Radio Text problem with changing answers
"""
shard = 24
@ddt.data(['correct', '1/1 point (ungraded)'], ['incorrect', '0/1 point (ungraded)'])
@ddt.unpack
@@ -1901,6 +1922,7 @@ class RadioTextProblemTypeTestNonRandomized(RadioTextProblemTypeBase, NonRandomi
"""
Tests for non-randomized Radio text problem
"""
shard = 24
def get_problem(self):
"""

View File

@@ -1,6 +1,4 @@
"""Acceptance tests for LMS-hosted Programs pages"""
import pytest
from common.test.acceptance.fixtures.catalog import CatalogFixture, CatalogIntegrationMixin
from common.test.acceptance.fixtures.course import CourseFixture
from common.test.acceptance.fixtures.programs import ProgramsConfigMixin
@@ -109,9 +107,10 @@ class ProgramListingPageTest(ProgramPageBase):
self.assertFalse(self.listing_page.are_cards_present)
@pytest.mark.a11y
class ProgramListingPageA11yTest(ProgramPageBase):
"""Test program listing page accessibility."""
a11y = True
def setUp(self):
super(ProgramListingPageA11yTest, self).setUp()
@@ -154,9 +153,10 @@ class ProgramListingPageA11yTest(ProgramPageBase):
self.listing_page.a11y_audit.check_for_accessibility_errors()
@pytest.mark.a11y
class ProgramDetailsPageA11yTest(ProgramPageBase):
"""Test program details page accessibility."""
a11y = True
def setUp(self):
super(ProgramDetailsPageA11yTest, self).setUp()

View File

@@ -6,7 +6,6 @@ progress page.
from contextlib import contextmanager
import ddt
import pytest
from ...fixtures.course import CourseFixture, XBlockFixtureDesc
from ...pages.common.logout import LogoutPage
@@ -276,15 +275,13 @@ class PersistentGradesTest(ProgressPageBaseTest):
self.assertEqual(self._get_section_score(), (0, 2))
class SubsectionGradingPolicyTest(ProgressPageBaseTest):
class SubsectionGradingPolicyBase(ProgressPageBaseTest):
"""
Tests changing a subsection's 'graded' field
and the effect it has on the progress page.
Base class for testing a subsection and its impact to
the progress page
"""
shard = 22
def setUp(self):
super(SubsectionGradingPolicyTest, self).setUp()
super(SubsectionGradingPolicyBase, self).setUp()
self._set_policy_for_subsection("Homework", 0)
self._set_policy_for_subsection("Lab", 1)
@@ -315,6 +312,41 @@ class SubsectionGradingPolicyTest(ProgressPageBaseTest):
self.assertEqual(sr_text, self.progress_page.x_tick_sr_text(index))
self.assertEqual([label, 'true' if label_hidden else None], self.progress_page.x_tick_label(index))
class SubsectionGradingPolicyTest(SubsectionGradingPolicyBase):
"""
Tests changing a subsection's 'graded' field
and the effect it has on the progress page.
"""
shard = 22
def test_subsection_grading_policy_on_progress_page(self):
with self._logged_in_session():
self._check_scores_and_page_text([(0, 1), (0, 1)], (0, 2), "Homework 1 - Test Subsection 1 - 0% (0/2)")
self.courseware_page.visit()
self._answer_problem_correctly()
self._check_scores_and_page_text([(1, 1), (0, 1)], (1, 2), "Homework 1 - Test Subsection 1 - 50% (1/2)")
self._set_policy_for_subsection("Not Graded")
with self._logged_in_session():
self.progress_page.visit()
self.assertEqual(self._get_problem_scores(), [(1, 1), (0, 1)])
self.assertEqual(self._get_section_score(), (1, 2))
self.assertFalse(self.progress_page.text_on_page("Homework 1 - Test Subsection 1"))
self._set_policy_for_subsection("Homework")
with self._logged_in_session():
self._check_scores_and_page_text([(1, 1), (0, 1)], (1, 2), "Homework 1 - Test Subsection 1 - 50% (1/2)")
class SubsectionGradingPolicyA11yTest(SubsectionGradingPolicyBase):
"""
Class to test the accessibility of subsection grading
"""
a11y = True
def test_axis_a11y(self):
"""
Tests that the progress chart axes have appropriate a11y (screenreader) markup.
@@ -326,6 +358,12 @@ class SubsectionGradingPolicyTest(ProgressPageBaseTest):
self.courseware_page.click_next_button_on_top()
# Answer the first Lab problem (unit only contains a single problem)
self._answer_problem_correctly()
self.progress_page.a11y_audit.config.set_rules({
"ignore": [
'aria-valid-attr', # TODO: LEARNER-6611 & LEARNER-6865
]
})
self.progress_page.visit()
# Verify the basic a11y of the progress page
@@ -398,32 +436,12 @@ class SubsectionGradingPolicyTest(ProgressPageBaseTest):
# second is the total text (including the sr-only text).
self.assertEqual(['Overall Score', 'Overall Score\n2%'], self.progress_page.graph_overall_score())
def test_subsection_grading_policy_on_progress_page(self):
with self._logged_in_session():
self._check_scores_and_page_text([(0, 1), (0, 1)], (0, 2), "Homework 1 - Test Subsection 1 - 0% (0/2)")
self.courseware_page.visit()
self._answer_problem_correctly()
self._check_scores_and_page_text([(1, 1), (0, 1)], (1, 2), "Homework 1 - Test Subsection 1 - 50% (1/2)")
self._set_policy_for_subsection("Not Graded")
with self._logged_in_session():
self.progress_page.visit()
self.assertEqual(self._get_problem_scores(), [(1, 1), (0, 1)])
self.assertEqual(self._get_section_score(), (1, 2))
self.assertFalse(self.progress_page.text_on_page("Homework 1 - Test Subsection 1"))
self._set_policy_for_subsection("Homework")
with self._logged_in_session():
self._check_scores_and_page_text([(1, 1), (0, 1)], (1, 2), "Homework 1 - Test Subsection 1 - 50% (1/2)")
@pytest.mark.a11y
class ProgressPageA11yTest(ProgressPageBaseTest):
"""
Class to test the accessibility of the progress page.
"""
a11y = True
def test_progress_page_a11y(self):
"""

View File

@@ -179,6 +179,7 @@ class AssetIndexTestStudioFrontend(StudioCourseTest):
class AssetIndexTestStudioFrontendPagination(StudioCourseTest):
"""Pagination tests for the Asset index page."""
shard = 23
def setUp(self, is_staff=False): # pylint: disable=arguments-differ
super(AssetIndexTestStudioFrontendPagination, self).setUp()

View File

@@ -13,6 +13,8 @@ class DiscussionComponentTest(ContainerBase):
Feature: CMS.Component Adding
As a course author, I want to be able to add and edit Discussion component
"""
shard = 14
def setUp(self, is_staff=True):
"""
Create a course with a section, subsection, and unit to which to add the component.

View File

@@ -11,6 +11,7 @@ class GradingPageTest(StudioCourseTest):
"""
Bockchoy tests to add/edit grade settings in studio.
"""
shard = 13
url = None
GRACE_FIELD_CSS = "#course-grading-graceperiod"

View File

@@ -14,6 +14,8 @@ class HTMLComponentEditorTests(ContainerBase):
Feature: CMS.Component Adding
As a course author, I want to be able to add and edit HTML component
"""
shard = 18
def setUp(self, is_staff=True):
"""
Create a course with a section, subsection, and unit to which to add the component.

View File

@@ -1870,6 +1870,7 @@ class SelfPacedOutlineTest(CourseOutlineTest):
class CourseStatusOutlineTest(CourseOutlineTest):
"""Test the course outline status section."""
shard = 6
def setUp(self):
super(CourseStatusOutlineTest, self).setUp()

View File

@@ -9,6 +9,7 @@ class PagesTest(StudioCourseTest):
"""
Test that Pages functionality is working properly on studio side
"""
shard = 23
def setUp(self, is_staff=True): # pylint: disable=arguments-differ
"""

View File

@@ -12,6 +12,8 @@ class TextbooksTest(StudioCourseTest):
"""
Test that textbook functionality is working properly on studio side
"""
shard = 8
def setUp(self, is_staff=True): # pylint: disable=arguments-differ
"""
Install a course with no content using a fixture.

View File

@@ -6,7 +6,6 @@ Acceptance tests for CMS Video Module.
import os
from unittest import skipIf
import pytest
from mock import patch
from bok_choy.promise import EmptyPromise
@@ -334,11 +333,11 @@ class CMSVideoTest(CMSVideoBaseTest):
self.video.click_player_button('play')
@pytest.mark.a11y
class CMSVideoA11yTest(CMSVideoBaseTest):
"""
CMS Video Accessibility Test Class
"""
a11y = True
def setUp(self):
browser = os.environ.get('SELENIUM_BROWSER', 'firefox')

View File

@@ -154,6 +154,7 @@ class VideoHLSEventsTest(VideoEventsTestMixin):
"""
Test video player event emission for HLS video
"""
shard = 16
def test_event_data_for_hls(self):
"""

View File

@@ -29,7 +29,7 @@ pipeline {
agent { label "jenkins-worker" }
options {
timestamps()
timeout(75)
timeout(60)
}
stages {
stage('Mark build as pending on Github') {
@@ -59,7 +59,7 @@ pipeline {
steps {
script {
def parallel_stages = [:]
for (int i = 1; i <= 22; i++) {
for (int i = 1; i <= 25; i++) {
int index = i
parallel_stages["${index}"] = {
node('jenkins-worker') {

View File

@@ -193,12 +193,12 @@ case "$TEST_SUITE" in
$TOX paver test_bokchoy $PAVER_ARGS
;;
[1-9]|1[0-9]|2[0-1])
[1-9]|1[0-9]|2[0-4])
$TOX paver test_bokchoy --eval-attr="shard==$SHARD and not a11y" $PAVER_ARGS
;;
22|"noshard")
$TOX paver test_bokchoy --eval-attr='not shard and not a11y' $PAVER_ARGS
25|"noshard")
$TOX paver test_bokchoy --eval-attr="(shard>=$SHARD or not shard) and not a11y" $PAVER_ARGS
;;
# Default case because if we later define another bok-choy shard on Jenkins