diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml new file mode 100644 index 0000000000..a9245bc9c1 --- /dev/null +++ b/.github/workflows/pylint-checks.yml @@ -0,0 +1,64 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + - open-release/lilac.master + +jobs: + run_pylint: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + include: + - module-name: lms-1 + path: "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/ lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/dashboard/ lms/djangoapps/debug/ lms/djangoapps/courseware/ lms/djangoapps/course_goals/ lms/djangoapps/rss_proxy/" + - module-name: lms-2 + path: "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/ lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/ lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py" + - module-name: openedx-1 + path: "openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/" + - module-name: openedx-2 + path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/" + - module-name: common + path: "common" + - module-name: cms + path: "cms" + + + name: pylint ${{ matrix.module-name }} + steps: + + - uses: actions/checkout@v2 + + - name: Install Required System Packages + run: sudo apt-get update && sudo apt-get install libxmlsec1-dev + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install Required Python Dependencies + run: | + pip install -r requirements/pip.txt + pip install -r requirements/edx/development.txt --src ${{ runner.temp }} + + - name: Run Quality Tests + run: | + pylint ${{ matrix.path }} diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml new file mode 100644 index 0000000000..ecaac2ac88 --- /dev/null +++ b/.github/workflows/quality-checks.yml @@ -0,0 +1,78 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + - open-release/lilac.master + +jobs: + run_tests: + name: Quality Others + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] + node-version: [ 12 ] + + steps: + + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Fetch master for comparison + run: git fetch --depth=1 origin master + + - name: Install Required System Packages + run: sudo apt-get update && sudo apt-get install libxmlsec1-dev + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install Required Python Dependencies + env: + PIP_SRC_DIR: ${{ runner.temp }} + run: | + pip install -r requirements/pip.txt + pip install -r requirements/edx/testing.txt -r requirements/edx/django.txt --src $PIP_SRC_DIR + + - name: Run Quality Tests + env: + TEST_SUITE: quality + SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh + SHARD: 4 + PIP_SRC_DIR: ${{ runner.temp }} + run: | + ./scripts/all-tests.sh + + - name: Save Job Artifacts + uses: actions/upload-artifact@v2 + with: + name: Build-Artifacts + path: | + **/reports/**/* + test_root/log/**/*.log + *.log diff --git a/openedx/tests/ci/__init__.py b/openedx/tests/ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/tests/ci/test_ci_matrix.py b/openedx/tests/ci/test_ci_matrix.py new file mode 100644 index 0000000000..a038d9a840 --- /dev/null +++ b/openedx/tests/ci/test_ci_matrix.py @@ -0,0 +1,41 @@ +""" +Since we have defined the test matrix manually in quality checks to achieve an optimized test build time, +this file is added to keep track if the ci matrices are up-to-date +""" +import os + +import yaml + +DIRS_TO_EXCLUDE = [ + 'lms/djangoapps', + 'lms/static', + 'lms/templates', + 'openedx/core', + 'openedx/core/djangoapps' +] + + +def valid_directory(path): + if path not in DIRS_TO_EXCLUDE and '__pycache__' not in path: + return True + + return False + + +def test_quality_matrix_is_complete(): + """ + This test should fail if a new python directory/file is added in code but quality checks don't cover it + """ + quality_ci_yml = f'{os.getcwd()}/.github/workflows/pylint-checks.yml' + with open(quality_ci_yml) as fp: + quality_yaml = yaml.safe_load(fp) + + matrix_dirs = [] + for matrix_item in quality_yaml['jobs']['run_pylint']['strategy']['matrix']['include']: + matrix_dirs.extend(matrix_item['path'].split(' ')) + + for module in ['lms', 'lms/djangoapps', 'openedx', 'openedx/core', 'openedx/core/djangoapps']: + directories = [f.path for f in os.scandir(module) if f.is_dir() and valid_directory(f.path)] + + for sub_dir in directories: + assert f'{sub_dir}/' in matrix_dirs, f'Please add {sub_dir} in quality matrix CI or exclude list' diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py index d56825cd99..4b670397d8 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs.py @@ -168,7 +168,11 @@ def python_prereqs_installation(): def pip_install_req_file(req_file): """Pip install the requirements file.""" pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - sh(f"{pip_cmd} -r {req_file}") + + if Env.PIP_SRC_DIR: + sh(f"{pip_cmd} -r {req_file} --src {Env.PIP_SRC_DIR}") + else: + sh(f"{pip_cmd} -r {req_file}") @task @@ -304,7 +308,10 @@ def install_python_prereqs(): files_to_fingerprint.append(sysconfig.get_python_lib()) # In a virtualenv, "-e installs" get put in a src directory. - src_dir = os.path.join(sys.prefix, "src") + if Env.PIP_SRC_DIR: + src_dir = Env.PIP_SRC_DIR + else: + src_dir = os.path.join(sys.prefix, "src") if os.path.isdir(src_dir): files_to_fingerprint.append(src_dir) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 8f7e528ddd..152160b5ab 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -222,6 +222,9 @@ class Env: # Directory for i18n test reports I18N_REPORT_DIR = REPORT_DIR / 'i18n' + # Directory for keeping src folder that comes with pip installation + PIP_SRC_DIR = os.environ.get("PIP_SRC_DIR") + # Service variant (lms, cms, etc.) configured with an environment variable # We use this to determine which envs.json file to load. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)