build: Quality on GH Actions (#28561)

.github/workflows/pylint-checks.yml

@@ -0,0 +1,64 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - open-release/lilac.master
+
+jobs:
+  run_pylint:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - module-name: lms-1
+            path: "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/ lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/dashboard/ lms/djangoapps/debug/ lms/djangoapps/courseware/ lms/djangoapps/course_goals/ lms/djangoapps/rss_proxy/"
+          - module-name: lms-2
+            path: "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/ lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/ lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py"
+          - module-name: openedx-1
+            path: "openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/"
+          - module-name: openedx-2
+            path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/"
+          - module-name: common
+            path: "common"
+          - module-name: cms
+            path: "cms"
+
+
+    name: pylint ${{ matrix.module-name }}
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: Install Required System Packages
+      run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Get pip cache dir
+      id: pip-cache-dir
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+
+    - name: Cache pip dependencies
+      id: cache-dependencies
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
+        restore-keys: ${{ runner.os }}-pip-
+
+    - name: Install Required Python Dependencies
+      run: |
+        pip install -r requirements/pip.txt
+        pip install -r requirements/edx/development.txt --src ${{ runner.temp }}
+
+    - name: Run Quality Tests
+      run: |
+        pylint  ${{ matrix.path }}

.github/workflows/quality-checks.yml

@@ -0,0 +1,78 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - open-release/lilac.master
+
+jobs:
+  run_tests:
+    name: Quality Others
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-20.04 ]
+        python-version: [ 3.8 ]
+        node-version: [ 12 ]
+
+    steps:
+
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+
+    - name: Fetch master for comparison
+      run: git fetch --depth=1 origin master
+
+    - name: Install Required System Packages
+      run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Setup Node
+      uses: actions/setup-node@v2
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Get pip cache dir
+      id: pip-cache-dir
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+
+    - name: Cache pip dependencies
+      id: cache-dependencies
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
+        restore-keys: ${{ runner.os }}-pip-
+
+    - name: Install Required Python Dependencies
+      env:
+        PIP_SRC_DIR: ${{ runner.temp }}
+      run: |
+        pip install -r requirements/pip.txt
+        pip install -r requirements/edx/testing.txt -r requirements/edx/django.txt --src $PIP_SRC_DIR
+
+    - name: Run Quality Tests
+      env:
+        TEST_SUITE: quality
+        SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
+        SHARD: 4
+        PIP_SRC_DIR: ${{ runner.temp }}
+      run: |
+        ./scripts/all-tests.sh
+
+    - name: Save Job Artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: Build-Artifacts
+        path: |
+          **/reports/**/*
+          test_root/log/**/*.log
+          *.log

openedx/tests/ci/test_ci_matrix.py

@@ -0,0 +1,41 @@
+"""
+Since we have defined the test matrix manually in quality checks to achieve an optimized test build time,
+this file is added to keep track if the ci matrices are up-to-date
+"""
+import os
+
+import yaml
+
+DIRS_TO_EXCLUDE = [
+    'lms/djangoapps',
+    'lms/static',
+    'lms/templates',
+    'openedx/core',
+    'openedx/core/djangoapps'
+]
+
+
+def valid_directory(path):
+    if path not in DIRS_TO_EXCLUDE and '__pycache__' not in path:
+        return True
+
+    return False
+
+
+def test_quality_matrix_is_complete():
+    """
+    This test should fail if a new python directory/file is added in code but quality checks don't cover it
+    """
+    quality_ci_yml = f'{os.getcwd()}/.github/workflows/pylint-checks.yml'
+    with open(quality_ci_yml) as fp:
+        quality_yaml = yaml.safe_load(fp)
+
+        matrix_dirs = []
+        for matrix_item in quality_yaml['jobs']['run_pylint']['strategy']['matrix']['include']:
+            matrix_dirs.extend(matrix_item['path'].split(' '))
+
+        for module in ['lms', 'lms/djangoapps', 'openedx', 'openedx/core', 'openedx/core/djangoapps']:
+            directories = [f.path for f in os.scandir(module) if f.is_dir() and valid_directory(f.path)]
+
+            for sub_dir in directories:
+                assert f'{sub_dir}/' in matrix_dirs, f'Please add {sub_dir} in quality matrix CI or exclude list'

pavelib/prereqs.py

@@ -168,7 +168,11 @@ def python_prereqs_installation():
 def pip_install_req_file(req_file):
     """Pip install the requirements file."""
     pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w'
-    sh(f"{pip_cmd} -r {req_file}")
+
+    if Env.PIP_SRC_DIR:
+        sh(f"{pip_cmd} -r {req_file} --src {Env.PIP_SRC_DIR}")
+    else:
+        sh(f"{pip_cmd} -r {req_file}")
 
 
 @task
@@ -304,7 +308,10 @@ def install_python_prereqs():
     files_to_fingerprint.append(sysconfig.get_python_lib())
 
     # In a virtualenv, "-e installs" get put in a src directory.
-    src_dir = os.path.join(sys.prefix, "src")
+    if Env.PIP_SRC_DIR:
+        src_dir = Env.PIP_SRC_DIR
+    else:
+        src_dir = os.path.join(sys.prefix, "src")
     if os.path.isdir(src_dir):
         files_to_fingerprint.append(src_dir)
 

pavelib/utils/envs.py

@@ -222,6 +222,9 @@ class Env:
     # Directory for i18n test reports
     I18N_REPORT_DIR = REPORT_DIR / 'i18n'
 
+    # Directory for keeping src folder that comes with pip installation
+    PIP_SRC_DIR = os.environ.get("PIP_SRC_DIR")
+
     # Service variant (lms, cms, etc.) configured with an environment variable
     # We use this to determine which envs.json file to load.
     SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)