diff --git a/.dockerignore b/.dockerignore index 1665783ba4..c3873d33a5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -102,7 +102,6 @@ common/test/data/badges/*.png ### Static assets pipeline artifacts **/*.scssc lms/static/css/ -!lms/static/css/vendor lms/static/certificates/css/ cms/static/css/ common/static/common/js/vendor/ diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml index 2a5429baff..7d217cd26f 100644 --- a/.github/actions/unit-tests/action.yml +++ b/.github/actions/unit-tests/action.yml @@ -1,5 +1,5 @@ -name: 'Run unit tests' -description: 'shared steps to run unit tests on both Github hosted and self hosted runners.' +name: "Run unit tests" +description: "shared steps to run unit tests on both Github hosted and self hosted runners." runs: using: "composite" steps: @@ -27,8 +27,9 @@ runs: - name: save pytest warnings json file if: success() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pytest-warnings-json path: | test_root/log/pytest_warnings*.json + overwrite: true diff --git a/.github/actions/verify-tests-count/action.yml b/.github/actions/verify-tests-count/action.yml index 11afb73671..d299f1ca11 100644 --- a/.github/actions/verify-tests-count/action.yml +++ b/.github/actions/verify-tests-count/action.yml @@ -1,5 +1,5 @@ -name: 'Verify unit tests count' -description: 'shared steps to verify unit tests count on both Github hosted and self hosted runners.' +name: "Verify unit tests count" +description: "shared steps to verify unit tests count on both Github hosted and self hosted runners." runs: using: "composite" steps: @@ -15,14 +15,12 @@ runs: echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV - - name: collect tests from GHA unit test shards shell: bash run: | echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - - name: add unit tests count shell: bash run: | diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c8b4a34532..4a01922edd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,9 +8,8 @@ Use conventional commits to separate and summarize commits logically: https://open-edx-proposals.readthedocs.io/en/latest/oep-0051-bp-conventional-commits.html Use this template as a guide. Omit sections that don't apply. -You may link to information rather than copy it, but only if the link is publicly -readable. If you must linked information must be private (because it has secrets), -clearly label the link as private. +You may link to information rather than copy it, but only if the link is publicly readable. +If the linked information must be private (because it contains secrets), clearly label the link as private. --> @@ -22,6 +21,7 @@ Design decisions and their rationales should be documented in the repo (docstrin linked here. Useful information to include: + - Which edX user roles will this change impact? Common user roles are "Learner", "Course Author", "Developer", and "Operator". - Include screenshots for changes to the UI (ideally, both "before" and "after" screenshots, if applicable). @@ -44,6 +44,7 @@ Please provide detailed step-by-step instructions for testing this change. ## Other information Include anything else that will help reviewers and consumers understand the change. + - Does this change depend on other changes elsewhere? - Any special concerns or limitations? For example: deprecations, migrations, security, or accessibility. - If your [database migration](https://openedx.atlassian.net/wiki/spaces/AC/pages/23003228/Everything+About+Database+Migrations) can't be rolled back easily. diff --git a/.github/renovate.json b/.github/renovate.json deleted file mode 100644 index e27406fabe..0000000000 --- a/.github/renovate.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": [ - "config:base", - "schedule:weekdays", - ":preserveSemverRanges" - ], - "prConcurrentLimit": 5, - "includePaths": [ - "package.json" - ] -} diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 0000000000..4365ab9be5 --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,42 @@ +// This file is written in "JSON5" (https://json5.org/) so that we can use comments. +{ + "extends": [ + "config:base", + "schedule:weekly", + ":automergeLinters", + ":automergeMinor", + ":automergeTesters", + ":enableVulnerabilityAlerts", + ":semanticCommits", + ":updateNotScheduled" + ], + "packageRules": [ + { + "matchDepTypes": [ + "devDependencies" + ], + "matchUpdateTypes": [ + "lockFileMaintenance", + "minor", + "patch", + "pin" + ], + "automerge": true + }, + { + "matchPackagePatterns": ["@edx", "@openedx"], + "matchUpdateTypes": ["minor", "patch"], + "automerge": true + } + ], + // When adding an ignoreDep, please include a reason and a public link that we can use to follow up and ensure + // that the ignoreDep is removed. + "ignoreDeps": [ + // Latest moment-timezone version broke the legacy programs dashboard, which is deprecated and soon to be removed. + // https://github.com/openedx/edx-platform/pull/34928" + "moment-timezone", + ], + "timezone": "America/New_York", + "prConcurrentLimit": 3, + "enabledManagers": ["npm"] +} diff --git a/.github/workflows/add-remove-label-on-comment.yml b/.github/workflows/add-remove-label-on-comment.yml index 0f369db7d2..a658064f09 100644 --- a/.github/workflows/add-remove-label-on-comment.yml +++ b/.github/workflows/add-remove-label-on-comment.yml @@ -17,4 +17,3 @@ on: jobs: add_remove_labels: uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master - diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index 51a4d5f24f..0e4595a9d0 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -34,13 +34,13 @@ jobs: echo "RELEVANT=true" >> "$GITHUB_ENV" fi - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ env.RELEVANT == 'true' }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 if: ${{ env.RELEVANT == 'true' }} with: - python-version: '3.8' + python-version: "3.8" - name: "Recompile requirements" if: ${{ env.RELEVANT == 'true' }} diff --git a/.github/workflows/check-for-tutorial-prs.yml b/.github/workflows/check-for-tutorial-prs.yml index 5d64a50573..dc8d8557e5 100644 --- a/.github/workflows/check-for-tutorial-prs.yml +++ b/.github/workflows/check-for-tutorial-prs.yml @@ -14,7 +14,7 @@ on: pull_request: types: [opened] paths: - - 'lms/templates/dashboard.html' + - "lms/templates/dashboard.html" jobs: # Provide helpful bot comment @@ -23,7 +23,7 @@ jobs: name: provide helpful bot comment steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Comment PR uses: thollander/actions-comment-pull-request@v2 @@ -32,4 +32,4 @@ jobs: Thank you for your pull request! Congratulations on completing the Open edX tutorial! A team member will be by to take a look shortly. To those watching community pull requests: No need to worry about this one, a tCRIL team member will be taking care of it. For this PR's author: If this is a PR that is NOT coming from the Open edX tutorial, please comment and let us know to disregard this message. - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check_python_dependencies.yml b/.github/workflows/check_python_dependencies.yml new file mode 100644 index 0000000000..85a4e796ce --- /dev/null +++ b/.github/workflows/check_python_dependencies.yml @@ -0,0 +1,40 @@ +name: Check Python Dependencies + +on: + pull_request: + +jobs: + check_dependencies: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.12"] + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install repo-tools + run: pip install edx-repo-tools[find_dependencies] + + - name: Install setuptool + run: pip install setuptools + + - name: Run Python script + run: | + find_python_dependencies \ + --req-file requirements/edx/base.txt \ + --req-file requirements/edx/testing.txt \ + --ignore https://github.com/edx/codejail-includes \ + --ignore https://github.com/edx/braze-client \ + --ignore https://github.com/edx/edx-name-affirmation \ + --ignore https://github.com/mitodl/edx-sga \ + --ignore https://github.com/edx/token-utils \ + --ignore https://github.com/open-craft/xblock-poll + diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index 2e40e09e0a..0ae7363601 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -9,13 +9,13 @@ jobs: strategy: matrix: python-version: - - '3.11' - os: ['ubuntu-20.04'] + - "3.11" + os: ["ubuntu-20.04"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -31,7 +31,7 @@ jobs: - name: Cache pip dependencies id: cache-dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml index f8f945545d..0ff99b9c68 100644 --- a/.github/workflows/compile-python-requirements.yml +++ b/.github/workflows/compile-python-requirements.yml @@ -4,9 +4,9 @@ on: workflow_dispatch: inputs: branch: - description: 'Target branch to create requirements PR against' + description: "Target branch to create requirements PR against" required: true - default: 'master' + default: "master" type: string defaults: @@ -19,12 +19,12 @@ jobs: steps: - name: Check out target branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "${{ inputs.branch }}" - name: Set up Python environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" @@ -44,7 +44,7 @@ jobs: - name: Make a PR id: make-pr - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: branch: "${{ github.triggering_actor }}/compile-python-deps" branch-suffix: short-commit-hash diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 978e616ee6..98a80f1da3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,8 +9,8 @@ jobs: # See also https://docs.docker.com/docker-hub/builds/ push: runs-on: ubuntu-latest - if: github.event_name == 'push' - + if: github.event_name == 'push' + strategy: matrix: variant: @@ -21,23 +21,22 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build and push lms base docker image env: DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run : make docker_tag_build_push_${{matrix.variant}} - \ No newline at end of file + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + run: make docker_tag_build_push_${{matrix.variant}} diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 8b7c137c8b..243f56262a 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -12,73 +12,73 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] - node-version: [ 18 ] + os: [ubuntu-20.04] + node-version: [18] python-version: - - '3.11' + - "3.11" steps: + - uses: actions/checkout@v4 + - name: Fetch master to compare coverage + run: git fetch --depth=1 origin master - - uses: actions/checkout@v2 - - name: Fetch master to compare coverage - run: git fetch --depth=1 origin master + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} + - name: Setup npm + run: npm i -g npm@10.5.x - - name: Setup npm - run: npm i -g npm@10.5.x + - name: Install Firefox 123.0 + run: | + sudo apt-get purge firefox + wget "https://ftp.mozilla.org/pub/firefox/releases/123.0/linux-x86_64/en-US/firefox-123.0.tar.bz2" + tar -xjf firefox-123.0.tar.bz2 + sudo mv firefox /opt/firefox + sudo ln -s /opt/firefox/firefox /usr/bin/firefox - - name: Install Firefox 123.0 - run: | - sudo apt-get purge firefox - wget "https://ftp.mozilla.org/pub/firefox/releases/123.0/linux-x86_64/en-US/firefox-123.0.tar.bz2" - tar -xjf firefox-123.0.tar.bz2 - sudo mv firefox /opt/firefox - sudo ln -s /opt/firefox/firefox /usr/bin/firefox + - name: Install Required System Packages + run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} + restore-keys: ${{ runner.os }}-pip- - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} - restore-keys: ${{ runner.os }}-pip- + - name: Install Required Python Dependencies + run: | + make base-requirements - - name: Install Required Python Dependencies - run: | - make base-requirements + - uses: c-hive/gha-npm-cache@v1 + - name: Run JS Tests + env: + TEST_SUITE: js-unit + SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh + run: | + npm install -g jest + xvfb-run --auto-servernum ./scripts/all-tests.sh - - uses: c-hive/gha-npm-cache@v1 - - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh - run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh - - - name: Save Job Artifacts - uses: actions/upload-artifact@v3 - with: - name: Build-Artifacts - path: | - reports/**/* - test_root/log/*.png - test_root/log/*.log - **/TEST-*.xml + - name: Save Job Artifacts + uses: actions/upload-artifact@v4 + with: + name: Build-Artifacts + path: | + reports/**/* + test_root/log/*.png + test_root/log/*.log + **/TEST-*.xml + overwrite: true diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index 243b44d203..24f241016b 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -7,19 +7,18 @@ on: - master jobs: - lint-imports: name: Lint Python Imports runs-on: ubuntu-20.04 steps: - name: Check out branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.11" - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev @@ -33,7 +32,7 @@ jobs: - name: Cache pip dependencies id: cache-dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} diff --git a/.github/workflows/lockfileversion-check.yml b/.github/workflows/lockfileversion-check.yml index 916dcb40d2..736f1f98de 100644 --- a/.github/workflows/lockfileversion-check.yml +++ b/.github/workflows/lockfileversion-check.yml @@ -5,7 +5,7 @@ name: Lockfile Version check on: push: branches: - - master + - master pull_request: jobs: diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 7375583e04..45919c8f65 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -13,9 +13,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ubuntu-20.04] python-version: - - '3.11' + - "3.11" # 'pinned' is used to install the latest patch version of Django # within the global constraint i.e. Django==4.2.8 in current case # because we have global constraint of Django<4.2 @@ -50,73 +50,73 @@ jobs: --health-timeout 5s --health-retries 3 steps: - - name: Setup mongodb user - run: | - mongosh edxapp --eval ' - db.createUser( - { - user: "edxapp", - pwd: "password", - roles: [ - { role: "readWrite", db: "edxapp" }, - ] - } - ); - ' + - name: Setup mongodb user + run: | + mongosh edxapp --eval ' + db.createUser( + { + user: "edxapp", + pwd: "password", + roles: [ + { role: "readWrite", db: "edxapp" }, + ] + } + ); + ' - - name: Verify mongo and mysql db credentials - run: | - mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp - mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + - name: Verify mongo and mysql db credentials + run: | + mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp + mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp - - name: Checkout repo - uses: actions/checkout@v2 + - name: Checkout repo + uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Install system Packages - run: | - sudo apt-get update - make ubuntu-requirements + - name: Install system Packages + run: | + sudo apt-get update + make ubuntu-requirements - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + restore-keys: ${{ runner.os }}-pip- - - name: Install Python dependencies - run: | - make dev-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi + - name: Install Python dependencies + run: | + make dev-requirements + if [[ "${{ matrix.django-version }}" != "pinned" ]]; then + pip install "django~=${{ matrix.django-version }}.0" + pip check # fail if this test-reqs/Django combination is broken + fi - - name: list installed package versions - run: | - sudo pip freeze + - name: list installed package versions + run: | + sudo pip freeze - - name: Run Tests - env: - LMS_CFG: lms/envs/minimal.yml - # This is from the LMS dir on purpose since we don't need anything different for the CMS yet. - STUDIO_CFG: lms/envs/minimal.yml - run: | - echo "Running the LMS migrations." - ./manage.py lms migrate - echo "Running the CMS migrations." - ./manage.py cms migrate + - name: Run Tests + env: + LMS_CFG: lms/envs/minimal.yml + # This is from the LMS dir on purpose since we don't need anything different for the CMS yet. + STUDIO_CFG: lms/envs/minimal.yml + run: | + echo "Running the LMS migrations." + ./manage.py lms migrate + echo "Running the CMS migrations." + ./manage.py cms migrate # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml index 20b238c238..0a9f50f6da 100644 --- a/.github/workflows/publish-ci-docker-image.yml +++ b/.github/workflows/publish-ci-docker-image.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 # This has to happen after checkout in order for gh to work. - name: "Cancel scheduled job on forks" @@ -41,4 +41,3 @@ jobs: run: | docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f scripts/ci-runner.Dockerfile . docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 3e3c87568c..90e457bf0b 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -31,13 +31,13 @@ jobs: name: pylint ${{ matrix.module-name }} steps: - name: Check out repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install required system packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 @@ -48,7 +48,7 @@ jobs: - name: Cache pip dependencies id: cache-dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache-dir.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 6bf16c9f0b..b9e2250cb1 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -13,70 +13,70 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ubuntu-20.04] python-version: - - '3.11' - node-version: [ 18 ] + - "3.11" + node-version: [18] steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 - - uses: actions/checkout@v2 - with: - fetch-depth: 2 + - name: Fetch base branch for comparison + run: git fetch --depth=1 origin ${{ github.base_ref }} - - name: Fetch base branch for comparison - run: git fetch --depth=1 origin ${{ github.base_ref }} + - name: Install Required System Packages + run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} + - name: Setup npm + run: npm i -g npm@8.5.x - - name: Setup npm - run: npm i -g npm@8.5.x + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} + restore-keys: ${{ runner.os }}-pip- - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- + - name: Install Required Python Dependencies + env: + PIP_SRC: ${{ runner.temp }} + run: | + make test-requirements - - name: Install Required Python Dependencies - env: - PIP_SRC: ${{ runner.temp }} - run: | - make test-requirements + - name: Run Quality Tests + env: + TEST_SUITE: quality + SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh + PIP_SRC: ${{ runner.temp }} + TARGET_BRANCH: ${{ github.base_ref }} + run: | + ./scripts/all-tests.sh - - name: Run Quality Tests - env: - TEST_SUITE: quality - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh - PIP_SRC: ${{ runner.temp }} - TARGET_BRANCH: ${{ github.base_ref }} - run: | - ./scripts/all-tests.sh - - - name: Save Job Artifacts - if: always() - uses: actions/upload-artifact@v3 - with: - name: Build-Artifacts - path: | - **/reports/**/* - test_root/log/**/*.log - *.log + - name: Save Job Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: Build-Artifacts + path: | + **/reports/**/* + test_root/log/**/*.log + *.log + overwrite: true diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index ec6b99e796..7f2b4925af 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -17,16 +17,16 @@ jobs: runs-on: "${{ matrix.os }}" strategy: matrix: - os: [ "ubuntu-20.04" ] + os: ["ubuntu-20.04"] python-version: - - '3.11' + - "3.11" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "${{ matrix.python-version }}" diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 704ff778a1..2e5b04bcc2 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -9,7 +9,7 @@ on: pull_request: push: branches: - - master + - master permissions: contents: read diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index e6837101aa..e6cb1a3b44 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -12,11 +12,11 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ubuntu-20.04] python-version: - - '3.11' - node-version: [ 18 ] - npm-version: [ 10.5.x ] + - "3.11" + node-version: [18] + npm-version: [10.5.x] mongo-version: - "7.0" @@ -34,73 +34,73 @@ jobs: --health-retries 3 steps: - - name: Checkout repo - uses: actions/checkout@v2 + - name: Checkout repo + uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install libxmlsec1-dev pkg-config + - name: Install system Packages + run: | + sudo apt-get update + sudo apt-get install libxmlsec1-dev pkg-config - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} - - name: Setup npm - run: npm i -g npm@${{ matrix.npm-version }} + - name: Setup npm + run: npm i -g npm@${{ matrix.npm-version }} - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} + restore-keys: ${{ runner.os }}-pip- - - name: Install Limited Python Deps for Build - run: | - pip install -r requirements/edx/assets.txt + - name: Install Limited Python Deps for Build + run: | + pip install -r requirements/edx/assets.txt - - name: Initiate Mongo DB Service - run: sudo systemctl start mongod + - name: Initiate Mongo DB Service + run: sudo systemctl start mongod - - name: Add node_modules bin to $Path - run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH + - name: Add node_modules bin to $Path + run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH - - name: Check Dev Assets Build - env: - COMPREHENSIVE_THEMES_DIR: ./themes - run: | - npm clean-install --dev - npm run build-dev + - name: Check Dev Assets Build + env: + COMPREHENSIVE_THEMES_DIR: ./themes + run: | + npm clean-install --dev + npm run build-dev - - name: Check Prod Assets Build - env: - COMPREHENSIVE_THEMES_DIR: ./themes - run: | - npm clean-install - npm run build + - name: Check Prod Assets Build + env: + COMPREHENSIVE_THEMES_DIR: ./themes + run: | + npm clean-install + npm run build - - name: Install Full Python Deps for Collection - run: | - pip install -r requirements/edx/base.txt -e . + - name: Install Full Python Deps for Collection + run: | + pip install -r requirements/edx/base.txt -e . - - name: Check Assets Collection - env: - LMS_CFG: lms/envs/minimal.yml - CMS_CFG: lms/envs/minimal.yml - DJANGO_SETTINGS_MODULE: lms.envs.production - run: | - ./manage.py lms collectstatic --noinput - ./manage.py cms collectstatic --noinput + - name: Check Assets Collection + env: + LMS_CFG: lms/envs/minimal.yml + CMS_CFG: lms/envs/minimal.yml + DJANGO_SETTINGS_MODULE: lms.envs.production + run: | + ./manage.py lms collectstatic --noinput + ./manage.py cms collectstatic --noinput diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index a96f6a9f67..8b6c1694da 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -239,7 +239,6 @@ "paths": [ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", - "cms/djangoapps/coursegraph/", "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/maintenance/", diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml deleted file mode 100644 index c7e8868f6b..0000000000 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: unit-tests-gh-hosted - -on: - pull_request: - push: - branches: - - master - - open-release/lilac.master - -jobs: - run-test: - name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: - - "3.11" - django-version: - - "pinned" - # When updating the shards, remember to make the same changes in - # .github/workflows/unit-tests.yml - shard_name: - - "lms-1" - - "lms-2" - - "lms-3" - - "lms-4" - - "lms-5" - - "lms-6" - - "openedx-1-with-lms" - - "openedx-2-with-lms" - - "openedx-1-with-cms" - - "openedx-2-with-cms" - - "cms-1" - - "cms-2" - - "common-with-lms" - - "common-with-cms" - - "xmodule-with-lms" - - "xmodule-with-cms" - mongo-version: - - "7.0" - - # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore) - # This code is left here as an example for future refernce in case we need to reduce the number of shards we're - # testing but still have good confidence with older versions of mongo. We use Mongo 4.4 in the example. - # - # exclude: - # - mongo-version: "4.4" - # include: - # - shard_name: "xmodule-with-cms" - # python-version: "3.11" - # django-version: "pinned" - # mongo-version: "4.4" - # - shard_name: "xmodule-with-lms" - # python-version: "3.11" - # django-version: "pinned" - # mongo-version: "4.4" - steps: - - uses: actions/checkout@v2 - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev lynx - - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.7.0 - with: - mongodb-version: ${{ matrix.mongo-version}} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - env: - PIP_SRC: ${{ runner.temp }} - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: Setup and run tests - uses: ./.github/actions/unit-tests - - collect-and-verify: - if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [ '3.11' ] - django-version: - - "pinned" - steps: - - uses: actions/checkout@v2 - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: verify unit tests count - uses: ./.github/actions/verify-tests-count diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b65e0bcd0b..c6513baa0f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -6,11 +6,16 @@ on: branches: - master +concurrency: + # We only need to be running tests for the latest commit on each PR + # (however, we fully test every commit on master, even as new ones land). + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + jobs: run-tests: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] + runs-on: ubuntu-20.04 strategy: matrix: python-version: @@ -56,14 +61,16 @@ jobs: # mongo-version: "4.4" steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* + - name: checkout repo + uses: actions/checkout@v4 + + - name: install system requirements + run: | + sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx - name: install mongo version run: | if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then - sudo apt-get purge -y "mongodb-org*" - sudo apt-get remove -y mongodb-org wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add - echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*" @@ -76,24 +83,21 @@ jobs: mongod & - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: checkout repo - uses: actions/checkout@v3 - - name: install requirements run: | - sudo make test-requirements + make test-requirements if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - sudo pip install "django~=${{ matrix.django-version }}.0" - sudo pip check # fail if this test-reqs/Django combination is broken + pip install "django~=${{ matrix.django-version }}.0" + pip check # fail if this test-reqs/Django combination is broken fi - name: list installed package versions run: | - sudo pip freeze + pip freeze - name: Setup and run tests uses: ./.github/actions/unit-tests @@ -103,20 +107,40 @@ jobs: mv reports/.coverage reports/${{ matrix.shard_name }}.coverage - name: Upload coverage - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage path: reports/${{matrix.shard_name}}.coverage + overwrite: true + + collect-and-verify: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: install system requirements + run: | + sudo apt-get update && sudo apt-get install libxmlsec1-dev + + - name: install requirements + run: | + make test-requirements + + - name: verify unit tests count + uses: ./.github/actions/verify-tests-count # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why # https://github.com/orgs/community/discussions/33579 success: name: Unit tests successful - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - needs: - - run-tests - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + if: always() + needs: [run-tests] steps: - name: Decide whether the needed jobs succeeded or failed # uses: re-actors/alls-green@v1.2.1 @@ -125,14 +149,12 @@ jobs: jobs: ${{ toJSON(needs) }} compile-warnings-report: - runs-on: [ edx-platform-runner ] - needs: [ run-tests ] + runs-on: ubuntu-20.04 + needs: [run-tests] steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: collect pytest warnings files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pytest-warnings-json path: test_root/log @@ -146,31 +168,33 @@ jobs: - name: save warning report if: success() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pytest-warning-report-html path: | reports/pytest_warnings/warning_report_all.html + overwrite: true # Combine and upload coverage reports. coverage: - needs: run-tests - runs-on: ubuntu-latest + if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) + runs-on: ubuntu-20.04 + needs: [run-tests] strategy: matrix: python-version: - 3.11 steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Download all artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: coverage path: reports @@ -184,4 +208,4 @@ jobs: coverage combine reports/* coverage report coverage xml - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index b87d27a884..cbf9da8f5c 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -13,14 +13,14 @@ jobs: strategy: matrix: python-version: - - '3.12' + - "3.12" steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index 0ab9bfb2da..f1b2b2c539 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -13,14 +13,14 @@ jobs: strategy: matrix: python-version: - - '3.12' + - "3.12" steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/update-geolite-database.yml b/.github/workflows/update-geolite-database.yml index 3289d31a22..08716b8ad9 100644 --- a/.github/workflows/update-geolite-database.yml +++ b/.github/workflows/update-geolite-database.yml @@ -6,18 +6,18 @@ on: workflow_dispatch: inputs: branch: - description: 'Target branch against which to create PR' + description: "Target branch against which to create PR" required: false - default: 'master' + default: "master" env: - MAXMIND_URL: 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz' - MAXMIND_SHA256_URL: 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz.sha256' - TAR_FILE_NAME: 'GeoLite2-Country.tar.gz' - TAR_SHA256_FILE_NAME: 'GeoLite2-Country.tar.gz.sha256' - TAR_UNZIPPED_ROOT_PATTERN: 'GeoLite2-Country_*' - DB_FILE: 'GeoLite2-Country.mmdb' - DB_DESTINATION_PATH: 'common/static/data/geoip' + MAXMIND_URL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz" + MAXMIND_SHA256_URL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz.sha256" + TAR_FILE_NAME: "GeoLite2-Country.tar.gz" + TAR_SHA256_FILE_NAME: "GeoLite2-Country.tar.gz.sha256" + TAR_UNZIPPED_ROOT_PATTERN: "GeoLite2-Country_*" + DB_FILE: "GeoLite2-Country.mmdb" + DB_DESTINATION_PATH: "common/static/data/geoip" jobs: download-and-replace: diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index a5d0aa9e48..6ca5dfcb35 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -4,22 +4,22 @@ on: workflow_dispatch: inputs: branch: - description: 'Target branch to create requirements PR against' + description: "Target branch to create requirements PR against" required: true - default: 'master' + default: "master" type: string package: - description: 'Name of package to upgrade' + description: "Name of package to upgrade" required: true type: string version: - description: 'Version number to upgrade to in constraints.txt (only needed if pinned)' - default: '' + description: "Version number to upgrade to in constraints.txt (only needed if pinned)" + default: "" type: string change_desc: description: | Description of change, for commit message and PR. (What does the new version add or fix?) - default: '' + default: "" type: string defaults: @@ -32,12 +32,12 @@ jobs: steps: - name: Check out target branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "${{ inputs.branch }}" - name: Set up Python environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index fcacf81929..ed6caf7590 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -2,26 +2,25 @@ name: Upgrade Requirements on: schedule: - - cron: "0 2 * * 2" + - cron: "0 2 * * 2" workflow_dispatch: - inputs: - branch: - description: 'Target branch to create requirements PR against' - required: true - default: 'master' + inputs: + branch: + description: "Target branch to create requirements PR against" + required: true + default: "master" jobs: call-upgrade-python-requirements-workflow: # Don't run the weekly upgrade job on forks -- it will send a weekly failure email. if: github.repository == 'openedx/edx-platform' || github.event_name != 'schedule' uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master with: - branch: ${{ github.event.inputs.branch }} - team_reviewers: "arbi-bom" - email_address: arbi-bom@edx.org - send_success_notification: false + branch: ${{ github.event.inputs.branch }} + team_reviewers: "arbi-bom" + email_address: arbi-bom@edx.org + send_success_notification: false secrets: - requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} - requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} - edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} - edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} - + requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} + requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} + edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} + edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml index aefc0f53b6..611fc0afc6 100644 --- a/.github/workflows/verify-dunder-init.yml +++ b/.github/workflows/verify-dunder-init.yml @@ -6,21 +6,18 @@ on: - master jobs: - verify_dunder_init: - name: Verify __init__.py Files runs-on: ubuntu-20.04 steps: + - name: Check out branch + uses: actions/checkout@v4 - - name: Check out branch - uses: actions/checkout@v2 + - name: Ensure git is installed + run: | + sudo apt-get update && sudo apt-get install git - - name: Ensure git is installed - run: | - sudo apt-get update && sudo apt-get install git - - - name: Verify __init__.py files exist - run: | - scripts/verify-dunder-init.sh + - name: Verify __init__.py files exist + run: | + scripts/verify-dunder-init.sh diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml deleted file mode 100644 index 1f6f91829c..0000000000 --- a/.github/workflows/verify-gha-unit-tests-count.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: verify unit tests count - -on: - pull_request: - push: - branches: - - master - -jobs: - collect-and-verify: - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] - steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - uses: actions/checkout@v2 - - name: install requirements - run: | - sudo make test-requirements - - - name: verify unit tests count - uses: ./.github/actions/verify-tests-count diff --git a/.gitignore b/.gitignore index 0d020e45fc..aaeb22eef1 100644 --- a/.gitignore +++ b/.gitignore @@ -102,11 +102,11 @@ bin/ ### Static assets pipeline artifacts *.scssc -lms/static/css/ -lms/static/certificates/css/ -cms/static/css/ -common/static/common/js/vendor/ -common/static/common/css/vendor/ +lms/static/css +lms/static/certificates/css +cms/static/css +common/static/common/js/vendor +common/static/common/css/vendor common/static/bundles webpack-stats.json @@ -115,7 +115,6 @@ lms/static/sass/*.css lms/static/sass/*.css.map lms/static/certificates/sass/*.css lms/static/themed_sass/ -cms/static/css/ cms/static/sass/*.css cms/static/sass/*.css.map cms/static/themed_sass/ diff --git a/Makefile b/Makefile index 6fc0192900..39b101907e 100644 --- a/Makefile +++ b/Makefile @@ -114,8 +114,8 @@ REQ_FILES = \ requirements/edx/base \ requirements/edx/doc \ requirements/edx/testing \ - requirements/edx/development \ requirements/edx/assets \ + requirements/edx/development \ requirements/edx/semgrep \ scripts/xblock/requirements \ scripts/user_retirement/requirements/base \ diff --git a/catalog-info.yaml b/catalog-info.yaml index 3a25eb3804..885b879833 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -11,6 +11,6 @@ metadata: title: "Documentation" icon: "Web" spec: - owner: group:2u-arch-bom + owner: group:wg-maintenance-edx-platform type: 'service' lifecycle: 'production' diff --git a/cms/djangoapps/contentstore/courseware_index.py b/cms/djangoapps/contentstore/courseware_index.py index dc28774c50..48647bf47b 100644 --- a/cms/djangoapps/contentstore/courseware_index.py +++ b/cms/djangoapps/contentstore/courseware_index.py @@ -278,7 +278,7 @@ class SearchIndexerBase(metaclass=ABCMeta): (Re)index all content within the given structure (course or library), tracking the fact that a full reindex has taken place """ - indexed_count = cls.index(modulestore, structure_key) + indexed_count = cls.index(modulestore, structure_key, timeout=180) if indexed_count: cls._track_index_request(cls.INDEX_EVENT['name'], cls.INDEX_EVENT['category'], indexed_count) return indexed_count diff --git a/cms/djangoapps/contentstore/rest_api/v0/serializers/__init__.py b/cms/djangoapps/contentstore/rest_api/v0/serializers/__init__.py index e90409ca86..33931a4a19 100644 --- a/cms/djangoapps/contentstore/rest_api/v0/serializers/__init__.py +++ b/cms/djangoapps/contentstore/rest_api/v0/serializers/__init__.py @@ -3,6 +3,7 @@ Serializers for v0 contentstore API. """ from .advanced_settings import AdvancedSettingsFieldSerializer, CourseAdvancedSettingsSerializer from .assets import AssetSerializer +from .authoring_grading import CourseGradingModelSerializer from .tabs import CourseTabSerializer, CourseTabUpdateSerializer, TabIDLocatorSerializer from .transcripts import TranscriptSerializer, YoutubeTranscriptCheckSerializer, YoutubeTranscriptUploadSerializer from .xblock import XblockSerializer diff --git a/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py b/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py new file mode 100644 index 0000000000..e3dd070573 --- /dev/null +++ b/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py @@ -0,0 +1,20 @@ +""" +API Serializers for course grading +""" + +from rest_framework import serializers + + +class GradersSerializer(serializers.Serializer): + """ Serializer for graders """ + type = serializers.CharField() + min_count = serializers.IntegerField() + drop_count = serializers.IntegerField() + short_label = serializers.CharField(required=False, allow_null=True, allow_blank=True) + weight = serializers.IntegerField() + id = serializers.IntegerField() + + +class CourseGradingModelSerializer(serializers.Serializer): + """ Serializer for course grading model data """ + graders = GradersSerializer(many=True, allow_null=True, allow_empty=True) diff --git a/cms/djangoapps/contentstore/rest_api/v0/urls.py b/cms/djangoapps/contentstore/rest_api/v0/urls.py index 0232854710..cc1e13b092 100644 --- a/cms/djangoapps/contentstore/rest_api/v0/urls.py +++ b/cms/djangoapps/contentstore/rest_api/v0/urls.py @@ -7,6 +7,7 @@ from openedx.core.constants import COURSE_ID_PATTERN from .views import ( AdvancedCourseSettingsView, + AuthoringGradingView, CourseTabSettingsView, CourseTabListView, CourseTabReorderView, @@ -46,8 +47,8 @@ urlpatterns = [ ), # Authoring API - re_path( - r'^heartbeat$', APIHeartBeatView.as_view(), name='heartbeat' + path( + 'heartbeat', APIHeartBeatView.as_view(), name='heartbeat' ), re_path( fr'^file_assets/{settings.COURSE_ID_PATTERN}$', @@ -61,6 +62,10 @@ urlpatterns = [ fr'^videos/encodings/{settings.COURSE_ID_PATTERN}$', authoring_videos.VideoEncodingsDownloadView.as_view(), name='cms_api_videos_encodings' ), + re_path( + fr'grading/{settings.COURSE_ID_PATTERN}', + AuthoringGradingView.as_view(), name='cms_api_update_grading' + ), path( 'videos/features', authoring_videos.VideoFeaturesView.as_view(), name='cms_api_videos_features' diff --git a/cms/djangoapps/contentstore/rest_api/v0/views/__init__.py b/cms/djangoapps/contentstore/rest_api/v0/views/__init__.py index 85b7b7d93b..00d22a1ea7 100644 --- a/cms/djangoapps/contentstore/rest_api/v0/views/__init__.py +++ b/cms/djangoapps/contentstore/rest_api/v0/views/__init__.py @@ -2,6 +2,7 @@ Views for v0 contentstore API. """ from .advanced_settings import AdvancedCourseSettingsView +from .authoring_grading import AuthoringGradingView from .tabs import CourseTabSettingsView, CourseTabListView, CourseTabReorderView from .transcripts import TranscriptView, YoutubeTranscriptCheckView, YoutubeTranscriptUploadView from .api_heartbeat import APIHeartBeatView diff --git a/cms/djangoapps/contentstore/rest_api/v0/views/authoring_grading.py b/cms/djangoapps/contentstore/rest_api/v0/views/authoring_grading.py new file mode 100644 index 0000000000..ef965bc3d4 --- /dev/null +++ b/cms/djangoapps/contentstore/rest_api/v0/views/authoring_grading.py @@ -0,0 +1,88 @@ +""" API Views for course advanced settings """ + +import edx_api_doc_tools as apidocs +from opaque_keys.edx.keys import CourseKey +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + +from cms.djangoapps.models.settings.course_grading import CourseGradingModel +from common.djangoapps.student.auth import has_studio_read_access +from openedx.core.djangoapps.credit.tasks import update_credit_course_requirements +from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, verify_course_exists, view_auth_classes +from ..serializers import CourseGradingModelSerializer + + +@view_auth_classes(is_authenticated=True) +class AuthoringGradingView(DeveloperErrorViewMixin, APIView): + """ + View for getting and setting the advanced settings for a course. + """ + @apidocs.schema( + body=CourseGradingModelSerializer, + parameters=[ + apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID"), + ], + responses={ + 200: CourseGradingModelSerializer, + 401: "The requester is not authenticated.", + 403: "The requester cannot access the specified course.", + 404: "The requested course does not exist.", + }, + ) + @verify_course_exists() + def post(self, request: Request, course_id: str): + """ + Update a course's grading. + + **Example Request** + + POST /api/contentstore/v0/course_grading/{course_id} + + **POST Parameters** + + The data sent for a post request should follow next object. + Here is an example request data that updates the ``course_grading`` + + ```json + { + "graders": [ + { + "type": "Homework", + "min_count": 1, + "drop_count": 0, + "short_label": "", + "weight": 100, + "id": 0 + } + ], + "grade_cutoffs": { + "A": 0.75, + "B": 0.63, + "C": 0.57, + "D": 0.5 + }, + "grace_period": { + "hours": 12, + "minutes": 0 + }, + "minimum_grade_credit": 0.7, + "is_credit_course": true + } + ``` + + **Response Values** + + If the request is successful, an HTTP 200 "OK" response is returned, + """ + course_key = CourseKey.from_string(course_id) + + if not has_studio_read_access(request.user, course_key): + self.permission_denied(request) + + if 'minimum_grade_credit' in request.data: + update_credit_course_requirements.delay(str(course_key)) + + updated_data = CourseGradingModel.update_from_json(course_key, request.data, request.user) + serializer = CourseGradingModelSerializer(updated_data) + return Response(serializer.data) diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index e0bc9fcc95..d756424bcc 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -123,9 +123,6 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= update_search_index, update_special_exams_and_publish ) - from cms.djangoapps.coursegraph.tasks import ( - dump_course_to_neo4j - ) # DEVELOPER README: probably all tasks here should use transaction.on_commit # to avoid stale data, but the tasks are owned by many teams and are often @@ -146,10 +143,6 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= # Push the course outline to learning_sequences asynchronously. update_outline_from_modulestore_task.delay(course_key_str) - if settings.COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: - # Push the course out to CourseGraph asynchronously. - dump_course_to_neo4j.delay(course_key_str) - # Kick off a courseware indexing action after the data is ready if CoursewareSearchIndexer.indexing_is_enabled() and CourseAboutSearchIndexer.indexing_is_enabled(): transaction.on_commit(lambda: update_search_index.delay(course_key_str, datetime.now(UTC).isoformat())) diff --git a/cms/djangoapps/contentstore/tests/test_course_create_rerun.py b/cms/djangoapps/contentstore/tests/test_course_create_rerun.py index d9023bfce2..a03e8e9cd2 100644 --- a/cms/djangoapps/contentstore/tests/test_course_create_rerun.py +++ b/cms/djangoapps/contentstore/tests/test_course_create_rerun.py @@ -104,10 +104,9 @@ class TestCourseListing(ModuleStoreTestCase): dest_course_key = CourseKey.from_string(data['destination_course_key']) self.assertEqual(dest_course_key.run, 'copy') - source_course = self.store.get_course(self.source_course_key) dest_course = self.store.get_course(dest_course_key) self.assertEqual(dest_course.start, CourseFields.start.default) - self.assertEqual(dest_course.end, source_course.end) + self.assertEqual(dest_course.end, None) self.assertEqual(dest_course.enrollment_start, None) self.assertEqual(dest_course.enrollment_end, None) course_orgs = get_course_organizations(dest_course_key) diff --git a/cms/djangoapps/contentstore/video_storage_handlers.py b/cms/djangoapps/contentstore/video_storage_handlers.py index 8d5e639e25..b84d7f0f7c 100644 --- a/cms/djangoapps/contentstore/video_storage_handlers.py +++ b/cms/djangoapps/contentstore/video_storage_handlers.py @@ -690,7 +690,10 @@ def _get_index_videos(course, pagination_conf=None): for attr in attrs: if attr == 'courses': current_course = [c for c in video['courses'] if course_id in c] - (__, values['course_video_image_url']), = list(current_course[0].items()) + if current_course: + values['course_video_image_url'] = current_course[0][course_id] + else: + values['course_video_image_url'] = None elif attr == 'encoded_videos': values['download_link'] = '' values['file_size'] = 0 diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index dce82809df..9f6cfb7c43 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -878,6 +878,7 @@ def _create_or_rerun_course(request): display_name = request.json.get('display_name') # force the start date for reruns and allow us to override start via the client start = request.json.get('start', CourseFields.start.default) + end = request.json.get('end', CourseFields.end.default) run = request.json.get('run') has_course_creator_role = is_content_creator(request.user, org) @@ -892,7 +893,7 @@ def _create_or_rerun_course(request): status=400 ) - fields = {'start': start} + fields = {'start': start, 'end': end} if display_name is not None: fields['display_name'] = display_name diff --git a/cms/djangoapps/coursegraph/README.rst b/cms/djangoapps/coursegraph/README.rst deleted file mode 100644 index 420eb8326e..0000000000 --- a/cms/djangoapps/coursegraph/README.rst +++ /dev/null @@ -1,120 +0,0 @@ - -CourseGraph Support -------------------- - -This app exists to write data to "CourseGraph", a tool enabling Open edX developers and support specialists to inspect their platform instance's learning content. CourseGraph itself is simply an instance of `Neo4j`_, which is an open-source graph database with a Web interface. - -.. _Neo4j: https://neo4j.com - -Deploying Coursegraph -===================== - -There are two ways to deploy CourseGraph: - -* For operators using Tutor, there is a `CourseGraph plugin for Tutor`_ that is currently released as "Beta". Nutmeg is the earliest Open edX release that the plugin will work alongside. - -* For operators still using the old Ansible installation pathway, there exists a `neo4j Ansible playbook`_. Be warned that this method is not well-documented nor officially supported. - -In order for CourseGraph to have queryable, up-to-date data, learning content from CMS must be written to CourseGraph regularly. That is where this Django app comes into play. For details on the various ways to write CMS data to CourseGraph, visit the `operations section of the CourseGraph Tutor plugin docs`_. - -**Please note**: Access to a populated CourseGraph instance confers access to all the learning content in the associated Open edX CMS (Studio). The basic authentication provided by Neo4j may or may not be sufficient for your security needs. Consider taking additional security measures, such as restricting CourseGraph access to only users on a private VPN. - -.. _neo4j Ansible playbook: https://github.com/openedx/configuration/blob/master/playbooks/neo4j.yml - -.. _CourseGraph plugin for Tutor: https://github.com/openedx/tutor-contrib-coursegraph/ - -.. _operations section of the CourseGraph Tutor plugin docs: https://github.com/openedx/tutor-contrib-coursegraph/#managing-data - -Running CourseGraph locally -=========================== - -In some circumstances, you may want to run CourseGraph locally, connected to a development-mode Open edX instance. You can do this in both Tutor and Devstack. - -Tutor -***** - -The `CourseGraph plugin for Tutor`_ makes it easy to install, configure, and run CourseGraph for local development. - -Devstack -******** - -CourseGraph is included as an "extra" component in the `Open edX Devstack`_. That is, it is not run or provisioned by default, but can be enabled on-demand. - -To provision Devstack CourseGraph with data from Devstack LMS, run:: - - make dev.provision.coursegraph - -CourseGraph should now be accessible at http://localhost:7474 with the username ``neo4j`` and the password ``edx``. - -Under the hood, the provisioning command just invokes ``dump_to_neo4j`` on your LMS, pointed at your CourseGraph. The provisioning command can be run again at any point in the future to refresh CourseGraph with new LMS data. The data in CourseGraph will persist unless you explicitly destroy it (as noted below). - -Other Devstack CourseGraph commands include:: - - make dev.up.coursegraph # Bring up the container (without re-provisioning). - make dev.down.coursegraph # Stop and remove the container. - make dev.shell.coursegraph # Start a shell session in the container. - make dev.attach.coursegraph # Attach to the container. - make dev.destroy.coursegraph # Stop the container and destroy its database. - -The above commands should be run in your ``devstack`` folder, and they assume that LMS is already properly provisioned. See the `Devstack interface`_ for more details. - -.. _Open edX Devstack: https://github.com/openedx/devstack/ -.. _Devstack interface: https://edx.readthedocs.io/projects/open-edx-devstack/en/latest/devstack_interface.html - - -Querying Coursegraph -==================== - -CourseGraph is queryable using the `Cypher`_ query language. Open edX learning content is represented in Neo4j using a straightforward scheme: - -* A node is an XBlock usage. - -* Nodes are tagged with their ``block_type``, such as: - - * ``course`` - * ``chapter`` - * ``sequential`` - * ``vertical`` - * ``problem`` - * ``html`` - * etc. - -* Every node is also tagged with ``item``. - -* Parent-child relationships in the course hierarchy are reflected in the ``PARENT_OF`` relationship. - -* Ordered sibling relationships in the course hierarchy are reflected in the ``PRECEDES`` relationship. - -* Fields on each XBlock usage (``.display_name``, ``.data``, etc) are available on the corresponding node. - -.. _Cypher: https://neo4j.com/developer/cypher/ - - -Example Queries -*************** - -How many XBlocks exist in the LMS, by type? :: - - MATCH - (c:course) -[:PARENT_OF*]-> (n:item) - RETURN - distinct(n.block_type) as block_type, - count(n) as number - order by - number DESC - - -In a given course, which units contain problems with custom Python grading code? :: - - MATCH - (c:course) -[:PARENT_OF*]-> (u:vertical) -[:PARENT_OF*]-> (p:problem) - WHERE - p.data CONTAINS 'loncapa/python' - AND - c.course_key = '' - RETURN - u.location - -You can see many more examples of useful CourseGraph queries on the `query archive wiki page`_. - -.. _query archive wiki page: https://openedx.atlassian.net/wiki/spaces/COMM/pages/3273228388/Useful+CourseGraph+Queries diff --git a/cms/djangoapps/coursegraph/__init__.py b/cms/djangoapps/coursegraph/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/coursegraph/admin.py b/cms/djangoapps/coursegraph/admin.py deleted file mode 100644 index f79fa909d2..0000000000 --- a/cms/djangoapps/coursegraph/admin.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Admin site bindings for coursegraph -""" -import logging - -from django.contrib import admin, messages -from django.utils.translation import gettext as _ -from edx_django_utils.admin.mixins import ReadOnlyAdminMixin - -from .models import CourseGraphCourseDump -from .tasks import ModuleStoreSerializer - -log = logging.getLogger(__name__) - - -@admin.action( - permissions=['change'], - description=_("Dump courses to CourseGraph (respect cache)"), -) -def dump_courses(modeladmin, request, queryset): - """ - Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses, - excluding courses that haven't been published since they were last dumped. - - queryset is a QuerySet of CourseGraphCourseDump objects, which are just - CourseOverview objects under the hood. - """ - all_course_keys = queryset.values_list('id', flat=True) - serializer = ModuleStoreSerializer(all_course_keys) - try: - submitted, skipped = serializer.dump_courses_to_neo4j() - # Unfortunately there is no unified base class for the reasonable - # exceptions we could expect from py2neo (connection unavailable, bolt protocol - # error, and so on), so we just catch broadly, show a generic error banner, - # and then log the exception for site operators to look at. - except Exception as err: # pylint: disable=broad-except - log.exception( - "Failed to enqueue CourseGraph dumps to Neo4j (respecting cache): %s", - ", ".join(str(course_key) for course_key in all_course_keys), - ) - modeladmin.message_user( - request, - _("Error enqueueing dumps for {} course(s): {}").format( - len(all_course_keys), str(err) - ), - level=messages.ERROR, - ) - return - if submitted: - modeladmin.message_user( - request, - _( - "Enqueued dumps for {} course(s). Skipped {} unchanged course(s)." - ).format(len(submitted), len(skipped)), - level=messages.SUCCESS, - ) - else: - modeladmin.message_user( - request, - _( - "Skipped all {} course(s), as they were unchanged.", - ).format(len(skipped)), - level=messages.WARNING, - ) - - -@admin.action( - permissions=['change'], - description=_("Dump courses to CourseGraph (override cache)") -) -def dump_courses_overriding_cache(modeladmin, request, queryset): - """ - Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses - (whether or not they have been published recently). - - queryset is a QuerySet of CourseGraphCourseDump objects, which are just - CourseOverview objects under the hood. - """ - all_course_keys = queryset.values_list('id', flat=True) - serializer = ModuleStoreSerializer(all_course_keys) - try: - submitted, _skipped = serializer.dump_courses_to_neo4j(override_cache=True) - # Unfortunately there is no unified base class for the reasonable - # exceptions we could expect from py2neo (connection unavailable, bolt protocol - # error, and so on), so we just catch broadly, show a generic error banner, - # and then log the exception for site operators to look at. - except Exception as err: # pylint: disable=broad-except - log.exception( - "Failed to enqueue CourseGraph Neo4j course dumps (overriding cache): %s", - ", ".join(str(course_key) for course_key in all_course_keys), - ) - modeladmin.message_user( - request, - _("Error enqueueing dumps for {} course(s): {}").format( - len(all_course_keys), str(err) - ), - level=messages.ERROR, - ) - return - modeladmin.message_user( - request, - _("Enqueued dumps for {} course(s).").format(len(submitted)), - level=messages.SUCCESS, - ) - - -@admin.register(CourseGraphCourseDump) -class CourseGraphCourseDumpAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): - """ - Model admin for "Course graph course dumps". - - Just a read-only table with some useful metadata, allowing admin users to - select courses to be dumped to CourseGraph. - """ - list_display = [ - 'id', - 'display_name', - 'modified', - 'enrollment_start', - 'enrollment_end', - ] - search_fields = ['id', 'display_name'] - actions = [dump_courses, dump_courses_overriding_cache] diff --git a/cms/djangoapps/coursegraph/apps.py b/cms/djangoapps/coursegraph/apps.py deleted file mode 100644 index 71ae91ad49..0000000000 --- a/cms/djangoapps/coursegraph/apps.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Coursegraph Application Configuration - -Signal handlers are connected here. -""" -import warnings - -from django.apps import AppConfig - - -class CoursegraphConfig(AppConfig): - """ - AppConfig for courseware app - """ - name = 'cms.djangoapps.coursegraph' - - from cms.djangoapps.coursegraph import tasks - - def ready(self) -> None: - warnings.warn( - "Neo4j support is going to be dropped after Sumac release," - "to read more here is a github issue https://github.com/openedx/edx-platform/issues/34342", - DeprecationWarning, - stacklevel=2 - ) diff --git a/cms/djangoapps/coursegraph/management/__init__.py b/cms/djangoapps/coursegraph/management/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/coursegraph/management/commands/__init__.py b/cms/djangoapps/coursegraph/management/commands/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py deleted file mode 100644 index 40afe7ffbe..0000000000 --- a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -This file contains a management command for exporting the modulestore to -Neo4j, a graph database. - -Example usages: - - # Dump all courses published since last dump. - # Use connection parameters from `settings.COURSEGRAPH_SETTINGS`. - python manage.py cms dump_to_neo4j - - # Dump all courses published since last dump. - # Use custom connection parameters. - python manage.py cms dump_to_neo4j --host localhost --port 7473 \ - --secure --user user --password password - - # Specify certain courses instead of dumping all of them. - # Use connection parameters from `settings.COURSEGRAPH_SETTINGS`. - python manage.py cms dump_to_neo4j --courses 'course-v1:A+B+1' 'course-v1:A+B+2' -""" - - -import logging -from textwrap import dedent - -from django.core.management.base import BaseCommand - -from cms.djangoapps.coursegraph.tasks import ModuleStoreSerializer - -log = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Dump recently-published course(s) over to a CourseGraph (Neo4j) instance. - """ - help = dedent(__doc__).strip() - - def add_arguments(self, parser): - parser.add_argument( - '--host', - type=str, - help="the hostname of the Neo4j server", - ) - parser.add_argument( - '--port', - type=int, - help="the port on the Neo4j server that accepts Bolt requests", - ) - parser.add_argument( - '--secure', - action='store_true', - help="connect to server over Bolt/TLS instead of plain unencrypted Bolt", - ) - parser.add_argument( - '--user', - type=str, - help="the username of the Neo4j user", - ) - parser.add_argument( - '--password', - type=str, - help="the password of the Neo4j user", - ) - parser.add_argument( - '--courses', - metavar='KEY', - type=str, - nargs='*', - help="keys of courses to serialize; if omitted all courses in system are serialized", - ) - parser.add_argument( - '--skip', - metavar='KEY', - type=str, - nargs='*', - help="keys of courses to NOT to serialize", - ) - parser.add_argument( - '--override', - action='store_true', - help="dump all courses regardless of when they were last published", - ) - - def handle(self, *args, **options): - """ - Iterates through each course, serializes them into graphs, and saves - those graphs to neo4j. - """ - - mss = ModuleStoreSerializer.create(options['courses'], options['skip']) - connection_overrides = { - key: options[key] - for key in ["host", "port", "secure", "user", "password"] - } - submitted_courses, skipped_courses = mss.dump_courses_to_neo4j( - connection_overrides=connection_overrides, - override_cache=options['override'], - ) - - log.info( - "%d courses submitted for export to neo4j. %d courses skipped.", - len(submitted_courses), - len(skipped_courses), - ) - - if not submitted_courses: - print("No courses submitted for export to neo4j at all!") - return - - if submitted_courses: - print( - "These courses were submitted for export to neo4j successfully:\n\t" + - "\n\t".join(submitted_courses) - ) diff --git a/cms/djangoapps/coursegraph/management/commands/tests/__init__.py b/cms/djangoapps/coursegraph/management/commands/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py deleted file mode 100644 index 24595098d3..0000000000 --- a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ /dev/null @@ -1,596 +0,0 @@ -""" -Tests for the dump_to_neo4j management command. -""" - - -from datetime import datetime - -from unittest import mock -import ddt -from django.core.management import call_command -from django.test.utils import override_settings -from edx_toggles.toggles.testutils import override_waffle_switch -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory - -import openedx.core.djangoapps.content.block_structure.config as block_structure_config -from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish -from cms.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer -from cms.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher -from cms.djangoapps.coursegraph.tasks import ( - coerce_types, - serialize_course, - serialize_item, - should_dump_course, - strip_branch_and_version -) -from openedx.core.djangolib.testing.utils import skip_unless_lms - - -class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): - """ - Base class for the test suites in this file. Sets up a couple courses. - """ - @classmethod - def setUpClass(cls): - r""" - Creates two courses; one that's just a course block, and one that - looks like: - course - | - chapter - | - sequential - | - vertical - / | \ \ - / | \ ---------- - / | \ \ - / | --- \ - / | \ \ - html -> problem -> video -> video2 - - The side-pointing arrows (->) are PRECEDES relationships; the more - vertical lines are PARENT_OF relationships. - - The vertical in this course and the first video have the same - display_name, so that their block_ids are the same. This is to - test for a bug where xblocks with the same block_ids (but different - locations) pointed to themselves erroneously. - """ - super().setUpClass() - cls.course = CourseFactory.create() - cls.chapter = BlockFactory.create(parent=cls.course, category='chapter') - cls.sequential = BlockFactory.create(parent=cls.chapter, category='sequential') - cls.vertical = BlockFactory.create(parent=cls.sequential, category='vertical', display_name='subject') - cls.html = BlockFactory.create(parent=cls.vertical, category='html') - cls.problem = BlockFactory.create(parent=cls.vertical, category='problem') - cls.video = BlockFactory.create(parent=cls.vertical, category='video', display_name='subject') - cls.video2 = BlockFactory.create(parent=cls.vertical, category='video') - - cls.course2 = CourseFactory.create() - - cls.course_strings = [str(cls.course.id), str(cls.course2.id)] - - @staticmethod - def setup_mock_graph(mock_matcher_class, mock_graph_class, transaction_errors=False): - """ - Replaces the py2neo Graph object with a MockGraph; similarly replaces - NodeMatcher with MockNodeMatcher. - - Arguments: - mock_matcher_class: a mocked NodeMatcher class - mock_graph_class: a mocked Graph class - transaction_errors: a bool for whether we should get errors - when transactions try to commit - - Returns: an instance of MockGraph - """ - - mock_graph = MockGraph(transaction_errors=transaction_errors) - mock_graph_class.return_value = mock_graph - - mock_node_matcher = MockNodeMatcher(mock_graph) - mock_matcher_class.return_value = mock_node_matcher - return mock_graph - - def assertCourseDump(self, mock_graph, number_of_courses, number_commits, number_rollbacks): - """ - Asserts that we have the expected number of courses, commits, and - rollbacks after we dump the modulestore to neo4j - Arguments: - mock_graph: a MockGraph backend - number_of_courses: number of courses we expect to find - number_commits: number of commits we expect against the graph - number_rollbacks: number of commit rollbacks we expect - """ - courses = {node['course_key'] for node in mock_graph.nodes} - assert len(courses) == number_of_courses - assert mock_graph.number_commits == number_commits - assert mock_graph.number_rollbacks == number_rollbacks - - -@ddt.ddt -class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): - """ - Tests for the dump to neo4j management command - """ - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') - @ddt.data(1, 2) - def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_matcher_class): - """ - Test that you can specify which courses you want to dump. - """ - mock_graph = self.setup_mock_graph(mock_matcher_class, mock_graph_class) - - call_command( - 'dump_to_neo4j', - courses=self.course_strings[:number_of_courses], - host='mock_host', - port=7687, - user='mock_user', - password='mock_password', - ) - - self.assertCourseDump( - mock_graph, - number_of_courses=number_of_courses, - number_commits=number_of_courses, - number_rollbacks=0 - ) - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') - def test_dump_skip_course(self, mock_graph_class, mock_matcher_class): - """ - Test that you can skip courses. - """ - mock_graph = self.setup_mock_graph( - mock_matcher_class, mock_graph_class - ) - - call_command( - 'dump_to_neo4j', - skip=self.course_strings[:1], - host='mock_host', - port=7687, - user='mock_user', - password='mock_password', - ) - - self.assertCourseDump( - mock_graph, - number_of_courses=1, - number_commits=1, - number_rollbacks=0, - ) - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') - def test_dump_skip_beats_specifying(self, mock_graph_class, mock_matcher_class): - """ - Test that if you skip and specify the same course, you'll skip it. - """ - mock_graph = self.setup_mock_graph( - mock_matcher_class, mock_graph_class - ) - - call_command( - 'dump_to_neo4j', - skip=self.course_strings[:1], - courses=self.course_strings[:1], - host='mock_host', - port=7687, - user='mock_user', - password='mock_password', - ) - - self.assertCourseDump( - mock_graph, - number_of_courses=0, - number_commits=0, - number_rollbacks=0, - ) - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') - def test_dump_all_courses(self, mock_graph_class, mock_matcher_class): - """ - Test if you don't specify which courses to dump, then you'll dump - all of them. - """ - mock_graph = self.setup_mock_graph( - mock_matcher_class, mock_graph_class - ) - - call_command( - 'dump_to_neo4j', - host='mock_host', - port=7687, - user='mock_user', - password='mock_password' - ) - - self.assertCourseDump( - mock_graph, - number_of_courses=2, - number_commits=2, - number_rollbacks=0, - ) - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.Graph', autospec=True) - @override_settings( - COURSEGRAPH_CONNECTION=dict( - protocol='bolt', - host='coursegraph.example.edu', - port=7777, - secure=True, - user="neo4j", - password="default-password", - ) - ) - def test_dump_to_neo4j_connection_defaults(self, mock_graph_class, mock_matcher_class): - """ - Test that user can override individual settings.COURSEGRAPH_CONNECTION parameters - by passing them to `dump_to_neo4j`, whilst falling back to the ones that they - don't override. - """ - self.setup_mock_graph( - mock_matcher_class, mock_graph_class - ) - call_command( - 'dump_to_neo4j', - courses=self.course_strings[:1], - port=7788, - secure=False, - password="overridden-password", - ) - assert mock_graph_class.call_args.args == () - assert mock_graph_class.call_args.kwargs == dict( - - # From settings: - protocol='bolt', - host='coursegraph.example.edu', - user="neo4j", - - # Overriden by command: - port=7788, - secure=False, - password="overridden-password", - ) - - -class SomeThing: - """Just to test the stringification of an object.""" - def __str__(self): - return "" - - -@skip_unless_lms -@ddt.ddt -class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): - """ - Tests for the ModuleStoreSerializer - """ - @classmethod - def setUpClass(cls): - """Any ModuleStore course/content operations can go here.""" - super().setUpClass() - cls.mss = ModuleStoreSerializer.create() - - def test_serialize_item(self): - """ - Tests the serialize_item method. - """ - fields, label = serialize_item(self.course) - assert label == 'course' - assert 'edited_on' in list(fields.keys()) - assert 'display_name' in list(fields.keys()) - assert 'org' in list(fields.keys()) - assert 'course' in list(fields.keys()) - assert 'run' in list(fields.keys()) - assert 'course_key' in list(fields.keys()) - assert 'location' in list(fields.keys()) - assert 'block_type' in list(fields.keys()) - assert 'detached' in list(fields.keys()) - assert 'checklist' not in list(fields.keys()) - - def test_serialize_course(self): - """ - Tests the serialize_course method. - """ - nodes, relationships = serialize_course(self.course.id) - assert len(nodes) == 9 - # the course has 7 "PARENT_OF" relationships and 3 "PRECEDES" - assert len(relationships) == 10 - - def test_strip_version_and_branch(self): - """ - Tests that the _strip_version_and_branch function strips the version - and branch from a location - """ - location = self.course.id.make_usage_key( - 'test_block_type', 'test_block_id' - ).for_branch( - 'test_branch' - ).for_version(b'test_version') - - assert location.branch is not None - assert location.version_guid is not None - - stripped_location = strip_branch_and_version(location) - - assert stripped_location.branch is None - assert stripped_location.version_guid is None - - @staticmethod - def _extract_relationship_pairs(relationships, relationship_type): - """ - Extracts a list of XBlock location tuples from a list of Relationships. - - Arguments: - relationships: list of py2neo `Relationship` objects - relationship_type: the type of relationship to filter `relationships` - by. - Returns: - List of tuples of the locations of of the relationships' - constituent nodes. - """ - relationship_pairs = [ - (rel.start_node["location"], rel.end_node["location"]) - for rel in relationships if type(rel).__name__ == relationship_type - ] - return relationship_pairs - - @staticmethod - def _extract_location_pair(xblock1, xblock2): - """ - Returns a tuple of locations from two XBlocks. - - Arguments: - xblock1: an xblock - xblock2: also an xblock - - Returns: - A tuple of the string representations of those XBlocks' locations. - """ - return (str(xblock1.location), str(xblock2.location)) - - def assertBlockPairIsRelationship(self, xblock1, xblock2, relationships, relationship_type): - """ - Helper assertion that a pair of xblocks have a certain kind of - relationship with one another. - """ - relationship_pairs = self._extract_relationship_pairs(relationships, relationship_type) - location_pair = self._extract_location_pair(xblock1, xblock2) - assert location_pair in relationship_pairs - - def assertBlockPairIsNotRelationship(self, xblock1, xblock2, relationships, relationship_type): - """ - The opposite of `assertBlockPairIsRelationship`: asserts that a pair - of xblocks do NOT have a certain kind of relationship. - """ - relationship_pairs = self._extract_relationship_pairs(relationships, relationship_type) - location_pair = self._extract_location_pair(xblock1, xblock2) - assert location_pair not in relationship_pairs - - def test_precedes_relationship(self): - """ - Tests that two nodes that should have a precedes relationship have it. - """ - __, relationships = serialize_course(self.course.id) - self.assertBlockPairIsRelationship(self.video, self.video2, relationships, "PRECEDES") - self.assertBlockPairIsNotRelationship(self.video2, self.video, relationships, "PRECEDES") - self.assertBlockPairIsNotRelationship(self.vertical, self.video, relationships, "PRECEDES") - self.assertBlockPairIsNotRelationship(self.html, self.video, relationships, "PRECEDES") - - def test_parent_relationship(self): - """ - Test that two nodes that should have a parent_of relationship have it. - """ - __, relationships = serialize_course(self.course.id) - self.assertBlockPairIsRelationship(self.vertical, self.video, relationships, "PARENT_OF") - self.assertBlockPairIsRelationship(self.vertical, self.html, relationships, "PARENT_OF") - self.assertBlockPairIsRelationship(self.course, self.chapter, relationships, "PARENT_OF") - self.assertBlockPairIsNotRelationship(self.course, self.video, relationships, "PARENT_OF") - self.assertBlockPairIsNotRelationship(self.video, self.vertical, relationships, "PARENT_OF") - self.assertBlockPairIsNotRelationship(self.video, self.html, relationships, "PARENT_OF") - - def test_nodes_have_indices(self): - """ - Test that we add index values on nodes - """ - nodes, relationships = serialize_course(self.course.id) # lint-amnesty, pylint: disable=unused-variable - - # the html node should have 0 index, and the problem should have 1 - html_nodes = [node for node in nodes if node['block_type'] == 'html'] - assert len(html_nodes) == 1 - problem_nodes = [node for node in nodes if node['block_type'] == 'problem'] - assert len(problem_nodes) == 1 - html_node = html_nodes[0] - problem_node = problem_nodes[0] - - assert html_node['index'] == 0 - assert problem_node['index'] == 1 - - @ddt.data( - (1, 1), - (SomeThing(), ""), - (1.5, 1.5), - ("úñîçø∂é", "úñîçø∂é"), - (b"plain string", b"plain string"), - (True, True), - (None, "None"), - ((1,), "(1,)"), - # list of elements should be coerced into a list of the - # string representations of those elements - ([SomeThing(), SomeThing()], ["", ""]), - ([1, 2], ["1", "2"]), - ) - @ddt.unpack - def test_coerce_types(self, original_value, coerced_expected): - """ - Tests the coerce_types helper - """ - coerced_value = coerce_types(original_value) - assert coerced_value == coerced_expected - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - def test_dump_to_neo4j(self, mock_graph_constructor, mock_matcher_class): - """ - Tests the dump_to_neo4j method works against a mock - py2neo Graph - """ - mock_graph = MockGraph() - mock_graph_constructor.return_value = mock_graph - mock_matcher_class.return_value = MockNodeMatcher(mock_graph) - # mocking is thorwing error in kombu serialzier and its not require here any more. - credentials = {} - - submitted, skipped = self.mss.dump_courses_to_neo4j(credentials) # lint-amnesty, pylint: disable=unused-variable - - self.assertCourseDump( - mock_graph, - number_of_courses=2, - number_commits=2, - number_rollbacks=0, - ) - - # 9 nodes + 7 relationships from the first course - # 2 nodes and no relationships from the second - - assert len(mock_graph.nodes) == 11 - self.assertCountEqual(submitted, self.course_strings) - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - def test_dump_to_neo4j_rollback(self, mock_graph_constructor, mock_matcher_class): - """ - Tests that the the dump_to_neo4j method handles the case where there's - an exception trying to write to the neo4j database. - """ - mock_graph = MockGraph(transaction_errors=True) - mock_graph_constructor.return_value = mock_graph - mock_matcher_class.return_value = MockNodeMatcher(mock_graph) - # mocking is thorwing error in kombu serialzier and its not require here any more. - credentials = {} - - submitted, skipped = self.mss.dump_courses_to_neo4j(credentials) # lint-amnesty, pylint: disable=unused-variable - - self.assertCourseDump( - mock_graph, - number_of_courses=0, - number_commits=0, - number_rollbacks=2, - ) - - self.assertCountEqual(submitted, self.course_strings) - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - @ddt.data((True, 2), (False, 0)) - @ddt.unpack - def test_dump_to_neo4j_cache( - self, - override_cache, - expected_number_courses, - mock_graph_constructor, - mock_matcher_class, - ): - """ - Tests the caching mechanism and override to make sure we only publish - recently updated courses. - """ - mock_graph = MockGraph() - mock_graph_constructor.return_value = mock_graph - mock_matcher_class.return_value = MockNodeMatcher(mock_graph) - # mocking is thorwing error in kombu serialzier and its not require here any more. - credentials = {} - - # run once to warm the cache - self.mss.dump_courses_to_neo4j( - credentials, override_cache=override_cache - ) - - # when run the second time, only dump courses if the cache override - # is enabled - submitted, __ = self.mss.dump_courses_to_neo4j( - credentials, override_cache=override_cache - ) - assert len(submitted) == expected_number_courses - - @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_matcher_class): - """ - Tests that we only dump those courses that have been published after - the last time the command was been run. - """ - mock_graph = MockGraph() - mock_graph_constructor.return_value = mock_graph - mock_matcher_class.return_value = MockNodeMatcher(mock_graph) - # mocking is thorwing error in kombu serialzier and its not require here any more. - credentials = {} - - # run once to warm the cache - submitted, skipped = self.mss.dump_courses_to_neo4j(credentials) # lint-amnesty, pylint: disable=unused-variable - assert len(submitted) == len(self.course_strings) - - # simulate one of the courses being published - with override_waffle_switch(block_structure_config.STORAGE_BACKING_FOR_CACHE, True): - update_block_structure_on_course_publish(None, self.course.id) - - # make sure only the published course was dumped - submitted, __ = self.mss.dump_courses_to_neo4j(credentials) - assert len(submitted) == 1 - assert submitted[0] == str(self.course.id) - - @mock.patch('cms.djangoapps.coursegraph.tasks.get_course_last_published') - @mock.patch('cms.djangoapps.coursegraph.tasks.get_command_last_run') - @ddt.data( - ( - str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), - (True, ( - 'course has been published since last neo4j update time - ' - 'update date 2016-03-30 00:00:00 < published date 2016-03-31 00:00:00' - )) - ), - ( - str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)), - (False, None) - ), - ( - str(datetime(2016, 3, 31)), None, - (False, None) - ), - ( - None, str(datetime(2016, 3, 30)), - (True, 'no record of the last neo4j update time for the course') - ), - ( - None, None, - (True, 'no record of the last neo4j update time for the course') - ), - ) - @ddt.unpack - def test_should_dump_course( - self, - last_command_run, - last_course_published, - should_dump, - mock_get_command_last_run, - mock_get_course_last_published, - ): - """ - Tests whether a course should be dumped given the last time it was - dumped and the last time it was published. - """ - mock_get_command_last_run.return_value = last_command_run - mock_get_course_last_published.return_value = last_course_published - mock_course_key = mock.Mock() - mock_graph = mock.Mock() - assert should_dump_course(mock_course_key, mock_graph) == should_dump diff --git a/cms/djangoapps/coursegraph/management/commands/tests/utils.py b/cms/djangoapps/coursegraph/management/commands/tests/utils.py deleted file mode 100644 index c1b776b7bf..0000000000 --- a/cms/djangoapps/coursegraph/management/commands/tests/utils.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Utilities for testing the dump_to_neo4j management command -""" - - -from py2neo import Node - - -class MockGraph: - """ - A stubbed out version of py2neo's Graph object, used for testing. - Args: - transaction_errors: a bool for whether transactions should throw - an error. - """ - def __init__(self, transaction_errors=False, **kwargs): # pylint: disable=unused-argument - self.nodes = set() - self.number_commits = 0 - self.number_rollbacks = 0 - self.transaction_errors = transaction_errors - - def begin(self): - """ - A stub of the method that generates transactions - Returns: a MockTransaction object (instead of a py2neo Transaction) - """ - return MockTransaction(self) - - def commit(self, transaction): - """ - Takes elements in the mock transaction's temporary storage and adds them - to this mock graph's storage. Throws an error if this graph's - transaction_errors param is set to True. - """ - if self.transaction_errors: - raise Exception("fake exception while trying to commit") - for element in transaction.temp: - self.nodes.add(element) - transaction.temp.clear() - self.number_commits += 1 - - def rollback(self, transaction): - """ - Clears the transactions temporary storage - """ - transaction.temp.clear() - self.number_rollbacks += 1 - - -class MockTransaction: - """ - A stubbed out version of py2neo's Transaction object, used for testing. - """ - def __init__(self, graph): - self.temp = set() - self.graph = graph - - def run(self, query): - """ - Deletes all nodes associated with a course. Normally `run` executes - an arbitrary query, but in our code, we only use it to delete nodes - associated with a course. - Args: - query: query string to be executed (in this case, to delete all - nodes associated with a course) - """ - start_string = "WHERE n.course_key='" - start = query.index(start_string) + len(start_string) - query = query[start:] - end = query.find("'") - course_key = query[:end] - - self.graph.nodes = { - node for node in self.graph.nodes if node['course_key'] != course_key - } - - def create(self, element): - """ - Adds elements to the transaction's temporary backend storage - Args: - element: a py2neo Node object - """ - if isinstance(element, Node): - self.temp.add(element) - - -class MockNodeMatcher: - """ - Mocks out py2neo's NodeMatcher class. Used to match a node from a graph. - py2neo's NodeMatcher expects a real graph object to run queries against, - so, rather than have to mock out MockGraph to accommodate those queries, - it seemed simpler to mock out NodeMatcher as well. - """ - def __init__(self, graph): - self.graph = graph - - def match(self, label, course_key): - """ - Selects nodes that match a label and course_key - Args: - label: the string of the label we're selecting nodes by - course_key: the string of the course key we're selecting node by - - Returns: a MockResult of matching nodes - """ - nodes = [] - for node in self.graph.nodes: - if node.has_label(label) and node["course_key"] == course_key: - nodes.append(node) - return MockNodeMatch(nodes) - - -class MockNodeMatch(list): - """ - Mocks out py2neo's NodeMatch class: this is the type of what - MockNodeMatcher's `match` method returns. - """ - def first(self): - """ - Returns: the first element of a list if the list has elements. - Otherwise, None. - """ - return self[0] if self else None diff --git a/cms/djangoapps/coursegraph/models.py b/cms/djangoapps/coursegraph/models.py deleted file mode 100644 index f053dc9993..0000000000 --- a/cms/djangoapps/coursegraph/models.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -(Proxy) models supporting CourseGraph. -""" - -from openedx.core.djangoapps.content.course_overviews.models import CourseOverview - - -class CourseGraphCourseDump(CourseOverview): - """ - Proxy model for CourseOverview. - - Does *not* create/update/delete CourseOverview objects - only reads the objects. - Uses the course IDs of the CourseOverview objects to determine which courses - can be dumped to CourseGraph. - """ - class Meta: - proxy = True - - def __str__(self): - """Represent ourselves with the course key.""" - return str(self.id) diff --git a/cms/djangoapps/coursegraph/tasks.py b/cms/djangoapps/coursegraph/tasks.py deleted file mode 100644 index e2d4bf5b09..0000000000 --- a/cms/djangoapps/coursegraph/tasks.py +++ /dev/null @@ -1,420 +0,0 @@ -""" -This file contains a management command for exporting the modulestore to -neo4j, a graph database. -""" - - -import logging - -from celery import shared_task -from django.conf import settings -from django.utils import timezone -from edx_django_utils.cache import RequestCache -from edx_django_utils.monitoring import set_code_owner_attribute -from opaque_keys.edx.keys import CourseKey - -import py2neo # pylint: disable=unused-import -from py2neo import Graph, Node, Relationship - -try: - from py2neo.matching import NodeMatcher -except ImportError: - from py2neo import NodeMatcher -else: - pass - - -log = logging.getLogger(__name__) -celery_log = logging.getLogger('edx.celery.task') - -# When testing locally, neo4j's bolt logger was noisy, so we'll only have it -# emit logs if there's an error. -bolt_log = logging.getLogger('neo4j.bolt') # pylint: disable=invalid-name -bolt_log.setLevel(logging.ERROR) - -PRIMITIVE_NEO4J_TYPES = (int, bytes, str, float, bool) - - -def serialize_item(item): - """ - Args: - item: an XBlock - - Returns: - fields: a dictionary of an XBlock's field names and values - block_type: the name of the XBlock's type (i.e. 'course' - or 'problem') - """ - from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES - - # convert all fields to a dict and filter out parent and children field - fields = { - field: field_value.read_from(item) - for (field, field_value) in item.fields.items() - if field not in ['parent', 'children'] - } - - course_key = item.scope_ids.usage_id.course_key - block_type = item.scope_ids.block_type - - # set or reset some defaults - fields['edited_on'] = str(getattr(item, 'edited_on', '')) - fields['display_name'] = item.display_name_with_default - fields['org'] = course_key.org - fields['course'] = course_key.course - fields['run'] = course_key.run - fields['course_key'] = str(course_key) - fields['location'] = str(item.location) - fields['block_type'] = block_type - fields['detached'] = block_type in DETACHED_XBLOCK_TYPES - - if block_type == 'course': - # prune the checklists field - if 'checklists' in fields: - del fields['checklists'] - - # record the time this command was run - fields['time_last_dumped_to_neo4j'] = str(timezone.now()) - - return fields, block_type - - -def coerce_types(value): - """ - Args: - value: the value of an xblock's field - - Returns: either the value, a text version of the value, or, if the - value is a list, a list where each element is converted to text. - """ - coerced_value = value - if isinstance(value, list): - coerced_value = [str(element) for element in coerced_value] - - # if it's not one of the types that neo4j accepts, - # just convert it to text - elif not isinstance(value, PRIMITIVE_NEO4J_TYPES): - coerced_value = str(value) - - return coerced_value - - -def add_to_transaction(neo4j_entities, transaction): - """ - Args: - neo4j_entities: a list of Nodes or Relationships - transaction: a neo4j transaction - """ - for entity in neo4j_entities: - transaction.create(entity) - - -def get_command_last_run(course_key, graph): - """ - This information is stored on the course node of a course in neo4j - Args: - course_key: a CourseKey - graph: a py2neo Graph - - Returns: The datetime that the command was last run, converted into - text, or None, if there's no record of this command last being run. - """ - matcher = NodeMatcher(graph) - course_node = matcher.match( - "course", - course_key=str(course_key) - ).first() - - last_this_command_was_run = None - if course_node: - last_this_command_was_run = course_node['time_last_dumped_to_neo4j'] - - return last_this_command_was_run - - -def get_course_last_published(course_key): - """ - Approximately when was a course last published? - - We use the 'modified' column in the CourseOverview table as a quick and easy - (although perhaps inexact) way of determining when a course was last - published. This works because CourseOverview rows are re-written upon - course publish. - - Args: - course_key: a CourseKey - - Returns: The datetime the course was last published at, stringified. - Uses Python's default str(...) implementation for datetimes, which - is sortable and similar to ISO 8601: - https://docs.python.org/3/library/datetime.html#datetime.date.__str__ - """ - # Import is placed here to avoid model import at project startup. - from openedx.core.djangoapps.content.course_overviews.models import CourseOverview - - approx_last_published = CourseOverview.get_from_id(course_key).modified - return str(approx_last_published) - - -def strip_branch_and_version(location): - """ - Removes the branch and version information from a location. - Args: - location: an xblock's location. - Returns: that xblock's location without branch and version information. - """ - return location.for_branch(None) - - -def serialize_course(course_id): - """ - Serializes a course into py2neo Nodes and Relationships - Args: - course_id: CourseKey of the course we want to serialize - - Returns: - nodes: a list of py2neo Node objects - relationships: a list of py2neo Relationships objects - """ - # Import is placed here to avoid model import at project startup. - from xmodule.modulestore.django import modulestore - - # create a location to node mapping we'll need later for - # writing relationships - location_to_node = {} - items = modulestore().get_items(course_id) - - # create nodes - for item in items: - fields, block_type = serialize_item(item) - - for field_name, value in fields.items(): - fields[field_name] = coerce_types(value) - - node = Node(block_type, 'item', **fields) - location_to_node[strip_branch_and_version(item.location)] = node - - # create relationships - relationships = [] - for item in items: - previous_child_node = None - for index, child in enumerate(item.get_children()): - parent_node = location_to_node.get(strip_branch_and_version(item.location)) - child_node = location_to_node.get(strip_branch_and_version(child.location)) - - if parent_node is not None and child_node is not None: - child_node["index"] = index - - relationship = Relationship(parent_node, "PARENT_OF", child_node) - relationships.append(relationship) - - if previous_child_node: - ordering_relationship = Relationship( - previous_child_node, - "PRECEDES", - child_node, - ) - relationships.append(ordering_relationship) - previous_child_node = child_node - - nodes = list(location_to_node.values()) - return nodes, relationships - - -def should_dump_course(course_key, graph): - """ - Only dump the course if it's been changed since the last time it's been - dumped. - Args: - course_key: a CourseKey object. - graph: a py2neo Graph object. - - Returns: - - whether this course should be dumped to neo4j (bool) - - reason why course needs to be dumped (string, None if doesn't need to be dumped) - """ - - last_this_command_was_run = get_command_last_run(course_key, graph) - - course_last_published_date = get_course_last_published(course_key) - - # if we don't have a record of the last time this command was run, - # we should serialize the course and dump it - if last_this_command_was_run is None: - return ( - True, - "no record of the last neo4j update time for the course" - ) - - # if we've serialized the course recently and we have no published - # events, we will not dump it, and so we can skip serializing it - # again here - if last_this_command_was_run and course_last_published_date is None: - return (False, None) - - # otherwise, serialize and dump the course if the command was run - # before the course's last published event - needs_update = last_this_command_was_run < course_last_published_date - update_reason = None - if needs_update: - update_reason = ( - f"course has been published since last neo4j update time - " - f"update date {last_this_command_was_run} < published date {course_last_published_date}" - ) - return (needs_update, update_reason) - - -@shared_task -@set_code_owner_attribute -def dump_course_to_neo4j(course_key_string, connection_overrides=None): - """ - Serializes a course and writes it to neo4j. - - Arguments: - course_key_string: course key for the course to be exported - connection_overrides (dict): overrides to Neo4j connection - parameters specified in `settings.COURSEGRAPH_CONNECTION`. - """ - course_key = CourseKey.from_string(course_key_string) - nodes, relationships = serialize_course(course_key) - celery_log.info( - "Now dumping %s to neo4j: %d nodes and %d relationships", - course_key, - len(nodes), - len(relationships), - ) - - graph = authenticate_and_create_graph( - connection_overrides=connection_overrides - ) - - transaction = graph.begin() - course_string = str(course_key) - try: - # first, delete existing course - transaction.run( - "MATCH (n:item) WHERE n.course_key='{}' DETACH DELETE n".format( - course_string - ) - ) - - # now, re-add it - add_to_transaction(nodes, transaction) - add_to_transaction(relationships, transaction) - graph.commit(transaction) - celery_log.info("Completed dumping %s to neo4j", course_key) - - except Exception: # pylint: disable=broad-except - celery_log.exception( - "Error trying to dump course %s to neo4j, rolling back", - course_string - ) - graph.rollback(transaction) - - -class ModuleStoreSerializer: - """ - Class with functionality to serialize a modulestore into subgraphs, - one graph per course. - """ - - def __init__(self, course_keys): - self.course_keys = course_keys - - @classmethod - def create(cls, courses=None, skip=None): - """ - Sets the object's course_keys attribute from the `courses` parameter. - If that parameter isn't furnished, loads all course_keys from the - modulestore. - Filters out course_keys in the `skip` parameter, if provided. - Args: - courses: A list of string serializations of course keys. - For example, ["course-v1:org+course+run"]. - skip: Also a list of string serializations of course keys. - """ - # Import is placed here to avoid model import at project startup. - from xmodule.modulestore.django import modulestore - if courses: - course_keys = [CourseKey.from_string(course.strip()) for course in courses] - else: - course_keys = [ - course.id for course in modulestore().get_course_summaries() - ] - if skip is not None: - skip_keys = [CourseKey.from_string(course.strip()) for course in skip] - course_keys = [course_key for course_key in course_keys if course_key not in skip_keys] - return cls(course_keys) - - def dump_courses_to_neo4j(self, connection_overrides=None, override_cache=False): - """ - Method that iterates through a list of courses in a modulestore, - serializes them, then submits tasks to write them to neo4j. - Arguments: - connection_overrides (dict): overrides to Neo4j connection - parameters specified in `settings.COURSEGRAPH_CONNECTION`. - override_cache: serialize the courses even if they'be been recently - serialized - - Returns: two lists--one of the courses that were successfully written - to neo4j and one of courses that were not. - """ - - total_number_of_courses = len(self.course_keys) - - submitted_courses = [] - skipped_courses = [] - - graph = authenticate_and_create_graph(connection_overrides) - - for index, course_key in enumerate(self.course_keys): - # first, clear the request cache to prevent memory leaks - RequestCache.clear_all_namespaces() - - (needs_dump, reason) = should_dump_course(course_key, graph) - if not (override_cache or needs_dump): - log.info("skipping submitting %s, since it hasn't changed", course_key) - skipped_courses.append(str(course_key)) - continue - - if override_cache: - reason = "override_cache is True" - - log.info( - "Now submitting %s for export to neo4j, because %s: course %d of %d total courses", - course_key, - reason, - index + 1, - total_number_of_courses, - ) - - dump_course_to_neo4j.apply_async( - kwargs=dict( - course_key_string=str(course_key), - connection_overrides=connection_overrides, - ) - ) - submitted_courses.append(str(course_key)) - - return submitted_courses, skipped_courses - - -def authenticate_and_create_graph(connection_overrides=None): - """ - This function authenticates with neo4j and creates a py2neo graph object - - Arguments: - connection_overrides (dict): overrides to Neo4j connection - parameters specified in `settings.COURSEGRAPH_CONNECTION`. - - Returns: a py2neo `Graph` object. - """ - provided_overrides = { - key: value - for key, value in (connection_overrides or {}).items() - # Drop overrides whose values are `None`. Note that `False` is a - # legitimate override value that we don't want to drop here. - if value is not None - } - connection_with_overrides = {**settings.COURSEGRAPH_CONNECTION, **provided_overrides} - return Graph(**connection_with_overrides) diff --git a/cms/djangoapps/coursegraph/tests/__init__.py b/cms/djangoapps/coursegraph/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cms/djangoapps/coursegraph/tests/test_admin.py b/cms/djangoapps/coursegraph/tests/test_admin.py deleted file mode 100644 index 21a26d8450..0000000000 --- a/cms/djangoapps/coursegraph/tests/test_admin.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Shallow tests for CourseGraph dump-queueing Django admin interface. - -See ..management.commands.tests.test_dump_to_neo4j for more comprehensive -tests of dump_course_to_neo4j. -""" - -from unittest import mock - -import py2neo -from django.test import TestCase -from django.test.utils import override_settings -from freezegun import freeze_time - -from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory -from openedx.core.djangoapps.content.course_overviews.models import CourseOverview - -from .. import admin, tasks - - -_coursegraph_connection = { - "protocol": "bolt", - "secure": True, - "host": "example.edu", - "port": 7687, - "user": "neo4j", - "password": "fake-coursegraph-password", -} - -_configure_coursegraph_connection = override_settings( - COURSEGRAPH_CONNECTION=_coursegraph_connection, -) - -_patch_log_exception = mock.patch.object( - admin.log, 'exception', autospec=True -) - -_patch_apply_dump_task = mock.patch.object( - tasks.dump_course_to_neo4j, 'apply_async' -) - -_pretend_last_course_dump_was_may_2020 = mock.patch.object( - tasks, - 'get_command_last_run', - new=(lambda _key, _graph: "2020-05-01"), -) - -_patch_neo4j_graph = mock.patch.object( - tasks, 'Graph', autospec=True -) - -_make_neo4j_graph_raise = mock.patch.object( - tasks, 'Graph', side_effect=py2neo.ConnectionUnavailable( - 'we failed to connect or something!' - ) -) - - -class CourseGraphAdminActionsTestCase(TestCase): - """ - Test CourseGraph Django admin actions. - """ - - @classmethod - def setUpTestData(cls): - """ - Make course overviews with varying modification dates. - """ - super().setUpTestData() - cls.course_updated_in_april = CourseOverviewFactory(run='april_update') - cls.course_updated_in_june = CourseOverviewFactory(run='june_update') - cls.course_updated_in_july = CourseOverviewFactory(run='july_update') - cls.course_updated_in_august = CourseOverviewFactory(run='august_update') - - # For each course overview, make an arbitrary update and then save() - # so that its `.modified` date is set. - with freeze_time("2020-04-01"): - cls.course_updated_in_april.marketing_url = "https://example.org" - cls.course_updated_in_april.save() - with freeze_time("2020-06-01"): - cls.course_updated_in_june.marketing_url = "https://example.org" - cls.course_updated_in_june.save() - with freeze_time("2020-07-01"): - cls.course_updated_in_july.marketing_url = "https://example.org" - cls.course_updated_in_july.save() - with freeze_time("2020-08-01"): - cls.course_updated_in_august.marketing_url = "https://example.org" - cls.course_updated_in_august.save() - - @_configure_coursegraph_connection - @_pretend_last_course_dump_was_may_2020 - @_patch_neo4j_graph - @_patch_apply_dump_task - @_patch_log_exception - def test_dump_courses(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): - """ - Test that dump_courses admin action dumps requested courses iff they have - been modified since the last dump to coursegraph. - """ - modeladmin_mock = mock.MagicMock() - - # Request all courses except the August-updated one - requested_course_keys = { - str(self.course_updated_in_april.id), - str(self.course_updated_in_june.id), - str(self.course_updated_in_july.id), - } - admin.dump_courses( - modeladmin=modeladmin_mock, - request=mock.MagicMock(), - queryset=CourseOverview.objects.filter(id__in=requested_course_keys), - ) - - # User should have been messaged - assert modeladmin_mock.message_user.call_count == 1 - assert modeladmin_mock.message_user.call_args.args[1] == ( - "Enqueued dumps for 2 course(s). Skipped 1 unchanged course(s)." - ) - - # For enqueueing, graph should've been authenticated once, using configured settings. - assert mock_neo4j_graph.call_count == 1 - assert mock_neo4j_graph.call_args.args == () - assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection - - # No errors should've been logged. - assert mock_log_exception.call_count == 0 - - # April course should have been skipped because the command was last run in May. - # Dumps for June and July courses should have been enqueued. - assert mock_apply_dump_task.call_count == 2 - actual_dumped_course_keys = { - call_args.kwargs['kwargs']['course_key_string'] - for call_args in mock_apply_dump_task.call_args_list - } - expected_dumped_course_keys = { - str(self.course_updated_in_june.id), - str(self.course_updated_in_july.id), - } - assert actual_dumped_course_keys == expected_dumped_course_keys - - @_configure_coursegraph_connection - @_pretend_last_course_dump_was_may_2020 - @_patch_neo4j_graph - @_patch_apply_dump_task - @_patch_log_exception - def test_dump_courses_overriding_cache(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): - """ - Test that dump_coursese_overriding_cach admin action dumps requested courses - whether or not they been modified since the last dump to coursegraph. - """ - modeladmin_mock = mock.MagicMock() - - # Request all courses except the August-updated one - requested_course_keys = { - str(self.course_updated_in_april.id), - str(self.course_updated_in_june.id), - str(self.course_updated_in_july.id), - } - admin.dump_courses_overriding_cache( - modeladmin=modeladmin_mock, - request=mock.MagicMock(), - queryset=CourseOverview.objects.filter(id__in=requested_course_keys), - ) - - # User should have been messaged - assert modeladmin_mock.message_user.call_count == 1 - assert modeladmin_mock.message_user.call_args.args[1] == ( - "Enqueued dumps for 3 course(s)." - ) - - # For enqueueing, graph should've been authenticated once, using configured settings. - assert mock_neo4j_graph.call_count == 1 - assert mock_neo4j_graph.call_args.args == () - assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection - - # No errors should've been logged. - assert mock_log_exception.call_count == 0 - - # April, June, and July courses should have all been dumped. - assert mock_apply_dump_task.call_count == 3 - actual_dumped_course_keys = { - call_args.kwargs['kwargs']['course_key_string'] - for call_args in mock_apply_dump_task.call_args_list - } - expected_dumped_course_keys = { - str(self.course_updated_in_april.id), - str(self.course_updated_in_june.id), - str(self.course_updated_in_july.id), - } - assert actual_dumped_course_keys == expected_dumped_course_keys - - @_configure_coursegraph_connection - @_pretend_last_course_dump_was_may_2020 - @_make_neo4j_graph_raise - @_patch_apply_dump_task - @_patch_log_exception - def test_dump_courses_error(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): - """ - Test that the dump_courses admin action dumps messages the user if an error - occurs when trying to enqueue course dumps. - """ - modeladmin_mock = mock.MagicMock() - - # Request dump of all four courses. - admin.dump_courses( - modeladmin=modeladmin_mock, - request=mock.MagicMock(), - queryset=CourseOverview.objects.all() - ) - - # Admin user should have been messaged about failure. - assert modeladmin_mock.message_user.call_count == 1 - assert modeladmin_mock.message_user.call_args.args[1] == ( - "Error enqueueing dumps for 4 course(s): we failed to connect or something!" - ) - - # For enqueueing, graph should've been authenticated once, using configured settings. - assert mock_neo4j_graph.call_count == 1 - assert mock_neo4j_graph.call_args.args == () - assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection - - # Exception should have been logged. - assert mock_log_exception.call_count == 1 - assert "Failed to enqueue" in mock_log_exception.call_args.args[0] - - # No courses should have been dumped. - assert mock_apply_dump_task.call_count == 0 diff --git a/cms/envs/common.py b/cms/envs/common.py index 45e3ccb42c..24a63fb7d9 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1756,9 +1756,6 @@ INSTALLED_APPS = [ # edx-milestones service 'milestones', - # Coursegraph - 'cms.djangoapps.coursegraph.apps.CoursegraphConfig', - # Credit courses 'openedx.core.djangoapps.credit.apps.CreditConfig', @@ -2447,40 +2444,6 @@ POLICY_CHANGE_TASK_RATE_LIMIT = '900/h' # 11 grade designations are used by the UI, so it's advisable to restrict the list to 11 items. DEFAULT_GRADE_DESIGNATIONS = ['A', 'B', 'C', 'D'] -############## Settings for CourseGraph ############################ - -# .. setting_name: COURSEGRAPH_JOB_QUEUE -# .. setting_default: value of LOW_PRIORITY_QUEUE -# .. setting_description: The name of the Celery queue to which CourseGraph refresh -# tasks will be sent -COURSEGRAPH_JOB_QUEUE: str = LOW_PRIORITY_QUEUE - -# .. setting_name: COURSEGRAPH_CONNECTION -# .. setting_default: 'bolt+s://localhost:7687', in dictionary form. -# .. setting_description: Dictionary specifying Neo4j connection parameters for -# CourseGraph refresh. Accepted keys are protocol ('bolt' or 'http'), -# secure (bool), host (str), port (int), user (str), and password (str). -# See https://py2neo.org/2021.1/profiles.html#individual-settings for a -# a description of each of those keys. -COURSEGRAPH_CONNECTION: dict = { - "protocol": "bolt", - "secure": True, - "host": "localhost", - "port": 7687, - "user": "neo4j", - "password": None, -} - -# .. toggle_name: COURSEGRAPH_DUMP_COURSE_ON_PUBLISH -# .. toggle_implementation: DjangoSetting -# .. toggle_creation_date: 2022-01-27 -# .. toggle_use_cases: open_edx -# .. toggle_default: False -# .. toggle_description: Whether, upon publish, a course should automatically -# be exported to Neo4j via the connection parameters specified in -# `COURSEGRAPH_CONNECTION`. -COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: bool = False - ########## Settings for video transcript migration tasks ############ VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index faacbc1548..e944d67eda 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -261,17 +261,6 @@ FEATURES['ENABLE_PREREQUISITE_COURSES'] = True # (ref MST-637) PROCTORING_USER_OBFUSCATION_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' -############## CourseGraph devstack settings ############################ - -COURSEGRAPH_CONNECTION: dict = { - "protocol": "bolt", - "secure": False, - "host": "edx.devstack.coursegraph", - "port": 7687, - "user": "neo4j", - "password": "edx", -} - #################### Webpack Configuration Settings ############################## WEBPACK_LOADER['DEFAULT']['TIMEOUT'] = 5 diff --git a/cms/envs/production.py b/cms/envs/production.py index f65b220432..cf2a7d2f3f 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -517,9 +517,6 @@ JWT_AUTH.update(AUTH_TOKENS.get('JWT_AUTH', {})) if FEATURES.get('CUSTOM_COURSES_EDX'): INSTALLED_APPS.append('openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig') -############## Settings for CourseGraph ############################ -COURSEGRAPH_JOB_QUEUE = ENV_TOKENS.get('COURSEGRAPH_JOB_QUEUE', LOW_PRIORITY_QUEUE) - ########## Settings for video transcript migration tasks ############ VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = ENV_TOKENS.get('VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE) @@ -616,8 +613,6 @@ EXPLICIT_QUEUES = { 'queue': SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY}, 'cms.djangoapps.contentstore.tasks.update_search_index': { 'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE}, - 'cms.djangoapps.coursegraph.tasks.dump_course_to_neo4j': { - 'queue': COURSEGRAPH_JOB_QUEUE}, } LOGO_IMAGE_EXTRA_TEXT = ENV_TOKENS.get('LOGO_IMAGE_EXTRA_TEXT', '') diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index 1e8e5e8b0c..7e1f6de3b6 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -367,7 +367,7 @@ function($, _, Backbone, gettext, BasePage, event.preventDefault(); if (!options || options.view !== 'visibility_view') { - const primaryHeader = $(event.target).closest('.xblock-header-primary'); + const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions'); var useNewTextEditor = primaryHeader.attr('use-new-editor-text'), useNewVideoEditor = primaryHeader.attr('use-new-editor-video'), diff --git a/cms/templates/container.html b/cms/templates/container.html index 488ad2a3b7..058587e6bb 100644 --- a/cms/templates/container.html +++ b/cms/templates/container.html @@ -13,6 +13,8 @@ from django.urls import reverse from django.utils.translation import gettext as _ from cms.djangoapps.contentstore.helpers import xblock_studio_url, xblock_type_display_name +from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow +from cms.djangoapps.contentstore.utils import get_editor_page_base_url from openedx.core.djangolib.js_utils import ( dump_js_escaped_json, js_escaped_string ) @@ -111,6 +113,13 @@ from openedx.core.djangolib.markup import HTML, Text <%block name="content"> +<% +use_new_editor_text = use_new_text_editor() +use_new_editor_video = use_new_video_editor() +use_new_editor_problem = use_new_problem_editor() +use_new_video_gallery_flow = use_video_gallery_flow() +%> + @@ -161,7 +170,15 @@ from openedx.core.djangolib.markup import HTML, Text -