Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into bleach#33209

Browse Source
This commit is contained in:
Deborah Kaplan
2024-06-10 11:27:12 -04:00
committed by GitHub
parent d496411f78 6fc7e3b1e9
commit 68b052620a
121 changed files with 2150 additions and 15357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@@ -102,7 +102,6 @@ common/test/data/badges/*.png
### Static assets pipeline artifacts
**/*.scssc
lms/static/css/
!lms/static/css/vendor
lms/static/certificates/css/
cms/static/css/
common/static/common/js/vendor/

7
.github/actions/unit-tests/action.yml vendored
View File

@@ -1,5 +1,5 @@
name: 'Run unit tests'
description: 'shared steps to run unit tests on both Github hosted and self hosted runners.'
name: "Run unit tests"
description: "shared steps to run unit tests on both Github hosted and self hosted runners."
runs:
using: "composite"
steps:
@@ -27,8 +27,9 @@ runs:
- name: save pytest warnings json file
if: success()
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: pytest-warnings-json
path: |
test_root/log/pytest_warnings*.json
overwrite: true

6
.github/actions/verify-tests-count/action.yml vendored
View File

@@ -1,5 +1,5 @@
name: 'Verify unit tests count'
description: 'shared steps to verify unit tests count on both Github hosted and self hosted runners.'
name: "Verify unit tests count"
description: "shared steps to verify unit tests count on both Github hosted and self hosted runners."
runs:
using: "composite"
steps:
@@ -15,14 +15,12 @@ runs:
echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV
echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV
- name: collect tests from GHA unit test shards
shell: bash
run: |
echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
- name: add unit tests count
shell: bash
run: |

7
.github/pull_request_template.md vendored
View File

@@ -8,9 +8,8 @@ Use conventional commits to separate and summarize commits logically:
https://open-edx-proposals.readthedocs.io/en/latest/oep-0051-bp-conventional-commits.html
Use this template as a guide. Omit sections that don't apply.
You may link to information rather than copy it, but only if the link is publicly
readable. If you must linked information must be private (because it has secrets),
clearly label the link as private.
You may link to information rather than copy it, but only if the link is publicly readable.
If the linked information must be private (because it contains secrets), clearly label the link as private.
-->
@@ -22,6 +21,7 @@ Design decisions and their rationales should be documented in the repo (docstrin
linked here.
Useful information to include:
- Which edX user roles will this change impact? Common user roles are "Learner", "Course Author",
"Developer", and "Operator".
- Include screenshots for changes to the UI (ideally, both "before" and "after" screenshots, if applicable).
@@ -44,6 +44,7 @@ Please provide detailed step-by-step instructions for testing this change.
## Other information
Include anything else that will help reviewers and consumers understand the change.
- Does this change depend on other changes elsewhere?
- Any special concerns or limitations? For example: deprecations, migrations, security, or accessibility.
- If your [database migration](https://openedx.atlassian.net/wiki/spaces/AC/pages/23003228/Everything+About+Database+Migrations) can't be rolled back easily.

11
.github/renovate.json vendored
View File

@@ -1,11 +0,0 @@
{
"extends": [
"config:base",
"schedule:weekdays",
":preserveSemverRanges"
],
"prConcurrentLimit": 5,
"includePaths": [
"package.json"
]
}

42
.github/renovate.json5 vendored Normal file
View File

@@ -0,0 +1,42 @@
// This file is written in "JSON5" (https://json5.org/) so that we can use comments.
{
"extends": [
"config:base",
"schedule:weekly",
":automergeLinters",
":automergeMinor",
":automergeTesters",
":enableVulnerabilityAlerts",
":semanticCommits",
":updateNotScheduled"
],
"packageRules": [
{
"matchDepTypes": [
"devDependencies"
],
"matchUpdateTypes": [
"lockFileMaintenance",
"minor",
"patch",
"pin"
],
"automerge": true
},
{
"matchPackagePatterns": ["@edx", "@openedx"],
"matchUpdateTypes": ["minor", "patch"],
"automerge": true
}
],
// When adding an ignoreDep, please include a reason and a public link that we can use to follow up and ensure
// that the ignoreDep is removed.
"ignoreDeps": [
// Latest moment-timezone version broke the legacy programs dashboard, which is deprecated and soon to be removed.
// https://github.com/openedx/edx-platform/pull/34928"
"moment-timezone",
],
"timezone": "America/New_York",
"prConcurrentLimit": 3,
"enabledManagers": ["npm"]
}

1
.github/workflows/add-remove-label-on-comment.yml vendored
View File

@@ -17,4 +17,3 @@ on:
jobs:
add_remove_labels:
uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master

6
.github/workflows/check-consistent-dependencies.yml vendored
View File

@@ -34,13 +34,13 @@ jobs:
echo "RELEVANT=true" >> "$GITHUB_ENV"
fi
- uses: actions/checkout@v3
- uses: actions/checkout@v4
if: ${{ env.RELEVANT == 'true' }}
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
if: ${{ env.RELEVANT == 'true' }}
with:
python-version: '3.8'
python-version: "3.8"
- name: "Recompile requirements"
if: ${{ env.RELEVANT == 'true' }}

6
.github/workflows/check-for-tutorial-prs.yml vendored
View File

@@ -14,7 +14,7 @@ on:
pull_request:
types: [opened]
paths:
- 'lms/templates/dashboard.html'
- "lms/templates/dashboard.html"
jobs:
# Provide helpful bot comment
@@ -23,7 +23,7 @@ jobs:
name: provide helpful bot comment
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Comment PR
uses: thollander/actions-comment-pull-request@v2
@@ -32,4 +32,4 @@ jobs:
Thank you for your pull request! Congratulations on completing the Open edX tutorial! A team member will be by to take a look shortly.
To those watching community pull requests: No need to worry about this one, a tCRIL team member will be taking care of it.
For this PR's author: If this is a PR that is NOT coming from the Open edX tutorial, please comment and let us know to disregard this message.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

40
.github/workflows/check_python_dependencies.yml vendored Normal file
View File

@@ -0,0 +1,40 @@
name: Check Python Dependencies
on:
pull_request:
jobs:
check_dependencies:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.12"]
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install repo-tools
run: pip install edx-repo-tools[find_dependencies]
- name: Install setuptool
run: pip install setuptools
- name: Run Python script
run: |
find_python_dependencies \
--req-file requirements/edx/base.txt \
--req-file requirements/edx/testing.txt \
--ignore https://github.com/edx/codejail-includes \
--ignore https://github.com/edx/braze-client \
--ignore https://github.com/edx/edx-name-affirmation \
--ignore https://github.com/mitodl/edx-sga \
--ignore https://github.com/edx/token-utils \
--ignore https://github.com/open-craft/xblock-poll

10
.github/workflows/ci-static-analysis.yml vendored
View File

@@ -9,13 +9,13 @@ jobs:
strategy:
matrix:
python-version:
- '3.11'
os: ['ubuntu-20.04']
- "3.11"
os: ["ubuntu-20.04"]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
@@ -31,7 +31,7 @@ jobs:
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}

10
.github/workflows/compile-python-requirements.yml vendored
View File

@@ -4,9 +4,9 @@ on:
workflow_dispatch:
inputs:
branch:
description: 'Target branch to create requirements PR against'
description: "Target branch to create requirements PR against"
required: true
default: 'master'
default: "master"
type: string
defaults:
@@ -19,12 +19,12 @@ jobs:
steps:
- name: Check out target branch
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
ref: "${{ inputs.branch }}"
- name: Set up Python environment
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: "3.11"
@@ -44,7 +44,7 @@ jobs:
- name: Make a PR
id: make-pr
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v6
with:
branch: "${{ github.triggering_actor }}/compile-python-deps"
branch-suffix: short-commit-hash

19
.github/workflows/docker-publish.yml vendored
View File

@@ -9,8 +9,8 @@ jobs:
# See also https://docs.docker.com/docker-hub/builds/
push:
runs-on: ubuntu-latest
if: github.event_name == 'push'
if: github.event_name == 'push'
strategy:
matrix:
variant:
@@ -21,23 +21,22 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build and push lms base docker image
env:
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
run : make docker_tag_build_push_${{matrix.variant}}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
run: make docker_tag_build_push_${{matrix.variant}}

114
.github/workflows/js-tests.yml vendored
View File

@@ -12,73 +12,73 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
node-version: [ 18 ]
os: [ubuntu-20.04]
node-version: [18]
python-version:
- '3.11'
- "3.11"
steps:
- uses: actions/checkout@v4
- name: Fetch master to compare coverage
run: git fetch --depth=1 origin master
- uses: actions/checkout@v2
- name: Fetch master to compare coverage
run: git fetch --depth=1 origin master
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Setup npm
run: npm i -g npm@10.5.x
- name: Setup npm
run: npm i -g npm@10.5.x
- name: Install Firefox 123.0
run: |
sudo apt-get purge firefox
wget "https://ftp.mozilla.org/pub/firefox/releases/123.0/linux-x86_64/en-US/firefox-123.0.tar.bz2"
tar -xjf firefox-123.0.tar.bz2
sudo mv firefox /opt/firefox
sudo ln -s /opt/firefox/firefox /usr/bin/firefox
- name: Install Firefox 123.0
run: |
sudo apt-get purge firefox
wget "https://ftp.mozilla.org/pub/firefox/releases/123.0/linux-x86_64/en-US/firefox-123.0.tar.bz2"
tar -xjf firefox-123.0.tar.bz2
sudo mv firefox /opt/firefox
sudo ln -s /opt/firefox/firefox /usr/bin/firefox
- name: Install Required System Packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb
- name: Install Required System Packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install Required Python Dependencies
run: |
make base-requirements
- name: Install Required Python Dependencies
run: |
make base-requirements
- uses: c-hive/gha-npm-cache@v1
- name: Run JS Tests
env:
TEST_SUITE: js-unit
SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
run: |
npm install -g jest
xvfb-run --auto-servernum ./scripts/all-tests.sh
- uses: c-hive/gha-npm-cache@v1
- name: Run JS Tests
env:
TEST_SUITE: js-unit
SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
run: |
npm install -g jest
xvfb-run --auto-servernum ./scripts/all-tests.sh
- name: Save Job Artifacts
uses: actions/upload-artifact@v3
with:
name: Build-Artifacts
path: |
reports/**/*
test_root/log/*.png
test_root/log/*.log
**/TEST-*.xml
- name: Save Job Artifacts
uses: actions/upload-artifact@v4
with:
name: Build-Artifacts
path: |
reports/**/*
test_root/log/*.png
test_root/log/*.log
**/TEST-*.xml
overwrite: true

9
.github/workflows/lint-imports.yml vendored
View File

@@ -7,19 +7,18 @@ on:
- master
jobs:
lint-imports:
name: Lint Python Imports
runs-on: ubuntu-20.04
steps:
- name: Check out branch
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: '3.11'
python-version: "3.11"
- name: Install system requirements
run: sudo apt update && sudo apt install -y libxmlsec1-dev
@@ -33,7 +32,7 @@ jobs:
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}

2
.github/workflows/lockfileversion-check.yml vendored
View File

@@ -5,7 +5,7 @@ name: Lockfile Version check
on:
push:
branches:
- master
- master
pull_request:
jobs:

120
.github/workflows/migrations-check.yml vendored
View File

@@ -13,9 +13,9 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
os: [ubuntu-20.04]
python-version:
- '3.11'
- "3.11"
# 'pinned' is used to install the latest patch version of Django
# within the global constraint i.e. Django==4.2.8 in current case
# because we have global constraint of Django<4.2
@@ -50,73 +50,73 @@ jobs:
--health-timeout 5s
--health-retries 3
steps:
- name: Setup mongodb user
run: |
mongosh edxapp --eval '
db.createUser(
{
user: "edxapp",
pwd: "password",
roles: [
{ role: "readWrite", db: "edxapp" },
]
}
);
'
- name: Setup mongodb user
run: |
mongosh edxapp --eval '
db.createUser(
{
user: "edxapp",
pwd: "password",
roles: [
{ role: "readWrite", db: "edxapp" },
]
}
);
'
- name: Verify mongo and mysql db credentials
run: |
mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp
mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp
- name: Verify mongo and mysql db credentials
run: |
mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp
mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp
- name: Checkout repo
uses: actions/checkout@v2
- name: Checkout repo
uses: actions/checkout@v4
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install system Packages
run: |
sudo apt-get update
make ubuntu-requirements
- name: Install system Packages
run: |
sudo apt-get update
make ubuntu-requirements
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install Python dependencies
run: |
make dev-requirements
if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
pip install "django~=${{ matrix.django-version }}.0"
pip check # fail if this test-reqs/Django combination is broken
fi
- name: Install Python dependencies
run: |
make dev-requirements
if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
pip install "django~=${{ matrix.django-version }}.0"
pip check # fail if this test-reqs/Django combination is broken
fi
- name: list installed package versions
run: |
sudo pip freeze
- name: list installed package versions
run: |
sudo pip freeze
- name: Run Tests
env:
LMS_CFG: lms/envs/minimal.yml
# This is from the LMS dir on purpose since we don't need anything different for the CMS yet.
STUDIO_CFG: lms/envs/minimal.yml
run: |
echo "Running the LMS migrations."
./manage.py lms migrate
echo "Running the CMS migrations."
./manage.py cms migrate
- name: Run Tests
env:
LMS_CFG: lms/envs/minimal.yml
# This is from the LMS dir on purpose since we don't need anything different for the CMS yet.
STUDIO_CFG: lms/envs/minimal.yml
run: |
echo "Running the LMS migrations."
./manage.py lms migrate
echo "Running the CMS migrations."
./manage.py cms migrate
# This job aggregates test results. It's the required check for branch protection.
# https://github.com/marketplace/actions/alls-green#why

3
.github/workflows/publish-ci-docker-image.yml vendored
View File

@@ -11,7 +11,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v4
# This has to happen after checkout in order for gh to work.
- name: "Cancel scheduled job on forks"
@@ -41,4 +41,3 @@ jobs:
run: |
docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f scripts/ci-runner.Dockerfile .
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

6
.github/workflows/pylint-checks.yml vendored
View File

@@ -31,13 +31,13 @@ jobs:
name: pylint ${{ matrix.module-name }}
steps:
- name: Check out repo
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Install required system packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: 3.11
@@ -48,7 +48,7 @@ jobs:
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}

108
.github/workflows/quality-checks.yml vendored
View File

@@ -13,70 +13,70 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
os: [ubuntu-20.04]
python-version:
- '3.11'
node-version: [ 18 ]
- "3.11"
node-version: [18]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Fetch base branch for comparison
run: git fetch --depth=1 origin ${{ github.base_ref }}
- name: Fetch base branch for comparison
run: git fetch --depth=1 origin ${{ github.base_ref }}
- name: Install Required System Packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
- name: Install Required System Packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Setup npm
run: npm i -g npm@8.5.x
- name: Setup npm
run: npm i -g npm@8.5.x
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install Required Python Dependencies
env:
PIP_SRC: ${{ runner.temp }}
run: |
make test-requirements
- name: Install Required Python Dependencies
env:
PIP_SRC: ${{ runner.temp }}
run: |
make test-requirements
- name: Run Quality Tests
env:
TEST_SUITE: quality
SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
PIP_SRC: ${{ runner.temp }}
TARGET_BRANCH: ${{ github.base_ref }}
run: |
./scripts/all-tests.sh
- name: Run Quality Tests
env:
TEST_SUITE: quality
SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
PIP_SRC: ${{ runner.temp }}
TARGET_BRANCH: ${{ github.base_ref }}
run: |
./scripts/all-tests.sh
- name: Save Job Artifacts
if: always()
uses: actions/upload-artifact@v3
with:
name: Build-Artifacts
path: |
**/reports/**/*
test_root/log/**/*.log
*.log
- name: Save Job Artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: Build-Artifacts
path: |
**/reports/**/*
test_root/log/**/*.log
*.log
overwrite: true

8
.github/workflows/semgrep.yml vendored
View File

@@ -17,16 +17,16 @@ jobs:
runs-on: "${{ matrix.os }}"
strategy:
matrix:
os: [ "ubuntu-20.04" ]
os: ["ubuntu-20.04"]
python-version:
- '3.11'
- "3.11"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 1
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: "${{ matrix.python-version }}"

2
.github/workflows/shellcheck.yml vendored
View File

@@ -9,7 +9,7 @@ on:
pull_request:
push:
branches:
- master
- master
permissions:
contents: read

122
.github/workflows/static-assets-check.yml vendored
View File

@@ -12,11 +12,11 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
os: [ubuntu-20.04]
python-version:
- '3.11'
node-version: [ 18 ]
npm-version: [ 10.5.x ]
- "3.11"
node-version: [18]
npm-version: [10.5.x]
mongo-version:
- "7.0"
@@ -34,73 +34,73 @@ jobs:
--health-retries 3
steps:
- name: Checkout repo
uses: actions/checkout@v2
- name: Checkout repo
uses: actions/checkout@v4
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install system Packages
run: |
sudo apt-get update
sudo apt-get install libxmlsec1-dev pkg-config
- name: Install system Packages
run: |
sudo apt-get update
sudo apt-get install libxmlsec1-dev pkg-config
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Setup npm
run: npm i -g npm@${{ matrix.npm-version }}
- name: Setup npm
run: npm i -g npm@${{ matrix.npm-version }}
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install Limited Python Deps for Build
run: |
pip install -r requirements/edx/assets.txt
- name: Install Limited Python Deps for Build
run: |
pip install -r requirements/edx/assets.txt
- name: Initiate Mongo DB Service
run: sudo systemctl start mongod
- name: Initiate Mongo DB Service
run: sudo systemctl start mongod
- name: Add node_modules bin to $Path
run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH
- name: Add node_modules bin to $Path
run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH
- name: Check Dev Assets Build
env:
COMPREHENSIVE_THEMES_DIR: ./themes
run: |
npm clean-install --dev
npm run build-dev
- name: Check Dev Assets Build
env:
COMPREHENSIVE_THEMES_DIR: ./themes
run: |
npm clean-install --dev
npm run build-dev
- name: Check Prod Assets Build
env:
COMPREHENSIVE_THEMES_DIR: ./themes
run: |
npm clean-install
npm run build
- name: Check Prod Assets Build
env:
COMPREHENSIVE_THEMES_DIR: ./themes
run: |
npm clean-install
npm run build
- name: Install Full Python Deps for Collection
run: |
pip install -r requirements/edx/base.txt -e .
- name: Install Full Python Deps for Collection
run: |
pip install -r requirements/edx/base.txt -e .
- name: Check Assets Collection
env:
LMS_CFG: lms/envs/minimal.yml
CMS_CFG: lms/envs/minimal.yml
DJANGO_SETTINGS_MODULE: lms.envs.production
run: |
./manage.py lms collectstatic --noinput
./manage.py cms collectstatic --noinput
- name: Check Assets Collection
env:
LMS_CFG: lms/envs/minimal.yml
CMS_CFG: lms/envs/minimal.yml
DJANGO_SETTINGS_MODULE: lms.envs.production
run: |
./manage.py lms collectstatic --noinput
./manage.py cms collectstatic --noinput

1
.github/workflows/unit-test-shards.json vendored
View File

@@ -239,7 +239,6 @@
"paths": [
"cms/djangoapps/api/",
"cms/djangoapps/cms_user_tasks/",
"cms/djangoapps/coursegraph/",
"cms/djangoapps/course_creators/",
"cms/djangoapps/export_course_metadata/",
"cms/djangoapps/maintenance/",

141
.github/workflows/unit-tests-gh-hosted.yml vendored
View File

@@ -1,141 +0,0 @@
name: unit-tests-gh-hosted
on:
pull_request:
push:
branches:
- master
- open-release/lilac.master
jobs:
run-test:
name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true))
runs-on: ubuntu-20.04
strategy:
matrix:
python-version:
- "3.11"
django-version:
- "pinned"
# When updating the shards, remember to make the same changes in
# .github/workflows/unit-tests.yml
shard_name:
- "lms-1"
- "lms-2"
- "lms-3"
- "lms-4"
- "lms-5"
- "lms-6"
- "openedx-1-with-lms"
- "openedx-2-with-lms"
- "openedx-1-with-cms"
- "openedx-2-with-cms"
- "cms-1"
- "cms-2"
- "common-with-lms"
- "common-with-cms"
- "xmodule-with-lms"
- "xmodule-with-cms"
mongo-version:
- "7.0"
# We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore)
# This code is left here as an example for future refernce in case we need to reduce the number of shards we're
# testing but still have good confidence with older versions of mongo. We use Mongo 4.4 in the example.
#
# exclude:
# - mongo-version: "4.4"
# include:
# - shard_name: "xmodule-with-cms"
# python-version: "3.11"
# django-version: "pinned"
# mongo-version: "4.4"
# - shard_name: "xmodule-with-lms"
# python-version: "3.11"
# django-version: "pinned"
# mongo-version: "4.4"
steps:
- uses: actions/checkout@v2
- name: Install Required System Packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev lynx
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.7.0
with:
mongodb-version: ${{ matrix.mongo-version}}
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install Required Python Dependencies
env:
PIP_SRC: ${{ runner.temp }}
run: |
make test-requirements
if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
pip install "django~=${{ matrix.django-version }}.0"
pip check # fail if this test-reqs/Django combination is broken
fi
- name: Setup and run tests
uses: ./.github/actions/unit-tests
collect-and-verify:
if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true))
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [ '3.11' ]
django-version:
- "pinned"
steps:
- uses: actions/checkout@v2
- name: Install Required System Packages
run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install Required Python Dependencies
run: |
make test-requirements
if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
pip install "django~=${{ matrix.django-version }}.0"
pip check # fail if this test-reqs/Django combination is broken
fi
- name: verify unit tests count
uses: ./.github/actions/verify-tests-count

88
.github/workflows/unit-tests.yml vendored
View File

@@ -6,11 +6,16 @@ on:
branches:
- master
concurrency:
# We only need to be running tests for the latest commit on each PR
# (however, we fully test every commit on master, even as new ones land).
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
jobs:
run-tests:
name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
runs-on: [ edx-platform-runner ]
runs-on: ubuntu-20.04
strategy:
matrix:
python-version:
@@ -56,14 +61,16 @@ jobs:
# mongo-version: "4.4"
steps:
- name: sync directory owner
run: sudo chown runner:runner -R .*
- name: checkout repo
uses: actions/checkout@v4
- name: install system requirements
run: |
sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx
- name: install mongo version
run: |
if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then
sudo apt-get purge -y "mongodb-org*"
sudo apt-get remove -y mongodb-org
wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add -
echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list
sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
@@ -76,24 +83,21 @@ jobs:
mongod &
- name: Setup Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: checkout repo
uses: actions/checkout@v3
- name: install requirements
run: |
sudo make test-requirements
make test-requirements
if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
sudo pip install "django~=${{ matrix.django-version }}.0"
sudo pip check # fail if this test-reqs/Django combination is broken
pip install "django~=${{ matrix.django-version }}.0"
pip check # fail if this test-reqs/Django combination is broken
fi
- name: list installed package versions
run: |
sudo pip freeze
pip freeze
- name: Setup and run tests
uses: ./.github/actions/unit-tests
@@ -103,20 +107,40 @@ jobs:
mv reports/.coverage reports/${{ matrix.shard_name }}.coverage
- name: Upload coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: coverage
path: reports/${{matrix.shard_name}}.coverage
overwrite: true
collect-and-verify:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: 3.11
- name: install system requirements
run: |
sudo apt-get update && sudo apt-get install libxmlsec1-dev
- name: install requirements
run: |
make test-requirements
- name: verify unit tests count
uses: ./.github/actions/verify-tests-count
# This job aggregates test results. It's the required check for branch protection.
# https://github.com/marketplace/actions/alls-green#why
# https://github.com/orgs/community/discussions/33579
success:
name: Unit tests successful
if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
needs:
- run-tests
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
if: always()
needs: [run-tests]
steps:
- name: Decide whether the needed jobs succeeded or failed
# uses: re-actors/alls-green@v1.2.1
@@ -125,14 +149,12 @@ jobs:
jobs: ${{ toJSON(needs) }}
compile-warnings-report:
runs-on: [ edx-platform-runner ]
needs: [ run-tests ]
runs-on: ubuntu-20.04
needs: [run-tests]
steps:
- name: sync directory owner
run: sudo chown runner:runner -R .*
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: collect pytest warnings files
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: pytest-warnings-json
path: test_root/log
@@ -146,31 +168,33 @@ jobs:
- name: save warning report
if: success()
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: pytest-warning-report-html
path: |
reports/pytest_warnings/warning_report_all.html
overwrite: true
# Combine and upload coverage reports.
coverage:
needs: run-tests
runs-on: ubuntu-latest
if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
runs-on: ubuntu-20.04
needs: [run-tests]
strategy:
matrix:
python-version:
- 3.11
steps:
- name: Checkout repo
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Download all artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: coverage
path: reports
@@ -184,4 +208,4 @@ jobs:
coverage combine reports/*
coverage report
coverage xml
- uses: codecov/codecov-action@v3
- uses: codecov/codecov-action@v4

4
.github/workflows/units-test-scripts-structures-pruning.yml vendored
View File

@@ -13,14 +13,14 @@ jobs:
strategy:
matrix:
python-version:
- '3.12'
- "3.12"
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

4
.github/workflows/units-test-scripts-user-retirement.yml vendored
View File

@@ -13,14 +13,14 @@ jobs:
strategy:
matrix:
python-version:
- '3.12'
- "3.12"
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

18
.github/workflows/update-geolite-database.yml vendored
View File

@@ -6,18 +6,18 @@ on:
workflow_dispatch:
inputs:
branch:
description: 'Target branch against which to create PR'
description: "Target branch against which to create PR"
required: false
default: 'master'
default: "master"
env:
MAXMIND_URL: 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz'
MAXMIND_SHA256_URL: 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz.sha256'
TAR_FILE_NAME: 'GeoLite2-Country.tar.gz'
TAR_SHA256_FILE_NAME: 'GeoLite2-Country.tar.gz.sha256'
TAR_UNZIPPED_ROOT_PATTERN: 'GeoLite2-Country_*'
DB_FILE: 'GeoLite2-Country.mmdb'
DB_DESTINATION_PATH: 'common/static/data/geoip'
MAXMIND_URL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz"
MAXMIND_SHA256_URL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz.sha256"
TAR_FILE_NAME: "GeoLite2-Country.tar.gz"
TAR_SHA256_FILE_NAME: "GeoLite2-Country.tar.gz.sha256"
TAR_UNZIPPED_ROOT_PATTERN: "GeoLite2-Country_*"
DB_FILE: "GeoLite2-Country.mmdb"
DB_DESTINATION_PATH: "common/static/data/geoip"
jobs:
download-and-replace:

16
.github/workflows/upgrade-one-python-dependency.yml vendored
View File

@@ -4,22 +4,22 @@ on:
workflow_dispatch:
inputs:
branch:
description: 'Target branch to create requirements PR against'
description: "Target branch to create requirements PR against"
required: true
default: 'master'
default: "master"
type: string
package:
description: 'Name of package to upgrade'
description: "Name of package to upgrade"
required: true
type: string
version:
description: 'Version number to upgrade to in constraints.txt (only needed if pinned)'
default: ''
description: "Version number to upgrade to in constraints.txt (only needed if pinned)"
default: ""
type: string
change_desc:
description: |
Description of change, for commit message and PR. (What does the new version add or fix?)
default: ''
default: ""
type: string
defaults:
@@ -32,12 +32,12 @@ jobs:
steps:
- name: Check out target branch
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
ref: "${{ inputs.branch }}"
- name: Set up Python environment
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: "3.11"

29
.github/workflows/upgrade-python-requirements.yml vendored
View File

@@ -2,26 +2,25 @@ name: Upgrade Requirements
on:
schedule:
- cron: "0 2 * * 2"
- cron: "0 2 * * 2"
workflow_dispatch:
inputs:
branch:
description: 'Target branch to create requirements PR against'
required: true
default: 'master'
inputs:
branch:
description: "Target branch to create requirements PR against"
required: true
default: "master"
jobs:
call-upgrade-python-requirements-workflow:
# Don't run the weekly upgrade job on forks -- it will send a weekly failure email.
if: github.repository == 'openedx/edx-platform' || github.event_name != 'schedule'
uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master
with:
branch: ${{ github.event.inputs.branch }}
team_reviewers: "arbi-bom"
email_address: arbi-bom@edx.org
send_success_notification: false
branch: ${{ github.event.inputs.branch }}
team_reviewers: "arbi-bom"
email_address: arbi-bom@edx.org
send_success_notification: false
secrets:
requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }}
requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }}
edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }}
edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }}
requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }}
requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }}
edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }}
edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }}

19
.github/workflows/verify-dunder-init.yml vendored
View File

@@ -6,21 +6,18 @@ on:
- master
jobs:
verify_dunder_init:
name: Verify __init__.py Files
runs-on: ubuntu-20.04
steps:
- name: Check out branch
uses: actions/checkout@v4
- name: Check out branch
uses: actions/checkout@v2
- name: Ensure git is installed
run: |
sudo apt-get update && sudo apt-get install git
- name: Ensure git is installed
run: |
sudo apt-get update && sudo apt-get install git
- name: Verify __init__.py files exist
run: |
scripts/verify-dunder-init.sh
- name: Verify __init__.py files exist
run: |
scripts/verify-dunder-init.sh

28
.github/workflows/verify-gha-unit-tests-count.yml vendored
View File

@@ -1,28 +0,0 @@
name: verify unit tests count
on:
pull_request:
push:
branches:
- master
jobs:
collect-and-verify:
if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
runs-on: [ edx-platform-runner ]
steps:
- name: sync directory owner
run: sudo chown runner:runner -R .*
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- uses: actions/checkout@v2
- name: install requirements
run: |
sudo make test-requirements
- name: verify unit tests count
uses: ./.github/actions/verify-tests-count

11
.gitignore vendored
View File

@@ -102,11 +102,11 @@ bin/
### Static assets pipeline artifacts
*.scssc
lms/static/css/
lms/static/certificates/css/
cms/static/css/
common/static/common/js/vendor/
common/static/common/css/vendor/
lms/static/css
lms/static/certificates/css
cms/static/css
common/static/common/js/vendor
common/static/common/css/vendor
common/static/bundles
webpack-stats.json
@@ -115,7 +115,6 @@ lms/static/sass/*.css
lms/static/sass/*.css.map
lms/static/certificates/sass/*.css
lms/static/themed_sass/
cms/static/css/
cms/static/sass/*.css
cms/static/sass/*.css.map
cms/static/themed_sass/

2
Makefile
View File

@@ -114,8 +114,8 @@ REQ_FILES = \
requirements/edx/base \
requirements/edx/doc \
requirements/edx/testing \
requirements/edx/development \
requirements/edx/assets \
requirements/edx/development \
requirements/edx/semgrep \
scripts/xblock/requirements \
scripts/user_retirement/requirements/base \

2
catalog-info.yaml
View File

@@ -11,6 +11,6 @@ metadata:
title: "Documentation"
icon: "Web"
spec:
owner: group:2u-arch-bom
owner: group:wg-maintenance-edx-platform
type: 'service'
lifecycle: 'production'

2
cms/djangoapps/contentstore/courseware_index.py
View File

@@ -278,7 +278,7 @@ class SearchIndexerBase(metaclass=ABCMeta):
(Re)index all content within the given structure (course or library),
tracking the fact that a full reindex has taken place
"""
indexed_count = cls.index(modulestore, structure_key)
indexed_count = cls.index(modulestore, structure_key, timeout=180)
if indexed_count:
cls._track_index_request(cls.INDEX_EVENT['name'], cls.INDEX_EVENT['category'], indexed_count)
return indexed_count

1
cms/djangoapps/contentstore/rest_api/v0/serializers/__init__.py
View File

@@ -3,6 +3,7 @@ Serializers for v0 contentstore API.
"""
from .advanced_settings import AdvancedSettingsFieldSerializer, CourseAdvancedSettingsSerializer
from .assets import AssetSerializer
from .authoring_grading import CourseGradingModelSerializer
from .tabs import CourseTabSerializer, CourseTabUpdateSerializer, TabIDLocatorSerializer
from .transcripts import TranscriptSerializer, YoutubeTranscriptCheckSerializer, YoutubeTranscriptUploadSerializer
from .xblock import XblockSerializer

20
cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py Normal file
View File

@@ -0,0 +1,20 @@
"""
API Serializers for course grading
"""
from rest_framework import serializers
class GradersSerializer(serializers.Serializer):
""" Serializer for graders """
type = serializers.CharField()
min_count = serializers.IntegerField()
drop_count = serializers.IntegerField()
short_label = serializers.CharField(required=False, allow_null=True, allow_blank=True)
weight = serializers.IntegerField()
id = serializers.IntegerField()
class CourseGradingModelSerializer(serializers.Serializer):
""" Serializer for course grading model data """
graders = GradersSerializer(many=True, allow_null=True, allow_empty=True)

9
cms/djangoapps/contentstore/rest_api/v0/urls.py
View File

@@ -7,6 +7,7 @@ from openedx.core.constants import COURSE_ID_PATTERN
from .views import (
AdvancedCourseSettingsView,
AuthoringGradingView,
CourseTabSettingsView,
CourseTabListView,
CourseTabReorderView,
@@ -46,8 +47,8 @@ urlpatterns = [
),
# Authoring API
re_path(
r'^heartbeat$', APIHeartBeatView.as_view(), name='heartbeat'
path(
'heartbeat', APIHeartBeatView.as_view(), name='heartbeat'
),
re_path(
fr'^file_assets/{settings.COURSE_ID_PATTERN}$',
@@ -61,6 +62,10 @@ urlpatterns = [
fr'^videos/encodings/{settings.COURSE_ID_PATTERN}$',
authoring_videos.VideoEncodingsDownloadView.as_view(), name='cms_api_videos_encodings'
),
re_path(
fr'grading/{settings.COURSE_ID_PATTERN}',
AuthoringGradingView.as_view(), name='cms_api_update_grading'
),
path(
'videos/features',
authoring_videos.VideoFeaturesView.as_view(), name='cms_api_videos_features'

1
cms/djangoapps/contentstore/rest_api/v0/views/__init__.py
View File

@@ -2,6 +2,7 @@
Views for v0 contentstore API.
"""
from .advanced_settings import AdvancedCourseSettingsView
from .authoring_grading import AuthoringGradingView
from .tabs import CourseTabSettingsView, CourseTabListView, CourseTabReorderView
from .transcripts import TranscriptView, YoutubeTranscriptCheckView, YoutubeTranscriptUploadView
from .api_heartbeat import APIHeartBeatView

88
cms/djangoapps/contentstore/rest_api/v0/views/authoring_grading.py Normal file
View File

@@ -0,0 +1,88 @@
""" API Views for course advanced settings """
import edx_api_doc_tools as apidocs
from opaque_keys.edx.keys import CourseKey
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.views import APIView
from cms.djangoapps.models.settings.course_grading import CourseGradingModel
from common.djangoapps.student.auth import has_studio_read_access
from openedx.core.djangoapps.credit.tasks import update_credit_course_requirements
from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, verify_course_exists, view_auth_classes
from ..serializers import CourseGradingModelSerializer
@view_auth_classes(is_authenticated=True)
class AuthoringGradingView(DeveloperErrorViewMixin, APIView):
"""
View for getting and setting the advanced settings for a course.
"""
@apidocs.schema(
body=CourseGradingModelSerializer,
parameters=[
apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID"),
],
responses={
200: CourseGradingModelSerializer,
401: "The requester is not authenticated.",
403: "The requester cannot access the specified course.",
404: "The requested course does not exist.",
},
)
@verify_course_exists()
def post(self, request: Request, course_id: str):
"""
Update a course's grading.
**Example Request**
POST /api/contentstore/v0/course_grading/{course_id}
**POST Parameters**
The data sent for a post request should follow next object.
Here is an example request data that updates the ``course_grading``
```json
{
"graders": [
{
"type": "Homework",
"min_count": 1,
"drop_count": 0,
"short_label": "",
"weight": 100,
"id": 0
}
],
"grade_cutoffs": {
"A": 0.75,
"B": 0.63,
"C": 0.57,
"D": 0.5
},
"grace_period": {
"hours": 12,
"minutes": 0
},
"minimum_grade_credit": 0.7,
"is_credit_course": true
}
```
**Response Values**
If the request is successful, an HTTP 200 "OK" response is returned,
"""
course_key = CourseKey.from_string(course_id)
if not has_studio_read_access(request.user, course_key):
self.permission_denied(request)
if 'minimum_grade_credit' in request.data:
update_credit_course_requirements.delay(str(course_key))
updated_data = CourseGradingModel.update_from_json(course_key, request.data, request.user)
serializer = CourseGradingModelSerializer(updated_data)
return Response(serializer.data)

7
cms/djangoapps/contentstore/signals/handlers.py
View File

@@ -123,9 +123,6 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=
update_search_index,
update_special_exams_and_publish
)
from cms.djangoapps.coursegraph.tasks import (
dump_course_to_neo4j
)
# DEVELOPER README: probably all tasks here should use transaction.on_commit
# to avoid stale data, but the tasks are owned by many teams and are often
@@ -146,10 +143,6 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=
# Push the course outline to learning_sequences asynchronously.
update_outline_from_modulestore_task.delay(course_key_str)
if settings.COURSEGRAPH_DUMP_COURSE_ON_PUBLISH:
# Push the course out to CourseGraph asynchronously.
dump_course_to_neo4j.delay(course_key_str)
# Kick off a courseware indexing action after the data is ready
if CoursewareSearchIndexer.indexing_is_enabled() and CourseAboutSearchIndexer.indexing_is_enabled():
transaction.on_commit(lambda: update_search_index.delay(course_key_str, datetime.now(UTC).isoformat()))

3
cms/djangoapps/contentstore/tests/test_course_create_rerun.py
View File

@@ -104,10 +104,9 @@ class TestCourseListing(ModuleStoreTestCase):
dest_course_key = CourseKey.from_string(data['destination_course_key'])
self.assertEqual(dest_course_key.run, 'copy')
source_course = self.store.get_course(self.source_course_key)
dest_course = self.store.get_course(dest_course_key)
self.assertEqual(dest_course.start, CourseFields.start.default)
self.assertEqual(dest_course.end, source_course.end)
self.assertEqual(dest_course.end, None)
self.assertEqual(dest_course.enrollment_start, None)
self.assertEqual(dest_course.enrollment_end, None)
course_orgs = get_course_organizations(dest_course_key)

5
cms/djangoapps/contentstore/video_storage_handlers.py
View File

@@ -690,7 +690,10 @@ def _get_index_videos(course, pagination_conf=None):
for attr in attrs:
if attr == 'courses':
current_course = [c for c in video['courses'] if course_id in c]
(__, values['course_video_image_url']), = list(current_course[0].items())
if current_course:
values['course_video_image_url'] = current_course[0][course_id]
else:
values['course_video_image_url'] = None
elif attr == 'encoded_videos':
values['download_link'] = ''
values['file_size'] = 0

3
cms/djangoapps/contentstore/views/course.py
View File

@@ -878,6 +878,7 @@ def _create_or_rerun_course(request):
display_name = request.json.get('display_name')
# force the start date for reruns and allow us to override start via the client
start = request.json.get('start', CourseFields.start.default)
end = request.json.get('end', CourseFields.end.default)
run = request.json.get('run')
has_course_creator_role = is_content_creator(request.user, org)
@@ -892,7 +893,7 @@ def _create_or_rerun_course(request):
status=400
)
fields = {'start': start}
fields = {'start': start, 'end': end}
if display_name is not None:
fields['display_name'] = display_name

120
cms/djangoapps/coursegraph/README.rst
View File

@@ -1,120 +0,0 @@
CourseGraph Support
-------------------
This app exists to write data to "CourseGraph", a tool enabling Open edX developers and support specialists to inspect their platform instance's learning content. CourseGraph itself is simply an instance of `Neo4j`_, which is an open-source graph database with a Web interface.
.. _Neo4j: https://neo4j.com
Deploying Coursegraph
=====================
There are two ways to deploy CourseGraph:
* For operators using Tutor, there is a `CourseGraph plugin for Tutor`_ that is currently released as "Beta". Nutmeg is the earliest Open edX release that the plugin will work alongside.
* For operators still using the old Ansible installation pathway, there exists a `neo4j Ansible playbook`_. Be warned that this method is not well-documented nor officially supported.
In order for CourseGraph to have queryable, up-to-date data, learning content from CMS must be written to CourseGraph regularly. That is where this Django app comes into play. For details on the various ways to write CMS data to CourseGraph, visit the `operations section of the CourseGraph Tutor plugin docs`_.
**Please note**: Access to a populated CourseGraph instance confers access to all the learning content in the associated Open edX CMS (Studio). The basic authentication provided by Neo4j may or may not be sufficient for your security needs. Consider taking additional security measures, such as restricting CourseGraph access to only users on a private VPN.
.. _neo4j Ansible playbook: https://github.com/openedx/configuration/blob/master/playbooks/neo4j.yml
.. _CourseGraph plugin for Tutor: https://github.com/openedx/tutor-contrib-coursegraph/
.. _operations section of the CourseGraph Tutor plugin docs: https://github.com/openedx/tutor-contrib-coursegraph/#managing-data
Running CourseGraph locally
===========================
In some circumstances, you may want to run CourseGraph locally, connected to a development-mode Open edX instance. You can do this in both Tutor and Devstack.
Tutor
*****
The `CourseGraph plugin for Tutor`_ makes it easy to install, configure, and run CourseGraph for local development.
Devstack
********
CourseGraph is included as an "extra" component in the `Open edX Devstack`_. That is, it is not run or provisioned by default, but can be enabled on-demand.
To provision Devstack CourseGraph with data from Devstack LMS, run::
make dev.provision.coursegraph
CourseGraph should now be accessible at http://localhost:7474 with the username ``neo4j`` and the password ``edx``.
Under the hood, the provisioning command just invokes ``dump_to_neo4j`` on your LMS, pointed at your CourseGraph. The provisioning command can be run again at any point in the future to refresh CourseGraph with new LMS data. The data in CourseGraph will persist unless you explicitly destroy it (as noted below).
Other Devstack CourseGraph commands include::
make dev.up.coursegraph # Bring up the container (without re-provisioning).
make dev.down.coursegraph # Stop and remove the container.
make dev.shell.coursegraph # Start a shell session in the container.
make dev.attach.coursegraph # Attach to the container.
make dev.destroy.coursegraph # Stop the container and destroy its database.
The above commands should be run in your ``devstack`` folder, and they assume that LMS is already properly provisioned. See the `Devstack interface`_ for more details.
.. _Open edX Devstack: https://github.com/openedx/devstack/
.. _Devstack interface: https://edx.readthedocs.io/projects/open-edx-devstack/en/latest/devstack_interface.html
Querying Coursegraph
====================
CourseGraph is queryable using the `Cypher`_ query language. Open edX learning content is represented in Neo4j using a straightforward scheme:
* A node is an XBlock usage.
* Nodes are tagged with their ``block_type``, such as:
* ``course``
* ``chapter``
* ``sequential``
* ``vertical``
* ``problem``
* ``html``
* etc.
* Every node is also tagged with ``item``.
* Parent-child relationships in the course hierarchy are reflected in the ``PARENT_OF`` relationship.
* Ordered sibling relationships in the course hierarchy are reflected in the ``PRECEDES`` relationship.
* Fields on each XBlock usage (``.display_name``, ``.data``, etc) are available on the corresponding node.
.. _Cypher: https://neo4j.com/developer/cypher/
Example Queries
***************
How many XBlocks exist in the LMS, by type? ::
MATCH
(c:course) -[:PARENT_OF*]-> (n:item)
RETURN
distinct(n.block_type) as block_type,
count(n) as number
order by
number DESC
In a given course, which units contain problems with custom Python grading code? ::
MATCH
(c:course) -[:PARENT_OF*]-> (u:vertical) -[:PARENT_OF*]-> (p:problem)
WHERE
p.data CONTAINS 'loncapa/python'
AND
c.course_key = '<course_key>'
RETURN
u.location
You can see many more examples of useful CourseGraph queries on the `query archive wiki page`_.
.. _query archive wiki page: https://openedx.atlassian.net/wiki/spaces/COMM/pages/3273228388/Useful+CourseGraph+Queries

0
cms/djangoapps/coursegraph/__init__.py
View File

123
cms/djangoapps/coursegraph/admin.py
View File

@@ -1,123 +0,0 @@
"""
Admin site bindings for coursegraph
"""
import logging
from django.contrib import admin, messages
from django.utils.translation import gettext as _
from edx_django_utils.admin.mixins import ReadOnlyAdminMixin
from .models import CourseGraphCourseDump
from .tasks import ModuleStoreSerializer
log = logging.getLogger(__name__)
@admin.action(
permissions=['change'],
description=_("Dump courses to CourseGraph (respect cache)"),
)
def dump_courses(modeladmin, request, queryset):
"""
Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses,
excluding courses that haven't been published since they were last dumped.
queryset is a QuerySet of CourseGraphCourseDump objects, which are just
CourseOverview objects under the hood.
"""
all_course_keys = queryset.values_list('id', flat=True)
serializer = ModuleStoreSerializer(all_course_keys)
try:
submitted, skipped = serializer.dump_courses_to_neo4j()
# Unfortunately there is no unified base class for the reasonable
# exceptions we could expect from py2neo (connection unavailable, bolt protocol
# error, and so on), so we just catch broadly, show a generic error banner,
# and then log the exception for site operators to look at.
except Exception as err: # pylint: disable=broad-except
log.exception(
"Failed to enqueue CourseGraph dumps to Neo4j (respecting cache): %s",
", ".join(str(course_key) for course_key in all_course_keys),
)
modeladmin.message_user(
request,
_("Error enqueueing dumps for {} course(s): {}").format(
len(all_course_keys), str(err)
),
level=messages.ERROR,
)
return
if submitted:
modeladmin.message_user(
request,
_(
"Enqueued dumps for {} course(s). Skipped {} unchanged course(s)."
).format(len(submitted), len(skipped)),
level=messages.SUCCESS,
)
else:
modeladmin.message_user(
request,
_(
"Skipped all {} course(s), as they were unchanged.",
).format(len(skipped)),
level=messages.WARNING,
)
@admin.action(
permissions=['change'],
description=_("Dump courses to CourseGraph (override cache)")
)
def dump_courses_overriding_cache(modeladmin, request, queryset):
"""
Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses
(whether or not they have been published recently).
queryset is a QuerySet of CourseGraphCourseDump objects, which are just
CourseOverview objects under the hood.
"""
all_course_keys = queryset.values_list('id', flat=True)
serializer = ModuleStoreSerializer(all_course_keys)
try:
submitted, _skipped = serializer.dump_courses_to_neo4j(override_cache=True)
# Unfortunately there is no unified base class for the reasonable
# exceptions we could expect from py2neo (connection unavailable, bolt protocol
# error, and so on), so we just catch broadly, show a generic error banner,
# and then log the exception for site operators to look at.
except Exception as err: # pylint: disable=broad-except
log.exception(
"Failed to enqueue CourseGraph Neo4j course dumps (overriding cache): %s",
", ".join(str(course_key) for course_key in all_course_keys),
)
modeladmin.message_user(
request,
_("Error enqueueing dumps for {} course(s): {}").format(
len(all_course_keys), str(err)
),
level=messages.ERROR,
)
return
modeladmin.message_user(
request,
_("Enqueued dumps for {} course(s).").format(len(submitted)),
level=messages.SUCCESS,
)
@admin.register(CourseGraphCourseDump)
class CourseGraphCourseDumpAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
"""
Model admin for "Course graph course dumps".
Just a read-only table with some useful metadata, allowing admin users to
select courses to be dumped to CourseGraph.
"""
list_display = [
'id',
'display_name',
'modified',
'enrollment_start',
'enrollment_end',
]
search_fields = ['id', 'display_name']
actions = [dump_courses, dump_courses_overriding_cache]

25
cms/djangoapps/coursegraph/apps.py
View File

@@ -1,25 +0,0 @@
"""
Coursegraph Application Configuration
Signal handlers are connected here.
"""
import warnings
from django.apps import AppConfig
class CoursegraphConfig(AppConfig):
"""
AppConfig for courseware app
"""
name = 'cms.djangoapps.coursegraph'
from cms.djangoapps.coursegraph import tasks
def ready(self) -> None:
warnings.warn(
"Neo4j support is going to be dropped after Sumac release,"
"to read more here is a github issue https://github.com/openedx/edx-platform/issues/34342",
DeprecationWarning,
stacklevel=2
)

0
cms/djangoapps/coursegraph/management/__init__.py
View File

0
cms/djangoapps/coursegraph/management/commands/__init__.py
View File

114
cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py
View File

@@ -1,114 +0,0 @@
"""
This file contains a management command for exporting the modulestore to
Neo4j, a graph database.
Example usages:
# Dump all courses published since last dump.
# Use connection parameters from `settings.COURSEGRAPH_SETTINGS`.
python manage.py cms dump_to_neo4j
# Dump all courses published since last dump.
# Use custom connection parameters.
python manage.py cms dump_to_neo4j --host localhost --port 7473 \
--secure --user user --password password
# Specify certain courses instead of dumping all of them.
# Use connection parameters from `settings.COURSEGRAPH_SETTINGS`.
python manage.py cms dump_to_neo4j --courses 'course-v1:A+B+1' 'course-v1:A+B+2'
"""
import logging
from textwrap import dedent
from django.core.management.base import BaseCommand
from cms.djangoapps.coursegraph.tasks import ModuleStoreSerializer
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Dump recently-published course(s) over to a CourseGraph (Neo4j) instance.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument(
'--host',
type=str,
help="the hostname of the Neo4j server",
)
parser.add_argument(
'--port',
type=int,
help="the port on the Neo4j server that accepts Bolt requests",
)
parser.add_argument(
'--secure',
action='store_true',
help="connect to server over Bolt/TLS instead of plain unencrypted Bolt",
)
parser.add_argument(
'--user',
type=str,
help="the username of the Neo4j user",
)
parser.add_argument(
'--password',
type=str,
help="the password of the Neo4j user",
)
parser.add_argument(
'--courses',
metavar='KEY',
type=str,
nargs='*',
help="keys of courses to serialize; if omitted all courses in system are serialized",
)
parser.add_argument(
'--skip',
metavar='KEY',
type=str,
nargs='*',
help="keys of courses to NOT to serialize",
)
parser.add_argument(
'--override',
action='store_true',
help="dump all courses regardless of when they were last published",
)
def handle(self, *args, **options):
"""
Iterates through each course, serializes them into graphs, and saves
those graphs to neo4j.
"""
mss = ModuleStoreSerializer.create(options['courses'], options['skip'])
connection_overrides = {
key: options[key]
for key in ["host", "port", "secure", "user", "password"]
}
submitted_courses, skipped_courses = mss.dump_courses_to_neo4j(
connection_overrides=connection_overrides,
override_cache=options['override'],
)
log.info(
"%d courses submitted for export to neo4j. %d courses skipped.",
len(submitted_courses),
len(skipped_courses),
)
if not submitted_courses:
print("No courses submitted for export to neo4j at all!")
return
if submitted_courses:
print(
"These courses were submitted for export to neo4j successfully:\n\t" +
"\n\t".join(submitted_courses)
)

0
cms/djangoapps/coursegraph/management/commands/tests/__init__.py
View File

596
cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py
View File

@@ -1,596 +0,0 @@
"""
Tests for the dump_to_neo4j management command.
"""
from datetime import datetime
from unittest import mock
import ddt
from django.core.management import call_command
from django.test.utils import override_settings
from edx_toggles.toggles.testutils import override_waffle_switch
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory
import openedx.core.djangoapps.content.block_structure.config as block_structure_config
from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish
from cms.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer
from cms.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher
from cms.djangoapps.coursegraph.tasks import (
coerce_types,
serialize_course,
serialize_item,
should_dump_course,
strip_branch_and_version
)
from openedx.core.djangolib.testing.utils import skip_unless_lms
class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase):
"""
Base class for the test suites in this file. Sets up a couple courses.
"""
@classmethod
def setUpClass(cls):
r"""
Creates two courses; one that's just a course block, and one that
looks like:
course
|
chapter
|
sequential
|
vertical
/ | \ \
/ | \ ----------
/ | \ \
/ | --- \
/ | \ \
html -> problem -> video -> video2
The side-pointing arrows (->) are PRECEDES relationships; the more
vertical lines are PARENT_OF relationships.
The vertical in this course and the first video have the same
display_name, so that their block_ids are the same. This is to
test for a bug where xblocks with the same block_ids (but different
locations) pointed to themselves erroneously.
"""
super().setUpClass()
cls.course = CourseFactory.create()
cls.chapter = BlockFactory.create(parent=cls.course, category='chapter')
cls.sequential = BlockFactory.create(parent=cls.chapter, category='sequential')
cls.vertical = BlockFactory.create(parent=cls.sequential, category='vertical', display_name='subject')
cls.html = BlockFactory.create(parent=cls.vertical, category='html')
cls.problem = BlockFactory.create(parent=cls.vertical, category='problem')
cls.video = BlockFactory.create(parent=cls.vertical, category='video', display_name='subject')
cls.video2 = BlockFactory.create(parent=cls.vertical, category='video')
cls.course2 = CourseFactory.create()
cls.course_strings = [str(cls.course.id), str(cls.course2.id)]
@staticmethod
def setup_mock_graph(mock_matcher_class, mock_graph_class, transaction_errors=False):
"""
Replaces the py2neo Graph object with a MockGraph; similarly replaces
NodeMatcher with MockNodeMatcher.
Arguments:
mock_matcher_class: a mocked NodeMatcher class
mock_graph_class: a mocked Graph class
transaction_errors: a bool for whether we should get errors
when transactions try to commit
Returns: an instance of MockGraph
"""
mock_graph = MockGraph(transaction_errors=transaction_errors)
mock_graph_class.return_value = mock_graph
mock_node_matcher = MockNodeMatcher(mock_graph)
mock_matcher_class.return_value = mock_node_matcher
return mock_graph
def assertCourseDump(self, mock_graph, number_of_courses, number_commits, number_rollbacks):
"""
Asserts that we have the expected number of courses, commits, and
rollbacks after we dump the modulestore to neo4j
Arguments:
mock_graph: a MockGraph backend
number_of_courses: number of courses we expect to find
number_commits: number of commits we expect against the graph
number_rollbacks: number of commit rollbacks we expect
"""
courses = {node['course_key'] for node in mock_graph.nodes}
assert len(courses) == number_of_courses
assert mock_graph.number_commits == number_commits
assert mock_graph.number_rollbacks == number_rollbacks
@ddt.ddt
class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase):
"""
Tests for the dump to neo4j management command
"""
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.Graph')
@ddt.data(1, 2)
def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_matcher_class):
"""
Test that you can specify which courses you want to dump.
"""
mock_graph = self.setup_mock_graph(mock_matcher_class, mock_graph_class)
call_command(
'dump_to_neo4j',
courses=self.course_strings[:number_of_courses],
host='mock_host',
port=7687,
user='mock_user',
password='mock_password',
)
self.assertCourseDump(
mock_graph,
number_of_courses=number_of_courses,
number_commits=number_of_courses,
number_rollbacks=0
)
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.Graph')
def test_dump_skip_course(self, mock_graph_class, mock_matcher_class):
"""
Test that you can skip courses.
"""
mock_graph = self.setup_mock_graph(
mock_matcher_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
skip=self.course_strings[:1],
host='mock_host',
port=7687,
user='mock_user',
password='mock_password',
)
self.assertCourseDump(
mock_graph,
number_of_courses=1,
number_commits=1,
number_rollbacks=0,
)
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.Graph')
def test_dump_skip_beats_specifying(self, mock_graph_class, mock_matcher_class):
"""
Test that if you skip and specify the same course, you'll skip it.
"""
mock_graph = self.setup_mock_graph(
mock_matcher_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
skip=self.course_strings[:1],
courses=self.course_strings[:1],
host='mock_host',
port=7687,
user='mock_user',
password='mock_password',
)
self.assertCourseDump(
mock_graph,
number_of_courses=0,
number_commits=0,
number_rollbacks=0,
)
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.Graph')
def test_dump_all_courses(self, mock_graph_class, mock_matcher_class):
"""
Test if you don't specify which courses to dump, then you'll dump
all of them.
"""
mock_graph = self.setup_mock_graph(
mock_matcher_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
host='mock_host',
port=7687,
user='mock_user',
password='mock_password'
)
self.assertCourseDump(
mock_graph,
number_of_courses=2,
number_commits=2,
number_rollbacks=0,
)
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.Graph', autospec=True)
@override_settings(
COURSEGRAPH_CONNECTION=dict(
protocol='bolt',
host='coursegraph.example.edu',
port=7777,
secure=True,
user="neo4j",
password="default-password",
)
)
def test_dump_to_neo4j_connection_defaults(self, mock_graph_class, mock_matcher_class):
"""
Test that user can override individual settings.COURSEGRAPH_CONNECTION parameters
by passing them to `dump_to_neo4j`, whilst falling back to the ones that they
don't override.
"""
self.setup_mock_graph(
mock_matcher_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
courses=self.course_strings[:1],
port=7788,
secure=False,
password="overridden-password",
)
assert mock_graph_class.call_args.args == ()
assert mock_graph_class.call_args.kwargs == dict(
# From settings:
protocol='bolt',
host='coursegraph.example.edu',
user="neo4j",
# Overriden by command:
port=7788,
secure=False,
password="overridden-password",
)
class SomeThing:
"""Just to test the stringification of an object."""
def __str__(self):
return "<SomeThing>"
@skip_unless_lms
@ddt.ddt
class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase):
"""
Tests for the ModuleStoreSerializer
"""
@classmethod
def setUpClass(cls):
"""Any ModuleStore course/content operations can go here."""
super().setUpClass()
cls.mss = ModuleStoreSerializer.create()
def test_serialize_item(self):
"""
Tests the serialize_item method.
"""
fields, label = serialize_item(self.course)
assert label == 'course'
assert 'edited_on' in list(fields.keys())
assert 'display_name' in list(fields.keys())
assert 'org' in list(fields.keys())
assert 'course' in list(fields.keys())
assert 'run' in list(fields.keys())
assert 'course_key' in list(fields.keys())
assert 'location' in list(fields.keys())
assert 'block_type' in list(fields.keys())
assert 'detached' in list(fields.keys())
assert 'checklist' not in list(fields.keys())
def test_serialize_course(self):
"""
Tests the serialize_course method.
"""
nodes, relationships = serialize_course(self.course.id)
assert len(nodes) == 9
# the course has 7 "PARENT_OF" relationships and 3 "PRECEDES"
assert len(relationships) == 10
def test_strip_version_and_branch(self):
"""
Tests that the _strip_version_and_branch function strips the version
and branch from a location
"""
location = self.course.id.make_usage_key(
'test_block_type', 'test_block_id'
).for_branch(
'test_branch'
).for_version(b'test_version')
assert location.branch is not None
assert location.version_guid is not None
stripped_location = strip_branch_and_version(location)
assert stripped_location.branch is None
assert stripped_location.version_guid is None
@staticmethod
def _extract_relationship_pairs(relationships, relationship_type):
"""
Extracts a list of XBlock location tuples from a list of Relationships.
Arguments:
relationships: list of py2neo `Relationship` objects
relationship_type: the type of relationship to filter `relationships`
by.
Returns:
List of tuples of the locations of of the relationships'
constituent nodes.
"""
relationship_pairs = [
(rel.start_node["location"], rel.end_node["location"])
for rel in relationships if type(rel).__name__ == relationship_type
]
return relationship_pairs
@staticmethod
def _extract_location_pair(xblock1, xblock2):
"""
Returns a tuple of locations from two XBlocks.
Arguments:
xblock1: an xblock
xblock2: also an xblock
Returns:
A tuple of the string representations of those XBlocks' locations.
"""
return (str(xblock1.location), str(xblock2.location))
def assertBlockPairIsRelationship(self, xblock1, xblock2, relationships, relationship_type):
"""
Helper assertion that a pair of xblocks have a certain kind of
relationship with one another.
"""
relationship_pairs = self._extract_relationship_pairs(relationships, relationship_type)
location_pair = self._extract_location_pair(xblock1, xblock2)
assert location_pair in relationship_pairs
def assertBlockPairIsNotRelationship(self, xblock1, xblock2, relationships, relationship_type):
"""
The opposite of `assertBlockPairIsRelationship`: asserts that a pair
of xblocks do NOT have a certain kind of relationship.
"""
relationship_pairs = self._extract_relationship_pairs(relationships, relationship_type)
location_pair = self._extract_location_pair(xblock1, xblock2)
assert location_pair not in relationship_pairs
def test_precedes_relationship(self):
"""
Tests that two nodes that should have a precedes relationship have it.
"""
__, relationships = serialize_course(self.course.id)
self.assertBlockPairIsRelationship(self.video, self.video2, relationships, "PRECEDES")
self.assertBlockPairIsNotRelationship(self.video2, self.video, relationships, "PRECEDES")
self.assertBlockPairIsNotRelationship(self.vertical, self.video, relationships, "PRECEDES")
self.assertBlockPairIsNotRelationship(self.html, self.video, relationships, "PRECEDES")
def test_parent_relationship(self):
"""
Test that two nodes that should have a parent_of relationship have it.
"""
__, relationships = serialize_course(self.course.id)
self.assertBlockPairIsRelationship(self.vertical, self.video, relationships, "PARENT_OF")
self.assertBlockPairIsRelationship(self.vertical, self.html, relationships, "PARENT_OF")
self.assertBlockPairIsRelationship(self.course, self.chapter, relationships, "PARENT_OF")
self.assertBlockPairIsNotRelationship(self.course, self.video, relationships, "PARENT_OF")
self.assertBlockPairIsNotRelationship(self.video, self.vertical, relationships, "PARENT_OF")
self.assertBlockPairIsNotRelationship(self.video, self.html, relationships, "PARENT_OF")
def test_nodes_have_indices(self):
"""
Test that we add index values on nodes
"""
nodes, relationships = serialize_course(self.course.id) # lint-amnesty, pylint: disable=unused-variable
# the html node should have 0 index, and the problem should have 1
html_nodes = [node for node in nodes if node['block_type'] == 'html']
assert len(html_nodes) == 1
problem_nodes = [node for node in nodes if node['block_type'] == 'problem']
assert len(problem_nodes) == 1
html_node = html_nodes[0]
problem_node = problem_nodes[0]
assert html_node['index'] == 0
assert problem_node['index'] == 1
@ddt.data(
(1, 1),
(SomeThing(), "<SomeThing>"),
(1.5, 1.5),
("úñîçø∂é", "úñîçø∂é"),
(b"plain string", b"plain string"),
(True, True),
(None, "None"),
((1,), "(1,)"),
# list of elements should be coerced into a list of the
# string representations of those elements
([SomeThing(), SomeThing()], ["<SomeThing>", "<SomeThing>"]),
([1, 2], ["1", "2"]),
)
@ddt.unpack
def test_coerce_types(self, original_value, coerced_expected):
"""
Tests the coerce_types helper
"""
coerced_value = coerce_types(original_value)
assert coerced_value == coerced_expected
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph')
def test_dump_to_neo4j(self, mock_graph_constructor, mock_matcher_class):
"""
Tests the dump_to_neo4j method works against a mock
py2neo Graph
"""
mock_graph = MockGraph()
mock_graph_constructor.return_value = mock_graph
mock_matcher_class.return_value = MockNodeMatcher(mock_graph)
# mocking is thorwing error in kombu serialzier and its not require here any more.
credentials = {}
submitted, skipped = self.mss.dump_courses_to_neo4j(credentials) # lint-amnesty, pylint: disable=unused-variable
self.assertCourseDump(
mock_graph,
number_of_courses=2,
number_commits=2,
number_rollbacks=0,
)
# 9 nodes + 7 relationships from the first course
# 2 nodes and no relationships from the second
assert len(mock_graph.nodes) == 11
self.assertCountEqual(submitted, self.course_strings)
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph')
def test_dump_to_neo4j_rollback(self, mock_graph_constructor, mock_matcher_class):
"""
Tests that the the dump_to_neo4j method handles the case where there's
an exception trying to write to the neo4j database.
"""
mock_graph = MockGraph(transaction_errors=True)
mock_graph_constructor.return_value = mock_graph
mock_matcher_class.return_value = MockNodeMatcher(mock_graph)
# mocking is thorwing error in kombu serialzier and its not require here any more.
credentials = {}
submitted, skipped = self.mss.dump_courses_to_neo4j(credentials) # lint-amnesty, pylint: disable=unused-variable
self.assertCourseDump(
mock_graph,
number_of_courses=0,
number_commits=0,
number_rollbacks=2,
)
self.assertCountEqual(submitted, self.course_strings)
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph')
@ddt.data((True, 2), (False, 0))
@ddt.unpack
def test_dump_to_neo4j_cache(
self,
override_cache,
expected_number_courses,
mock_graph_constructor,
mock_matcher_class,
):
"""
Tests the caching mechanism and override to make sure we only publish
recently updated courses.
"""
mock_graph = MockGraph()
mock_graph_constructor.return_value = mock_graph
mock_matcher_class.return_value = MockNodeMatcher(mock_graph)
# mocking is thorwing error in kombu serialzier and its not require here any more.
credentials = {}
# run once to warm the cache
self.mss.dump_courses_to_neo4j(
credentials, override_cache=override_cache
)
# when run the second time, only dump courses if the cache override
# is enabled
submitted, __ = self.mss.dump_courses_to_neo4j(
credentials, override_cache=override_cache
)
assert len(submitted) == expected_number_courses
@mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher')
@mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph')
def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_matcher_class):
"""
Tests that we only dump those courses that have been published after
the last time the command was been run.
"""
mock_graph = MockGraph()
mock_graph_constructor.return_value = mock_graph
mock_matcher_class.return_value = MockNodeMatcher(mock_graph)
# mocking is thorwing error in kombu serialzier and its not require here any more.
credentials = {}
# run once to warm the cache
submitted, skipped = self.mss.dump_courses_to_neo4j(credentials) # lint-amnesty, pylint: disable=unused-variable
assert len(submitted) == len(self.course_strings)
# simulate one of the courses being published
with override_waffle_switch(block_structure_config.STORAGE_BACKING_FOR_CACHE, True):
update_block_structure_on_course_publish(None, self.course.id)
# make sure only the published course was dumped
submitted, __ = self.mss.dump_courses_to_neo4j(credentials)
assert len(submitted) == 1
assert submitted[0] == str(self.course.id)
@mock.patch('cms.djangoapps.coursegraph.tasks.get_course_last_published')
@mock.patch('cms.djangoapps.coursegraph.tasks.get_command_last_run')
@ddt.data(
(
str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)),
(True, (
'course has been published since last neo4j update time - '
'update date 2016-03-30 00:00:00 < published date 2016-03-31 00:00:00'
))
),
(
str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)),
(False, None)
),
(
str(datetime(2016, 3, 31)), None,
(False, None)
),
(
None, str(datetime(2016, 3, 30)),
(True, 'no record of the last neo4j update time for the course')
),
(
None, None,
(True, 'no record of the last neo4j update time for the course')
),
)
@ddt.unpack
def test_should_dump_course(
self,
last_command_run,
last_course_published,
should_dump,
mock_get_command_last_run,
mock_get_course_last_published,
):
"""
Tests whether a course should be dumped given the last time it was
dumped and the last time it was published.
"""
mock_get_command_last_run.return_value = last_command_run
mock_get_course_last_published.return_value = last_course_published
mock_course_key = mock.Mock()
mock_graph = mock.Mock()
assert should_dump_course(mock_course_key, mock_graph) == should_dump

123
cms/djangoapps/coursegraph/management/commands/tests/utils.py
View File

@@ -1,123 +0,0 @@
"""
Utilities for testing the dump_to_neo4j management command
"""
from py2neo import Node
class MockGraph:
"""
A stubbed out version of py2neo's Graph object, used for testing.
Args:
transaction_errors: a bool for whether transactions should throw
an error.
"""
def __init__(self, transaction_errors=False, **kwargs): # pylint: disable=unused-argument
self.nodes = set()
self.number_commits = 0
self.number_rollbacks = 0
self.transaction_errors = transaction_errors
def begin(self):
"""
A stub of the method that generates transactions
Returns: a MockTransaction object (instead of a py2neo Transaction)
"""
return MockTransaction(self)
def commit(self, transaction):
"""
Takes elements in the mock transaction's temporary storage and adds them
to this mock graph's storage. Throws an error if this graph's
transaction_errors param is set to True.
"""
if self.transaction_errors:
raise Exception("fake exception while trying to commit")
for element in transaction.temp:
self.nodes.add(element)
transaction.temp.clear()
self.number_commits += 1
def rollback(self, transaction):
"""
Clears the transactions temporary storage
"""
transaction.temp.clear()
self.number_rollbacks += 1
class MockTransaction:
"""
A stubbed out version of py2neo's Transaction object, used for testing.
"""
def __init__(self, graph):
self.temp = set()
self.graph = graph
def run(self, query):
"""
Deletes all nodes associated with a course. Normally `run` executes
an arbitrary query, but in our code, we only use it to delete nodes
associated with a course.
Args:
query: query string to be executed (in this case, to delete all
nodes associated with a course)
"""
start_string = "WHERE n.course_key='"
start = query.index(start_string) + len(start_string)
query = query[start:]
end = query.find("'")
course_key = query[:end]
self.graph.nodes = {
node for node in self.graph.nodes if node['course_key'] != course_key
}
def create(self, element):
"""
Adds elements to the transaction's temporary backend storage
Args:
element: a py2neo Node object
"""
if isinstance(element, Node):
self.temp.add(element)
class MockNodeMatcher:
"""
Mocks out py2neo's NodeMatcher class. Used to match a node from a graph.
py2neo's NodeMatcher expects a real graph object to run queries against,
so, rather than have to mock out MockGraph to accommodate those queries,
it seemed simpler to mock out NodeMatcher as well.
"""
def __init__(self, graph):
self.graph = graph
def match(self, label, course_key):
"""
Selects nodes that match a label and course_key
Args:
label: the string of the label we're selecting nodes by
course_key: the string of the course key we're selecting node by
Returns: a MockResult of matching nodes
"""
nodes = []
for node in self.graph.nodes:
if node.has_label(label) and node["course_key"] == course_key:
nodes.append(node)
return MockNodeMatch(nodes)
class MockNodeMatch(list):
"""
Mocks out py2neo's NodeMatch class: this is the type of what
MockNodeMatcher's `match` method returns.
"""
def first(self):
"""
Returns: the first element of a list if the list has elements.
Otherwise, None.
"""
return self[0] if self else None

21
cms/djangoapps/coursegraph/models.py
View File

@@ -1,21 +0,0 @@
"""
(Proxy) models supporting CourseGraph.
"""
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
class CourseGraphCourseDump(CourseOverview):
"""
Proxy model for CourseOverview.
Does *not* create/update/delete CourseOverview objects - only reads the objects.
Uses the course IDs of the CourseOverview objects to determine which courses
can be dumped to CourseGraph.
"""
class Meta:
proxy = True
def __str__(self):
"""Represent ourselves with the course key."""
return str(self.id)

420
cms/djangoapps/coursegraph/tasks.py
View File

@@ -1,420 +0,0 @@
"""
This file contains a management command for exporting the modulestore to
neo4j, a graph database.
"""
import logging
from celery import shared_task
from django.conf import settings
from django.utils import timezone
from edx_django_utils.cache import RequestCache
from edx_django_utils.monitoring import set_code_owner_attribute
from opaque_keys.edx.keys import CourseKey
import py2neo # pylint: disable=unused-import
from py2neo import Graph, Node, Relationship
try:
from py2neo.matching import NodeMatcher
except ImportError:
from py2neo import NodeMatcher
else:
pass
log = logging.getLogger(__name__)
celery_log = logging.getLogger('edx.celery.task')
# When testing locally, neo4j's bolt logger was noisy, so we'll only have it
# emit logs if there's an error.
bolt_log = logging.getLogger('neo4j.bolt') # pylint: disable=invalid-name
bolt_log.setLevel(logging.ERROR)
PRIMITIVE_NEO4J_TYPES = (int, bytes, str, float, bool)
def serialize_item(item):
"""
Args:
item: an XBlock
Returns:
fields: a dictionary of an XBlock's field names and values
block_type: the name of the XBlock's type (i.e. 'course'
or 'problem')
"""
from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES
# convert all fields to a dict and filter out parent and children field
fields = {
field: field_value.read_from(item)
for (field, field_value) in item.fields.items()
if field not in ['parent', 'children']
}
course_key = item.scope_ids.usage_id.course_key
block_type = item.scope_ids.block_type
# set or reset some defaults
fields['edited_on'] = str(getattr(item, 'edited_on', ''))
fields['display_name'] = item.display_name_with_default
fields['org'] = course_key.org
fields['course'] = course_key.course
fields['run'] = course_key.run
fields['course_key'] = str(course_key)
fields['location'] = str(item.location)
fields['block_type'] = block_type
fields['detached'] = block_type in DETACHED_XBLOCK_TYPES
if block_type == 'course':
# prune the checklists field
if 'checklists' in fields:
del fields['checklists']
# record the time this command was run
fields['time_last_dumped_to_neo4j'] = str(timezone.now())
return fields, block_type
def coerce_types(value):
"""
Args:
value: the value of an xblock's field
Returns: either the value, a text version of the value, or, if the
value is a list, a list where each element is converted to text.
"""
coerced_value = value
if isinstance(value, list):
coerced_value = [str(element) for element in coerced_value]
# if it's not one of the types that neo4j accepts,
# just convert it to text
elif not isinstance(value, PRIMITIVE_NEO4J_TYPES):
coerced_value = str(value)
return coerced_value
def add_to_transaction(neo4j_entities, transaction):
"""
Args:
neo4j_entities: a list of Nodes or Relationships
transaction: a neo4j transaction
"""
for entity in neo4j_entities:
transaction.create(entity)
def get_command_last_run(course_key, graph):
"""
This information is stored on the course node of a course in neo4j
Args:
course_key: a CourseKey
graph: a py2neo Graph
Returns: The datetime that the command was last run, converted into
text, or None, if there's no record of this command last being run.
"""
matcher = NodeMatcher(graph)
course_node = matcher.match(
"course",
course_key=str(course_key)
).first()
last_this_command_was_run = None
if course_node:
last_this_command_was_run = course_node['time_last_dumped_to_neo4j']
return last_this_command_was_run
def get_course_last_published(course_key):
"""
Approximately when was a course last published?
We use the 'modified' column in the CourseOverview table as a quick and easy
(although perhaps inexact) way of determining when a course was last
published. This works because CourseOverview rows are re-written upon
course publish.
Args:
course_key: a CourseKey
Returns: The datetime the course was last published at, stringified.
Uses Python's default str(...) implementation for datetimes, which
is sortable and similar to ISO 8601:
https://docs.python.org/3/library/datetime.html#datetime.date.__str__
"""
# Import is placed here to avoid model import at project startup.
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
approx_last_published = CourseOverview.get_from_id(course_key).modified
return str(approx_last_published)
def strip_branch_and_version(location):
"""
Removes the branch and version information from a location.
Args:
location: an xblock's location.
Returns: that xblock's location without branch and version information.
"""
return location.for_branch(None)
def serialize_course(course_id):
"""
Serializes a course into py2neo Nodes and Relationships
Args:
course_id: CourseKey of the course we want to serialize
Returns:
nodes: a list of py2neo Node objects
relationships: a list of py2neo Relationships objects
"""
# Import is placed here to avoid model import at project startup.
from xmodule.modulestore.django import modulestore
# create a location to node mapping we'll need later for
# writing relationships
location_to_node = {}
items = modulestore().get_items(course_id)
# create nodes
for item in items:
fields, block_type = serialize_item(item)
for field_name, value in fields.items():
fields[field_name] = coerce_types(value)
node = Node(block_type, 'item', **fields)
location_to_node[strip_branch_and_version(item.location)] = node
# create relationships
relationships = []
for item in items:
previous_child_node = None
for index, child in enumerate(item.get_children()):
parent_node = location_to_node.get(strip_branch_and_version(item.location))
child_node = location_to_node.get(strip_branch_and_version(child.location))
if parent_node is not None and child_node is not None:
child_node["index"] = index
relationship = Relationship(parent_node, "PARENT_OF", child_node)
relationships.append(relationship)
if previous_child_node:
ordering_relationship = Relationship(
previous_child_node,
"PRECEDES",
child_node,
)
relationships.append(ordering_relationship)
previous_child_node = child_node
nodes = list(location_to_node.values())
return nodes, relationships
def should_dump_course(course_key, graph):
"""
Only dump the course if it's been changed since the last time it's been
dumped.
Args:
course_key: a CourseKey object.
graph: a py2neo Graph object.
Returns:
- whether this course should be dumped to neo4j (bool)
- reason why course needs to be dumped (string, None if doesn't need to be dumped)
"""
last_this_command_was_run = get_command_last_run(course_key, graph)
course_last_published_date = get_course_last_published(course_key)
# if we don't have a record of the last time this command was run,
# we should serialize the course and dump it
if last_this_command_was_run is None:
return (
True,
"no record of the last neo4j update time for the course"
)
# if we've serialized the course recently and we have no published
# events, we will not dump it, and so we can skip serializing it
# again here
if last_this_command_was_run and course_last_published_date is None:
return (False, None)
# otherwise, serialize and dump the course if the command was run
# before the course's last published event
needs_update = last_this_command_was_run < course_last_published_date
update_reason = None
if needs_update:
update_reason = (
f"course has been published since last neo4j update time - "
f"update date {last_this_command_was_run} < published date {course_last_published_date}"
)
return (needs_update, update_reason)
@shared_task
@set_code_owner_attribute
def dump_course_to_neo4j(course_key_string, connection_overrides=None):
"""
Serializes a course and writes it to neo4j.
Arguments:
course_key_string: course key for the course to be exported
connection_overrides (dict): overrides to Neo4j connection
parameters specified in `settings.COURSEGRAPH_CONNECTION`.
"""
course_key = CourseKey.from_string(course_key_string)
nodes, relationships = serialize_course(course_key)
celery_log.info(
"Now dumping %s to neo4j: %d nodes and %d relationships",
course_key,
len(nodes),
len(relationships),
)
graph = authenticate_and_create_graph(
connection_overrides=connection_overrides
)
transaction = graph.begin()
course_string = str(course_key)
try:
# first, delete existing course
transaction.run(
"MATCH (n:item) WHERE n.course_key='{}' DETACH DELETE n".format(
course_string
)
)
# now, re-add it
add_to_transaction(nodes, transaction)
add_to_transaction(relationships, transaction)
graph.commit(transaction)
celery_log.info("Completed dumping %s to neo4j", course_key)
except Exception: # pylint: disable=broad-except
celery_log.exception(
"Error trying to dump course %s to neo4j, rolling back",
course_string
)
graph.rollback(transaction)
class ModuleStoreSerializer:
"""
Class with functionality to serialize a modulestore into subgraphs,
one graph per course.
"""
def __init__(self, course_keys):
self.course_keys = course_keys
@classmethod
def create(cls, courses=None, skip=None):
"""
Sets the object's course_keys attribute from the `courses` parameter.
If that parameter isn't furnished, loads all course_keys from the
modulestore.
Filters out course_keys in the `skip` parameter, if provided.
Args:
courses: A list of string serializations of course keys.
For example, ["course-v1:org+course+run"].
skip: Also a list of string serializations of course keys.
"""
# Import is placed here to avoid model import at project startup.
from xmodule.modulestore.django import modulestore
if courses:
course_keys = [CourseKey.from_string(course.strip()) for course in courses]
else:
course_keys = [
course.id for course in modulestore().get_course_summaries()
]
if skip is not None:
skip_keys = [CourseKey.from_string(course.strip()) for course in skip]
course_keys = [course_key for course_key in course_keys if course_key not in skip_keys]
return cls(course_keys)
def dump_courses_to_neo4j(self, connection_overrides=None, override_cache=False):
"""
Method that iterates through a list of courses in a modulestore,
serializes them, then submits tasks to write them to neo4j.
Arguments:
connection_overrides (dict): overrides to Neo4j connection
parameters specified in `settings.COURSEGRAPH_CONNECTION`.
override_cache: serialize the courses even if they'be been recently
serialized
Returns: two lists--one of the courses that were successfully written
to neo4j and one of courses that were not.
"""
total_number_of_courses = len(self.course_keys)
submitted_courses = []
skipped_courses = []
graph = authenticate_and_create_graph(connection_overrides)
for index, course_key in enumerate(self.course_keys):
# first, clear the request cache to prevent memory leaks
RequestCache.clear_all_namespaces()
(needs_dump, reason) = should_dump_course(course_key, graph)
if not (override_cache or needs_dump):
log.info("skipping submitting %s, since it hasn't changed", course_key)
skipped_courses.append(str(course_key))
continue
if override_cache:
reason = "override_cache is True"
log.info(
"Now submitting %s for export to neo4j, because %s: course %d of %d total courses",
course_key,
reason,
index + 1,
total_number_of_courses,
)
dump_course_to_neo4j.apply_async(
kwargs=dict(
course_key_string=str(course_key),
connection_overrides=connection_overrides,
)
)
submitted_courses.append(str(course_key))
return submitted_courses, skipped_courses
def authenticate_and_create_graph(connection_overrides=None):
"""
This function authenticates with neo4j and creates a py2neo graph object
Arguments:
connection_overrides (dict): overrides to Neo4j connection
parameters specified in `settings.COURSEGRAPH_CONNECTION`.
Returns: a py2neo `Graph` object.
"""
provided_overrides = {
key: value
for key, value in (connection_overrides or {}).items()
# Drop overrides whose values are `None`. Note that `False` is a
# legitimate override value that we don't want to drop here.
if value is not None
}
connection_with_overrides = {**settings.COURSEGRAPH_CONNECTION, **provided_overrides}
return Graph(**connection_with_overrides)

0
cms/djangoapps/coursegraph/tests/__init__.py
View File

227
cms/djangoapps/coursegraph/tests/test_admin.py
View File

@@ -1,227 +0,0 @@
"""
Shallow tests for CourseGraph dump-queueing Django admin interface.
See ..management.commands.tests.test_dump_to_neo4j for more comprehensive
tests of dump_course_to_neo4j.
"""
from unittest import mock
import py2neo
from django.test import TestCase
from django.test.utils import override_settings
from freezegun import freeze_time
from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from .. import admin, tasks
_coursegraph_connection = {
"protocol": "bolt",
"secure": True,
"host": "example.edu",
"port": 7687,
"user": "neo4j",
"password": "fake-coursegraph-password",
}
_configure_coursegraph_connection = override_settings(
COURSEGRAPH_CONNECTION=_coursegraph_connection,
)
_patch_log_exception = mock.patch.object(
admin.log, 'exception', autospec=True
)
_patch_apply_dump_task = mock.patch.object(
tasks.dump_course_to_neo4j, 'apply_async'
)
_pretend_last_course_dump_was_may_2020 = mock.patch.object(
tasks,
'get_command_last_run',
new=(lambda _key, _graph: "2020-05-01"),
)
_patch_neo4j_graph = mock.patch.object(
tasks, 'Graph', autospec=True
)
_make_neo4j_graph_raise = mock.patch.object(
tasks, 'Graph', side_effect=py2neo.ConnectionUnavailable(
'we failed to connect or something!'
)
)
class CourseGraphAdminActionsTestCase(TestCase):
"""
Test CourseGraph Django admin actions.
"""
@classmethod
def setUpTestData(cls):
"""
Make course overviews with varying modification dates.
"""
super().setUpTestData()
cls.course_updated_in_april = CourseOverviewFactory(run='april_update')
cls.course_updated_in_june = CourseOverviewFactory(run='june_update')
cls.course_updated_in_july = CourseOverviewFactory(run='july_update')
cls.course_updated_in_august = CourseOverviewFactory(run='august_update')
# For each course overview, make an arbitrary update and then save()
# so that its `.modified` date is set.
with freeze_time("2020-04-01"):
cls.course_updated_in_april.marketing_url = "https://example.org"
cls.course_updated_in_april.save()
with freeze_time("2020-06-01"):
cls.course_updated_in_june.marketing_url = "https://example.org"
cls.course_updated_in_june.save()
with freeze_time("2020-07-01"):
cls.course_updated_in_july.marketing_url = "https://example.org"
cls.course_updated_in_july.save()
with freeze_time("2020-08-01"):
cls.course_updated_in_august.marketing_url = "https://example.org"
cls.course_updated_in_august.save()
@_configure_coursegraph_connection
@_pretend_last_course_dump_was_may_2020
@_patch_neo4j_graph
@_patch_apply_dump_task
@_patch_log_exception
def test_dump_courses(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph):
"""
Test that dump_courses admin action dumps requested courses iff they have
been modified since the last dump to coursegraph.
"""
modeladmin_mock = mock.MagicMock()
# Request all courses except the August-updated one
requested_course_keys = {
str(self.course_updated_in_april.id),
str(self.course_updated_in_june.id),
str(self.course_updated_in_july.id),
}
admin.dump_courses(
modeladmin=modeladmin_mock,
request=mock.MagicMock(),
queryset=CourseOverview.objects.filter(id__in=requested_course_keys),
)
# User should have been messaged
assert modeladmin_mock.message_user.call_count == 1
assert modeladmin_mock.message_user.call_args.args[1] == (
"Enqueued dumps for 2 course(s). Skipped 1 unchanged course(s)."
)
# For enqueueing, graph should've been authenticated once, using configured settings.
assert mock_neo4j_graph.call_count == 1
assert mock_neo4j_graph.call_args.args == ()
assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection
# No errors should've been logged.
assert mock_log_exception.call_count == 0
# April course should have been skipped because the command was last run in May.
# Dumps for June and July courses should have been enqueued.
assert mock_apply_dump_task.call_count == 2
actual_dumped_course_keys = {
call_args.kwargs['kwargs']['course_key_string']
for call_args in mock_apply_dump_task.call_args_list
}
expected_dumped_course_keys = {
str(self.course_updated_in_june.id),
str(self.course_updated_in_july.id),
}
assert actual_dumped_course_keys == expected_dumped_course_keys
@_configure_coursegraph_connection
@_pretend_last_course_dump_was_may_2020
@_patch_neo4j_graph
@_patch_apply_dump_task
@_patch_log_exception
def test_dump_courses_overriding_cache(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph):
"""
Test that dump_coursese_overriding_cach admin action dumps requested courses
whether or not they been modified since the last dump to coursegraph.
"""
modeladmin_mock = mock.MagicMock()
# Request all courses except the August-updated one
requested_course_keys = {
str(self.course_updated_in_april.id),
str(self.course_updated_in_june.id),
str(self.course_updated_in_july.id),
}
admin.dump_courses_overriding_cache(
modeladmin=modeladmin_mock,
request=mock.MagicMock(),
queryset=CourseOverview.objects.filter(id__in=requested_course_keys),
)
# User should have been messaged
assert modeladmin_mock.message_user.call_count == 1
assert modeladmin_mock.message_user.call_args.args[1] == (
"Enqueued dumps for 3 course(s)."
)
# For enqueueing, graph should've been authenticated once, using configured settings.
assert mock_neo4j_graph.call_count == 1
assert mock_neo4j_graph.call_args.args == ()
assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection
# No errors should've been logged.
assert mock_log_exception.call_count == 0
# April, June, and July courses should have all been dumped.
assert mock_apply_dump_task.call_count == 3
actual_dumped_course_keys = {
call_args.kwargs['kwargs']['course_key_string']
for call_args in mock_apply_dump_task.call_args_list
}
expected_dumped_course_keys = {
str(self.course_updated_in_april.id),
str(self.course_updated_in_june.id),
str(self.course_updated_in_july.id),
}
assert actual_dumped_course_keys == expected_dumped_course_keys
@_configure_coursegraph_connection
@_pretend_last_course_dump_was_may_2020
@_make_neo4j_graph_raise
@_patch_apply_dump_task
@_patch_log_exception
def test_dump_courses_error(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph):
"""
Test that the dump_courses admin action dumps messages the user if an error
occurs when trying to enqueue course dumps.
"""
modeladmin_mock = mock.MagicMock()
# Request dump of all four courses.
admin.dump_courses(
modeladmin=modeladmin_mock,
request=mock.MagicMock(),
queryset=CourseOverview.objects.all()
)
# Admin user should have been messaged about failure.
assert modeladmin_mock.message_user.call_count == 1
assert modeladmin_mock.message_user.call_args.args[1] == (
"Error enqueueing dumps for 4 course(s): we failed to connect or something!"
)
# For enqueueing, graph should've been authenticated once, using configured settings.
assert mock_neo4j_graph.call_count == 1
assert mock_neo4j_graph.call_args.args == ()
assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection
# Exception should have been logged.
assert mock_log_exception.call_count == 1
assert "Failed to enqueue" in mock_log_exception.call_args.args[0]
# No courses should have been dumped.
assert mock_apply_dump_task.call_count == 0

37
cms/envs/common.py
View File

@@ -1756,9 +1756,6 @@ INSTALLED_APPS = [
# edx-milestones service
'milestones',
# Coursegraph
'cms.djangoapps.coursegraph.apps.CoursegraphConfig',
# Credit courses
'openedx.core.djangoapps.credit.apps.CreditConfig',
@@ -2447,40 +2444,6 @@ POLICY_CHANGE_TASK_RATE_LIMIT = '900/h'
# 11 grade designations are used by the UI, so it's advisable to restrict the list to 11 items.
DEFAULT_GRADE_DESIGNATIONS = ['A', 'B', 'C', 'D']
############## Settings for CourseGraph ############################
# .. setting_name: COURSEGRAPH_JOB_QUEUE
# .. setting_default: value of LOW_PRIORITY_QUEUE
# .. setting_description: The name of the Celery queue to which CourseGraph refresh
# tasks will be sent
COURSEGRAPH_JOB_QUEUE: str = LOW_PRIORITY_QUEUE
# .. setting_name: COURSEGRAPH_CONNECTION
# .. setting_default: 'bolt+s://localhost:7687', in dictionary form.
# .. setting_description: Dictionary specifying Neo4j connection parameters for
# CourseGraph refresh. Accepted keys are protocol ('bolt' or 'http'),
# secure (bool), host (str), port (int), user (str), and password (str).
# See https://py2neo.org/2021.1/profiles.html#individual-settings for a
# a description of each of those keys.
COURSEGRAPH_CONNECTION: dict = {
"protocol": "bolt",
"secure": True,
"host": "localhost",
"port": 7687,
"user": "neo4j",
"password": None,
}
# .. toggle_name: COURSEGRAPH_DUMP_COURSE_ON_PUBLISH
# .. toggle_implementation: DjangoSetting
# .. toggle_creation_date: 2022-01-27
# .. toggle_use_cases: open_edx
# .. toggle_default: False
# .. toggle_description: Whether, upon publish, a course should automatically
# be exported to Neo4j via the connection parameters specified in
# `COURSEGRAPH_CONNECTION`.
COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: bool = False
########## Settings for video transcript migration tasks ############
VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE

11
cms/envs/devstack.py
View File

@@ -261,17 +261,6 @@ FEATURES['ENABLE_PREREQUISITE_COURSES'] = True
# (ref MST-637)
PROCTORING_USER_OBFUSCATION_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
############## CourseGraph devstack settings ############################
COURSEGRAPH_CONNECTION: dict = {
"protocol": "bolt",
"secure": False,
"host": "edx.devstack.coursegraph",
"port": 7687,
"user": "neo4j",
"password": "edx",
}
#################### Webpack Configuration Settings ##############################
WEBPACK_LOADER['DEFAULT']['TIMEOUT'] = 5

5
cms/envs/production.py
View File

@@ -517,9 +517,6 @@ JWT_AUTH.update(AUTH_TOKENS.get('JWT_AUTH', {}))
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS.append('openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig')
############## Settings for CourseGraph ############################
COURSEGRAPH_JOB_QUEUE = ENV_TOKENS.get('COURSEGRAPH_JOB_QUEUE', LOW_PRIORITY_QUEUE)
########## Settings for video transcript migration tasks ############
VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = ENV_TOKENS.get('VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE)
@@ -616,8 +613,6 @@ EXPLICIT_QUEUES = {
'queue': SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY},
'cms.djangoapps.contentstore.tasks.update_search_index': {
'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE},
'cms.djangoapps.coursegraph.tasks.dump_course_to_neo4j': {
'queue': COURSEGRAPH_JOB_QUEUE},
}
LOGO_IMAGE_EXTRA_TEXT = ENV_TOKENS.get('LOGO_IMAGE_EXTRA_TEXT', '')

2
cms/static/js/views/pages/container.js
View File

@@ -367,7 +367,7 @@ function($, _, Backbone, gettext, BasePage,
event.preventDefault();
if (!options || options.view !== 'visibility_view') {
const primaryHeader = $(event.target).closest('.xblock-header-primary');
const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions');
var useNewTextEditor = primaryHeader.attr('use-new-editor-text'),
useNewVideoEditor = primaryHeader.attr('use-new-editor-video'),

19
cms/templates/container.html
View File

@@ -13,6 +13,8 @@ from django.urls import reverse
from django.utils.translation import gettext as _
from cms.djangoapps.contentstore.helpers import xblock_studio_url, xblock_type_display_name
from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow
from cms.djangoapps.contentstore.utils import get_editor_page_base_url
from openedx.core.djangolib.js_utils import (
dump_js_escaped_json, js_escaped_string
)
@@ -111,6 +113,13 @@ from openedx.core.djangolib.markup import HTML, Text
<%block name="content">
<%
use_new_editor_text = use_new_text_editor()
use_new_editor_video = use_new_video_editor()
use_new_editor_problem = use_new_problem_editor()
use_new_video_gallery_flow = use_video_gallery_flow()
%>
<script type="text/javascript">
window.STUDIO_FRONTEND_IN_CONTEXT_IMAGE_SELECTION = true;
</script>
@@ -161,7 +170,15 @@ from openedx.core.djangolib.markup import HTML, Text
</div>
</div>
<nav class="nav-actions" aria-label="${_('Page Actions')}">
<nav class="nav-actions" aria-label="${_('Page Actions')}"
use-new-editor-text = ${use_new_editor_text}
use-new-editor-video = ${use_new_editor_video}
use-new-editor-problem = ${use_new_editor_problem}
use-video-gallery-flow = ${use_new_video_gallery_flow}
authoring_MFE_base_url = ${get_editor_page_base_url(xblock_locator.course_key)}
data-block-type = ${xblock.scope_ids.block_type}
data-usage-id = ${xblock.scope_ids.usage_id}
>
<h3 class="sr">${_("Page Actions")}</h3>
<ul>
## Hide the sequence navigation when we've browsed into a child of the unit, e.g. showing the child blocks of a problem-builder xblock

9
common/djangoapps/student/management/commands/recover_account.py
View File

@@ -16,6 +16,7 @@ from django.urls import reverse
from django.utils.http import int_to_base36
from edx_ace import ace
from edx_ace.recipient import Recipient
from eventtracking import tracker
from common.djangoapps.student.models import AccountRecoveryConfiguration
from openedx.core.djangoapps.user_authn.toggles import should_redirect_to_authn_microfrontend
@@ -27,6 +28,7 @@ from openedx.core.djangoapps.user_authn.message_types import PasswordReset
from openedx.core.lib.celery.task_utils import emulate_http_request
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
PASSWORD_RESET_INITIATED = 'edx.user.passwordreset.initiated'
class Command(BaseCommand):
@@ -84,6 +86,13 @@ class Command(BaseCommand):
user = get_user_model().objects.get(Q(username__iexact=username) | Q(email__iexact=current_email))
user.email = desired_email
user.save()
tracker.emit(
PASSWORD_RESET_INITIATED,
{
"user_id": user.id,
"source": "Account Recovery Management Command",
}
)
self.send_password_reset_email(user, site)
successful_updates.append(desired_email)
except Exception as exc: # pylint: disable=broad-except

7
common/djangoapps/student/models/course_enrollment.py
View File

@@ -1016,18 +1016,22 @@ class CourseEnrollment(models.Model):
self.course_id
)
if certificate and not CertificateStatuses.is_refundable_status(certificate.status):
log.info(f"{self.user} has already been given a certificate therefore cannot be refunded.")
return False
# If it is after the refundable cutoff date they should not be refunded.
refund_cutoff_date = self.refund_cutoff_date()
# `refund_cuttoff_date` will be `None` if there is no order. If there is no order return `False`.
if refund_cutoff_date is None:
log.info("Refund cutoff date is null")
return False
if datetime.now(UTC) > refund_cutoff_date:
log.info(f"Refund cutoff date: {refund_cutoff_date} has passed")
return False
course_mode = CourseMode.mode_for_course(self.course_id, 'verified', include_expired=True)
if course_mode is None:
log.info(f"Course mode for {self.course_id} doesn't exist.")
return False
else:
return True
@@ -1037,14 +1041,17 @@ class CourseEnrollment(models.Model):
# NOTE: This is here to avoid circular references
from openedx.core.djangoapps.commerce.utils import ECOMMERCE_DATE_FORMAT
date_placed = self.get_order_attribute_value('date_placed')
log.info(f"Successfully retrieved date_placed: {date_placed} from order")
if not date_placed:
order_number = self.get_order_attribute_value('order_number')
if not order_number:
log.info("Failed to get order number")
return None
date_placed = self.get_order_attribute_from_ecommerce('date_placed')
if not date_placed:
log.info("Failed to get date_placed attribute")
return None
# also save the attribute so that we don't need to call ecommerce again.

24
common/djangoapps/student/views/management.py
View File

@@ -8,6 +8,7 @@ import logging
import urllib.parse
import uuid
from collections import namedtuple
import re
from django.conf import settings
from django.contrib import messages
@@ -55,7 +56,10 @@ from openedx.core.djangoapps.site_configuration import helpers as configuration_
from openedx.core.djangoapps.theming import helpers as theming_helpers
from openedx.core.djangoapps.user_api.preferences import api as preferences_api
from openedx.core.djangoapps.user_authn.tasks import send_activation_email
from openedx.core.djangoapps.user_authn.toggles import should_redirect_to_authn_microfrontend
from openedx.core.djangoapps.user_authn.toggles import (
should_redirect_to_authn_microfrontend,
is_auto_generated_username_enabled
)
from openedx.core.djangolib.markup import HTML, Text
from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser
from openedx.features.enterprise_support.utils import is_enterprise_learner
@@ -172,6 +176,23 @@ def index(request, extra_context=None, user=AnonymousUser()):
return render_to_response('index.html', context)
def show_auto_generated_username(username):
"""
Check if the auto-generated username is valid based on the specified pattern.
Parameters:
- username (str): The username to be checked.
Returns:
- bool: True if the username is valid and the auto-generated username check is enabled, False otherwise.
"""
if not is_auto_generated_username_enabled():
return False
pattern = r'^[A-Z]{1,2}_\d{4}_[A-Z0-9]+$'
return bool(re.match(pattern, username))
def compose_activation_email(
user, user_registration=None, route_enabled=False, profile_name='', redirect_url=None, registration_flow=False
):
@@ -191,6 +212,7 @@ def compose_activation_email(
'routed_profile_name': profile_name,
'registration_flow': registration_flow,
'is_enterprise_learner': is_enterprise_learner(user),
'show_auto_generated_username': show_auto_generated_username(user.username),
})
if route_enabled:

0
lms/static/css/vendor/images/treeview-default-line.gif → common/static/css/vendor/images/treeview-default-line.gif vendored
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 848 B

After

Width:  |  Height:  |  Size: 848 B

0
lms/static/css/vendor/images/treeview-default.gif → common/static/css/vendor/images/treeview-default.gif vendored
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 405 B

After

Width:  |  Height:  |  Size: 405 B

0
lms/static/css/vendor/indicator.gif → common/static/css/vendor/indicator.gif vendored
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.4 KiB

After

Width:  |  Height:  |  Size: 1.4 KiB

0
lms/static/css/vendor/jquery.autocomplete.css → common/static/css/vendor/jquery.autocomplete.css vendored
View File

0
lms/static/css/vendor/jquery.treeview.css → common/static/css/vendor/jquery.treeview.css vendored
View File

BIN
common/static/data/geoip/GeoLite2-Country.mmdb
View File

Binary file not shown.

2
lms/djangoapps/course_home_api/urls.py
View File

@@ -18,7 +18,7 @@ from lms.djangoapps.course_home_api.outline.views import (
from lms.djangoapps.course_home_api.progress.views import ProgressTabView
# This API is a BFF ("backend for frontend") designed for the learning MFE. It's not versioned because there is no
# guarantee of stability over time. It may change from one open edx release to another. Don't write any scripts
# guarantee of stability over time. It may change from one Open edX release to another. Don't write any scripts
# that depend on it.
urlpatterns = []

15
lms/djangoapps/courseware/views/views.py
View File

@@ -1977,13 +1977,14 @@ class PublicVideoXBlockEmbedView(BasePublicVideoXBlockView):
# Translators: "percent_sign" is the symbol "%". "platform_name" is a
# string identifying the name of this installation, such as "edX".
FINANCIAL_ASSISTANCE_HEADER = _(
'We plan to use this information to evaluate your application for financial assistance and to further develop our'
' financial assistance program. Please note that while \nassistance is available in most courses that offer'
' verified certificates, a few courses and programs are not eligible. You must complete a separate application'
' \nfor each course you take. You may be approved for financial assistance five (5) times each year'
' (based on 12-month period from you first approval). \nTo apply for financial assistance: \n'
'1. Enroll in the audit track for an eligible course that offers Verified Certificates \n2. Complete this'
' application \n3. Check your email, your application will be reviewed in 3-4 business days'
'We plan to use this information to evaluate your application for financial assistance and to further develop our '
'financial assistance program. \nPlease note that while assistance is available in most courses that offer '
'verified certificates, a few courses and programs are not eligible. You must complete a separate application '
'for each course you take. You may be approved for financial assistance five (5) times each year '
'(based on 12-month period from you first approval). \nTo apply for financial assistance: '
'\n1. Enroll in the audit track for an eligible course that offers Verified Certificates. '
'\n2. Complete this application. '
'\n3. Check your email, please allow 4 weeks for your application to be processed.'
)

17
lms/djangoapps/edxnotes/README.rst
View File

@@ -2,15 +2,24 @@ Status: Maintenance
Responsibilities
================
The edxnotes app is responsible for displaying parts of the Notes UI to students in different parts of the LMS, as well as figuring out whether Notes is enabled for a particular situation. The bulk of the actual work in storing the notes is done by a separate service (see the edx-notes-api repo).
The edxnotes app is responsible for displaying parts of the Notes UI to students in different parts of the LMS, as well as figuring out whether Notes is enabled for a particular situation. The bulk of the actual work in storing the notes is done by a separate service (see the `edx-notes-api`_ repo).
.. _edx-notes-api: https://github.com/openedx/edx-notes-api/
Direction: Extract into Plugin
==============================
Notes needs to insert a new tab into the LMS courseware, as well as wrap/decorate the courseware XBlock output so that it can add annotation capability to it. Both of these can now be done via plugins (decorating XBlock output can be done with XBlock Asides), and this app should be extracted into a separate repository.
This app is also currently proxying some requests through the LMS instead of hitting its service endpoint directly. It should instead always let the user's browser hit the edx-notes-api service directly.
Notes needs to insert a new tab into the LMS courseware, as well as wrap/decorate the courseware XBlock output so that it can add annotation capability to it.
Both of these can now be done via plugins (decorating XBlock output can be done with XBlock Asides), and this app should be extracted into a separate repository.
The edxnotes app also has an endpoint to get JWT tokens that the edx-notes-api will accept. This should be removed, and the edx-notes-api service converted to use the OAuth2 + JWT Cookie approach detailed in the `Transport JWT in HTTP Cookies <https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0009-jwt-in-session-cookie.rst>`_ decision record for ``oauth_dispatch``.
This app is also currently proxying some requests through the LMS instead of hitting its service endpoint directly.
It should instead always let the user's browser hit the edx-notes-api service directly.
The edxnotes app also has an endpoint to get JWT tokens that the edx-notes-api will accept.
This should be removed, and the edx-notes-api service converted to use the OAuth2 + JWT Cookie approach detailed in the `Transport JWT in HTTP Cookies`_ decision record for ``oauth_dispatch``.
.. _Transport JWT in HTTP Cookies: https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0009-jwt-in-session-cookie.rst
Glossary
========

35
lms/djangoapps/verify_student/api.py
View File

@@ -0,0 +1,35 @@
"""
API module.
"""
from django.conf import settings
from django.utils.translation import gettext as _
from lms.djangoapps.verify_student.emails import send_verification_approved_email
from lms.djangoapps.verify_student.tasks import send_verification_status_email
def send_approval_email(attempt):
"""
Send an approval email to the learner associated with the IDV attempt.
"""
verification_status_email_vars = {
'platform_name': settings.PLATFORM_NAME,
}
expiration_datetime = attempt.expiration_datetime.date()
if settings.VERIFY_STUDENT.get('USE_DJANGO_MAIL'):
verification_status_email_vars['expiration_datetime'] = expiration_datetime.strftime("%m/%d/%Y")
verification_status_email_vars['full_name'] = attempt.user.profile.name
subject = _("Your {platform_name} ID verification was approved!").format(
platform_name=settings.PLATFORM_NAME
)
context = {
'subject': subject,
'template': 'emails/passed_verification_email.txt',
'email': attempt.user.email,
'email_vars': verification_status_email_vars
}
send_verification_status_email.delay(context)
else:
email_context = {'user': attempt.user, 'expiration_datetime': expiration_datetime.strftime("%m/%d/%Y")}
send_verification_approved_email(context=email_context)

6
lms/djangoapps/verify_student/management/commands/approve_id_verifications.py
View File

@@ -11,6 +11,7 @@ from pprint import pformat
from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user
from django.core.management.base import BaseCommand, CommandError
from lms.djangoapps.verify_student.api import send_approval_email
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification
from lms.djangoapps.verify_student.utils import earliest_allowed_verification_date
@@ -125,8 +126,8 @@ class Command(BaseCommand):
def _approve_id_verifications(self, user_ids):
"""
This command manually approves ID verification attempts for a provided set of learners whose ID verification
attempt is in the submitted or must_retry state.
This method manually approves ID verification attempts for a provided set of user IDs so long as the attempt
is in the submitted or must_retry state. This method also send an IDV approval email to the user.
Arguments:
user_ids (list): user IDs of the users whose ID verification attempt should be manually approved
@@ -148,5 +149,6 @@ class Command(BaseCommand):
for verification in existing_id_verifications:
verification.approve(service='idv_verifications command')
send_approval_email(verification)
return list(failed_user_ids)

30
lms/djangoapps/verify_student/management/commands/tests/test_approve_id_verifications.py
View File

@@ -8,6 +8,7 @@ import os
import tempfile
import pytest
from django.core import mail
from django.core.management import CommandError, call_command
from django.test import TestCase
from testfixtures import LogCapture
@@ -70,6 +71,35 @@ class TestApproveIDVerificationsCommand(TestCase):
assert SoftwareSecurePhotoVerification.objects.filter(status='approved').count() == 3
@ddt.data('submitted', 'must_retry')
def test_approve_id_verifications_email(self, status):
"""
Tests that the approve_id_verifications management command correctly sends approval emails.
"""
# Create SoftwareSecurePhotoVerification instances for the users.
for user in [self.user1_profile, self.user2_profile]:
SoftwareSecurePhotoVerification.objects.create(
user=user.user,
name=user.name,
status=status,
)
SoftwareSecurePhotoVerification.objects.create(
user=self.user3_profile.user,
name=self.user3_profile.name,
status='denied',
)
call_command('approve_id_verifications', self.tmp_file_path)
assert len(mail.outbox) == 2
# All three emails should have equal expiration dates, so just pick one from an attempt.
expiration_date = SoftwareSecurePhotoVerification.objects.first().expiration_datetime
for email in mail.outbox:
assert email.subject == 'Your édX ID verification was approved!'
assert 'Your édX ID verification photos have been approved' in email.body
assert expiration_date.strftime("%m/%d/%Y") in email.body
def test_user_does_not_exist_log(self):
"""
Tests that the approve_id_verifications management command logs an error when an invalid user ID is

43
lms/djangoapps/verify_student/tests/test_api.py Normal file
View File

@@ -0,0 +1,43 @@
"""
Tests of API module.
"""
from unittest.mock import patch
import ddt
from django.conf import settings
from django.core import mail
from django.test import TestCase
from common.djangoapps.student.tests.factories import UserFactory
from lms.djangoapps.verify_student.api import send_approval_email
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification
@ddt.ddt
class TestSendApprovalEmail(TestCase):
"""
Test cases for the send_approval_email API method.
"""
def setUp(self):
super().setUp()
self.user = UserFactory.create()
self.attempt = SoftwareSecurePhotoVerification(
status="submitted",
user=self.user
)
self.attempt.save()
def _assert_verification_approved_email(self, expiration_date):
"""Check that a verification approved email was sent."""
assert len(mail.outbox) == 1
email = mail.outbox[0]
assert email.subject == 'Your édX ID verification was approved!'
assert 'Your édX ID verification photos have been approved' in email.body
assert expiration_date.strftime("%m/%d/%Y") in email.body
@ddt.data(True, False)
def test_send_approval(self, use_ace):
with patch.dict(settings.VERIFY_STUDENT, {'USE_DJANGO_MAIL': use_ace}):
send_approval_email(self.attempt)
self._assert_verification_approved_email(self.attempt.expiration_datetime)

22
lms/djangoapps/verify_student/views.py
View File

@@ -36,7 +36,8 @@ from common.djangoapps.util.db import outer_atomic
from common.djangoapps.util.json_request import JsonResponse
from common.djangoapps.util.views import require_global_staff
from lms.djangoapps.commerce.utils import EcommerceService, is_account_activation_requirement_disabled
from lms.djangoapps.verify_student.emails import send_verification_approved_email, send_verification_confirmation_email
from lms.djangoapps.verify_student.api import send_approval_email
from lms.djangoapps.verify_student.emails import send_verification_confirmation_email
from lms.djangoapps.verify_student.image import InvalidImageData, decode_image_data
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification, VerificationDeadline
from lms.djangoapps.verify_student.tasks import send_verification_status_email
@@ -1117,24 +1118,7 @@ def results_callback(request): # lint-amnesty, pylint: disable=too-many-stateme
log.info("[COSMO-184] Approved verification for receipt_id={receipt_id}.".format(receipt_id=receipt_id))
attempt.approve()
expiration_datetime = attempt.expiration_datetime.date()
if settings.VERIFY_STUDENT.get('USE_DJANGO_MAIL'):
verification_status_email_vars['expiration_datetime'] = expiration_datetime.strftime("%m/%d/%Y")
verification_status_email_vars['full_name'] = user.profile.name
subject = _("Your {platform_name} ID verification was approved!").format(
platform_name=settings.PLATFORM_NAME
)
context = {
'subject': subject,
'template': 'emails/passed_verification_email.txt',
'email': user.email,
'email_vars': verification_status_email_vars
}
send_verification_status_email.delay(context)
else:
email_context = {'user': user, 'expiration_datetime': expiration_datetime.strftime("%m/%d/%Y")}
send_verification_approved_email(context=email_context)
send_approval_email(attempt)
elif result == "FAIL":
log.debug("Denying verification for %s", receipt_id)

27
lms/envs/common.py
View File

@@ -290,7 +290,7 @@ FEATURES = {
# sandbox, for testing whether it's enabled properly.
'ENABLE_DEBUG_RUN_PYTHON': False,
# Enable URL that shows information about the status of variuous services
# Enable URL that shows information about the status of various services
'ENABLE_SERVICE_STATUS': False,
# Don't autoplay videos for students
@@ -458,7 +458,7 @@ FEATURES = {
# .. toggle_name: FEATURES['ENABLE_THIRD_PARTY_AUTH']
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
# .. toggle_description: Turn on third-party auth. Disabled for now because full mplementations are not yet
# .. toggle_description: Turn on third-party auth. Disabled for now because full implementations are not yet
# available. Remember to run migrations if you enable this; we don't create tables by default. This feature can
# be enabled on a per-site basis. When enabling this feature, remember to define the allowed authentication
# backends with the AUTHENTICATION_BACKENDS setting.
@@ -699,7 +699,7 @@ FEATURES = {
# .. toggle_default: False
# .. toggle_description: When set to True, Open edX site can be used as an LTI Provider to other systems
# and applications.
# .. toggle_warning: After enabling this feature flag there are multiple steps invloved to configure edX
# .. toggle_warning: After enabling this feature flag there are multiple steps involved to configure edX
# as LTI provider. Full guide is available here:
# https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/lti/index.html
# .. toggle_use_cases: open_edx
@@ -983,7 +983,7 @@ FEATURES = {
# .. toggle_default: False
# .. toggle_description: When true, replaces the bulk email tool found on the
# instructor dashboard with a link to the new communications MFE version instead.
# Stting the tool to false will leave the old bulk email tool experience in place.
# Setting the tool to false will leave the old bulk email tool experience in place.
# .. toggle_use_cases: opt_in
# .. toggle_creation_date: 2022-03-21
# .. toggle_target_removal_date: None
@@ -1018,7 +1018,7 @@ FEATURES = {
# .. toggle_name: FEATURES['ENABLE_CERTIFICATES_IDV_REQUIREMENT']
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
# .. toggle_description: Whether to enforce ID Verification requirements for couse certificates generation
# .. toggle_description: Whether to enforce ID Verification requirements for course certificates generation
# .. toggle_use_cases: open_edx
# .. toggle_creation_date: 2022-04-26
# .. toggle_target_removal_date: None
@@ -1129,7 +1129,7 @@ MARKETING_EMAILS_OPT_IN = False
# .. toggle_name: ENABLE_COPPA_COMPLIANCE
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
# .. toggle_description: When True, inforces COPPA compliance and removes YOB field from registration form and accounnt
# .. toggle_description: When True, enforces COPPA compliance and removes YOB field from registration form and account
# .. settings page. Also hide YOB banner from profile page.
# .. toggle_use_cases: open_edx
# .. toggle_creation_date: 2021-10-27
@@ -1367,7 +1367,7 @@ CONTEXT_PROCESSORS = [
# Mobile App processor (Detects if request is from the mobile app)
'lms.djangoapps.mobile_api.context_processor.is_from_mobile_app',
# Context processor necesarry for the survey report message appear on the admin site
# Context processor necessary for the survey report message appear on the admin site
'openedx.features.survey_report.context_processors.admin_extra_context'
@@ -1458,7 +1458,7 @@ SEARCH_COURSEWARE_CONTENT_LOG_PARAMS = False
# .. setting_name: ELASTIC_SEARCH_INDEX_PREFIX
# .. setting_default: ''
# .. setting_description: Specifies the prefix used when namixng elasticsearch indexes related to edx-search.
# .. setting_description: Specifies the prefix used when naming elasticsearch indexes related to edx-search.
ELASTICSEARCH_INDEX_PREFIX = ""
VIDEO_CDN_URL = {
@@ -1845,7 +1845,7 @@ CODE_JAIL = {
# ]
COURSES_WITH_UNSAFE_CODE = []
# Cojail REST service
# Code jail REST service
ENABLE_CODEJAIL_REST_SERVICE = False
# .. setting_name: CODE_JAIL_REST_SERVICE_REMOTE_EXEC
# .. setting_default: 'xmodule.capa.safe_exec.remote_exec.send_safe_exec_request_v0'
@@ -3183,9 +3183,13 @@ INSTALLED_APPS = [
# Notes
'lms.djangoapps.edxnotes',
# User API
# Django Rest Framework
'rest_framework',
# REST framework JWT Auth
'rest_framework_jwt',
# User API
'openedx.core.djangoapps.user_api',
# Different Course Modes
@@ -3351,7 +3355,6 @@ INSTALLED_APPS = [
# Management of per-user schedules
'openedx.core.djangoapps.schedules',
'rest_framework_jwt',
# Learning Sequence Navigation
'openedx.core.djangoapps.content.learning_sequences.apps.LearningSequencesConfig',
@@ -4523,7 +4526,7 @@ FINANCIAL_ASSISTANCE_MAX_LENGTH = 2500
REGISTRATION_EXTENSION_FORM = None
# Identifier included in the User Agent from open edX mobile apps.
# Identifier included in the User Agent from Open edX mobile apps.
MOBILE_APP_USER_AGENT_REGEXES = [
r'edX/org.edx.mobile',
]

2
lms/envs/devstack-experimental.yml
View File

@@ -11,7 +11,7 @@
# A. You don't *have* to, because settings in devstack.py
# override these settings. But, it doesn't harm to also make them
# here in order to quell confusion. The hope is that we'll
# adpot OEP-45 eventually, which recommends against having
# adopt OEP-45 eventually, which recommends against having
# a devstack.py at all.
#
# This is part of the effort to move our dev tools off of Ansible and

2
lms/envs/devstack.py
View File

@@ -135,7 +135,7 @@ REQUIRE_DEBUG = DEBUG
PIPELINE['SASS_ARGUMENTS'] = '--debug-info'
# Load development webpack donfiguration
# Load development webpack configuration
WEBPACK_CONFIG_PATH = 'webpack.dev.config.js'
########################### VERIFIED CERTIFICATES #################################

2
lms/envs/devstack_with_worker.py
View File

@@ -1,5 +1,5 @@
"""
This config file follows the devstack enviroment, but adds the
This config file follows the devstack environment, but adds the
requirement of a celery worker running in the background to process
celery tasks.

28
lms/envs/docs/README.rst
View File

@@ -1,7 +1,7 @@
LMS Configuration Settings
==========================
##########################
The lms.envs module contains project-wide settings, defined in python modules
The ``lms.envs`` module contains project-wide settings, defined in python modules
using the standard `Django Settings`_ mechanism, plus some Open edX
particularities, which we describe below.
@@ -9,11 +9,14 @@ particularities, which we describe below.
YAML Configuration Files
------------------------
************************
In addition, there is a mechanism for reading and overriding configuration settings from YAML files on-disk. The :file:`/lms/envs/production.py` module loads settings from a YAML file. The location of the YAML file is pulled from the value of the ``LMS_CFG`` environment variable. Except for a limited set of exceptions, if a key exists in the YAML file, it will be injected into the settings module as it is defined in the YAML file.
In addition, there is a mechanism for reading and overriding configuration settings from YAML files on-disk.
The :file:`/lms/envs/production.py` module loads settings from a YAML file.
The location of the YAML file is pulled from the value of the ``LMS_CFG`` environment variable.
Except for a limited set of exceptions, if a key exists in the YAML file, it will be injected into the settings module as it is defined in the YAML file.
The YAML file allow open edX operators to configure the Django runtime
The YAML file allow Open edX operators to configure the Django runtime
without needing to make any changes to source-controlled python files in
edx-platform. Therefore, they are not checked into the edx-platform repo.
Rather, they are generated from the `edxapp playbook in the configuration
@@ -23,7 +26,7 @@ repo`_ and available in the ``/edx/etc/`` folder on edX servers.
Feature Flags and Settings Guidelines
-------------------------------------
*************************************
For guidelines on using Django settings and feature flag mechanisms in the edX
platform, please see `Feature Flags and Settings`_.
@@ -32,16 +35,19 @@ platform, please see `Feature Flags and Settings`_.
Derived Settings
----------------
****************
In cases where you need to define one or more settings relative to the value of
another setting, you can explicitly designate them as derived calculations.
This can let you override one setting (such as a path or a feature toggle) and
have it automatically propagate to other settings which are defined in terms of
that value, without needing to track down all potentially impacted settings and
explicitly override them as well. This can be useful for test setting overrides
explicitly override them as well. This can be useful for test setting overrides
even if you don't anticipate end users customizing the value very often.
For example::
For example:
.. code-block:: python
def _make_locale_paths(settings):
locale_paths = [settings.REPO_ROOT + '/conf/locale'] # edx-platform/conf/locale/
@@ -61,7 +67,9 @@ defined in ``lms/envs/common.py`` and you're using ``lms/envs/production.py`` wh
includes overrides both from that module and the JSON configuration files.
List entries and dictionary values can also be derived from other settings, even
when nested within each other::
when nested within each other:
.. code-block:: python
def _make_mako_template_dirs(settings):
"""

10
lms/envs/production.py
View File

@@ -143,8 +143,8 @@ if STATIC_URL_BASE:
REQUIRE_BUILD_PROFILE = ENV_TOKENS.get('REQUIRE_BUILD_PROFILE', REQUIRE_BUILD_PROFILE)
# The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text
# values when the varibale is an empty string. Therefore, setting these variable as empty text in related
# json files will make the system reads thier values from django translation files
# values when the variable is an empty string. Therefore, setting these variable as empty text in related
# json files will make the system reads their values from django translation files
PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME') or PLATFORM_NAME
PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION') or PLATFORM_DESCRIPTION
@@ -362,7 +362,7 @@ VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {})
# Determines whether the CSRF token can be transported on
# unencrypted channels. It is set to False here for backward compatibility,
# but it is highly recommended that this is True for enviroments accessed
# but it is highly recommended that this is True for environments accessed
# by end users.
CSRF_COOKIE_SECURE = ENV_TOKENS.get('CSRF_COOKIE_SECURE', False)
@@ -502,9 +502,9 @@ MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTOR
# After conversion above, the modulestore will have a "stores" list with all defined stores, for all stores, add the
# fs_root entry to derived collection so that if it's a callable it can be resolved. We need to do this because the
# `derived_collection_entry` takes an exact index value but the config file might have overidden the number of stores
# `derived_collection_entry` takes an exact index value but the config file might have overridden the number of stores
# and so we can't be sure that the 2 we define in common.py will be there when we try to derive settings. This could
# lead to execptions being thrown when the `derive_settings` call later in this file tries to update settings. We call
# lead to exceptions being thrown when the `derive_settings` call later in this file tries to update settings. We call
# the derived_collection_entry function here to ensure that we update the fs_root for any callables that remain after
# we've updated the MODULESTORE setting from our config file.
for idx, store in enumerate(MODULESTORE['default']['OPTIONS']['stores']):

2
lms/static/sass/views/_financial-assistance.scss
View File

@@ -32,7 +32,7 @@
p {
line-height: $base-line-height;
margin-top: 0;
margin-top: 1em;
color: $gray-700;
font-size: 1em;
}

3
lms/templates/financial-assistance/financial-assistance.html
View File

@@ -36,7 +36,8 @@ from common.djangoapps.edxmako.shortcuts import marketing_link
## Translators: This string will not be used in Open edX installations.
<p>${_("EdX is committed to making it possible for you to take high quality courses from leading institutions regardless of your financial situation, earn a Verified Certificate, and share your success with others.")}</p>
## Translators: This string will not be used in Open edX installations. Do not translate the name "Anant".
<p class="signature">${_("Sincerely, Anant")}</p>
<p class="signature">${_("Sincerely,")}</p>
<p class="signature">${_("Anant")}</p>
</div>
% endif

62
openedx/core/djangoapps/content/search/api.py
View File

@@ -13,6 +13,7 @@ from typing import Callable, Generator
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.core.paginator import Paginator
from meilisearch import Client as MeilisearchClient
from meilisearch.errors import MeilisearchError
from meilisearch.models.task import TaskInfo
@@ -21,10 +22,9 @@ from opaque_keys.edx.locator import LibraryLocatorV2
from common.djangoapps.student.roles import GlobalStaff
from rest_framework.request import Request
from common.djangoapps.student.role_helpers import get_course_roles
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from openedx.core.djangoapps.content.search.models import get_access_ids_for_request
from openedx.core.djangoapps.content_libraries import api as lib_api
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from .documents import (
@@ -292,9 +292,7 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None:
# Get the list of courses
status_cb("Counting courses...")
with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred):
all_courses = store.get_courses()
num_courses = len(all_courses)
num_courses = CourseOverview.objects.count()
# Some counters so we can track our progress as indexing progresses:
num_contexts = num_courses + num_libraries
@@ -327,11 +325,20 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None:
])
# Mark which attributes are used for keyword search, in order of importance:
client.index(temp_index_name).update_searchable_attributes([
# Keyword search does _not_ search the course name, course ID, breadcrumbs, block type, or other fields.
Fields.display_name,
Fields.block_id,
Fields.content,
Fields.tags,
# Keyword search does _not_ search the course name, course ID, breadcrumbs, block type, or other fields.
# If we don't list the following sub-fields _explicitly_, they're only sometimes searchable - that is, they
# are searchable only if at least one document in the index has a value. If we didn't list them here and,
# say, there were no tags.level3 tags in the index, the client would get an error if trying to search for
# these sub-fields: "Attribute `tags.level3` is not searchable."
Fields.tags + "." + Fields.tags_taxonomy,
Fields.tags + "." + Fields.tags_level0,
Fields.tags + "." + Fields.tags_level1,
Fields.tags + "." + Fields.tags_level2,
Fields.tags + "." + Fields.tags_level3,
])
############## Libraries ##############
@@ -358,30 +365,33 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None:
############## Courses ##############
status_cb("Indexing courses...")
for course in all_courses:
status_cb(
f"{num_contexts_done + 1}/{num_contexts}. Now indexing course {course.display_name} ({course.id})"
)
docs = []
# To reduce memory usage on large instances, split up the CourseOverviews into pages of 1,000 courses:
paginator = Paginator(CourseOverview.objects.only('id', 'display_name'), 1000)
for p in paginator.page_range:
for course in paginator.page(p).object_list:
status_cb(
f"{num_contexts_done + 1}/{num_contexts}. Now indexing course {course.display_name} ({course.id})"
)
docs = []
# Pre-fetch the course with all of its children:
course = store.get_course(course.id, depth=None)
# Pre-fetch the course with all of its children:
course = store.get_course(course.id, depth=None)
def add_with_children(block):
""" Recursively index the given XBlock/component """
doc = searchable_doc_for_course_block(block)
doc.update(searchable_doc_tags(block.usage_key))
docs.append(doc) # pylint: disable=cell-var-from-loop
_recurse_children(block, add_with_children) # pylint: disable=cell-var-from-loop
def add_with_children(block):
""" Recursively index the given XBlock/component """
doc = searchable_doc_for_course_block(block)
doc.update(searchable_doc_tags(block.usage_key))
docs.append(doc) # pylint: disable=cell-var-from-loop
_recurse_children(block, add_with_children) # pylint: disable=cell-var-from-loop
# Index course children
_recurse_children(course, add_with_children)
# Index course children
_recurse_children(course, add_with_children)
if docs:
# Add all the docs in this course at once (usually faster than adding one at a time):
_wait_for_meili_task(client.index(temp_index_name).add_documents(docs))
num_contexts_done += 1
num_blocks_done += len(docs)
if docs:
# Add all the docs in this course at once (usually faster than adding one at a time):
_wait_for_meili_task(client.index(temp_index_name).add_documents(docs))
num_contexts_done += 1
num_blocks_done += len(docs)
status_cb(f"Done! {num_blocks_done} blocks indexed across {num_contexts_done} courses and libraries.")

3
openedx/core/djangoapps/content/search/tests/test_api.py
View File

@@ -15,6 +15,7 @@ from organizations.tests.factories import OrganizationFactory
from common.djangoapps.student.tests.factories import UserFactory
from openedx.core.djangoapps.content_libraries import api as library_api
from openedx.core.djangoapps.content_tagging import api as tagging_api
from openedx.core.djangoapps.content.course_overviews.api import CourseOverview
from openedx.core.djangolib.testing.utils import skip_unless_cms
from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase
@@ -106,6 +107,8 @@ class TestSearchApi(ModuleStoreTestCase):
"content": {},
"access_id": course_access.id,
}
# Make sure the CourseOverview for the course is created:
CourseOverview.get_from_id(self.course.id)
# Create a content library:
self.library = library_api.create_library(

50
openedx/core/djangoapps/course_apps/docs/decisions/001-course-apps.rst
View File

@@ -1,12 +1,13 @@
Course Apps API
_______________
###############
Status
======
******
Proposal
Context
=======
*******
The new `Course Authoring MFE`_ includes a new UX called "Pages and Resources"
for configuring different aspects of the course experience such as progress,
@@ -27,7 +28,7 @@ enable/disable these apps using the API.
Decision
========
########
We propose to call such individual course features "Course Apps". They can be
introduced as a new type of Open edX plugin. Any functionality that can be
@@ -39,16 +40,29 @@ some bits of metadata, such as a name, a description etc. Additionally we will
need a common interface for such apps so they can be enabled/disabled using
a standard common interface.
To do this we can follow the example of existing plugins, [such as Course
Tabs](https://github.com/openedx/edx-platform/blob/636b2ca4c5add531cfce755fdb8965599acd79e0/common/lib/xmodule/xmodule/tabs.py#L24-L243),
To do this we can follow the example of existing plugins, `such as Course Tabs`_,
which provide a specific Python class that the plugin can inherit from, or
implement. The required metadata and features, can be implemented as class
attributes, and methods on this class.
We can then discover the installed apps using the existing tooling for plugins
using a subclass of PluginManager designed for this purpose. Here is an example
for [Course
Tabs](https://github.com/openedx/edx-platform/blob/636b2ca4c5add531cfce755fdb8965599acd79e0/openedx/core/lib/course_tabs.py#L13-L47)
using a subclass of ``PluginManager`` designed for this purpose.
Here is an example for `CourseTabs`_:
.. code-block:: python
class CourseTabPluginManager(PluginManager):
"""
Manager for all of the course tabs that have been made available.
All course tabs should implement `CourseTab`.
"""
NAMESPACE = COURSE_TAB_NAMESPACE
@classmethod
def get_tab_types(cls):
"""
Returns the list of available course tabs in their canonical order.
It might not always make sense for an app installed in this way to be
automatically show up for use on all courses. So each app will expose a method
@@ -85,14 +99,17 @@ In the case of Course Apps, the standard plugin API will automatically discover
all installed apps. Inactive apps will be filtered out during the availability
check.
Course App Plugin Class
-----------------------
.. _such as Course Tabs: https://github.com/openedx/edx-platform/blob/636b2ca4c5add531cfce755fdb8965599acd79e0/common/lib/xmodule/xmodule/tabs.py#L24-L243
.. _CourseTabs: https://github.com/openedx/edx-platform/blob/636b2ca4c5add531cfce755fdb8965599acd79e0/openedx/core/lib/course_tabs.py#L13-L47
To be loaded as a Course App, you need to provide an entrypoint in `setup.py`
with the namespace "openedx.course_app". The entry should point to a Python
Course App Plugin Class
=======================
To be loaded as a Course App, you need to provide an entrypoint in ``setup.py``
with the namespace ``openedx.course_app``. The entry should point to a Python
class with the following basic structure:
.. code-block :: python
.. code-block:: python
class CourseApp:
# The app id should match what is specified in the setup.py entrypoint
@@ -143,7 +160,7 @@ such a class and have these class methods call back to the existing code for
availability checks and enabled checks.
Course Apps API
---------------
===============
Each app has some associated metadata:
@@ -218,9 +235,8 @@ link should only be provided for Course Apps that don't have a UI in the course
authoring MFE yet. If a partial UI exists, the MFE settings view can always link
back to the old studio view from there.
Consequences
============
************
- A new Course Apps API that consistently uses a standard mechanism (a plugin
class) for discovering Course Apps, determining their availability and

2
openedx/core/djangoapps/oauth_dispatch/docs/decisions/0006-enforce-scopes-in-LMS-APIs.rst
View File

@@ -99,7 +99,7 @@ unprotected microservices.
4. Associate Available Scopes with Applications
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In order to allow open edX operators to a priori limit the
In order to allow Open edX operators to a priori limit the
types of access an Application can request, we will allow them
to configure Application-specific "available scopes".

8
openedx/core/djangoapps/oauth_dispatch/docs/decisions/0009-jwt-in-session-cookie.rst
View File

@@ -28,7 +28,7 @@ user interacts with the overall application, the user's experience may lead them
each accessing APIs on various backends. Stateless authentication (via self-contained JWTs) would allow scalable
interactions between microfrontends and microservices.
Note: User authentication for open edX mobile apps is outside the scope of this decision record. As a brief note, we
Note: User authentication for Open edX mobile apps is outside the scope of this decision record. As a brief note, we
believe any decisions in this record will neither affect the current authentication mechanisms used for mobile
apps nor impact forward compatibility when/if mobile apps are consolidated to use a similar (if not the same)
authentication mechanism as outlined here for web apps.
@@ -69,7 +69,7 @@ Login -> Cookie -> API
recombined JWT in a temporary cookie specified by JWT_AUTH_COOKIE_.
* The `Django Rest Framework JWT`_ library we use makes use of the JWT_AUTH_COOKIE_ configuration setting.
When set, the JSONWebTokenAuthentication_ class `automatically extracts the JWT from the cookie`_. Since all
open edX REST endpoints that support JWT-based authentication derive from this base class, their authentication
Open edX REST endpoints that support JWT-based authentication derive from this base class, their authentication
checks will make use of the JWTs provided in the JWT-related cookies.
#. **Introduce forgiving JWTs for backward compatibility.**
@@ -121,7 +121,7 @@ JWT Cookie Lifetime
* For simplicity and consistency, the cookies and their containing JWT will expire at the same time. There's
no need to have these be different values.
* Given this, JWT cookies will always have expiration values, unlike `current open edX session cookies that may
* Given this, JWT cookies will always have expiration values, unlike `current Open edX session cookies that may
have no expiration`_.
* A configuration setting, JWT_AUTH_COOKIE_EXPIRATION, will specify the expiration duration for JWTs and their
@@ -141,7 +141,7 @@ JWT Cookie Lifetime
which will remove them from the user's browser cookie jar. Thus, the user will be logged out of all the
microfrontends.
.. _`current open edX session cookies that may have no expiration`: https://github.com/openedx/edx-platform/blob/92030ea15216a6641c83dd7bb38a9b65112bf31a/common/djangoapps/student/cookies.py#L25-L27
.. _`current Open edX session cookies that may have no expiration`: https://github.com/openedx/edx-platform/blob/92030ea15216a6641c83dd7bb38a9b65112bf31a/common/djangoapps/student/cookies.py#L25-L27
.. _JWT blacklist: https://auth0.com/blog/blacklist-json-web-token-api-keys/
.. _`JWT ID (jti)`: http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html#jtiDef

2
openedx/core/djangoapps/oauth_dispatch/docs/how_tos/testing_manually.rst
View File

@@ -1,7 +1,7 @@
Manually Testing OAuth2 Provider implementation
-----------------------------------------------
This document explains how to manually test the open edX LMS' OAuth2 Provider
This document explains how to manually test the Open edX LMS' OAuth2 Provider
implementation. In order to verify that it correctly implements the
`OAuth2 standard`_, use a publicly available 3rd party standard OAuth2 client.
The steps here show how to use `Google's OAuth2 Playground`_ as the client for

8
openedx/core/djangoapps/safe_sessions/tests/test_middleware.py
View File

@@ -326,9 +326,9 @@ class TestSafeSessionMiddleware(TestSafeSessionsLogMixin, CacheIsolationTestCase
self.request.path = '/xblock/block-v1:org+course+run+type@html+block@block_id'
self.verify_success()
@override_settings(MOBILE_APP_USER_AGENT_REGEXES=[r'open edX Mobile App'])
@override_settings(MOBILE_APP_USER_AGENT_REGEXES=[r'Open edX Mobile App'])
def test_success_from_mobile_app(self):
self.request.META = {'HTTP_USER_AGENT': 'open edX Mobile App Version 2.1'}
self.request.META = {'HTTP_USER_AGENT': 'Open edX Mobile App Version 2.1'}
self.verify_success()
def verify_error(self, expected_response_status):
@@ -364,9 +364,9 @@ class TestSafeSessionMiddleware(TestSafeSessionsLogMixin, CacheIsolationTestCase
self.request.META['HTTP_ACCEPT'] = http_accept
self.verify_error(expected_response)
@override_settings(MOBILE_APP_USER_AGENT_REGEXES=[r'open edX Mobile App'])
@override_settings(MOBILE_APP_USER_AGENT_REGEXES=[r'Open edX Mobile App'])
def test_error_from_mobile_app(self):
self.request.META = {'HTTP_USER_AGENT': 'open edX Mobile App Version 2.1'}
self.request.META = {'HTTP_USER_AGENT': 'Open edX Mobile App Version 2.1'}
self.verify_error(401)
@override_settings(ENFORCE_SAFE_SESSIONS=False)

2
openedx/core/djangoapps/user_api/accounts/__init__.py
View File

@@ -61,7 +61,7 @@ EMAIL_CONFLICT_MSG = _(
"Try again with a different email address."
)
AUTHN_EMAIL_CONFLICT_MSG = _( # pylint: disable=translation-of-non-string
f'This email is already associated with an existing or previous {settings.PLATFORM_NAME} account')
"This email is already associated with an existing account")
RETIRED_EMAIL_MSG = _(
"This email is associated to a retired account."
)

13
openedx/core/djangoapps/user_authn/views/login.py
View File

@@ -24,6 +24,7 @@ from django.views.decorators.debug import sensitive_post_parameters
from django.views.decorators.http import require_http_methods
from django_ratelimit.decorators import ratelimit
from edx_django_utils.monitoring import set_custom_attribute
from eventtracking import tracker
from openedx_events.learning.data import UserData, UserPersonalData
from openedx_events.learning.signals import SESSION_LOGIN_COMPLETED
from openedx_filters.learning.filters import StudentLoginRequested
@@ -61,6 +62,7 @@ from openedx.features.enterprise_support.api import activate_learner_enterprise,
log = logging.getLogger("edx.student")
AUDIT_LOG = logging.getLogger("audit")
USER_MODEL = get_user_model()
PASSWORD_RESET_INITIATED = 'edx.user.passwordreset.initiated'
def _do_third_party_auth(request):
@@ -194,6 +196,13 @@ def _enforce_password_policy_compliance(request, user): # lint-amnesty, pylint:
LoginFailures.increment_lockout_counter(user)
AUDIT_LOG.info("Password reset initiated for email %s.", user.email)
tracker.emit(
PASSWORD_RESET_INITIATED,
{
"user_id": user.id,
"source": "Policy Compliance",
}
)
send_password_reset_email_for_user(user, request)
# Prevent the login attempt.
@@ -354,13 +363,15 @@ def _track_user_login(user, request):
'MailChimp': False
}
)
register_intent = request.POST.get('register_intent') == 'true'
segment.track(
user.id,
"edx.bi.user.account.authenticated",
{
'category': "conversion",
'label': request.POST.get('course_id'),
'provider': None
'provider': None,
'register_intent': register_intent,
},
)

15
openedx/core/djangoapps/user_authn/views/password_reset.py
View File

@@ -52,6 +52,7 @@ from common.djangoapps.util.password_policy_validators import normalize_password
POST_EMAIL_KEY = 'openedx.core.djangoapps.util.ratelimit.request_post_email'
REAL_IP_KEY = 'openedx.core.djangoapps.util.ratelimit.real_ip'
SETTING_CHANGE_INITIATED = 'edx.user.settings.change_initiated'
PASSWORD_RESET_INITIATED = 'edx.user.passwordreset.initiated'
# Maintaining this naming for backwards compatibility.
log = logging.getLogger("edx.student")
@@ -289,6 +290,13 @@ def password_reset(request):
user = request.user
# Prefer logged-in user's email
email = user.email if user.is_authenticated else request.POST.get('email')
tracker.emit(
PASSWORD_RESET_INITIATED,
{
"user_id": user.id,
"source": "Logistration Page",
}
)
AUDIT_LOG.info("Password reset initiated for email %s.", email)
if getattr(request, 'limited', False):
@@ -608,6 +616,13 @@ def password_change_request_handler(request):
# Prefer logged-in user's email
email = user.email if user.is_authenticated else request.POST.get('email')
AUDIT_LOG.info("Password reset initiated for email %s.", email)
tracker.emit(
PASSWORD_RESET_INITIATED,
{
"user_id": user.id,
"source": "Account API",
}
)
if getattr(request, 'limited', False) and not request_from_support_tools:
AUDIT_LOG.warning("Password reset rate limit exceeded for email %s.", email)

Some files were not shown because too many files have changed in this diff Show More