From ab60a0800d95ce916961cb27825593c2461e75c2 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 10:48:07 -0400 Subject: [PATCH 1/7] feat!: Unpin scipy. Scipy was constrained to prevent a numpy upgrade however the numpy upgrade happened a while ago and this never got updated. Now the scipy version is preventing us from going to an even newer numpy version that would be compatible with both Python 3.8 and Python 3.12 BREAKING CHANGE: This will update scipy in the codejail sandbox from 1.7.3 to a newer version. This contains many backwards incompatible changes that are all documented in the SciPy changelogs: https://docs.scipy.org/doc/scipy/release/1.8.0-notes.html --- requirements/constraints.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index ed7c0ee9b0..dd0b48e097 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -48,9 +48,6 @@ django-webpack-loader==0.7.0 # version of py2neo will work with Neo4j 3.5. py2neo<2022 -# scipy version 1.8 requires numpy>=1.17.3, we've pinned numpy to <1.17.0 in requirements/edx-sandbox/py38.in -scipy<1.8.0 - # edx-enterprise, snowflake-connector-python require charset-normalizer==2.0.0 # Can be removed once snowflake-connector-python>2.7.9 is released with the fix. charset-normalizer<2.1.0 From 05ec09fcdc28df190af10d9d9699cde8a43ae74f Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 11:59:42 -0400 Subject: [PATCH 2/7] chore: Upgrade scipy Ran `package=scipy make upgrade-package` to upgrade just this one dependency. --- requirements/edx-sandbox/base.txt | 3 +-- requirements/edx/base.txt | 4 ++-- requirements/edx/development.txt | 4 ++-- requirements/edx/doc.txt | 4 ++-- requirements/edx/testing.txt | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 53c7960778..b8651a0340 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -76,9 +76,8 @@ random2==1.0.2 # via -r requirements/edx-sandbox/base.in regex==2024.4.16 # via nltk -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/base.in # chem # openedx-calc diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 12e24dd019..23e6ccca45 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -57,6 +57,7 @@ backoff==1.10.0 # via analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # celery # django # edx-milestones @@ -1045,9 +1046,8 @@ s3transfer==0.10.0 # via boto3 sailthru-client==2.2.3 # via edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # chem # openedx-calc semantic-version==2.10.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 4c2d235073..b533ffb745 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -113,6 +113,7 @@ backoff==1.10.0 # analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # celery @@ -1797,9 +1798,8 @@ sailthru-client==2.2.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # chem diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 8be896b0b2..015694f99d 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -80,6 +80,7 @@ backoff==1.10.0 # analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery # django @@ -1239,9 +1240,8 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # chem # openedx-calc diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 4a0d690735..062a86b6a3 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -80,6 +80,7 @@ backoff==1.10.0 # analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery # django @@ -1360,9 +1361,8 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # chem # openedx-calc From fb727c101e049510aa9f7d755b1def8d249c2550 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 12:08:41 -0400 Subject: [PATCH 3/7] feat!: Upgrade numpy to be python 3.8 and 3.12 compatible. BREAKING CHANGE: Because numpy is available in the codejail sandbox, the update from 1.22.4 to 1.24.4 does have completed deprecations, and more details can be found in the Numpy Release Notes: https://numpy.org/doc/stable/release/1.23.0-notes.html#expired-deprecations --- requirements/edx-sandbox/base.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index b8651a0340..fbc05dac94 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -49,7 +49,7 @@ nltk==3.8.1 # via # -r requirements/edx-sandbox/base.in # chem -numpy==1.22.4 +numpy==1.24.4 # via # chem # contourpy diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 23e6ccca45..97dbdb1090 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -748,7 +748,7 @@ nltk==3.8.1 # via chem nodeenv==1.8.0 # via -r requirements/edx/kernel.in -numpy==1.22.4 +numpy==1.24.4 # via # chem # openedx-calc diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index b533ffb745..2c9482f606 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1247,7 +1247,7 @@ nodeenv==1.8.0 # -r requirements/edx/assets.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 015694f99d..d27ccebc4b 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -881,7 +881,7 @@ nltk==3.8.1 # chem nodeenv==1.8.0 # via -r requirements/edx/base.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/base.txt # chem diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 062a86b6a3..077ac742cb 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -936,7 +936,7 @@ nltk==3.8.1 # chem nodeenv==1.8.0 # via -r requirements/edx/base.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/base.txt # chem From 1d485ca2b84dca7006786679a754b018c53d34fa Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 12:41:22 -0400 Subject: [PATCH 4/7] build: Don't require the backports.zoneinfo package for newer python. We do this so that we can build requirements files that could work to `pip install` on both the old (3.8) and new (3.11) versions of python. --- requirements/constraints.txt | 3 +++ requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/user_retirement/requirements/base.txt | 3 ++- scripts/user_retirement/requirements/testing.txt | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index dd0b48e097..ad17ea9f3c 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -130,3 +130,6 @@ moto<5.0 # path==16.12.0 breaks the unit test collections check # needs to be investigated and fixed separately path<16.12.0 + +# Temporary to Support the python 3.11 Upgrade +backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 97dbdb1090..4e027596a0 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -55,7 +55,7 @@ babel==2.14.0 # enmerkar-underscore backoff==1.10.0 # via analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # celery diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 2c9482f606..10b88b4afd 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -111,7 +111,7 @@ backoff==1.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index d27ccebc4b..1a63a42e57 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -78,7 +78,7 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 077ac742cb..7dd6f913e9 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -78,7 +78,7 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index f66759b556..bd3fc4848f 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -10,8 +10,9 @@ attrs==23.2.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via + # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # django # pendulum boto3==1.34.26 diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index b61e49596e..37444f4694 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -14,7 +14,7 @@ attrs==23.2.0 # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -r scripts/user_retirement/requirements/base.txt # django From 85db730cef3f8ef819a6c96997f9a2ba70e6510d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 13:24:13 -0400 Subject: [PATCH 5/7] build: Run tests on python 3.11 as well as 3.8 --- .github/workflows/unit-tests-gh-hosted.yml | 2 +- .github/workflows/unit-tests.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index b9681db67f..5f4e3c5a68 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ '3.8' ] + python-version: [ '3.8', '3.11' ] django-version: - "pinned" # When updating the shards, remember to make the same changes in diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d08dc1aadc..8069ee8ef6 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,6 +15,7 @@ jobs: matrix: python-version: - "3.8" + - "3.11" django-version: - "pinned" # When updating the shards, remember to make the same changes in From e9d716375fafecf598fcc034ed6795195cc2d7d4 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 15:35:53 -0400 Subject: [PATCH 6/7] build: Update to allow tests to run on python 3.11 * Update so tox will allow running on the new version. * Update so the hosted runner actually respects the python version in the matrix --- .github/workflows/unit-tests.yml | 11 ++++++++--- tox.ini | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8069ee8ef6..9a50342024 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -57,15 +57,20 @@ jobs: sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*" fi - - name: checkout repo - uses: actions/checkout@v3 - - name: start mongod server for tests run: | sudo mkdir -p /data/db sudo chmod -R a+rw /data/db mongod & + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: checkout repo + uses: actions/checkout@v3 + - name: install requirements run: | sudo make test-requirements diff --git a/tox.ini b/tox.ini index 9b1937f364..1b4252fd19 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38, quality +envlist = py{38,311} quality # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages From 325a7b91b20e1a68a383bad250a8f32b8435f7b6 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 9 Apr 2024 11:32:43 -0400 Subject: [PATCH 7/7] build: Update more workflows to test on Python 3.11 --- .github/workflows/ci-static-analysis.yml | 2 +- .github/workflows/js-tests.yml | 2 +- .github/workflows/lint-imports.yml | 2 +- .github/workflows/migrations-check.yml | 6 +++--- .github/workflows/pylint-checks.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests-gh-hosted.yml | 2 +- .github/workflows/units-test-scripts-structures-pruning.yml | 2 +- .github/workflows/units-test-scripts-user-retirement.yml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index dd21eb3cac..33415601fe 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.8'] + python-version: ['3.8', '3.11'] os: ['ubuntu-20.04'] steps: diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 650ce9efb4..179b376b68 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -14,7 +14,7 @@ jobs: matrix: os: [ ubuntu-20.04 ] node-version: [ 18 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] steps: diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index 3bc7d2bf9d..243b44d203 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.11' - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 68023a6f4b..417371c646 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -14,10 +14,10 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] # 'pinned' is used to install the latest patch version of Django # within the global constraint i.e. Django==4.2.8 in current case - # because we have global constraint of Django<4.2 + # because we have global constraint of Django<4.2 django-version: ["pinned"] mongo-version: ["4", "7"] mysql-version: ["8"] @@ -115,7 +115,7 @@ jobs: ./manage.py lms migrate echo "Running the CMS migrations." ./manage.py cms migrate - + # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why # https://github.com/orgs/community/discussions/33579 diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 840dc985e3..c2e04fc191 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.11 - name: Get pip cache dir id: pip-cache-dir diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 8c8a248878..9206596319 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] node-version: [ 18 ] steps: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index f5ff2f63e5..870de732cf 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ "ubuntu-20.04" ] - python-version: [ "3.8" ] + python-version: [ "3.8", '3.11' ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 177d84af3d..db5ead905a 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] node-version: [ 18 ] npm-version: [ 10.5.x ] mongo-version: ["4.4", "7.0"] diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index 5f4e3c5a68..75b86d1dfa 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [ '3.8' ] + python-version: [ '3.11' ] django-version: - "pinned" steps: diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index 434b617c17..9fab62643c 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: [ '3.8', '3.12' ] + python-version: [ '3.8', '3.11', '3.12' ] steps: - name: Checkout code diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index e11d70193f..22bbdc3f61 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: [ '3.8' ] + python-version: [ '3.8', '3.11', '3.12'] steps: - name: Checkout code