From 872174e28d92556a2384525e674f0f809d2b2cc9 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 9 Sep 2024 09:58:19 -0400 Subject: [PATCH 1/4] build: Switch to ubuntu-latest for builds This code does not have any dependencies that are specific to any specific version of ubuntu. So instead of testing on a specific version and then needing to do work to keep the versions up-to-date, we switch to the ubuntu-latest target which should be sufficient for testing purposes. This work is being done as a part of https://github.com/openedx/platform-roadmap/issues/377 closes https://github.com/openedx/edx-platform/issues/35314 --- .github/workflows/ci-static-analysis.yml | 2 +- .github/workflows/compile-python-requirements.yml | 2 +- .github/workflows/js-tests.yml | 2 +- .github/workflows/lint-imports.yml | 2 +- .github/workflows/migrations-check.yml | 2 +- .github/workflows/publish-ci-docker-image.yml | 2 +- .github/workflows/pylint-checks.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests.yml | 12 ++++++------ .github/workflows/upgrade-one-python-dependency.yml | 2 +- .github/workflows/verify-dunder-init.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index 7e768a4564..a3b0527aad 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -10,7 +10,7 @@ jobs: matrix: python-version: - "3.11" - os: ["ubuntu-20.04"] + os: ["ubuntu-latest"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml index 0ff99b9c68..21cb80083f 100644 --- a/.github/workflows/compile-python-requirements.yml +++ b/.github/workflows/compile-python-requirements.yml @@ -15,7 +15,7 @@ defaults: jobs: recompile-python-dependencies: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out target branch diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 4d025e5401..c9d2d7ab11 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] node-version: [18, 20] python-version: - "3.11" diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index 8ead8396bf..e3c59ec093 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -9,7 +9,7 @@ on: jobs: lint-imports: name: Lint Python Imports - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out branch diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 183b90effa..ec3ff21e60 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: - "3.11" # 'pinned' is used to install the latest patch version of Django diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml index 0a9f50f6da..6a0f3768b7 100644 --- a/.github/workflows/publish-ci-docker-image.yml +++ b/.github/workflows/publish-ci-docker-image.yml @@ -7,7 +7,7 @@ on: jobs: push: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index eeb53c24ed..58560bf307 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -8,7 +8,7 @@ on: jobs: run-pylint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index cf8ffd5d29..5445d70e3b 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: - "3.11" node-version: [20] diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 7f2b4925af..d880d73517 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -17,7 +17,7 @@ jobs: runs-on: "${{ matrix.os }}" strategy: matrix: - os: ["ubuntu-20.04"] + os: ["ubuntu-latest"] python-version: - "3.11" diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 7bbfd3369b..1774167704 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: - "3.11" node-version: [18, 20] diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3e442b75d4..a697700898 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: run-tests: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python-version: @@ -142,7 +142,7 @@ jobs: overwrite: true collect-and-verify: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Setup Python @@ -207,7 +207,7 @@ jobs: # https://github.com/orgs/community/discussions/33579 success: name: Unit tests successful - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: always() needs: [run-tests] steps: @@ -218,7 +218,7 @@ jobs: jobs: ${{ toJSON(needs) }} compile-warnings-report: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [run-tests] steps: - uses: actions/checkout@v4 @@ -246,7 +246,7 @@ jobs: overwrite: true merge-artifacts: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [compile-warnings-report] steps: - name: Merge Pytest Warnings JSON Artifacts @@ -266,7 +266,7 @@ jobs: # Combine and upload coverage reports. coverage: if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [run-tests] strategy: matrix: diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index 6ca5dfcb35..84a00266e9 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -28,7 +28,7 @@ defaults: jobs: upgrade-one-python-dependency: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out target branch diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml index 611fc0afc6..9d920238eb 100644 --- a/.github/workflows/verify-dunder-init.yml +++ b/.github/workflows/verify-dunder-init.yml @@ -8,7 +8,7 @@ on: jobs: verify_dunder_init: name: Verify __init__.py Files - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out branch From d8eef6e347b36c61221c455fdf1931290939e298 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 11 Sep 2024 09:26:57 -0400 Subject: [PATCH 2/4] build: Run mongosh commands within the container. This is no longer installed by default on ubuntu and so we have to either manually install it or just run the relevant commands in the container here it's already available. This lets us do some of the test setup in a more robust way. --- .github/workflows/migrations-check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index ec3ff21e60..f253d48e4f 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -52,7 +52,7 @@ jobs: steps: - name: Setup mongodb user run: | - mongosh edxapp --eval ' + docker exec ${{ job.services.mongo.id }} mongosh edxapp --eval ' db.createUser( { user: "edxapp", @@ -67,7 +67,7 @@ jobs: - name: Verify mongo and mysql db credentials run: | mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp - mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + docker exec ${{ job.services.mongo.id }} mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp - name: Checkout repo uses: actions/checkout@v4 From e6e5bedf63fc393b95fc74501298fba72099d5bb Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 11 Sep 2024 10:05:23 -0400 Subject: [PATCH 3/4] fix: Don't start the mongo service. We stopped using mongo on the runner directly a while ago so this is just an errant start that should have been removed. --- .github/workflows/static-assets-check.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 1774167704..0a417f9b1c 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -72,9 +72,6 @@ jobs: run: | pip install -r requirements/edx/assets.txt - - name: Initiate Mongo DB Service - run: sudo systemctl start mongod - - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH From 1804fbb13107bd9681c5c51c5c20e90edf9e09d2 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 17 Sep 2024 15:10:12 -0400 Subject: [PATCH 4/4] build!: enable md4 for testing. Operators Note: In newer versions of ubuntu the MD4 hashing algorithm is disabled by default. To enable it the openssl config needs to be updated in a manner similar to what's being done here. Alternatively, you can set the `FEATURES['ENABLE_BLAKE2B_HASHING']` setting to `True` which will switch to a newer hashing algorithm where MD4 was previously used. Because this hashing is being used as a part of the edx-platform caching mechanism, this will effectively clear the cache for the items that use this hash. The will impact any items where the cache key might have been too big to store in memcache so it's hard to predict exactly which items will be impacted. BREAKING CHANGE: See the operator note above for more details as this may break for users transitioning from Ubuntu 20.04 to newer versions. --- .github/workflows/unit-tests.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a697700898..5fef1c8352 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -66,7 +66,29 @@ jobs: - name: install system requirements run: | - sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx + sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx openssl + + # This is needed until the ENABLE_BLAKE2B_HASHING can be removed and we + # can stop using MD4 by default. + - name: enable md4 hashing in libssl + run: | + cat <