From 2edbdcf7f48eddeff9fc6f1967e3f11be8c8e390 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 3 Oct 2022 20:26:40 +0000 Subject: [PATCH] build: Clear out stale dependencies when installing requirements (#31080) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We've been seeing some cross-version caching issues in the edx-platform-runner unit tests, which apparently run on a "dirty" environment—the virtualenv does not get cleared out between runs, which probably improves performance but allows installed dependencies to "leak" between runs. This results in errors between master and older open-releases but could also prevent us from noticing missing deps. By using pip-sync in the new CI Make targets (as we already do for the regular `make requirements` target) we can ensure that any stale dependencies from runs by other branches (or older versions of the code) are removed. Calling `make local-requirements` at the end of each `*-requirements` target rather than making it a prerequisite is necessary for using sync, since otherwise the local reqs would be wiped out. The `requirements` target is also deduplicated into the newer `dev-requirements` (aliased to it, with both installing private deps now.) Adding a prerequisite of `pre-requirements` allows us to simplify some workflow calls slightly. This ends up being the bulk of the commit by line count. The pip lockfile also wasn't being used in the Makefile, so I added that to pre-requirements as well. Also fix leading whitespace issue in Makefile. --- .github/workflows/js-tests.yml | 1 - .github/workflows/migrations-check-mysql8.yml | 1 - .github/workflows/migrations-check.yml | 1 - .github/workflows/pylint-checks.yml | 1 - .github/workflows/static-assets-check.yml | 1 - .github/workflows/unit-tests.yml | 1 - .../workflows/verify-gha-unit-tests-count.yml | 1 - Makefile | 26 ++++++++++--------- 8 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 4975253e28..743b5286d8 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -63,7 +63,6 @@ jobs: - name: Install Required Python Dependencies run: | - pip install -r requirements/pip.txt make base-requirements - uses: c-hive/gha-npm-cache@v1 diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml index 6c817abb0e..74c4e6cabd 100644 --- a/.github/workflows/migrations-check-mysql8.yml +++ b/.github/workflows/migrations-check-mysql8.yml @@ -50,7 +50,6 @@ jobs: - name: Install Python dependencies run: | - pip install -r requirements/pip.txt make dev-requirements pip uninstall -y mysqlclient pip install --no-binary mysqlclient mysqlclient diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 7fe2acad86..bd0a07680a 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -60,7 +60,6 @@ jobs: - name: Install Python dependencies run: | - pip install -r requirements/pip.txt make dev-requirements pip uninstall -y mysqlclient pip install --no-binary mysqlclient mysqlclient diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index eb069fa2ee..5212ce2d0e 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -57,7 +57,6 @@ jobs: - name: Install required Python dependencies run: | - pip install -r requirements/pip.txt make dev-requirements pip uninstall -y mysqlclient pip install --no-binary mysqlclient mysqlclient diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 4da597d69b..7c0f8a9563 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -49,7 +49,6 @@ jobs: - name: Install Required Python Dependencies run: | - pip install -r requirements/pip.txt make base-requirements - name: Initiate Mongo DB Service diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b2777e1893..07a9adebdf 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -61,7 +61,6 @@ jobs: - name: install requirements run: | - sudo pip install -r requirements/pip.txt sudo make test-requirements if [[ "${{ matrix.django-version }}" == "pinned" ]]; then sudo pip install -r requirements/edx/django.txt diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml index e1ae969678..039d45ad95 100644 --- a/.github/workflows/verify-gha-unit-tests-count.yml +++ b/.github/workflows/verify-gha-unit-tests-count.yml @@ -17,7 +17,6 @@ jobs: - uses: actions/checkout@v2 - name: install requirements run: | - sudo pip install -r requirements/pip.txt sudo make test-requirements - name: verify unit tests count diff --git a/Makefile b/Makefile index 0bc1766a40..39b0d48db0 100644 --- a/Makefile +++ b/Makefile @@ -64,26 +64,28 @@ pull: ## update the Docker image used by "make shell" docker pull edxops/edxapp:latest pre-requirements: ## install Python requirements for running pip-tools + pip install -qr requirements/pip.txt pip install -qr requirements/edx/pip-tools.txt local-requirements: # edx-platform installs some Python projects from within the edx-platform repo itself. pip install -e . -dev-requirements: local-requirements - pip install -qr requirements/edx/development.txt - -base-requirements: local-requirements - pip install -qr requirements/edx/base.txt - -test-requirements: local-requirements - pip install --exists-action='w' -qr requirements/edx/testing.txt - -requirements: pre-requirements ## install development environment requirements +dev-requirements: pre-requirements @# The "$(wildcard..)" is to include private.txt if it exists, and make no mention @# of it if it does not. Shell wildcarding can't do that with default options. pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt) - pip install -e . + make local-requirements + +base-requirements: pre-requirements + pip-sync requirements/edx/base.txt + make local-requirements + +test-requirements: pre-requirements + pip-sync --pip-args="--exists-action=w" requirements/edx/testing.txt + make local-requirements + +requirements: dev-requirements ## install development environment requirements shell: ## launch a bash shell in a Docker container with all edx-platform dependencies installed docker run -it -e "NO_PYTHON_UNINSTALL=1" -e "PIP_INDEX_URL=https://pypi.python.org/simple" -e TERM \ @@ -130,7 +132,7 @@ compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ export REBUILD=''; \ done - # Let tox control the Django version for tests + # Let tox control the Django version for tests grep -e "^django==" requirements/edx/base.txt > requirements/edx/django.txt sed '/^[dD]jango==/d' requirements/edx/testing.txt > requirements/edx/testing.tmp mv requirements/edx/testing.tmp requirements/edx/testing.txt