build: Set Django version for tests more safely; drop support for non-GHA (#31387)

We have a need to lock the version of Django for production and tests, but
also to test on newer versions of Django so that we can get the repo ready
for long-term-support releases.

We've been doing that by extracting the `django==x.y.z` from the
pip-compiled files and moving it to a django.txt that is then co-installed
but can be overridden during tests. The problem is that this can result
in broken packages.

The approach here is to have `make test-requirements` continue to
ensure a consistent set of packages, and then install a different
Django on top of that in the CI script -- and call `pip check` to make
sure that combination isn't broken.

Adding Django 4.0 to the unit-tests.yml matrix will now correctly
result in this error and a failing job:
`django-splash 1.2.1 has requirement Django<4.0, but you have django 4.0.8.`

The other half of this is to change other CI runners to remove their
ability to control the Django version, since it's complicated to make
this work, and we probably only need it in unit-tests.yml. Convert them
to just use `make test-requirements`.

Also:

- Simplify handling of `pip --src` by setting `PIP_SRC` (rather than our
  own `PIP_SRC_DIR`, which pip ignores because `--src-dir` isn't an option
  that it knows). This is needed to allow `make test-requirements` to do
  the pip calls. An alternative would be to set a pip-options env var for
  the make target to use, but `PIP_SRC` already exists.
- Remove outdated modifications to common_constraints
- Add comment explaining why pylint tests need dev-requirements

.github/workflows/pylint-checks.yml

@@ -56,9 +56,13 @@ jobs:
 
       - name: Install required Python dependencies
         run: |
+          # dev-requirements is needed because the linter will otherwise
+          # trip over some dev-only things like django-debug-toolbar
+          # (import debug_toolbar) that aren't in testing.txt.
           make dev-requirements
           pip uninstall -y mysqlclient
           pip install --no-binary mysqlclient mysqlclient
+          pip check
 
       - name: Run quality tests
         run: |

.github/workflows/quality-checks.yml

@@ -57,19 +57,16 @@ jobs:
 
     - name: Install Required Python Dependencies
       env:
-        PIP_SRC_DIR: ${{ runner.temp }}
+        PIP_SRC: ${{ runner.temp }}
       run: |
-        pip install -r requirements/pip.txt
-        pip install -r requirements/edx/testing.txt --src $PIP_SRC_DIR
-        pip install -e . --src $PIP_SRC_DIR
-        pip install -r requirements/edx/django.txt
+        make test-requirements
 
     - name: Run Quality Tests
       env:
         TEST_SUITE: quality
         SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
         SHARD: 4
-        PIP_SRC_DIR: ${{ runner.temp }}
+        PIP_SRC: ${{ runner.temp }}
         TARGET_BRANCH: ${{ github.base_ref }}
       run: |
         ./scripts/all-tests.sh

.github/workflows/unit-tests-gh-hosted.yml

@@ -15,7 +15,8 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ '3.8' ]
-        django-version: [ "3.2" ]
+        django-version:
+          - "pinned"
         shard_name: [
             "lms-1",
             "lms-2",
@@ -60,16 +61,18 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
           restore-keys: ${{ runner.os }}-pip-
 
       - name: Install Required Python Dependencies
+        env:
+          PIP_SRC: ${{ runner.temp }}
         run: |
-            pip install -r requirements/pip.txt
-            pip install -r requirements/edx/development.txt --src ${{ runner.temp }}
-            # edx-platform installs some Python projects from within the edx-platform repo itself.
-            pip install -e .
+          make test-requirements
+          if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
             pip install "django~=${{ matrix.django-version }}.0"
+            pip check # fail if this test-reqs/Django combination is broken
+          fi
 
       - name: Setup and run tests
         uses: ./.github/actions/unit-tests
@@ -80,7 +83,8 @@ jobs:
     strategy:
       matrix:
         python-version: [ '3.8' ]
-        django-version: [ "3.2" ]
+        django-version:
+          - "pinned"
     steps:
       - uses: actions/checkout@v2
 
@@ -102,14 +106,16 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
           restore-keys: ${{ runner.os }}-pip-
 
       - name: Install Required Python Dependencies
         run: |
-            pip install -r requirements/pip.txt
-            make dev-requirements
+          make test-requirements
+          if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
             pip install "django~=${{ matrix.django-version }}.0"
+            pip check # fail if this test-reqs/Django combination is broken
+          fi
 
       - name: verify unit tests count
         uses: ./.github/actions/verify-tests-count

.github/workflows/unit-tests.yml

@@ -62,10 +62,9 @@ jobs:
       - name: install requirements
         run: |
           sudo make test-requirements
-          if [[ "${{ matrix.django-version }}" == "pinned" ]]; then
-            sudo pip install -r requirements/edx/django.txt
-          else
+          if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
             sudo pip install "django~=${{ matrix.django-version }}.0"
+            sudo pip check # fail if this test-reqs/Django combination is broken
           fi
 
       - name: list installed package versions