build: Set Django version for tests more safely; drop support for non-GHA (#31387)

We have a need to lock the version of Django for production and tests, but
also to test on newer versions of Django so that we can get the repo ready
for long-term-support releases.

We've been doing that by extracting the `django==x.y.z` from the
pip-compiled files and moving it to a django.txt that is then co-installed
but can be overridden during tests. The problem is that this can result
in broken packages.

The approach here is to have `make test-requirements` continue to
ensure a consistent set of packages, and then install a different
Django on top of that in the CI script -- and call `pip check` to make
sure that combination isn't broken.

Adding Django 4.0 to the unit-tests.yml matrix will now correctly
result in this error and a failing job:
`django-splash 1.2.1 has requirement Django<4.0, but you have django 4.0.8.`

The other half of this is to change other CI runners to remove their
ability to control the Django version, since it's complicated to make
this work, and we probably only need it in unit-tests.yml. Convert them
to just use `make test-requirements`.

Also:

- Simplify handling of `pip --src` by setting `PIP_SRC` (rather than our
  own `PIP_SRC_DIR`, which pip ignores because `--src-dir` isn't an option
  that it knows). This is needed to allow `make test-requirements` to do
  the pip calls. An alternative would be to set a pip-options env var for
  the make target to use, but `PIP_SRC` already exists.
- Remove outdated modifications to common_constraints
- Add comment explaining why pylint tests need dev-requirements

scripts/ci-runner.Dockerfile

@@ -46,10 +46,10 @@ COPY openedx/core/lib openedx/core/lib
 COPY lms lms
 COPY cms cms
 COPY requirements/pip.txt requirements/pip.txt
+COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt
 COPY requirements/edx/testing.txt requirements/edx/testing.txt
-COPY requirements/edx/django.txt requirements/edx/django.txt
-RUN pip install -r requirements/pip.txt && \
-pip install -r requirements/edx/testing.txt -r requirements/edx/django.txt
+COPY Makefile Makefile
+RUN make test-requirements
 
 FROM base as runner
 

scripts/generic-ci-tests.sh

@@ -70,7 +70,7 @@ if [ -z ${TOX_ENV+x} ] || [[ ${TOX_ENV} == 'null' ]]; then
     echo "TOX_ENV: ${TOX_ENV}"
     TOX=""
 elif tox -l |grep -q "${TOX_ENV}"; then
-    if [[ "${TOX_ENV}" == 'quality-django32' ]]; then
+    if [[ "${TOX_ENV}" == 'quality' ]]; then
         TOX=""
     else
         TOX="tox -r -e ${TOX_ENV} --"

scripts/jenkins-common.sh

@@ -48,9 +48,7 @@ source $VENV_PATH/edx-venv/bin/activate
 # Hack to fix up egg-link files given that the virtualenv is not relocatable
 sed -i "s|^/home/jenkins/shallow-clone|`pwd`|" -- \
     $VENV_PATH/edx-venv/lib/python*/site-packages/*.egg-link
-pip install pip==21.3
-pip install -qr requirements/edx/pip-tools.txt
-pip-sync -q requirements/edx/testing.txt requirements/edx/django.txt
+make test-requirements
 
 # add the node packages dir to PATH
 PATH=$PATH:node_modules/.bin

scripts/xdist/prepare_xdist_nodes.sh

@@ -10,18 +10,13 @@ python scripts/xdist/pytest_worker_manager.py -a up -n ${XDIST_NUM_WORKERS} \
 -key ${XDIST_WORKER_KEY_NAME} \
 -iam ${XDIST_WORKER_IAM_PROFILE_ARN}
 
-# Install the correct version of Django depending on which tox environment (if any) is in use
-if [[ -z ${TOXENV+x} ]] || [[ ${TOXENV} == 'null' ]] || [[ ${TOXENV} == *'django32'* ]]; then
-    DJANGO_REQUIREMENT="requirements/edx/django.txt"
-fi
-
 ip_list=$(<pytest_worker_ips.txt)
 for ip in $(echo $ip_list | sed "s/,/ /g")
 do
     worker_reqs_cmd="ssh -o StrictHostKeyChecking=no jenkins@$ip
     'git clone --branch master --depth 1 -q https://github.com/openedx/edx-platform.git; cd edx-platform;
     git fetch -fq origin ${XDIST_GIT_REFSPEC}; git checkout -q ${XDIST_GIT_BRANCH};
-    scripts/xdist/setup_worker.sh -p $PYTHON_VERSION -d $DJANGO_REQUIREMENT' & "
+    scripts/xdist/setup_worker.sh -p $PYTHON_VERSION & "
 
     cmd=$cmd$worker_reqs_cmd
 done

scripts/xdist/setup_worker.sh

@@ -1,12 +1,11 @@
 #!/usr/bin/env bash
 # Set up worker node.
 
-while getopts 'p:d:' opt; do
+while getopts 'p:' opt; do
     case "$opt" in
 	p) PYTHON_VERSION="$OPTARG";;
-	d) DJANGO_REQUIREMENT="$OPTARG";;
 	[?])
-	    print >&2 "Usage: $0 -p python-version -d django-reqs-file"
+	    print >&2 "Usage: $0 -p python-version"
 	    exit 1
             ;;
     esac
@@ -21,7 +20,7 @@ source $venv/bin/activate
 # Hack to fix up egg-link files given that the virtualenv is not relocatable
 sed -i "s|\(^/home/jenkins\)/shallow-clone|\1/edx-platform|" -- \
     $venv/lib/python*/site-packages/*.egg-link
-pip install -qr requirements/edx/pip-tools.txt
-pip-sync -q requirements/edx/testing.txt "${DJANGO_REQUIREMENT}"
+
+make test-requirements
 
 mkdir reports