From b852344fcfaba49b7df47e050a691e72eff3649f Mon Sep 17 00:00:00 2001
From: Tim McCormack <tmccormack@edx.org>
Date: Mon, 17 Apr 2023 17:21:48 +0000
Subject: [PATCH] build: Deduplicate mismatched pip.txt files that caused build
 failure (#32081)

There was a `requirements/pip.txt` with old versions, and a newer
`requirements/edx/pip.txt` managed via a `pip.in` file. The old one was
used in most places, but came out of sync with pip-tools.txt, which was
managed properly. Eventually this caused a `pip check` failure due to the
mismatch.

This should resolve at least part of https://github.com/edx/edx-arch-experiments/issues/267

This PR moves pip.in and pip-tools.in and their corresponding pin files
up to the `requirements/` dir, since they should be shared between the edx
and sandbox environments. This also has the effect of upgrading pip to
match the version in the file we've been uselessly upgrading.

Other improvements:

- Remove `-q` option from pip and pip-sync calls, as it was hiding some
  debugging information that would have resolved this sooner.
- Depend on `pre-requirements` from `compile-requirements`, rather than
  from `upgrade`. (The base target is the one that actually needs it.)
  This also lets us remove the explicit `pip install pip-tools` line.
- Install the recompiled pip and pip-tools files right away, not after the
  loop. When we upgrade pip-tools, we want to use the upgraded version,
  not the previous version. This requires moving the pip-tools.txt
  recompilation outside of the loop and into its own explicit line.
- Don't upgrade pip if we're not running `make upgrade` (respect the
  compile options).
- Remove apparently-unneeded `--no-emit-trusted-host --no-emit-index-url`
  options (we don't pass trusted-host or index-url options).
---
 Makefile                             | 28 +++++++++++++++-------------
 requirements/edx/development.in      |  2 +-
 requirements/edx/development.txt     | 14 +++++++-------
 requirements/edx/pip.txt             | 14 --------------
 requirements/{edx => }/pip-tools.in  |  2 +-
 requirements/{edx => }/pip-tools.txt |  4 ++--
 requirements/{edx => }/pip.in        |  2 +-
 requirements/pip.txt                 | 16 ++++++++++++++--
 scripts/ci-runner.Dockerfile         |  2 +-
 9 files changed, 42 insertions(+), 42 deletions(-)
 delete mode 100644 requirements/edx/pip.txt
 rename requirements/{edx => }/pip-tools.in (95%)
 rename requirements/{edx => }/pip-tools.txt (82%)
 rename requirements/{edx => }/pip.in (77%)

diff --git a/Makefile b/Makefile
index 7aa7334fdf..fec1c2caf2 100644
--- a/Makefile
+++ b/Makefile
@@ -64,8 +64,8 @@ pull: ## update the Docker image used by "make shell"
 	docker pull edxops/edxapp:latest
 
 pre-requirements: ## install Python requirements for running pip-tools
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
+	pip install -r requirements/pip.txt
+	pip install -r requirements/pip-tools.txt
 
 local-requirements:
 # 	edx-platform installs some Python projects from within the edx-platform repo itself.
@@ -74,7 +74,7 @@ local-requirements:
 dev-requirements: pre-requirements
 	@# The "$(wildcard..)" is to include private.txt if it exists, and make no mention
 	@# of it if it does not.  Shell wildcarding can't do that with default options.
-	pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
+	pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
 	make local-requirements
 
 base-requirements: pre-requirements
@@ -96,7 +96,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend
 
 # Order is very important in this list: files must appear after everything they include!
 REQ_FILES = \
-	requirements/edx/pip-tools \
 	requirements/edx/coverage \
 	requirements/edx/doc \
 	requirements/edx/paver \
@@ -117,23 +116,26 @@ $(COMMON_CONSTRAINTS_TXT):
 	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
 
 compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
-compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
-	pip install -q pip-tools
-	pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in
+compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
+	@# Bootstrapping: Rebuild pip and pip-tools first, and then install them
+	@# so that if there are any failures we'll know now, rather than the next
+	@# time someone tries to use the outputs.
+	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
+	pip install -r requirements/pip.txt
+
+	pip-compile -v ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -r requirements/pip-tools.txt
 
 	@ export REBUILD='--rebuild'; \
 	for f in $(REQ_FILES); do \
 		echo ; \
 		echo "== $$f ===============================" ; \
-		echo "pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
-		pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
+		echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
+		pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
 		export REBUILD=''; \
 	done
 
-	pip install -qr requirements/edx/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
-
-upgrade: pre-requirements  ## update the pip requirements files to use the latest releases satisfying our constraints
+upgrade:  ## update the pip requirements files to use the latest releases satisfying our constraints
 	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"
 
 check-types: ## run static type-checking tests
diff --git a/requirements/edx/development.in b/requirements/edx/development.in
index 6b4cfa8317..1a857f2917 100644
--- a/requirements/edx/development.in
+++ b/requirements/edx/development.in
@@ -10,7 +10,7 @@
 
 -c ../constraints.txt
 
--r pip-tools.txt                        # pip-tools and its dependencies, for managing requirements files
+-r ../pip-tools.txt                     # pip-tools and its dependencies, for managing requirements files
 -r testing.txt                          # Dependencies for running the various test suites
 
 click                                   # Used for perf_tests utilities in modulestore
diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
index 1afd30cca0..7d13433308 100644
--- a/requirements/edx/development.txt
+++ b/requirements/edx/development.txt
@@ -125,7 +125,7 @@ bridgekeeper==0.9
     # via -r requirements/edx/testing.txt
 build==0.10.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   pip-tools
 celery==5.2.7
     # via
@@ -168,8 +168,8 @@ chem==1.2.0
 click==8.1.3
     # via
     #   -c requirements/edx/../constraints.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/development.in
-    #   -r requirements/edx/pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   celery
     #   click-didyoumean
@@ -1047,7 +1047,7 @@ outcome-surveys==2.4.0
     # via -r requirements/edx/testing.txt
 packaging==23.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   build
     #   drf-yasg
@@ -1093,7 +1093,7 @@ pillow==9.4.0
     #   edx-enterprise
     #   edx-organizations
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.txt
+    # via -r requirements/edx/../pip-tools.txt
 pkgutil-resolve-name==1.3.10
     # via
     #   -r requirements/edx/testing.txt
@@ -1237,7 +1237,7 @@ pyparsing==3.0.9
     #   openedx-calc
 pyproject-hooks==1.0.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   build
 pyquery==2.0.0
     # via -r requirements/edx/testing.txt
@@ -1584,7 +1584,7 @@ toml==0.10.2
     # via vulture
 tomli==2.0.1
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   -r requirements/edx/testing.txt
     #   build
     #   coverage
@@ -1691,7 +1691,7 @@ webob==1.8.7
     #   xblock
 wheel==0.40.0
     # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
     #   pip-tools
 wrapt==1.15.0
     # via
diff --git a/requirements/edx/pip.txt b/requirements/edx/pip.txt
deleted file mode 100644
index f0e5e9397c..0000000000
--- a/requirements/edx/pip.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.8
-# by the following command:
-#
-#    make upgrade
-#
-wheel==0.40.0
-    # via -r requirements/edx/pip.in
-
-# The following packages are considered to be unsafe in a requirements file:
-pip==23.0.1
-    # via -r requirements/edx/pip.in
-setuptools==67.6.0
-    # via -r requirements/edx/pip.in
diff --git a/requirements/edx/pip-tools.in b/requirements/pip-tools.in
similarity index 95%
rename from requirements/edx/pip-tools.in
rename to requirements/pip-tools.in
index 50f3de3e19..5d4419ea4b 100644
--- a/requirements/edx/pip-tools.in
+++ b/requirements/pip-tools.in
@@ -7,6 +7,6 @@
 #   * confirm that it has no system requirements beyond what we already install
 #   * run "make upgrade" to update the detailed requirements files
 
--c ../constraints.txt
+-c constraints.txt
 
 pip-tools                 # Contains pip-compile, used to generate pip requirements files
diff --git a/requirements/edx/pip-tools.txt b/requirements/pip-tools.txt
similarity index 82%
rename from requirements/edx/pip-tools.txt
rename to requirements/pip-tools.txt
index 15e167ee71..f434fbb8b9 100644
--- a/requirements/edx/pip-tools.txt
+++ b/requirements/pip-tools.txt
@@ -8,12 +8,12 @@ build==0.10.0
     # via pip-tools
 click==8.1.3
     # via
-    #   -c requirements/edx/../constraints.txt
+    #   -c requirements/constraints.txt
     #   pip-tools
 packaging==23.0
     # via build
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.in
+    # via -r requirements/pip-tools.in
 pyproject-hooks==1.0.0
     # via build
 tomli==2.0.1
diff --git a/requirements/edx/pip.in b/requirements/pip.in
similarity index 77%
rename from requirements/edx/pip.in
rename to requirements/pip.in
index 741969aac1..cc36db5a08 100644
--- a/requirements/edx/pip.in
+++ b/requirements/pip.in
@@ -1,4 +1,4 @@
--c ../constraints.txt
+-c constraints.txt
 # Core dependencies for installing other dependencies
 
 pip
diff --git a/requirements/pip.txt b/requirements/pip.txt
index a811cc82f1..45fb600be5 100644
--- a/requirements/pip.txt
+++ b/requirements/pip.txt
@@ -1,2 +1,14 @@
-pip==22.1
-wheel==0.37.1
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    make upgrade
+#
+wheel==0.40.0
+    # via -r requirements/pip.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==23.0.1
+    # via -r requirements/pip.in
+setuptools==67.6.0
+    # via -r requirements/pip.in
diff --git a/scripts/ci-runner.Dockerfile b/scripts/ci-runner.Dockerfile
index 999b89941e..291f1d33d3 100644
--- a/scripts/ci-runner.Dockerfile
+++ b/scripts/ci-runner.Dockerfile
@@ -46,7 +46,7 @@ COPY openedx/core/lib openedx/core/lib
 COPY lms lms
 COPY cms cms
 COPY requirements/pip.txt requirements/pip.txt
-COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt
+COPY requirements/pip-tools.txt requirements/pip-tools.txt
 COPY requirements/edx/testing.txt requirements/edx/testing.txt
 COPY Makefile Makefile
 RUN make test-requirements