build: Deduplicate mismatched pip.txt files that caused build failure (#32081)

There was a `requirements/pip.txt` with old versions, and a newer `requirements/edx/pip.txt` managed via a `pip.in` file. The old one was used in most places, but came out of sync with pip-tools.txt, which was managed properly. Eventually this caused a `pip check` failure due to the mismatch. This should resolve at least part of https://github.com/edx/edx-arch-experiments/issues/267 This PR moves pip.in and pip-tools.in and their corresponding pin files up to the `requirements/` dir, since they should be shared between the edx and sandbox environments. This also has the effect of upgrading pip to match the version in the file we've been uselessly upgrading. Other improvements: - Remove `-q` option from pip and pip-sync calls, as it was hiding some debugging information that would have resolved this sooner. - Depend on `pre-requirements` from `compile-requirements`, rather than from `upgrade`. (The base target is the one that actually needs it.) This also lets us remove the explicit `pip install pip-tools` line. - Install the recompiled pip and pip-tools files right away, not after the loop. When we upgrade pip-tools, we want to use the upgraded version, not the previous version. This requires moving the pip-tools.txt recompilation outside of the loop and into its own explicit line. - Don't upgrade pip if we're not running `make upgrade` (respect the compile options). - Remove apparently-unneeded `--no-emit-trusted-host --no-emit-index-url` options (we don't pass trusted-host or index-url options).
2023-04-17 17:21:48 +00:00
parent 2dc79bcab4
commit b852344fcf
9 changed files with 42 additions and 42 deletions
--- a/28
+++ b/28
@@ -64,8 +64,8 @@ pull: ## update the Docker image used by "make shell"
 	docker pull edxops/edxapp:latest

 pre-requirements: ## install Python requirements for running pip-tools
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
+	pip install -r requirements/pip.txt
+	pip install -r requirements/pip-tools.txt

 local-requirements:
 # 	edx-platform installs some Python projects from within the edx-platform repo itself.
@@ -74,7 +74,7 @@ local-requirements:
 dev-requirements: pre-requirements
 	@# The "$(wildcard..)" is to include private.txt if it exists, and make no mention
 	@# of it if it does not.  Shell wildcarding can't do that with default options.
-	pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
+	pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
 	make local-requirements

 base-requirements: pre-requirements
@@ -96,7 +96,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend

 # Order is very important in this list: files must appear after everything they include!
 REQ_FILES = \
-	requirements/edx/pip-tools \
 	requirements/edx/coverage \
 	requirements/edx/doc \
 	requirements/edx/paver \
@@ -117,23 +116,26 @@ $(COMMON_CONSTRAINTS_TXT):
 	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)

 compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
-compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
-	pip install -q pip-tools
-	pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in
+compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
+	@# Bootstrapping: Rebuild pip and pip-tools first, and then install them
+	@# so that if there are any failures we'll know now, rather than the next
+	@# time someone tries to use the outputs.
+	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
+	pip install -r requirements/pip.txt
+
+	pip-compile -v ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -r requirements/pip-tools.txt

 	@ export REBUILD='--rebuild'; \
 	for f in $(REQ_FILES); do \
 		echo ; \
 		echo "== $$f ===============================" ; \
-		echo "pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
-		pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
+		echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
+		pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
 		export REBUILD=''; \
 	done

-	pip install -qr requirements/edx/pip.txt
-	pip install -qr requirements/edx/pip-tools.txt
-
-upgrade: pre-requirements  ## update the pip requirements files to use the latest releases satisfying our constraints
+upgrade:  ## update the pip requirements files to use the latest releases satisfying our constraints
 	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"

 check-types: ## run static type-checking tests
--- a/requirements/edx/development.in
+++ b/requirements/edx/development.in
@@ -10,7 +10,7 @@

 -c ../constraints.txt

-r pip-tools.txt                        # pip-tools and its dependencies, for managing requirements files
+-r ../pip-tools.txt                     # pip-tools and its dependencies, for managing requirements files
 -r testing.txt                          # Dependencies for running the various test suites

 click                                   # Used for perf_tests utilities in modulestore
--- a/requirements/edx/development.txt
+++ b/requirements/edx/development.txt
@@ -125,7 +125,7 @@ bridgekeeper==0.9
    # via -r requirements/edx/testing.txt
 build==0.10.0
    # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
    #   pip-tools
 celery==5.2.7
    # via
@@ -168,8 +168,8 @@ chem==1.2.0
 click==8.1.3
    # via
    #   -c requirements/edx/../constraints.txt
+    #   -r requirements/edx/../pip-tools.txt
    #   -r requirements/edx/development.in
-    #   -r requirements/edx/pip-tools.txt
    #   -r requirements/edx/testing.txt
    #   celery
    #   click-didyoumean
@@ -1047,7 +1047,7 @@ outcome-surveys==2.4.0
    # via -r requirements/edx/testing.txt
 packaging==23.0
    # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
    #   -r requirements/edx/testing.txt
    #   build
    #   drf-yasg
@@ -1093,7 +1093,7 @@ pillow==9.4.0
    #   edx-enterprise
    #   edx-organizations
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.txt
+    # via -r requirements/edx/../pip-tools.txt
 pkgutil-resolve-name==1.3.10
    # via
    #   -r requirements/edx/testing.txt
@@ -1237,7 +1237,7 @@ pyparsing==3.0.9
    #   openedx-calc
 pyproject-hooks==1.0.0
    # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
    #   build
 pyquery==2.0.0
    # via -r requirements/edx/testing.txt
@@ -1584,7 +1584,7 @@ toml==0.10.2
    # via vulture
 tomli==2.0.1
    # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
    #   -r requirements/edx/testing.txt
    #   build
    #   coverage
@@ -1691,7 +1691,7 @@ webob==1.8.7
    #   xblock
 wheel==0.40.0
    # via
-    #   -r requirements/edx/pip-tools.txt
+    #   -r requirements/edx/../pip-tools.txt
    #   pip-tools
 wrapt==1.15.0
    # via
--- a/requirements/edx/pip.txt
+++ b/requirements/edx/pip.txt
@@ -1,14 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.8
-# by the following command:
-#
-#    make upgrade
-#
-wheel==0.40.0
-    # via -r requirements/edx/pip.in
-
-# The following packages are considered to be unsafe in a requirements file:
-pip==23.0.1
-    # via -r requirements/edx/pip.in
-setuptools==67.6.0
-    # via -r requirements/edx/pip.in
--- a/requirements/edx/pip-tools.in
+++ b/requirements/edx/pip-tools.in
@@ -7,6 +7,6 @@
 #   * confirm that it has no system requirements beyond what we already install
 #   * run "make upgrade" to update the detailed requirements files

-c ../constraints.txt
+-c constraints.txt

 pip-tools                 # Contains pip-compile, used to generate pip requirements files
--- a/requirements/edx/pip-tools.txt
+++ b/requirements/edx/pip-tools.txt
@@ -8,12 +8,12 @@ build==0.10.0
    # via pip-tools
 click==8.1.3
    # via
-    #   -c requirements/edx/../constraints.txt
+    #   -c requirements/constraints.txt
    #   pip-tools
 packaging==23.0
    # via build
 pip-tools==6.12.3
-    # via -r requirements/edx/pip-tools.in
+    # via -r requirements/pip-tools.in
 pyproject-hooks==1.0.0
    # via build
 tomli==2.0.1
--- a/requirements/edx/pip.in
+++ b/requirements/edx/pip.in
@@ -1,4 +1,4 @@
-c ../constraints.txt
+-c constraints.txt
 # Core dependencies for installing other dependencies

 pip
--- a/requirements/pip.txt
+++ b/requirements/pip.txt
@@ -1,2 +1,14 @@
-pip==22.1
-wheel==0.37.1
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    make upgrade
+#
+wheel==0.40.0
+    # via -r requirements/pip.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==23.0.1
+    # via -r requirements/pip.in
+setuptools==67.6.0
+    # via -r requirements/pip.in
--- a/scripts/ci-runner.Dockerfile
+++ b/scripts/ci-runner.Dockerfile
@@ -46,7 +46,7 @@ COPY openedx/core/lib openedx/core/lib
 COPY lms lms
 COPY cms cms
 COPY requirements/pip.txt requirements/pip.txt
-COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt
+COPY requirements/pip-tools.txt requirements/pip-tools.txt
 COPY requirements/edx/testing.txt requirements/edx/testing.txt
 COPY Makefile Makefile
 RUN make test-requirements