diff --git a/Makefile b/Makefile index 7aa7334fdf..fec1c2caf2 100644 --- a/Makefile +++ b/Makefile @@ -64,8 +64,8 @@ pull: ## update the Docker image used by "make shell" docker pull edxops/edxapp:latest pre-requirements: ## install Python requirements for running pip-tools - pip install -qr requirements/pip.txt - pip install -qr requirements/edx/pip-tools.txt + pip install -r requirements/pip.txt + pip install -r requirements/pip-tools.txt local-requirements: # edx-platform installs some Python projects from within the edx-platform repo itself. @@ -74,7 +74,7 @@ local-requirements: dev-requirements: pre-requirements @# The "$(wildcard..)" is to include private.txt if it exists, and make no mention @# of it if it does not. Shell wildcarding can't do that with default options. - pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt) + pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt) make local-requirements base-requirements: pre-requirements @@ -96,7 +96,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend # Order is very important in this list: files must appear after everything they include! REQ_FILES = \ - requirements/edx/pip-tools \ requirements/edx/coverage \ requirements/edx/doc \ requirements/edx/paver \ @@ -117,23 +116,26 @@ $(COMMON_CONSTRAINTS_TXT): echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@) compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade -compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt - pip install -q pip-tools - pip-compile --allow-unsafe --upgrade -o requirements/edx/pip.txt requirements/edx/pip.in +compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt + @# Bootstrapping: Rebuild pip and pip-tools first, and then install them + @# so that if there are any failures we'll know now, rather than the next + @# time someone tries to use the outputs. + pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in + pip install -r requirements/pip.txt + + pip-compile -v ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in + pip install -r requirements/pip-tools.txt @ export REBUILD='--rebuild'; \ for f in $(REQ_FILES); do \ echo ; \ echo "== $$f ===============================" ; \ - echo "pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \ - pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ + echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \ + pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ export REBUILD=''; \ done - pip install -qr requirements/edx/pip.txt - pip install -qr requirements/edx/pip-tools.txt - -upgrade: pre-requirements ## update the pip requirements files to use the latest releases satisfying our constraints +upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" check-types: ## run static type-checking tests diff --git a/requirements/edx/development.in b/requirements/edx/development.in index 6b4cfa8317..1a857f2917 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -10,7 +10,7 @@ -c ../constraints.txt --r pip-tools.txt # pip-tools and its dependencies, for managing requirements files +-r ../pip-tools.txt # pip-tools and its dependencies, for managing requirements files -r testing.txt # Dependencies for running the various test suites click # Used for perf_tests utilities in modulestore diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 1afd30cca0..7d13433308 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -125,7 +125,7 @@ bridgekeeper==0.9 # via -r requirements/edx/testing.txt build==0.10.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # pip-tools celery==5.2.7 # via @@ -168,8 +168,8 @@ chem==1.2.0 click==8.1.3 # via # -c requirements/edx/../constraints.txt + # -r requirements/edx/../pip-tools.txt # -r requirements/edx/development.in - # -r requirements/edx/pip-tools.txt # -r requirements/edx/testing.txt # celery # click-didyoumean @@ -1047,7 +1047,7 @@ outcome-surveys==2.4.0 # via -r requirements/edx/testing.txt packaging==23.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # -r requirements/edx/testing.txt # build # drf-yasg @@ -1093,7 +1093,7 @@ pillow==9.4.0 # edx-enterprise # edx-organizations pip-tools==6.12.3 - # via -r requirements/edx/pip-tools.txt + # via -r requirements/edx/../pip-tools.txt pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/testing.txt @@ -1237,7 +1237,7 @@ pyparsing==3.0.9 # openedx-calc pyproject-hooks==1.0.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # build pyquery==2.0.0 # via -r requirements/edx/testing.txt @@ -1584,7 +1584,7 @@ toml==0.10.2 # via vulture tomli==2.0.1 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # -r requirements/edx/testing.txt # build # coverage @@ -1691,7 +1691,7 @@ webob==1.8.7 # xblock wheel==0.40.0 # via - # -r requirements/edx/pip-tools.txt + # -r requirements/edx/../pip-tools.txt # pip-tools wrapt==1.15.0 # via diff --git a/requirements/edx/pip.txt b/requirements/edx/pip.txt deleted file mode 100644 index f0e5e9397c..0000000000 --- a/requirements/edx/pip.txt +++ /dev/null @@ -1,14 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# make upgrade -# -wheel==0.40.0 - # via -r requirements/edx/pip.in - -# The following packages are considered to be unsafe in a requirements file: -pip==23.0.1 - # via -r requirements/edx/pip.in -setuptools==67.6.0 - # via -r requirements/edx/pip.in diff --git a/requirements/edx/pip-tools.in b/requirements/pip-tools.in similarity index 95% rename from requirements/edx/pip-tools.in rename to requirements/pip-tools.in index 50f3de3e19..5d4419ea4b 100644 --- a/requirements/edx/pip-tools.in +++ b/requirements/pip-tools.in @@ -7,6 +7,6 @@ # * confirm that it has no system requirements beyond what we already install # * run "make upgrade" to update the detailed requirements files --c ../constraints.txt +-c constraints.txt pip-tools # Contains pip-compile, used to generate pip requirements files diff --git a/requirements/edx/pip-tools.txt b/requirements/pip-tools.txt similarity index 82% rename from requirements/edx/pip-tools.txt rename to requirements/pip-tools.txt index 15e167ee71..f434fbb8b9 100644 --- a/requirements/edx/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,12 +8,12 @@ build==0.10.0 # via pip-tools click==8.1.3 # via - # -c requirements/edx/../constraints.txt + # -c requirements/constraints.txt # pip-tools packaging==23.0 # via build pip-tools==6.12.3 - # via -r requirements/edx/pip-tools.in + # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via build tomli==2.0.1 diff --git a/requirements/edx/pip.in b/requirements/pip.in similarity index 77% rename from requirements/edx/pip.in rename to requirements/pip.in index 741969aac1..cc36db5a08 100644 --- a/requirements/edx/pip.in +++ b/requirements/pip.in @@ -1,4 +1,4 @@ --c ../constraints.txt +-c constraints.txt # Core dependencies for installing other dependencies pip diff --git a/requirements/pip.txt b/requirements/pip.txt index a811cc82f1..45fb600be5 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,2 +1,14 @@ -pip==22.1 -wheel==0.37.1 +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# make upgrade +# +wheel==0.40.0 + # via -r requirements/pip.in + +# The following packages are considered to be unsafe in a requirements file: +pip==23.0.1 + # via -r requirements/pip.in +setuptools==67.6.0 + # via -r requirements/pip.in diff --git a/scripts/ci-runner.Dockerfile b/scripts/ci-runner.Dockerfile index 999b89941e..291f1d33d3 100644 --- a/scripts/ci-runner.Dockerfile +++ b/scripts/ci-runner.Dockerfile @@ -46,7 +46,7 @@ COPY openedx/core/lib openedx/core/lib COPY lms lms COPY cms cms COPY requirements/pip.txt requirements/pip.txt -COPY requirements/edx/pip-tools.txt requirements/edx/pip-tools.txt +COPY requirements/pip-tools.txt requirements/pip-tools.txt COPY requirements/edx/testing.txt requirements/edx/testing.txt COPY Makefile Makefile RUN make test-requirements