From 1552ea076205cf54c83267747ee6b0c8e284d096 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 18 Sep 2023 13:32:51 -0400 Subject: [PATCH] Add compile-requirements GH workflow and update requirements README (#33126) Adds a GH workflow that will run `make compile-requirements` and create a PR for it and overhauls the requirements README with info on how to use the new workflows. README changes: - Group the workflow/makefile info under a new heading - Switch to imperative for headings - Move "Upgrade all dependencies" to end of list (uncommon for people to actually need this command, I suspect) - Add "Add a dependency" - Preamble about .in and .txt files and Mac vs. Linux, as well as about forks Also, some style fixes to upgrade-one-python-dependency GH workflow: - Fix indentation in inputs block - Remove "-workflow" from job name (probably copied from another workflow) - Wrap a long line --- .../workflows/compile-python-requirements.yml | 72 +++++++++++++++++++ .../upgrade-one-python-dependency.yml | 45 ++++++------ requirements/README.rst | 30 +++++--- 3 files changed, 115 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/compile-python-requirements.yml diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml new file mode 100644 index 0000000000..18e68aa0ad --- /dev/null +++ b/.github/workflows/compile-python-requirements.yml @@ -0,0 +1,72 @@ +name: Recompile Python dependencies + +on: + workflow_dispatch: + inputs: + branch: + description: 'Target branch to create requirements PR against' + required: true + default: 'master' + type: string + +defaults: + run: + shell: bash # making this explicit opts into -e -o pipefail + +jobs: + recompile-python-dependencies: + runs-on: ubuntu-20.04 + + steps: + - name: Check out target branch + uses: actions/checkout@v3 + with: + ref: "${{ inputs.branch }}" + + - name: Set up Python environment + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Run make compile-requirements + env: + PACKAGE: "${{ inputs.package }}" + run: | + make compile-requirements + + - name: PR preflight + run: | + if git diff --exit-code; then + # Fail early (and avoid quiet failure of create-pull-request action) + echo "Error: No changes, so not creating PR." | tee -a "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + - name: Make a PR + id: make-pr + uses: peter-evans/create-pull-request@v5 + with: + branch: "${{ github.triggering_actor }}/compile-python-deps" + branch-suffix: short-commit-hash + add-paths: requirements + commit-message: | + feat: Recompile Python dependencies + + Commit generated by workflow `${{ github.workflow_ref }}` + title: "chore: Recompile Python dependencies" + body: >- + PR generated by workflow `${{ github.workflow_ref }}` + on behalf of @${{ github.triggering_actor }}. + assignees: "${{ github.triggering_actor }}" + reviewers: "${{ github.triggering_actor }}" + + - name: Job summary + env: + PR_URL: "${{ steps.make-pr.outputs.pull-request-url }}" + run: | + if [[ -z "$PR_URL" ]]; then + echo "PR not created; see log for more information" | tee -a "$GITHUB_STEP_SUMMARY" + exit 1 + else + echo "PR created or updated: $PR_URL" | tee -a "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index 2ac704e999..dbc7eb9448 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -2,32 +2,32 @@ name: Upgrade one Python dependency on: workflow_dispatch: - inputs: - branch: - description: 'Target branch to create requirements PR against' - required: true - default: 'master' - type: string - package: - description: 'Name of package to upgrade' - required: true - type: string - version: - description: 'Version number to upgrade to in constraints.txt (only needed if pinned)' - default: '' - type: string - change_desc: - description: | - Description of change, for commit message and PR. (What does the new version add or fix?) - default: '' - type: string + inputs: + branch: + description: 'Target branch to create requirements PR against' + required: true + default: 'master' + type: string + package: + description: 'Name of package to upgrade' + required: true + type: string + version: + description: 'Version number to upgrade to in constraints.txt (only needed if pinned)' + default: '' + type: string + change_desc: + description: | + Description of change, for commit message and PR. (What does the new version add or fix?) + default: '' + type: string defaults: run: shell: bash # making this explicit opts into -e -o pipefail jobs: - upgrade-one-python-dependency-workflow: + upgrade-one-python-dependency: runs-on: ubuntu-20.04 steps: @@ -92,8 +92,9 @@ jobs: ${{ env.body_prefix }}Commit generated by workflow `${{ github.workflow_ref }}` title: "feat: Upgrade Python dependency ${{ inputs.package }}" - body: | - ${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}` on behalf of @${{ github.triggering_actor }}. + body: >- + ${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}` + on behalf of @${{ github.triggering_actor }}. assignees: "${{ github.triggering_actor }}" reviewers: "${{ github.triggering_actor }}" diff --git a/requirements/README.rst b/requirements/README.rst index 4e14554e00..c602cc8905 100644 --- a/requirements/README.rst +++ b/requirements/README.rst @@ -12,22 +12,29 @@ directly in the requirements directory.) .. _OEP-18: https://github.com/openedx/open-edx-proposals/blob/master/oeps/oep-0018-bp-python-dependencies.rst -Upgrading all dependencies -************************** +While the ``*.in`` files are intended to be updated manually, the ``*.txt`` files should only be manipulated using Makefile targets in a Linux environment (to match our build and deploy systems). For developers on Mac, this can be achieved by using the GitHub workflows or by running Make targets from inside devstack's lms-shell or another Linux environment. -You can use the `upgrade-requirements Github Workflow `_ to make a PR that upgrades as many packages as possible. +If you don't have write permissions to openedx/edx-platform, you'll need to run these workflows on a fork. -Upgrading just one dependency -***************************** +Workflows and Makefile targets +****************************** + +Add a dependency +================ + +To add a Python dependency, specify it in the appropriate ``requirements/edx/*.in`` file, push that up to a branch, and then use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. This will ensure the lockfiles are updated with any transitive dependencies and will ping you on a PR for updating your branch. + +Upgrade just one dependency +=========================== Want to upgrade just *one* dependency without pulling in other upgrades? You can `run the upgrade-one-python-dependency.yml workflow `_ to have a pull request made against a branch of your choice. -Or, if you need to do it locally, you can use the ``upgrade-package`` make target directly. For example, you could run ``make upgrade-package package=ecommerce``. But the GitHub workflow is likely easier. +Or, if you need to do it locally, you can use the ``upgrade-package`` make target directly. For example, you could run ``make upgrade-package package=ecommerce``. If your dependency is pinned in constraints.txt, you'll need to enter an explicit version number in the appropriate field when running the workflow; this will include an update to the constraint file in the resulting PR. -Downgrading a dependency -************************ +Downgrade a dependency +====================== If you instead need to surgically *downgrade* a dependency: @@ -36,9 +43,12 @@ If you instead need to surgically *downgrade* a dependency: # frobulator 2.x has breaking API changes; see https://github.com/openedx/edx-platform/issue/1234567 for fixing it frobulator<2.0.0 -2. Run ``make compile-requirements`` +2. After pushing that up to a branch, use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. -This is considerably safer than trying to manually edit the ``*.txt`` files, which can easily result in incompatible dependency versions. +Upgrade all dependencies +======================== + + You can use the `upgrade-requirements Github Workflow `_ to make a PR that upgrades as many packages as possible to newer versions. This is a wrapper around ``make upgrade`` and is run on a schedule to keep dependencies up to date. Inconsistent dependencies *************************