diff --git a/.cookiecutter-replay.json b/.cookiecutter-replay.json index add4eb4..b79463c 100644 --- a/.cookiecutter-replay.json +++ b/.cookiecutter-replay.json @@ -8,6 +8,7 @@ "keywords": "cloud, microgrid", "github_org": "frequenz-floss", "license": "MIT", + "private_repo": "no", "author_name": "Frequenz Energy-as-a-Service GmbH", "author_email": "floss@frequenz.com", "python_package": "frequenz.api.reporting", @@ -35,6 +36,10 @@ "MIT", "Proprietary" ], + "private_repo": [ + "{{ 'yes' if cookiecutter.license == 'Proprietary' else 'no' }}", + "{{ 'no' if cookiecutter.license == 'Proprietary' else 'yes' }}" + ], "author_name": "Frequenz Energy-as-a-Service GmbH", "author_email": "floss@frequenz.com", "python_package": "{{cookiecutter | python_package}}", diff --git a/.github/containers/nox-cross-arch/arm64-ubuntu-20.04-python-3.11.Dockerfile b/.github/containers/nox-cross-arch/arm64-ubuntu-20.04-python-3.11.Dockerfile deleted file mode 100644 index 4a29eec..0000000 --- a/.github/containers/nox-cross-arch/arm64-ubuntu-20.04-python-3.11.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# License: MIT -# Copyright © 2024 Frequenz Energy-as-a-Service GmbH -# This Dockerfile is used to run the tests in arm64, which is not supported by -# GitHub Actions at the moment. - -FROM docker.io/library/ubuntu:20.04 - -ENV DEBIAN_FRONTEND=noninteractive - -# Install Python 3.11 and curl to install pip later -RUN apt-get update -y && \ - apt-get install --no-install-recommends -y \ - software-properties-common && \ - add-apt-repository ppa:deadsnakes/ppa && \ - apt-get install --no-install-recommends -y \ - ca-certificates \ - curl \ - git \ - python3.11 \ - python3.11-distutils && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install pip -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 - -RUN update-alternatives --install \ - /usr/local/bin/python python /usr/bin/python3.11 1 && \ - python -m pip install --upgrade --no-cache-dir pip - -COPY entrypoint.bash /usr/bin/entrypoint.bash - -ENTRYPOINT ["/usr/bin/entrypoint.bash"] diff --git a/.github/containers/nox-cross-arch/entrypoint.bash b/.github/containers/nox-cross-arch/entrypoint.bash deleted file mode 100755 index f344deb..0000000 --- a/.github/containers/nox-cross-arch/entrypoint.bash +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# License: MIT -# Copyright © 2024 Frequenz Energy-as-a-Service GmbH -set -e - -echo "System details:" $(uname -a) -echo "Machine:" $(uname -m) - -exec "$@" diff --git a/.github/containers/test-installation/Dockerfile b/.github/containers/test-installation/Dockerfile deleted file mode 100644 index d5e2399..0000000 --- a/.github/containers/test-installation/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# License: MIT -# Copyright © 2023 Frequenz Energy-as-a-Service GmbH -# This Dockerfile is used to test the installation of the python package in -# multiple platforms in the CI. It is not used to build the package itself. - -FROM python:3.11-slim - -RUN apt-get update -y && \ - apt-get install --no-install-recommends -y \ - git && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - python -m pip install --upgrade --no-cache-dir pip - -COPY dist dist -# This git-credentials file is made available by the GitHub ci.yaml workflow -COPY git-credentials /root/.git-credentials -RUN git config --global credential.helper store && \ - pip install dist/*.whl && \ - rm -rf dist /root/.git-credentials diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2147f7e..8180d70 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,6 +29,12 @@ updates: exclude-patterns: # pydoclint has shipped breaking changes in patch updates often - "pydoclint" + # These need a migration script to fix Dependabot not updating the + # runtime dependencies + - "grpcio" + - "grpcio-tools" + - "protobuf" + - "isort" minor: update-types: - "minor" @@ -48,6 +54,7 @@ updates: - "protobuf" - "pydoclint" - "pytest-asyncio" + - "isort" # We group repo-config updates as it uses optional dependencies that are # considered different dependencies otherwise, and will create one PR for # each if we don't group them. @@ -63,10 +70,28 @@ updates: # We group grpcio and protobuf updates together, as they need special # handling on the pyproject.toml file because of the protobuf/grpcio # build/runtime cross-version guarantees - grpc: + # We group grpcio and protobuf updates together, as they need special + # handling on the pyproject.toml file because of the protobuf/grpcio + # build/runtime cross-version guarantees and wrong dependabot handling + # of build/runtime dependencies. + grpc-compatible: + update-types: + - "patch" + - "minor" + patterns: + - "grpcio" + - "grpcio-tools" + - "protobuf" + # For major updates we split it up. It was observed in the past that + # grpcio releases lag behind protobuf releases, and they are not + # compatible with a major protobuf update for a while, so we shouldn't + # block the update of one with the other. + grpcio-major: patterns: - "grpcio" - "grpcio-tools" + protobuf-major: + patterns: - "protobuf" - package-ecosystem: "github-actions" diff --git a/.github/workflows/auto-dependabot.yaml b/.github/workflows/auto-dependabot.yaml index 65935f7..ae9d968 100644 --- a/.github/workflows/auto-dependabot.yaml +++ b/.github/workflows/auto-dependabot.yaml @@ -12,7 +12,9 @@ on: pull_request_target: permissions: + # Read repository contents and Dependabot metadata used by the nested action. contents: read + # The nested action also uses `github.token` internally for PR operations. pull-requests: write jobs: @@ -20,7 +22,12 @@ jobs: name: Auto-merge Dependabot PR if: > github.actor == 'dependabot[bot]' && - !contains(github.event.pull_request.title, 'the repo-config group') + !contains(github.event.pull_request.title, 'the repo-config group') && + !contains(github.event.pull_request.title, 'the grpc-compatible group') && + !contains(github.event.pull_request.title, 'the grpcio-major group') && + !contains(github.event.pull_request.title, 'the protobuf-major group') && + !contains(github.event.pull_request.title, 'Bump black from ') && + !contains(github.event.pull_request.title, 'Bump isort from ') runs-on: ubuntu-slim steps: - name: Generate GitHub App token @@ -29,6 +36,12 @@ jobs: with: app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + # Merge Dependabot PRs. + permission-contents: write + # Create the auto-merged label if it does not exist. + permission-issues: write + # Approve PRs, add labels, and enable auto-merge. + permission-pull-requests: write - name: Auto-merge Dependabot PR uses: frequenz-floss/dependabot-auto-approve@e943399cc9d76fbb6d7faae446cd57301d110165 # v1.5.0 diff --git a/.github/workflows/black-migration.yaml b/.github/workflows/black-migration.yaml new file mode 100644 index 0000000..0013d70 --- /dev/null +++ b/.github/workflows/black-migration.yaml @@ -0,0 +1,89 @@ +# Automatic black formatting migration for Dependabot PRs +# +# When Dependabot upgrades black, this workflow installs the new version +# and runs `black .` so the PR already contains any formatting changes +# introduced by the upgrade, while leaving the PR open for review. +# +# Black uses calendar versioning. Only the first release of a new calendar +# year may introduce formatting changes (major bump in Dependabot's terms). +# Minor and patch updates within a year keep formatting stable, so they stay +# in the regular Dependabot groups and are auto-merged normally. +# +# The companion auto-dependabot workflow skips major black PRs so they're +# handled exclusively by this migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (the migration script is embedded +# in this workflow file on the base branch, not taken from the PR). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: Black Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + # Commit reformatted files back to the PR branch. + contents: write + # Create and normalize migration state labels. + issues: write + # Read/update pull request metadata and comments. + pull-requests: write + +jobs: + black-migration: + name: Migrate Black + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + github.actor == 'dependabot[bot]' && + contains(github.event.pull_request.title, 'Bump black from ') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + # Push reformatted files to the PR branch. + permission-contents: write + # Create and normalize migration state labels. + permission-issues: write + # Read/update pull request metadata and labels. + permission-pull-requests: write + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@27763fb5eb56476d91abe00132e8a0614171f92f # v1.2.0 + with: + migration-script: | + import os + import subprocess + import sys + + version = os.environ["MIGRATION_VERSION"].lstrip("v") + subprocess.run( + [sys.executable, "-Im", "pip", "install", f"black=={version}"], + check=True, + ) + subprocess.run([sys.executable, "-Im", "black", "."], check=True) + token: ${{ steps.create-app-token.outputs.token }} + auto-merge-on-changes: "false" + version-iteration: "false" + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:black:migration:executed" + intervention-pending-label: "tool:black:migration:intervention-pending" + intervention-done-label: "tool:black:migration:intervention-done" diff --git a/.github/workflows/ci-pr.yaml b/.github/workflows/ci-pr.yaml index 73fc391..c98925c 100644 --- a/.github/workflows/ci-pr.yaml +++ b/.github/workflows/ci-pr.yaml @@ -3,6 +3,10 @@ name: Test PR on: pull_request: +permissions: + # Read repository contents for checkout and dependency resolution only. + contents: read + env: # Please make sure this version is included in the `matrix`, as the # `matrix` section can't use `env`, so it must be entered manually @@ -17,10 +21,10 @@ jobs: steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Fetch sources - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -43,7 +47,7 @@ jobs: steps: - name: Run nox - uses: frequenz-floss/gh-action-nox@v1.1.1 + uses: frequenz-floss/gh-action-nox@e1351cf45e05e85afc1c79ab883e06322892d34c # v1.1.0 with: python-version: "3.11" nox-session: ci_checks_max @@ -53,15 +57,15 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Fetch sources - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.4 + uses: frequenz-floss/gh-action-setup-python-with-deps@e4d0b2ef8f5a1612d7827f3abaef17c931d2b946 # v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: .[dev-mkdocs] @@ -70,11 +74,14 @@ jobs: env: MIKE_VERSION: gh-${{ github.job }} run: | - mike deploy $MIKE_VERSION - mike set-default $MIKE_VERSION + # mike is installed as a console script, not a runnable module. + # Run the installed script under isolated mode to avoid importing from + # the workspace when building docs from checked-out code. + python -I "$(command -v mike)" deploy "$MIKE_VERSION" + python -I "$(command -v mike)" set-default "$MIKE_VERSION" - name: Upload site - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: docs-site path: site/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 185c768..71f9191 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,6 +15,10 @@ on: - 'dependabot/**' workflow_dispatch: +permissions: + # Read repository contents for checkout and dependency resolution only. + contents: read + env: # Please make sure this version is included in the `matrix`, as the # `matrix` section can't use `env`, so it must be entered manually @@ -29,10 +33,10 @@ jobs: steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Fetch sources - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -69,7 +73,7 @@ jobs: steps: - name: Run nox - uses: frequenz-floss/gh-action-nox@v1.1.1 + uses: frequenz-floss/gh-action-nox@e1351cf45e05e85afc1c79ab883e06322892d34c # v1.1.0 with: python-version: ${{ matrix.python }} nox-session: ${{ matrix.nox-session }} @@ -85,6 +89,8 @@ jobs: # We skip this job only if nox was also skipped if: always() && needs.nox.result != 'skipped' runs-on: ubuntu-slim + # Drop token permissions: this job only checks matrix status from `needs`. + permissions: {} env: DEPS_RESULT: ${{ needs.nox.result }} steps: @@ -100,24 +106,24 @@ jobs: steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Fetch sources - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.4 + uses: frequenz-floss/gh-action-setup-python-with-deps@e4d0b2ef8f5a1612d7827f3abaef17c931d2b946 # v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: build - name: Build the source and binary distribution - run: python -m build + run: python -Im build - name: Upload distribution files - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: dist-packages path: dist/ @@ -139,13 +145,13 @@ jobs: steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Print environment (debug) run: env - name: Download package - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: dist-packages path: dist @@ -165,13 +171,13 @@ jobs: > pyproject.toml - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.4 + uses: frequenz-floss/gh-action-setup-python-with-deps@e4d0b2ef8f5a1612d7827f3abaef17c931d2b946 # v1.0.2 with: python-version: ${{ matrix.python }} dependencies: dist/*.whl - name: Print installed packages (debug) - run: python -m pip freeze + run: python -Im pip freeze # This job runs if all the `test-installation` matrix jobs ran and succeeded. # It is only used to have a single job that we can require in branch @@ -184,6 +190,8 @@ jobs: # We skip this job only if test-installation was also skipped if: always() && needs.test-installation.result != 'skipped' runs-on: ubuntu-slim + # Drop token permissions: this job only checks matrix status from `needs`. + permissions: {} env: DEPS_RESULT: ${{ needs.test-installation.result }} steps: @@ -196,15 +204,15 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Fetch sources - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.4 + uses: frequenz-floss/gh-action-setup-python-with-deps@e4d0b2ef8f5a1612d7827f3abaef17c931d2b946 # v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: .[dev-mkdocs] @@ -213,11 +221,14 @@ jobs: env: MIKE_VERSION: gh-${{ github.job }} run: | - mike deploy $MIKE_VERSION - mike set-default $MIKE_VERSION + # mike is installed as a console script, not a runnable module. + # Run the installed script under isolated mode to avoid importing from + # the workspace when building docs from checked-out code. + python -I "$(command -v mike)" deploy "$MIKE_VERSION" + python -I "$(command -v mike)" set-default "$MIKE_VERSION" - name: Upload site - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: docs-site path: site/ @@ -229,18 +240,19 @@ jobs: if: github.event_name == 'push' runs-on: ubuntu-24.04 permissions: + # Push generated documentation updates to the `gh-pages` branch. contents: write steps: - name: Setup Git - uses: frequenz-floss/gh-action-setup-git@v1.0.0 + uses: frequenz-floss/gh-action-setup-git@f9d86a01228ee1cadaac5224d4d7626f1eb23f90 # v1.0.0 - name: Fetch sources - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - name: Setup Python - uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.4 + uses: frequenz-floss/gh-action-setup-python-with-deps@e4d0b2ef8f5a1612d7827f3abaef17c931d2b946 # v1.0.2 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} dependencies: .[dev-mkdocs] @@ -253,7 +265,7 @@ jobs: GIT_REF: ${{ github.ref }} GIT_SHA: ${{ github.sha }} run: | - python -m frequenz.repo.config.cli.version.mike.info + python -Im frequenz.repo.config.cli.version.mike.info - name: Fetch the gh-pages branch if: steps.mike-version.outputs.version @@ -274,13 +286,23 @@ jobs: GIT_REF: ${{ github.ref }} GIT_SHA: ${{ github.sha }} run: | - mike deploy --update-aliases --title "$TITLE" "$VERSION" $ALIASES + # Collect aliases into an array to avoid accidental (or malicious) + # shell injection when passing them to mike. + aliases=() + if test -n "$ALIASES"; then + read -r -a aliases <<<"$ALIASES" + fi + # mike is installed as a console script, not a runnable module. + # Run the installed script under isolated mode to avoid importing from + # the workspace when building docs from checked-out code. + python -I "$(command -v mike)" \ + deploy --update-aliases --title "$TITLE" "$VERSION" "${aliases[@]}" - name: Sort site versions if: steps.mike-version.outputs.version run: | git checkout gh-pages - python -m frequenz.repo.config.cli.version.mike.sort versions.json + python -Im frequenz.repo.config.cli.version.mike.sort versions.json git commit -a -m "Sort versions.json" - name: Publish site @@ -294,14 +316,12 @@ jobs: # Create a release only on tags creation if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') permissions: - # We need write permissions on contents to create GitHub releases and on - # discussions to create the release announcement in the discussion forums + # Create GitHub releases and upload distribution artifacts. contents: write - discussions: write runs-on: ubuntu-slim steps: - name: Download distribution files - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: dist-packages path: dist @@ -323,14 +343,14 @@ jobs: - name: Create GitHub release run: | set -ux - extra_opts= - if echo "$REF_NAME" | grep -- -; then extra_opts=" --prerelease"; fi + extra_opts=() + if echo "$REF_NAME" | grep -- -; then extra_opts+=(--prerelease); fi gh release create \ -R "$REPOSITORY" \ --notes-file RELEASE_NOTES.md \ --generate-notes \ - $extra_opts \ - $REF_NAME \ + "${extra_opts[@]}" \ + "$REF_NAME" \ dist/* env: REF_NAME: ${{ github.ref_name }} @@ -347,10 +367,10 @@ jobs: id-token: write steps: - name: Download distribution files - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: dist-packages path: dist - name: Publish the Python distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/dco-merge-queue.yml b/.github/workflows/dco-merge-queue.yml index d9597ad..7a4260d 100644 --- a/.github/workflows/dco-merge-queue.yml +++ b/.github/workflows/dco-merge-queue.yml @@ -3,6 +3,9 @@ name: DCO on: merge_group: +# Drop all token permissions: this workflow only runs a local echo command. +permissions: {} + jobs: DCO: runs-on: ubuntu-slim diff --git a/.github/workflows/grpc-migration.yaml b/.github/workflows/grpc-migration.yaml new file mode 100644 index 0000000..555bd24 --- /dev/null +++ b/.github/workflows/grpc-migration.yaml @@ -0,0 +1,87 @@ +# Automatic grpc/protobuf build/runtime sync for Dependabot PRs +# +# The template's `pyproject.toml` pins `protobuf`, `grpcio` and `grpcio-tools` +# in `[build-system].requires` as *exact* versions, and also declares +# `protobuf` and `grpcio` in `[project].dependencies` with a `>= ` +# lower bound. The lower bound must always match the exact pin, because the +# protobuf cross-version runtime guarantee requires the runtime to be at +# least the version used at generation time: +# https://protobuf.dev/support/cross-version-runtime-guarantee/ +# +# Dependabot correctly bumps `[build-system].requires`, but it does not bump +# the matching `>=` floor in `[project].dependencies`. This workflow runs +# after a Dependabot grpc/protobuf group PR, rewrites the `>=` floor to match +# the new build pins, and pushes the fix-up commit back onto the PR branch. +# +# The companion auto-dependabot workflow skips the `grpc-compatible`, +# `grpcio-major` and `protobuf-major` groups so those PRs are handled +# exclusively by this migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (the migration script is fetched +# from the repo-config branch configured below, not taken from the PR). +# - Gating migration steps on github.actor == 'dependabot[bot]' AND the +# PR title. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: gRPC Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + # Commit the sync-up to the PR branch. + contents: write + # Create and normalize migration state labels. + issues: write + # Read/update pull request metadata and comments. + pull-requests: write + +jobs: + grpc-migration: + name: Fix gRPC/protobuf runtime floors + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + github.actor == 'dependabot[bot]' && + (contains(github.event.pull_request.title, 'the grpc-compatible group') || + contains(github.event.pull_request.title, 'the grpcio-major group') || + contains(github.event.pull_request.title, 'the protobuf-major group')) + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + # Push the sync-up commit to the PR branch. + permission-contents: write + # Create and normalize migration state labels. + permission-issues: write + # Read/update pull request metadata and labels. + permission-pull-requests: write + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@27763fb5eb56476d91abe00132e8a0614171f92f # v1.2.0 + with: + script-url-template: >- # v0.18.0 + https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/529d30b554392e6d8b66e84e92c04ac9cd170da7/cookiecutter/scripts/dependabot-grpc-fixer.py + token: ${{ steps.create-app-token.outputs.token }} + version-iteration: "false" + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:grpc:migration:executed" + intervention-pending-label: "tool:grpc:migration:intervention-pending" + intervention-done-label: "tool:grpc:migration:intervention-done" diff --git a/.github/workflows/isort-migration.yaml b/.github/workflows/isort-migration.yaml new file mode 100644 index 0000000..fde6c0c --- /dev/null +++ b/.github/workflows/isort-migration.yaml @@ -0,0 +1,92 @@ +# Automatic isort migration for Dependabot PRs +# +# When Dependabot upgrades isort, this workflow installs the new version and +# runs `isort .` so the PR already contains any import-ordering changes +# introduced by the upgrade, while leaving the PR open for review. +# +# isort follows SemVer but its release policy +# (https://github.com/PyCQA/isort/blob/main/docs/major_releases/release_policy.md) +# explicitly allows intentional formatting changes in minor releases, and +# patch releases may also adjust output in smaller bug-fix ways. Because of +# that, isort is excluded from the regular `patch` and `minor` Dependabot +# groups: every isort bump produces an individual `Bump isort from …` PR and +# is routed through this migration workflow. +# +# The companion auto-dependabot workflow skips those PRs so they're handled +# exclusively by this migration workflow. +# +# XXX: !!! SECURITY WARNING !!! +# pull_request_target has write access to the repo, and can read secrets. +# This is required because Dependabot PRs are treated as fork PRs: the +# GITHUB_TOKEN is read-only and secrets are unavailable with a plain +# pull_request trigger. The action mitigates the risk by: +# - Never executing code from the PR (the migration script is embedded +# in this workflow file on the base branch, not taken from the PR). +# - Gating migration steps on github.actor == 'dependabot[bot]'. +# - Running checkout with persist-credentials: false and isolating +# push credentials from the migration script environment. +# For more details read: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + +name: isort Migration + +on: + merge_group: # To allow using this as a required check for merging + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled] + +permissions: + # Commit reformatted files back to the PR branch. + contents: write + # Create and normalize migration state labels. + issues: write + # Read/update pull request metadata and comments. + pull-requests: write + +jobs: + isort-migration: + name: Migrate isort + # Skip if it was triggered by the merge queue. We only need the workflow to + # be executed to meet the "Required check" condition for merging, but we + # don't need to actually run the job, having the job present as Skipped is + # enough. + if: | + github.event_name == 'pull_request_target' && + github.actor == 'dependabot[bot]' && + contains(github.event.pull_request.title, 'Bump isort from ') + runs-on: ubuntu-24.04 + steps: + - name: Generate token + id: create-app-token + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} + private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + # Push reformatted files to the PR branch. + permission-contents: write + # Create and normalize migration state labels. + permission-issues: write + # Read/update pull request metadata and labels. + permission-pull-requests: write + - name: Migrate + uses: frequenz-floss/gh-action-dependabot-migrate@27763fb5eb56476d91abe00132e8a0614171f92f # v1.2.0 + with: + migration-script: | + import os + import subprocess + import sys + + version = os.environ["MIGRATION_VERSION"].lstrip("v") + subprocess.run( + [sys.executable, "-Im", "pip", "install", f"isort=={version}"], + check=True, + ) + subprocess.run([sys.executable, "-Im", "isort", "."], check=True) + token: ${{ steps.create-app-token.outputs.token }} + auto-merge-on-changes: "false" + version-iteration: "false" + sign-commits: "true" + auto-merged-label: "tool:auto-merged" + migrated-label: "tool:isort:migration:executed" + intervention-pending-label: "tool:isort:migration:intervention-pending" + intervention-done-label: "tool:isort:migration:intervention-done" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index c327e7f..393ddfc 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -5,7 +5,9 @@ on: [pull_request_target] jobs: Label: permissions: + # Read the labeler configuration from the repository. contents: read + # Add labels to pull requests. pull-requests: write runs-on: ubuntu-slim steps: diff --git a/.github/workflows/release-notes-check.yml b/.github/workflows/release-notes-check.yml index ef59bef..f606c53 100644 --- a/.github/workflows/release-notes-check.yml +++ b/.github/workflows/release-notes-check.yml @@ -18,6 +18,7 @@ jobs: name: Check release notes are updated runs-on: ubuntu-slim permissions: + # Read pull request metadata to evaluate labels and changed files. pull-requests: read steps: - name: Check for a release notes update diff --git a/.github/workflows/repo-config-migration.yaml b/.github/workflows/repo-config-migration.yaml index 6ffaf60..eb1d9c1 100644 --- a/.github/workflows/repo-config-migration.yaml +++ b/.github/workflows/repo-config-migration.yaml @@ -24,8 +24,11 @@ on: types: [opened, synchronize, reopened, labeled, unlabeled] permissions: + # Commit migration changes back to the PR branch. contents: write + # Create and normalize migration state labels. issues: write + # Read/update pull request metadata and comments. pull-requests: write jobs: @@ -46,13 +49,23 @@ jobs: with: app-id: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_ID }} private-key: ${{ secrets.FREQUENZ_AUTO_DEPENDABOT_APP_PRIVATE_KEY }} + # Push migration commits to the PR branch. + permission-contents: write + # Manage labels when auto-merging patch-only updates. + permission-issues: write + # Approve pull requests and enable auto-merge. + permission-pull-requests: write + # Allow pushes when migration changes workflow files. + permission-workflows: write - name: Migrate - uses: frequenz-floss/gh-action-dependabot-migrate@45994e185a9040449304a470e8f02d0e197873b4 # v1.1.1 + uses: frequenz-floss/gh-action-dependabot-migrate@27763fb5eb56476d91abe00132e8a0614171f92f # v1.2.0 with: script-url-template: >- https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/{version}/cookiecutter/migrate.py token: ${{ steps.create-app-token.outputs.token }} migration-token: ${{ secrets.REPO_CONFIG_MIGRATION_TOKEN }} + version-iteration: "minor" + if-no-iterations: "pass" sign-commits: "true" auto-merged-label: "tool:auto-merged" migrated-label: "tool:repo-config:migration:executed" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed8a1b2..9cbd589 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,14 +96,14 @@ pytest tests/test_*.py Or you can use `nox`: ```sh -nox -R -s pytest -- test/test_*.py +nox -R -s pytest -- tests/test_*.py ``` The same appliest to `pylint` or `mypy` for example: ```sh -nox -R -s pylint -- test/test_*.py -nox -R -s mypy -- test/test_*.py +nox -R -s pylint -- tests/test_*.py +nox -R -s mypy -- tests/test_*.py ``` ### Building the documentation @@ -200,30 +200,3 @@ These are the steps to create a new release: eventually too). 7. Celebrate! - -## Cross-Arch Testing - -This project has built-in support for testing across multiple architectures. -Currently, our CI conducts tests on `arm64` machines using QEMU emulation. We -also have the flexibility to expand this support to include additional -architectures in the future. - -This project contains Dockerfiles that can be used in the CI to test the -python package in non-native machine architectures, e.g., `arm64`. The -Dockerfiles exist in the directory `.github/containers/nox-cross-arch`, and -follow a naming scheme so that they can be easily used in build matrices in the -CI, in `nox-cross-arch` job. The naming scheme is: - -``` ---python-.Dockerfile -``` - -E.g., - -``` -arm64-ubuntu-20.04-python-3.11.Dockerfile -``` - -If a Dockerfile for your desired target architecture, OS, and python version -does not exist here, please add one before proceeding to add your options to -the test matrix. diff --git a/pyproject.toml b/pyproject.toml index 5d6461a..20617ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "setuptools == 82.0.1", "setuptools_scm[toml] == 10.0.5", - "frequenz-repo-config[api] == 0.17.0", + "frequenz-repo-config[api] == 0.18.0", # We need to pin the protobuf, grpcio and grpcio-tools dependencies to make # sure the code is generated using the minimum supported versions, as older # versions can't work with code that was generated with newer versions. @@ -80,7 +80,7 @@ dev-mkdocs = [ "mkdocs-material == 9.7.6", "mkdocstrings[python] == 0.30.1", "mkdocstrings-python == 1.18.2", - "frequenz-repo-config[api] == 0.17.0", + "frequenz-repo-config[api] == 0.18.0", ] dev-mypy = [ "mypy == 2.0.0", @@ -89,7 +89,7 @@ dev-mypy = [ # For checking the noxfile, docs/ script, and tests "frequenz-api-reporting[dev-mkdocs,dev-noxfile,dev-pytest]", ] -dev-noxfile = ["nox == 2026.4.10", "frequenz-repo-config[api] == 0.17.0"] +dev-noxfile = ["nox == 2026.4.10", "frequenz-repo-config[api] == 0.18.0"] dev-pylint = [ # dev-pytest already defines a dependency to pylint because of the examples # For checking the noxfile, docs/ script, and tests @@ -98,7 +98,7 @@ dev-pylint = [ dev-pytest = [ "pytest == 9.0.3", "pylint == 4.0.5", # We need this to check for the examples - "frequenz-repo-config[extra-lint-examples] == 0.17.0", + "frequenz-repo-config[extra-lint-examples] == 0.18.0", ] dev = [ "frequenz-api-reporting[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]", @@ -114,11 +114,15 @@ Support = "https://github.com/frequenz-floss/frequenz-api-reporting/discussions/ [tool.black] line-length = 88 target-version = ['py311'] +# Submodules may contain external code that doesn't follow our formatting rules. +extend-exclude = '^/submodules/' [tool.isort] profile = "black" line_length = 88 src_paths = ["benchmarks", "examples", "src", "tests"] +# Submodules may contain external code that doesn't follow our import sorting rules. +skip_glob = ["submodules/*"] [tool.flake8] # We give some flexibility to go over 88, there are cases like long URLs or