diff --git a/.github/workflows/README.md b/.github/workflows/README.md index f6b1a440e175..e30da284aff6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -545,4 +545,5 @@ PostCommit Jobs run in a schedule against master branch and generally do not get | [ Infrastructure Policy Enforcer ](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_PolicyEnforcer.yml) | N/A | [![.github/workflows/beam_Infrastructure_PolicyEnforcer.yml](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_PolicyEnforcer.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_PolicyEnforcer.yml?query=event%3Aschedule) | | [ Modify the GCP User Roles according to the infra/users.yml file ](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_UsersPermissions.yml) | N/A | [![.github/workflows/beam_Infrastructure_UsersPermissions.yml](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_UsersPermissions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_UsersPermissions.yml?query=event%3Aschedule) | | [ Service Account Keys Management ](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_ServiceAccountKeys.yml) | N/A | [![.github/workflows/beam_Infrastructure_ServiceAccountKeys.yml](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_ServiceAccountKeys.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_ServiceAccountKeys.yml?query=event%3Aschedule) | +| [ Upgrade GCP Libraries BOM ](https://github.com/apache/beam/actions/workflows/beam_Upgrade_GCP_BOM.yml) | N/A | [![.github/workflows/beam_Upgrade_GCP_BOM.yml](https://github.com/apache/beam/actions/workflows/beam_Upgrade_GCP_BOM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Upgrade_GCP_BOM.yml?query=event%3Aschedule) | | [ Unmanaged Service Accounts Keys Audit ](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_AuditUnmanagedKeys.yml) | N/A | [![.github/workflows/beam_Infrastructure_AuditUnmanagedKeys.yml](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_AuditUnmanagedKeys.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Infrastructure_AuditUnmanagedKeys.yml?query=event%3Aschedule) | diff --git a/.github/workflows/beam_Upgrade_GCP_BOM.yml b/.github/workflows/beam_Upgrade_GCP_BOM.yml new file mode 100644 index 000000000000..ca7489f21b25 --- /dev/null +++ b/.github/workflows/beam_Upgrade_GCP_BOM.yml @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Upgrade GCP Libraries BOM + +on: + schedule: + - cron: "0 0 * * 0" # Weekly on Sundays at 00:00 UTC + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + checks: read + issues: read + statuses: read + +concurrency: + group: '${{ github.workflow }} @ ${{ github.ref }}' + cancel-in-progress: true + +jobs: + upgrade_gcp_bom: + runs-on: [self-hosted, ubuntu-24.04, main] + name: Upgrade GCP BOM + steps: + - name: Checkout code + uses: actions/checkout@v6 + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: 3.11 + java-version: default + go-version: default + - name: Check if new BOM is available + id: check_bom + run: python3 scripts/tools/bomupgrader.py --check + - name: Run bomupgrader + if: steps.check_bom.outputs.should_upgrade == 'true' + run: python3 scripts/tools/bomupgrader.py ${{ steps.check_bom.outputs.latest_version }} + - name: Install gh cli + if: steps.check_bom.outputs.should_upgrade == 'true' + uses: ./.github/actions/setup-gh-cli-linux + - name: Set git config + if: steps.check_bom.outputs.should_upgrade == 'true' + run: | + git config user.name $GITHUB_ACTOR + git config user.email actions@"$RUNNER_NAME".local + - name: Commit Changes and create PR + if: steps.check_bom.outputs.should_upgrade == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_EVENT: ${{ github.event_name }} + LATEST_VER: ${{ steps.check_bom.outputs.latest_version }} + CURRENT_VER: ${{ steps.check_bom.outputs.current_version }} + run: | + # Take the current date, subtract from a release cut date in the past (June 24, 2026), + # then get the num days % 42 (our release cadence is 42 days). + # This will ensure it only runs the week after a release branch has been cut. + days_diff=$(( ($(date +%s) - $(date --date="260624" +%s) )/(60*60*24)%42 )) + if [[ $GH_EVENT != 'workflow_dispatch' && $days_diff -gt 6 ]]; then + echo "Exiting early. We only update dependencies the week after we cut the release" + exit 0 + fi + branchName=upgrade_gcp_bom_${LATEST_VER//./_} + git checkout -b $branchName + git add -A + git diff-index --quiet HEAD || gitdiff=$? || echo $? + if [[ $gitDiff != 0 ]]; then + echo "Changes are ready to commit" + git commit -m "Upgrade GCP Libraries BOM to ${LATEST_VER}" --quiet + git push origin $branchName --quiet + + PR_BODY="This PR was created by automation. It upgrades the Google Cloud Platform Libraries BOM from **${CURRENT_VER}** to **${LATEST_VER}** and updates Netty, gRPC, Arrow, Gax, Protobuf, and OpenTelemetry versions to match. + + Please review the changes and merge if all tests pass." + + GITHUB_PR_URL=$(gh pr create --title "Upgrade GCP Libraries BOM to ${LATEST_VER}" --body "$PR_BODY" --label "dependencies" --base master) + echo "Link of the new PR: $GITHUB_PR_URL" + else + echo "No changes on the files" + fi diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md index 2024f1017975..7c96465c722b 100644 --- a/contributor-docs/release-guide.md +++ b/contributor-docs/release-guide.md @@ -1164,10 +1164,16 @@ At the end of the release, go to the GitHub milestones page and mark the recentl #### Update the Java BOM Google releases a BOM that pins compatible versions of their Java libraries. -After the release, try updating the BOM to the latest version. +After the release, an automated upgrade BOM PR is created or can be triggered manually. -To do so, create a draft PR and run test suites following the instructions at -https://github.com/apache/beam/blob/master/contributor-docs/java-dependency-upgrades.md. +A PR should have already been created (and possibly merged) by github-actions bot, you should verify that this was done correctly +by looking at open PRs from that bot - https://github.com/apache/beam/pulls/app%2Fgithub-actions + +If a PR has not been merged, drive it to completion. +If no PR was created, triage any failures in https://github.com/apache/beam/actions/workflows/beam_Upgrade_GCP_BOM.yml and manually update the BOM, +following https://github.com/apache/beam/blob/master/contributor-docs/java-dependency-upgrades.md or simply trigger the workflow again. + +##### Troubleshooting Triage the test failures and rerun any tests that seem potentially unrelated to the upgrade. If there are no test failures due to the BOM upgrade, request review and merge the PR as normal. diff --git a/scripts/tools/bomupgrader.py b/scripts/tools/bomupgrader.py index 4697c46a5568..715c19a899d1 100644 --- a/scripts/tools/bomupgrader.py +++ b/scripts/tools/bomupgrader.py @@ -20,6 +20,7 @@ import re import subprocess import sys +import urllib.request """ This Python script is used for upgrading the GCP-BOM in BeamModulePlugin. Specifically, it @@ -43,6 +44,59 @@ # To format: yapf --style sdks/python/setup.cfg --in-place scripts/tools/bomupgrader.py +def get_latest_bom(): + url = "https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/maven-metadata.xml" + with urllib.request.urlopen(url, timeout=15) as response: + xml = response.read().decode('utf-8') + match = re.search(r'([^<]+)', xml) + if match: + return match.group(1) + raise RuntimeError("Could not find latest release in Maven metadata") + + +def get_current_bom(): + path = "buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy" + with open(path) as f: + content = f.read() + match = re.search( + r'google_cloud_platform_libraries_bom\s*:\s*[\"\']com\.google\.cloud:libraries-bom:([0-9.]+)[\"\']', + content) + if match: + return match.group(1) + raise RuntimeError( + "Could not find current libraries-bom in BeamModulePlugin.groovy") + + +def to_tuple(version_str): + parts = [] + for part in version_str.split('.'): + match = re.match(r'^(\d+)', part) + parts.append(int(match.group(1)) if match else 0) + return tuple(parts) + + +def check_bom(): + latest = get_latest_bom() + current = get_current_bom() + print(f"Latest libraries-bom version: {latest}") + print(f"Current libraries-bom version: {current}") + + should_upgrade = to_tuple(latest) > to_tuple(current) + + github_output = os.getenv('GITHUB_OUTPUT') + if github_output: + with open(github_output, 'a') as f: + f.write(f"should_upgrade={str(should_upgrade).lower()}\n") + f.write(f"latest_version={latest}\n") + f.write(f"current_version={current}\n") + + if should_upgrade: + print("A newer version of libraries-bom is available. Upgrade needed.") + else: + print("libraries-bom is up-to-date.") + return should_upgrade, latest + + class BeamModulePluginProcessor: # Known dependencies managed by GCP BOM and also used by Beam. # We only need to have one dependency for each project to figure out the target version @@ -313,7 +367,16 @@ def run(self): if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) if len(sys.argv) < 2: - print("Usage: python scripts/tools/bomupgrader.py target_version") + print("Usage: python scripts/tools/bomupgrader.py [--check | latest | target_version]") exit(1) - processor = BeamModulePluginProcessor(sys.argv[1]) - processor.run() + + arg = sys.argv[1] + if arg in ['--check', '--check-only']: + check_bom() + else: + if arg in ['latest', '--latest']: + _, target_ver = check_bom() + else: + target_ver = arg + processor = BeamModulePluginProcessor(target_ver) + processor.run()