Skip to content

Commit fcbc37d

Browse files
damccormdamccorm
authored
Merge branch 'master' into updateDocsAndAddExamplesForCloudSQLHandler
2 parents 99fe5d7 + 0c509d3 commit fcbc37d

226 files changed

Lines changed: 3907 additions & 1379 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ github:
5151

5252
protected_branches:
5353
master: {}
54+
release-2.68: {}
5455
release-2.67.0-postrelease: {}
5556
release-2.67: {}
5657
release-2.66.0-postrelease: {}

‎.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Dataflow.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run",
3+
"modification": 1
4+
}

‎.github/trigger_files/beam_PostCommit_SQL.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run ",
3-
"modification": 3
3+
"modification": 4
44
}

‎.github/workflows/beam_IODatastoresCredentialsRotation.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,17 +82,17 @@ jobs:
8282
run: |
8383
date=$(date -u +"%Y-%m-%d")
8484
echo "date=$date" >> $GITHUB_ENV
85-
- name: Send email
86-
uses: dawidd6/action-send-mail@v3
87-
if: failure()
88-
with:
89-
server_address: smtp.gmail.com
90-
server_port: 465
91-
secure: true
92-
username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
93-
password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
94-
subject: Credentials Rotation Failure on IO-Datastores cluster (${{ env.date }})
95-
to: dev@beam.apache.org
96-
from: gactions@beam.apache.org
97-
body: |
98-
Something went wrong during the automatic credentials rotation for IO-Datastores Cluster, performed at ${{ env.date }}. It may be necessary to check the state of the cluster certificates. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_IODatastoresCredentialsRotation.yml \n * Cluster URL: https://pantheon.corp.google.com/kubernetes/clusters/details/us-central1-a/io-datastores/details?mods=dataflow_dev&project=apache-beam-testing
85+
# - name: Send email
86+
# uses: dawidd6/action-send-mail@v3
87+
# if: failure()
88+
# with:
89+
# server_address: smtp.gmail.com
90+
# server_port: 465
91+
# secure: true
92+
# username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
93+
# password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
94+
# subject: Credentials Rotation Failure on IO-Datastores cluster (${{ env.date }})
95+
# to: dev@beam.apache.org
96+
# from: gactions@beam.apache.org
97+
# body: |
98+
# Something went wrong during the automatic credentials rotation for IO-Datastores Cluster, performed at ${{ env.date }}. It may be necessary to check the state of the cluster certificates. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_IODatastoresCredentialsRotation.yml \n * Cluster URL: https://pantheon.corp.google.com/kubernetes/clusters/details/us-central1-a/io-datastores/details?mods=dataflow_dev&project=apache-beam-testing

‎.github/workflows/beam_Infrastructure_ServiceAccountKeys.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# This workflow modifies the GCP Service Account keys and manages the
19+
# storage, saving them onto Google Cloud Secret Manager. It also handles
20+
# the rotation of the keys.
21+
22+
name: Service Account Keys Management
23+
24+
on:
25+
workflow_dispatch:
26+
# Trigger when the keys.yaml file is modified on the main branch
27+
push:
28+
branches:
29+
- main
30+
paths:
31+
- 'infra/keys/keys.yaml'
32+
schedule:
33+
# Once a week at 9:00 AM on Monday
34+
- cron: '0 9 * * 1'
35+
36+
# This ensures that only one workflow run is running at a time, and others are queued.
37+
concurrency:
38+
group: ${{ github.workflow }}
39+
cancel-in-progress: false
40+
41+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
42+
permissions:
43+
contents: read
44+
45+
jobs:
46+
beam_UserRoles:
47+
name: Apply user roles changes
48+
runs-on: [self-hosted, ubuntu-20.04, main]
49+
timeout-minutes: 30
50+
steps:
51+
- uses: actions/checkout@v4
52+
- name: Setup gcloud
53+
uses: google-github-actions/setup-gcloud@v2
54+
55+
- name: Setup Python
56+
uses: actions/setup-python@v4
57+
with:
58+
python-version: '3.13'
59+
60+
- name: Install Python dependencies
61+
working-directory: ./infra/keys
62+
run: |
63+
python -m pip install --upgrade pip
64+
pip install -r requirements.txt
65+
66+
- name: Run Service Account Key Management
67+
working-directory: ./infra/keys
68+
run: python keys.py --cron-dry-run

‎.github/workflows/beam_MetricsCredentialsRotation.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,17 +82,17 @@ jobs:
8282
run: |
8383
date=$(date -u +"%Y-%m-%d")
8484
echo "date=$date" >> $GITHUB_ENV
85-
- name: Send email
86-
uses: dawidd6/action-send-mail@v3
87-
if: failure()
88-
with:
89-
server_address: smtp.gmail.com
90-
server_port: 465
91-
secure: true
92-
username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
93-
password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
94-
subject: Credentials Rotation Failure on Metrics cluster (${{ env.date }})
95-
to: dev@beam.apache.org
96-
from: gactions@beam.apache.org
97-
body: |
98-
Something went wrong during the automatic credentials rotation for Metrics Cluster, performed at ${{ env.date }}. It may be necessary to check the state of the cluster certificates. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_MetricsCredentialsRotation.yml \n * Cluster URL: https://pantheon.corp.google.com/kubernetes/clusters/details/us-central1-a/metrics/details?mods=dataflow_dev&project=apache-beam-testing
85+
# - name: Send email
86+
# uses: dawidd6/action-send-mail@v3
87+
# if: failure()
88+
# with:
89+
# server_address: smtp.gmail.com
90+
# server_port: 465
91+
# secure: true
92+
# username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
93+
# password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
94+
# subject: Credentials Rotation Failure on Metrics cluster (${{ env.date }})
95+
# to: dev@beam.apache.org
96+
# from: gactions@beam.apache.org
97+
# body: |
98+
# Something went wrong during the automatic credentials rotation for Metrics Cluster, performed at ${{ env.date }}. It may be necessary to check the state of the cluster certificates. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_MetricsCredentialsRotation.yml \n * Cluster URL: https://pantheon.corp.google.com/kubernetes/clusters/details/us-central1-a/metrics/details?mods=dataflow_dev&project=apache-beam-testing

‎.github/workflows/beam_Metrics_Report.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ jobs:
5858
(github.event_name == 'schedule' && github.repository == 'apache/beam')
5959
|| github.event_name == 'workflow_dispatch'
6060
)
61-
61+
6262
steps:
6363
- uses: actions/checkout@v4
6464
- name: Setup environment
@@ -82,15 +82,15 @@ jobs:
8282
run: |
8383
date=$(date -u +"%Y-%m-%d")
8484
echo "date=$date" >> $GITHUB_ENV
85-
- name: Send mail
86-
uses: dawidd6/action-send-mail@v3
87-
with:
88-
server_address: smtp.gmail.com
89-
server_port: 465
90-
secure: true
91-
username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
92-
password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
93-
subject: Beam Metrics Report ${{ env.date }}
94-
to: dev@beam.apache.org
95-
from: beamactions@gmail.com
96-
html_body: file://${{ github.workspace }}/.test-infra/jenkins/metrics_report/beam-metrics_report.html
85+
# - name: Send mail
86+
# uses: dawidd6/action-send-mail@v6
87+
# with:
88+
# server_address: smtp.gmail.com
89+
# server_port: 465
90+
# secure: true
91+
# username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
92+
# password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
93+
# subject: Beam Metrics Report ${{ env.date }}
94+
# to: dev@beam.apache.org
95+
# from: beamactions@gmail.com
96+
# html_body: file://${{ github.workspace }}/.test-infra/jenkins/metrics_report/beam-metrics_report.html

‎.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -83,20 +83,20 @@ jobs:
8383
run: |
8484
date=$(date -u +"%Y-%m-%d")
8585
echo "date=$date" >> $GITHUB_ENV
86-
- name: Send email
87-
uses: dawidd6/action-send-mail@v3
88-
if: failure()
89-
with:
90-
server_address: smtp.gmail.com
91-
server_port: 465
92-
secure: true
93-
username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
94-
password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
95-
subject: BigQueryEarlyRollout Beam Failure (${{ env.date }}) - investigate and escalate quickly
96-
to: datapls-plat-team@google.com # Team at Google responsible for escalating BQ failures
97-
from: gactions@beam.apache.org
98-
body: |
99-
PostCommit Java BigQueryEarlyRollout failed on ${{ env.date }}. This test monitors BigQuery rollouts impacting Beam and should be escalated immediately if a real issue is encountered to pause further rollouts. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml
86+
# - name: Send email
87+
# uses: dawidd6/action-send-mail@v3
88+
# if: failure()
89+
# with:
90+
# server_address: smtp.gmail.com
91+
# server_port: 465
92+
# secure: true
93+
# username: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
94+
# password: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
95+
# subject: BigQueryEarlyRollout Beam Failure (${{ env.date }}) - investigate and escalate quickly
96+
# to: datapls-plat-team@google.com # Team at Google responsible for escalating BQ failures
97+
# from: gactions@beam.apache.org
98+
# body: |
99+
# PostCommit Java BigQueryEarlyRollout failed on ${{ env.date }}. This test monitors BigQuery rollouts impacting Beam and should be escalated immediately if a real issue is encountered to pause further rollouts. For further details refer to the following links:\n * Failing job: https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml \n * Job configuration: https://github.com/apache/beam/blob/master/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml
100100
- name: Archive JUnit Test Results
101101
uses: actions/upload-artifact@v4
102102
if: ${{ !success() }}

‎.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ jobs:
5757
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
5858
github.event.comment.body == 'Run Python_Xlang_Gcp_Dataflow PostCommit'
5959
runs-on: [self-hosted, ubuntu-20.04, main]
60-
timeout-minutes: 180
60+
timeout-minutes: 240
6161
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
6262
strategy:
6363
matrix:
@@ -95,4 +95,4 @@ jobs:
9595
commit: '${{ env.prsha || env.GITHUB_SHA }}'
9696
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
9797
files: '**/pytest*.xml'
98-
large_files: true
98+
large_files: true

‎.github/workflows/beam_PreCommit_Python_Coverage.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ env:
5454
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
5555
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
5656
HF_INFERENCE_TOKEN: ${{ secrets.HF_INFERENCE_TOKEN }}
57+
ALLOYDB_PASSWORD: ${{ secrets.ALLOYDB_PASSWORD }}
5758

5859

5960
jobs:

0 commit comments

Comments
 (0)