Merge branch 'master' into 20260213_createYamlDynamicDestinationExample

Author: derrickaw

.agent/skills/gradle-build/SKILL.md

@@ -154,12 +154,32 @@ applyJavaNature(
 ./gradlew :sdks:java:container:java11:docker
 
 # Build Python SDK container
-./gradlew :sdks:python:container:py39:docker
+./gradlew :sdks:python:container:py312:docker
 
-# With custom repository
+# Build and push a container into a custom repository
 ./gradlew :sdks:java:container:java11:docker \
   -Pdocker-repository-root=gcr.io/project \
-  -Pdocker-tag=custom
+  -Pdocker-tag=custom \
+  -Ppush-containers
+```
+
+If a `:docker` task produces logs that contain the following:
+
+```
+WARNING: No output specified with docker-container driver.
+Build result will only remain in the build cache.
+```
+
+then you must use `-PuseDockerBuildx` when running `:docker` tasks
+in this environment. For example:
+
+```
+# Build and push a go container into a custom repository
+./gradlew :sdks:go:container:docker \
+  -Pdocker-repository-root=gcr.io/project \
+  -Pdocker-tag=custom \
+  -Ppush-containers \
+  -PuseDockerBuildx
 ```
 
 ## Dependency Management

.agent/skills/python-development/SKILL.md

@@ -123,12 +123,18 @@ pip install build && python -m build --sdist
 ./gradlew :sdks:python:bdistPy311linux  # For Python 3.11 on Linux
 ```
 
-### Build SDK Container
+### Build and Push SDK Container Image
 ```bash
-./gradlew :sdks:python:container:py39:docker \
-  -Pdocker-repository-root=gcr.io/your-project -Pdocker-tag=custom
+./gradlew :sdks:python:container:py311:docker \
+  -Pdocker-repository-root=gcr.io/your-project/your-name \
+  -Pdocker-tag=custom \
+  -Ppush-containers
+
+# Container image will be pushed to: gcr.io/your-project/your-name/beam_python3.11_sdk:custom
 ```
 
+To use this container image, supply it via `--sdk_container_image`.
+
 ## Running Pipelines with Modified Code
 
 ```bash

.asf.yaml

@@ -51,6 +51,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.72: {}
     release-2.71.0-postrelease: {}
     release-2.71: {}
     release-2.70.0-postrelease: {}

.github/gh-actions-self-hosted-runners/arc/environments/beam.env

@@ -47,6 +47,7 @@ main_runner = {
         cpu = "2"
         memory = "3Gi"
     }
+    labels = ["self-hosted", "ubuntu-20.04", "main"]
 }
 additional_runner_pools = [{
     name = "small-runner"
@@ -98,5 +99,74 @@ additional_runner_pools = [{
     labels = ["self-hosted", "ubuntu-20.04", "highmem22"]
     enable_selector = true
     enable_taint = true
+},
+{
+    name = "main-runner-2404"
+    machine_type = "e2-standard-16"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:latest"
+    min_node_count = "1"
+    max_node_count = "30"
+    min_replicas = "1"
+    max_replicas = "240"
+    webhook_scaling = false
+    disk_size_gb = 200
+    requests = {
+        cpu = "2"
+        memory = "3Gi"
+    }
+    labels = ["self-hosted", "ubuntu-24.04", "main"]
+    enable_selector = true
+    enable_taint = true
+},
+{
+    name = "small-runner-2404"
+    machine_type = "e2-standard-2"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:latest"
+    min_node_count = "1"
+    max_node_count = "15"
+    min_replicas = "1"
+    max_replicas = "15"
+    webhook_scaling = false
+    requests = {
+        cpu = "1500m"
+        memory = "5Gi"
+    }
+    labels = ["self-hosted", "ubuntu-24.04", "small"]
+    enable_selector = true
+    enable_taint = true
+},
+{
+    name = "highmem-runner-2404"
+    machine_type = "c3-highmem-8"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:latest"
+    min_node_count = "1"
+    max_node_count = "15"
+    min_replicas = "1"
+    max_replicas = "15"
+    webhook_scaling = false
+    requests = {
+        cpu = "7.5"
+        memory = "5Gi"
+    }
+    labels = ["self-hosted", "ubuntu-24.04", "highmem"]
+    enable_selector = true
+    enable_taint = true
+},
+{
+    name = "highmem-runner-22-2404"
+    machine_type = "c3-highmem-22"
+    runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:latest"
+    min_node_count = "0"
+    max_node_count = "8"
+    min_replicas = "0"
+    max_replicas = "8"
+    webhook_scaling = false
+    requests = {
+        cpu = "7.5"
+        memory = "100Gi"
+    }
+    labels = ["self-hosted", "ubuntu-24.04", "highmem22"]
+    enable_selector = true
+    enable_taint = true
 }]
 #state_bucket_name = "beam-arc-state"

.github/gh-actions-self-hosted-runners/arc/images/Dockerfile

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-FROM ghcr.io/actions-runner-controller/actions-runner-controller/actions-runner:ubuntu-20.04
+FROM ghcr.io/actions-runner-controller/actions-runner-controller/actions-runner:ubuntu-24.04
 #install BuildX
 COPY --from=docker/buildx-bin:0.11.1 /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 RUN docker buildx install && docker buildx version

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 1
+    "modification": 2
 }

.github/trigger_files/beam_PostCommit_Go_VR_Flink.json

@@ -1,6 +1,6 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 1,
+  "modification": 2,
   "https://github.com/apache/beam/pull/32440": "testing datastream optimizations",
   "https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
 }

.github/trigger_files/beam_PostCommit_Java.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 4
-}
+  "modification": 6
+} 

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow.json

@@ -2,3 +2,4 @@
   "comment": "Modify this file in a trivial way to cause this test suite to run!",
   "modification": 3,
 }
+ 

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run!",
+  "modification":  1
+}