Merge branch 'master' into fix-python-postcommit-dependency-1

Author: aIbrahiim

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -676,6 +676,7 @@ class BeamModulePlugin implements Plugin<Project> {
         activemq_junit                              : "org.apache.activemq.tooling:activemq-junit:$activemq_version",
         activemq_kahadb_store                       : "org.apache.activemq:activemq-kahadb-store:$activemq_version",
         activemq_mqtt                               : "org.apache.activemq:activemq-mqtt:$activemq_version",
+        aircompressor                               : "io.airlift:aircompressor:2.0.3",
         args4j                                      : "args4j:args4j:2.33",
         auto_value_annotations                      : "com.google.auto.value:auto-value-annotations:$autovalue_version",
         // TODO: https://github.com/apache/beam/issues/34993 after stopping supporting Java 8
@@ -1534,7 +1535,6 @@ class BeamModulePlugin implements Plugin<Project> {
             "AutoValueImmutableFields",
             "AutoValueSubclassLeaked",
             "BadImport",
-            "BadInstanceof",
             "BigDecimalEquals",
             "ComparableType",
             "DoNotMockAutoValue",
@@ -1544,7 +1544,6 @@ class BeamModulePlugin implements Plugin<Project> {
             "EqualsUnsafeCast",
             "EscapedEntity",
             "ExtendsAutoValue",
-            "InlineFormatString",
             "InlineMeSuggester",
             "InvalidBlockTag",
             "InvalidInlineTag",
@@ -1555,35 +1554,28 @@ class BeamModulePlugin implements Plugin<Project> {
             "JavaUtilDate",
             "JodaConstructors",
             "MalformedInlineTag",
-            "MissingSummary",
             "MixedMutabilityReturnType",
             "PreferJavaTimeOverload",
-            "MutablePublicArray",
             "NonCanonicalType",
-            "ProtectedMembersInFinalClass",
             "Slf4jFormatShouldBeConst",
             "Slf4jSignOnlyFormat",
             "StaticAssignmentInConstructor",
             "ThreadPriorityCheck",
             "TimeUnitConversionChecker",
             "UndefinedEquals",
             "UnescapedEntity",
-            "UnnecessaryLambda",
             "UnnecessaryMethodReference",
             "UnnecessaryParentheses",
             "UnrecognisedJavadocTag",
             "UnsafeReflectiveConstructionCast",
-            "UseCorrectAssertInTests",
             // errorprone 3.2.0+ checks
             "DirectInvocationOnMock",
             "Finalize",
             "JUnitIncompatibleType",
-            "LongDoubleConversion",
             "MockNotUsedInProduction",
             "NarrowCalculation",
             "NullableTypeParameter",
             "NullableWildcard",
-            "StringCharset",
             "SuperCallToObjectMethod",
             "UnnecessaryLongToIntConversion",
             "UnusedVariable",
@@ -3124,7 +3116,8 @@ class BeamModulePlugin implements Plugin<Project> {
             // pip 25.1 casues :sdks:python:installGcpTest stuck. Pin to 25.0.1 for now.
             args '-c', ". ${project.ext.envdir}/bin/activate && " +
                 "pip install --pre --retries 10 --upgrade pip==25.0.1 --no-cache-dir && " +
-                "pip install --pre --retries 10 --upgrade tox --no-cache-dir"
+                "pip install --pre --retries 10 --upgrade tox --no-cache-dir && " +
+                "pip install --pre --retries 10 --upgrade setuptools build --no-cache-dir"
           }
         }
         // Gradle will delete outputs whenever it thinks they are stale. Putting a
@@ -3165,14 +3158,16 @@ class BeamModulePlugin implements Plugin<Project> {
           def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz"
           def packages = "gcp,test,aws,azure,dataframe"
           def extra = project.findProperty('beamPythonExtra')
-          if (extra) {
-            packages += ",${extra}"
-          }
-
           project.exec {
             executable 'sh'
             args '-c', ". ${project.ext.envdir}/bin/activate && pip install --pre --retries 10 ${distTarBall}[${packages}]"
           }
+          if (extra) {
+            project.exec {
+              executable 'sh'
+              args '-c', ". ${project.ext.envdir}/bin/activate && pip install --pre --retries 10 ${distTarBall}[${extra}]"
+            }
+          }
         }
       }
 

examples/java/src/main/java/org/apache/beam/examples/complete/game/injector/Injector.java

@@ -17,6 +17,8 @@
  */
 package org.apache.beam.examples.complete.game.injector;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.google.api.services.pubsub.Pubsub;
 import com.google.api.services.pubsub.model.PublishRequest;
 import com.google.api.services.pubsub.model.PubsubMessage;
@@ -327,7 +329,7 @@ public static void publishData(int numMessages, int delayInMillis) throws IOExce
     for (int i = 0; i < Math.max(1, numMessages); i++) {
       Long currTime = System.currentTimeMillis();
       String message = generateEvent(currTime, delayInMillis);
-      PubsubMessage pubsubMessage = new PubsubMessage().encodeData(message.getBytes("UTF-8"));
+      PubsubMessage pubsubMessage = new PubsubMessage().encodeData(message.getBytes(UTF_8));
       pubsubMessage.setAttributes(
           ImmutableMap.of(
               GameConstants.TIMESTAMP_ATTRIBUTE,
@@ -350,7 +352,7 @@ public static void publishDataToFile(String fileName, int numMessages, int delay
     PrintWriter out =
         new PrintWriter(
             new OutputStreamWriter(
-                new BufferedOutputStream(new FileOutputStream(fileName, true)), "UTF-8"));
+                new BufferedOutputStream(new FileOutputStream(fileName, true)), UTF_8));
 
     try {
       for (int i = 0; i < Math.max(1, numMessages); i++) {

examples/terraform/envoy-ratelimiter/README.md

@@ -38,7 +38,7 @@ Example Beam Java Pipelines using it:
   - **Cloud NAT (Prerequisite)**: Allows private nodes to pull Docker images.
 - **Envoy Rate Limit Service**: A stateless Go/gRPC service that handles rate limit logic.
 - **Redis**: Stores the rate limit counters.
-- **StatsD Exporter**: Sidecar container that converts StatsD metrics to Prometheus format, exposed on port `9102`.
+- **Prometheus Metrics**: Exposes Prometheus metrics on port `9090`. These metrics are exported to Google Cloud Monitoring.
 - **Internal Load Balancer**: A Google Cloud TCP Load Balancer exposing the Rate Limit service internally within the VPC.
 
 ## Prerequisites:
@@ -82,7 +82,7 @@ cluster_name          = "ratelimit-cluster"         # Name of the GKE cluster
 deletion_protection   = true                        # Prevent accidental cluster deletion (set "true" for prod)
 control_plane_cidr    = "172.16.0.0/28"             # CIDR for GKE control plane (must not overlap with subnet)
 namespace             = "envoy-ratelimiter"         # Kubernetes namespace for deployment
-enable_metrics        = false                       # Deploy statsd-exporter sidecar
+enable_metrics        = true                        # Enable metrics export to Google Cloud Monitoring
 ratelimit_replicas    = 1                           # Initial number of Rate Limit pods
 min_replicas          = 1                           # Minimum HPA replicas
 max_replicas          = 5                           # Maximum HPA replicas
@@ -110,25 +110,34 @@ EOF
 ```
 
 # Deploy Envoy Rate Limiter:
-1. Initialize Terraform to download providers and modules:
+
+1. **Deploy Script (Recommended)**:
+Run the helper script to handle the deployment process automatically:
 ```bash
-terraform init
+./deploy.sh
 ```
+The script will provide the ip address of the load balancer once the deployment is complete.
 
-2. Plan and apply the changes:
+2. **Deploy (Manual Alternative)**:
+If you prefer running Terraform manually, you can use the following commands:
 ```bash
-terraform plan -out=tfplan
-terraform apply tfplan
+# Step 1: Initialize Terraform
+terraform init
+
+# Step 2: Create Cluster
+terraform apply -target=time_sleep.wait_for_cluster
+
+# Step 3: Create Resources
+terraform apply
 ```
 
-3. Connect to the service:
 After deployment, get the **Internal** IP address:
 ```bash
 terraform output load_balancer_ip
 ```
 The service is accessible **only from within the VPC** (e.g., via Dataflow workers or GCE instances in the same network) at `<INTERNAL_IP>:8081`.
 
-4. **Test with Dataflow Workflow**:
+3. **Test with Dataflow Workflow**:
    Verify connectivity and rate limiting logic by running the example Dataflow pipeline.
 
    ```bash
@@ -150,11 +159,40 @@ The service is accessible **only from within the VPC** (e.g., via Dataflow worke
    ```
 
 
+# Observability & Metrics:
+This module supports exporting native Prometheus metrics to **Google Cloud Monitoring**.
+
+ `enable_metrics` is set to `true` by default.
+
+### Sample Metrics
+| Metric Name | Description |
+| :--- | :--- |
+| `ratelimit_service_rate_limit_total_hits` | Total rate limit requests received. |
+| `ratelimit_service_rate_limit_over_limit` | Requests that exceeded the limit (HTTP 429). |
+| `ratelimit_service_rate_limit_near_limit` | Requests that are approaching the limit. |
+| `ratelimit_service_call_should_rate_limit` | Total valid gRPC calls to the service. |
+
+*Note: You will also see many other Go runtime metrics (`go_*`) and Redis client metrics (`redis_*`)
+
+### Viewing in Google Cloud Console
+1. Go to **Monitoring** > **Metrics Explorer**.
+2. Click **Select a metric**.
+3. Search for `ratelimit` and select **Prometheus Target** > **ratelimit**.
+4. Select a metric (e.g., `ratelimit_service_rate_limit_over_limit`) and click **Apply**.
+5. Use **Filters** to drill down by `domain`, `key`, and `value` (e.g., `key=database`, `value=users`).
+
 # Clean up resources:
 To destroy the cluster and all created resources:
+
+```bash
+./deploy.sh destroy
+```
+
+Alternatively:
 ```bash
 terraform destroy
 ```
+
 *Note: If `deletion_protection` was enabled, you must set it to `false` in `terraform.tfvars` before destroying.*
 
 # Variables description:
@@ -169,7 +207,7 @@ terraform destroy
 |control_plane_cidr     |CIDR block for GKE control plane                     |172.16.0.0/28                    |
 |cluster_name           |Name of the GKE cluster                              |ratelimit-cluster                |
 |namespace              |Kubernetes namespace to deploy resources into        |envoy-ratelimiter                |
-|enable_metrics         |Deploy statsd-exporter sidecar                       |false                            |
+|enable_metrics         |Enable metrics export to Google Cloud Monitoring     |true                             |
 |deletion_protection    |Prevent accidental cluster deletion                  |false                            |
 |ratelimit_replicas     |Initial number of Rate Limit pods                    |1                                |
 |min_replicas           |Minimum HPA replicas                                 |1                                |

examples/terraform/envoy-ratelimiter/deploy.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script deploys the Envoy Rate Limiter on GKE.
+
+set -e
+
+COMMAND=${1:-"apply"}
+
+# 1. Initialize Terraform
+if [ ! -d ".terraform" ]; then
+    echo "Initializing Terraform..."
+    terraform init
+else
+    # Verify terraform initialization is valid, or re-initialize
+    terraform init -upgrade=false >/dev/null 2>&1 || terraform init
+fi
+
+if [ "$COMMAND" = "destroy" ]; then
+    echo "Destroying Envoy Rate Limiter Resources..."
+    terraform destroy -auto-approve
+    exit $?
+fi
+
+if [ "$COMMAND" = "apply" ]; then
+    echo "Deploying Envoy Rate Limiter..."
+
+    echo "--------------------------------------------------"
+    echo "Creating/Updating GKE Cluster..."
+    echo "--------------------------------------------------"
+    # Deploy the cluster and wait for it to be ready.
+    terraform apply -target=time_sleep.wait_for_cluster -auto-approve
+
+    echo ""
+    echo "--------------------------------------------------"
+    echo "Deploying Application Resources..."
+    echo "--------------------------------------------------"
+    # Deploy the rest of the resources
+    terraform apply -auto-approve
+
+    echo ""
+    echo "Deployment Complete!"
+    echo "Cluster Name: $(terraform output -raw cluster_name)"
+    echo "Load Balancer IP: $(terraform output -raw load_balancer_ip)"
+    exit 0
+fi
+
+echo "Usage:"
+echo "  ./deploy.sh [apply]    # Initialize and deploy resources (Default)"
+echo "  ./deploy.sh destroy    # Destroy resources"
+exit 1

examples/terraform/envoy-ratelimiter/prerequisites.tf

@@ -21,6 +21,7 @@ resource "google_project_service" "required" {
     "container",
     "iam",
     "compute",
+    "monitoring",
   ])
 
   service            = "${each.key}.googleapis.com"