refactor: optimize Spanner batching and standardize OpenTelemetry deployment

This commit optimizes Spanner write performance and standardizes OpenTelemetry (OTel) deployment across the backend API, background workers, and ingestion jobs.

1. Spanner Batching:
* Refactored UpsertWPTRunFeatureMetrics in lib/gcpspanner/wpt_run_feature_metric.go to execute two batch queries at the transaction start instead of executing queries inside a loop, reducing database roundtrips.
* Extracted mutation building to buildWPTRunFeatureMetricMutations to reduce cognitive complexity.

2. Centralized Terraform Telemetry Configuration:
* Created root-level infra/telemetry.tf to define a single shared Secret Manager secret containing the OTel collector configuration.
* Swapped the custom in-tree OTel collector image for the official Google-managed image (otelcol-google:0.151.0). This is to match this new documentation: https://docs.cloud.google.com/stackdriver/docs/instrumentation/opentelemetry-collector-cloud-run
* Defined a centralized otel_collector_config_mount_path local variable in infra/telemetry.tf set to "/etc/otelcol-google" and propagated it to all submodules, replacing hardcoded paths.

3. OTel Sidecar Deployment (Go & Terraform):
* Added opentelemetry.MaybeSetup helper to lib/opentelemetry/setup.go to encapsulate environment checks and OTel SDK initialization.
* Refactored backend/cmd/server/main.go and all 11 background workers and daily ingestion scraper entrypoints to call MaybeSetup and defer shutdown.
* Deployed the OTel sidecar container and mounted the shared config secret across all 4 worker pools and the reusable job module.
* Granted roles/cloudtrace.agent, roles/monitoring.metricWriter, and roles/logging.logWriter to the backend and worker service accounts.

4. GCP Error Reporting & Structured Logging:
* Updated the custom slog handler in lib/opentelemetry/slog.go to capture and append runtime/debug.Stack() to ERROR logs.
* Structured logs to enable automatic GCP Error Reporting aggregation and trace-log linking using the trace field.

5. Go Startup Logging & Refactoring:
* Added verbose BOOT: log statements before each client initialization phase (Datastore, Spanner, Valkey, OTel) in backend/cmd/server/main.go to provide startup phase visibility.
* Refactored the inline OpenTelemetry setup block in main.go to use the new opentelemetry.MaybeSetup helper, keeping the startup sequence synchronous.

6. Repository Cleanup:
* Deleted the unused custom in-tree OTel collector Dockerfile (otel/Dockerfile).
* Removed the /otel Docker package-ecosystem update entry from .github/dependabot.yml.

BUG=526562255

Author: jcscottiii

.github/dependabot.yml

@@ -78,11 +78,6 @@ updates:
           - '>=23'
           - '<24'
 
-  - package-ecosystem: 'docker'
-    directory: '/otel'
-    schedule:
-      interval: 'weekly'
-
   - package-ecosystem: 'docker'
     directory: '/.dev/auth'
     schedule:

backend/cmd/server/main.go

@@ -59,6 +59,7 @@ func main() {
 		datastoreDB = &value
 	}
 	projectID := os.Getenv("PROJECT_ID")
+	slog.InfoContext(context.TODO(), "BOOT: Creating Datastore client...")
 	fs, err := gds.NewDatastoreClient(projectID, datastoreDB)
 	if err != nil {
 		slog.ErrorContext(context.TODO(), "failed to create datastore client", "error", err.Error())
@@ -69,6 +70,7 @@ func main() {
 
 	spannerDB := os.Getenv("SPANNER_DATABASE")
 	spannerInstance := os.Getenv("SPANNER_INSTANCE")
+	slog.InfoContext(ctx, "BOOT: Creating Spanner client...")
 	spannerClient, err := gcpspanner.NewSpannerClient(projectID, spannerInstance, spannerDB)
 	if err != nil {
 		slog.ErrorContext(ctx, "failed to create spanner client", "error", err.Error())
@@ -106,6 +108,7 @@ func main() {
 	slog.InfoContext(ctx, "cache settings", "duration", cacheTTL, "prefix", cacheKeyPrefix,
 		"aggregatedFeaturesStatsTTL", aggregatedFeaturesStatsTTL)
 
+	slog.InfoContext(ctx, "BOOT: Creating Valkey cache...")
 	cache, err := valkeycache.NewValkeyDataCache[string, []byte](
 		cacheKeyPrefix,
 		valkeyHost,
@@ -165,25 +168,22 @@ func main() {
 			}),
 	}
 
-	if os.Getenv("OTEL_SERVICE_NAME") != "" {
-		slog.InfoContext(ctx, "opentelemetry settings detected.")
-		otelProjectID := os.Getenv("OTEL_GCP_PROJECT_ID")
-		if otelProjectID == "" {
-			slog.ErrorContext(ctx, "missing project id for opentelemetry")
-			os.Exit(1)
-		}
-		shutdown, err := opentelemetry.SetupOpenTelemetry(ctx, otelProjectID)
-		if err != nil {
-			slog.ErrorContext(ctx, "failed to setup opentelemetry", "error", err.Error())
-			os.Exit(1)
-		}
+	slog.InfoContext(ctx, "BOOT: Running OTel setup...")
+	shutdown, err := opentelemetry.MaybeSetup(ctx)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to setup opentelemetry", "error", err.Error())
+		os.Exit(1)
+	}
+	if shutdown != nil {
 		defer func() {
-			err := shutdown(ctx)
-			if err != nil {
-				slog.ErrorContext(ctx, "unable to shutdown opentelemetry")
+			if err := shutdown(ctx); err != nil {
+				slog.ErrorContext(ctx, "unable to shutdown opentelemetry", "error", err)
 			}
 		}()
-		// Prepend the opentelemtry middleware
+	}
+
+	if os.Getenv("OTEL_SERVICE_NAME") != "" {
+		// Prepend the opentelemetry middleware
 		preRequestMiddlewares = slices.Insert(preRequestMiddlewares, 0, opentelemetry.NewOpenTelemetryChiMiddleware())
 	}
 

infra/backend/service.tf

@@ -46,13 +46,11 @@ resource "docker_registry_image" "backend_remote_image" {
 data "google_project" "host_project" {
 }
 
-module "otel_sidecar" {
-  source = "../modules/otel"
-  providers = {
-    google.project = google.public_project
-  }
-  docker_repository_details = var.docker_repository_details
-  service_account           = google_service_account.backend.member
+resource "google_secret_manager_secret_iam_member" "backend_otel_config_secret_access" {
+  provider  = google.internal_project
+  secret_id = var.otel_config_secret_id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.backend.email}"
 }
 
 resource "google_cloud_run_v2_service" "service" {
@@ -125,7 +123,7 @@ resource "google_cloud_run_v2_service" "service" {
       }
       env {
         name  = "OTEL_EXPORTER_OTLP_ENDPOINT"
-        value = "http://localhost:4318"
+        value = var.otel_collector_endpoint
       }
       env {
         name  = "OTEL_SERVICE_NAME"
@@ -150,7 +148,16 @@ resource "google_cloud_run_v2_service" "service" {
     }
     containers {
       name  = "otel"
-      image = module.otel_sidecar.otel_image
+      image = var.otel_collector_image
+      args  = ["--config=${var.otel_collector_config_mount_path}/config.yaml"]
+      env {
+        name  = "OTEL_COLLECTOR_REGION"
+        value = each.key
+      }
+      volume_mounts {
+        name       = "otel-config"
+        mount_path = var.otel_collector_config_mount_path
+      }
       liveness_probe {
         http_get {
           port = 4319
@@ -162,7 +169,7 @@ resource "google_cloud_run_v2_service" "service" {
         timeout_seconds       = 10
       }
       startup_probe {
-        initial_delay_seconds = 0
+        initial_delay_seconds = 5
         timeout_seconds       = 1
         period_seconds        = 3
         failure_threshold     = 10
@@ -178,6 +185,16 @@ resource "google_cloud_run_v2_service" "service" {
       }
       egress = "PRIVATE_RANGES_ONLY"
     }
+    volumes {
+      name = "otel-config"
+      secret {
+        secret = var.otel_config_secret_id
+        items {
+          version = "latest"
+          path    = "config.yaml"
+        }
+      }
+    }
     service_account = google_service_account.backend.email
   }
 
@@ -188,6 +205,7 @@ resource "google_cloud_run_v2_service" "service" {
 
   depends_on = [
     google_project_iam_member.gcp_datastore_user,
+    google_secret_manager_secret_iam_member.backend_otel_config_secret_access,
   ]
 
   deletion_protection = var.deletion_protection
@@ -327,3 +345,29 @@ resource "google_compute_managed_ssl_certificate" "lb_default" {
     domains = var.domains
   }
 }
+
+# --- Telemetry IAM Permissions for Backend Service Account ---
+
+# Grant Cloud Trace Agent role to allow exporting traces directly or via OTel sidecar.
+resource "google_project_iam_member" "backend_trace_agent" {
+  provider = google.public_project
+  project  = var.projects.public
+  role     = "roles/cloudtrace.agent"
+  member   = "serviceAccount:${google_service_account.backend.email}"
+}
+
+# Grant Monitoring Metric Writer role to allow exporting metrics.
+resource "google_project_iam_member" "backend_metric_writer" {
+  provider = google.public_project
+  project  = var.projects.public
+  role     = "roles/monitoring.metricWriter"
+  member   = "serviceAccount:${google_service_account.backend.email}"
+}
+
+# Grant Logging Log Writer role to allow exporting structured logs.
+resource "google_project_iam_member" "backend_log_writer" {
+  provider = google.public_project
+  project  = var.projects.public
+  role     = "roles/logging.logWriter"
+  member   = "serviceAccount:${google_service_account.backend.email}"
+}

infra/backend/variables.tf

@@ -110,3 +110,23 @@ variable "backend_api_url" {
 
 variable "pubsub_project_id" { type = string }
 variable "ingestion_topic_id" { type = string }
+
+variable "otel_config_secret_id" {
+  type        = string
+  description = "The Secret Manager secret ID containing the OTel collector configuration"
+}
+
+variable "otel_collector_image" {
+  type        = string
+  description = "The container image to use for the OTel collector sidecar"
+}
+
+variable "otel_collector_config_mount_path" {
+  type        = string
+  description = "The volume mount path for the OTel collector configuration"
+}
+
+variable "otel_collector_endpoint" {
+  type        = string
+  description = "The endpoint for the application to export OTLP metrics/traces to the local collector"
+}

infra/ingestion/variables.tf

@@ -94,3 +94,28 @@ variable "notification_channel_ids" {
   description = "A list of notification channel ids to send alerts to."
   type        = list(string)
 }
+
+variable "otel_config_secret_id" {
+  type        = string
+  description = "The Secret Manager secret ID containing the OTel collector configuration"
+}
+
+variable "otel_project_id" {
+  type        = string
+  description = "The GCP project ID where telemetry traces/metrics will be exported"
+}
+
+variable "otel_collector_image" {
+  type        = string
+  description = "The container image to use for the OTel collector sidecar"
+}
+
+variable "otel_collector_config_mount_path" {
+  type        = string
+  description = "The volume mount path for the OTel collector configuration"
+}
+
+variable "otel_collector_endpoint" {
+  type        = string
+  description = "The endpoint for the application to export OTLP metrics/traces to the local collector"
+}

infra/ingestion/workflows.tf

@@ -59,6 +59,11 @@ module "web_features_workflow" {
       value = "262980h" # 30 years in hours (365.25*30*24)
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "wpt_workflow" {
@@ -105,9 +110,14 @@ module "wpt_workflow" {
     },
     {
       name  = "DATA_WINDOW_DURATION"
-      value = "17520h" # 2 years
+      value = "720h" # 30 days
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "bcd_workflow" {
@@ -144,6 +154,11 @@ module "bcd_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "chromium_enum_workflow" {
@@ -180,6 +195,11 @@ module "chromium_enum_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "uma_export_workflow" {
@@ -216,6 +236,11 @@ module "uma_export_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "developer_signals_workflow" {
@@ -252,6 +277,11 @@ module "developer_signals_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }
 
 module "web_features_mapping_workflow" {
@@ -288,4 +318,9 @@ module "web_features_mapping_workflow" {
       value = var.spanner_datails.instance
     }
   ]
+  otel_config_secret_id            = var.otel_config_secret_id
+  otel_project_id                  = var.otel_project_id
+  otel_collector_image             = var.otel_collector_image
+  otel_collector_config_mount_path = var.otel_collector_config_mount_path
+  otel_collector_endpoint          = var.otel_collector_endpoint
 }

infra/main.tf

@@ -96,6 +96,11 @@ module "ingestion" {
   developer_signals_region_schedules    = var.developer_signals_region_schedules
   web_features_mapping_region_schedules = var.web_features_mapping_region_schedules
   notification_channel_ids              = var.notification_channel_ids
+  otel_config_secret_id                 = google_secret_manager_secret.otel_config.id
+  otel_project_id                       = var.projects.public
+  otel_collector_image                  = local.otel_collector_image
+  otel_collector_config_mount_path      = local.otel_collector_config_mount_path
+  otel_collector_endpoint               = local.otel_collector_endpoint
 }
 
 module "backend" {
@@ -124,8 +129,12 @@ module "backend" {
   firebase_settings = {
     tenant_id = module.auth.tenant_id
   }
-  pubsub_project_id  = var.projects.internal
-  ingestion_topic_id = module.pubsub.ingestion_topic_id
+  pubsub_project_id                = var.projects.internal
+  ingestion_topic_id               = module.pubsub.ingestion_topic_id
+  otel_config_secret_id            = google_secret_manager_secret.otel_config.id
+  otel_collector_image             = local.otel_collector_image
+  otel_collector_config_mount_path = local.otel_collector_config_mount_path
+  otel_collector_endpoint          = local.otel_collector_endpoint
 }
 
 module "frontend" {
@@ -205,6 +214,11 @@ module "workers" {
 
   chime_details = var.chime_details
 
-  email_service_account_email = var.email_service_account_email
-  deletion_protection         = var.deletion_protection
+  email_service_account_email      = var.email_service_account_email
+  deletion_protection              = var.deletion_protection
+  otel_config_secret_id            = google_secret_manager_secret.otel_config.id
+  otel_project_id                  = var.projects.public
+  otel_collector_image             = local.otel_collector_image
+  otel_collector_config_mount_path = local.otel_collector_config_mount_path
+  otel_collector_endpoint          = local.otel_collector_endpoint
 }