OTel/DDog updates 2

Signed-off-by: Lukasz Gryglicki <lgryglicki@cncf.io>

Assisted by [OpenAI](https://platform.openai.com/)

Assisted by [GitHub Copilot](https://github.com/features/copilot)

Author: lukaszgryglicki

.gitignore

@@ -276,7 +276,7 @@ cla-backend-go/golang-api.log
 utils/otel_dd_go/otel_dd
 audit.json
 spans*.json
-*api_usage.csv
+*api_usage*.csv
 
 *.exe
 *.exe~

cla-backend-legacy/internal/server/server.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/linuxfoundation/easycla/cla-backend-legacy/internal/api"
 	"github.com/linuxfoundation/easycla/cla-backend-legacy/internal/logging"
@@ -116,6 +117,13 @@ func wrapHTTPHandlerWithTelemetryBestEffort(next http.Handler) (wrapped http.Han
 	return telemetry.WrapHTTPHandler(next)
 }
 
+func flushTelemetryAfterResponse(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		next.ServeHTTP(w, r)
+		telemetry.ForceFlushBestEffort(250 * time.Millisecond)
+	})
+}
+
 // NewHTTPHandler builds the HTTP handler for both Lambda (via adapter) and local runs.
 //
 // Note: router-level middleware already handles request logging and CORS.
@@ -126,5 +134,5 @@ func NewHTTPHandler() http.Handler {
 	if !otelEnabled {
 		return router
 	}
-	return wrapHTTPHandlerWithTelemetryBestEffort(router)
+	return flushTelemetryAfterResponse(wrapHTTPHandlerWithTelemetryBestEffort(router))
 }

cla-backend-legacy/internal/telemetry/datadog_otlp.go

@@ -138,6 +138,32 @@ func InitDatadogOTel(cfg DatadogOTelConfig) (err error) {
 	return ddInitErr
 }
 
+type forceFlusher interface {
+	ForceFlush(context.Context) error
+}
+
+// ForceFlushBestEffort asks the global tracer provider to flush queued spans.
+// This is intentionally fail-open for Lambda: flushing must never affect API behavior.
+func ForceFlushBestEffort(timeout time.Duration) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Warnf("LG:otel-datadog-forceflush-panic recovered=%v", r)
+		}
+	}()
+
+	ff, ok := otel.GetTracerProvider().(forceFlusher)
+	if !ok {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	if err := ff.ForceFlush(ctx); err != nil {
+		log.Debugf("LG:otel-datadog-forceflush-err err=%v", err)
+	}
+}
+
 // NewHTTPClient returns an HTTP client whose outbound requests are instrumented
 // and propagate trace context when OTel is enabled.
 //

utils/otel_dd/api_usage_stats_ddog.py

@@ -20,7 +20,7 @@
 Example:
   ./utils/otel_dd/api_usage_stats_ddog.py --from now-60m --to now > api_usage.csv
   ./utils/otel_dd/api_usage_stats_ddog.py --no-skip-e2e | head
-  ./utils/otel_dd/api_usage_stats_ddog.py --from now-24h --to now > api_usage.csv
+  ./utils/otel_dd/api_usage_stats_ddog.py --from now-24h --to now --env prod > api_usage.csv
   ./utils/otel_dd/api_usage_stats_ddog.py --verbose | head
 """
 
@@ -34,6 +34,7 @@
 import re
 import sys
 import urllib.error
+import urllib.parse
 import urllib.request
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -143,6 +144,24 @@ def is_e2e_true(span: Dict[str, Any]) -> bool:
     return str(v).strip().lower() == "true"
 
 
+def api_path_from_candidate(value: Any) -> Optional[str]:
+    if not isinstance(value, str):
+        return None
+    raw = value.strip()
+    if not raw:
+        return None
+
+    if raw.startswith("http://") or raw.startswith("https://"):
+        parsed = urllib.parse.urlparse(raw)
+        path = parsed.path or ""
+    else:
+        path = raw.split("?", 1)[0]
+
+    if not path.startswith("/v"):
+        return None
+    return path
+
+
 def extract_route(span: Dict[str, Any], *, sanitize_routes: bool = False) -> Optional[str]:
     """
     Prefer templated HTTP route:
@@ -160,17 +179,28 @@ def extract_route(span: Dict[str, Any], *, sanitize_routes: bool = False) -> Opt
         value = route.strip()
         return sanitize_api_path(value) if sanitize_routes else value
 
+    for candidate in (
+        http.get("target"),
+        http.get("path"),
+        (http.get("url_details") or {}).get("path") if isinstance(http.get("url_details"), dict) else None,
+        http.get("url"),
+    ):
+        path = api_path_from_candidate(candidate)
+        if path:
+            return sanitize_api_path(path) if sanitize_routes else path
+
     resource_name = attrs.get("resource_name")
     if isinstance(resource_name, str):
         rn = resource_name.strip()
         # Often "METHOD /path"
         parts = rn.split(None, 1)
         if len(parts) == 2 and parts[1].startswith("/"):
-            value = parts[1].strip()
+            value = parts[1].strip().split("?", 1)[0]
             return sanitize_api_path(value) if sanitize_routes else value
         # Sometimes just "/path"
         if rn.startswith("/"):
-            return sanitize_api_path(rn) if sanitize_routes else rn
+            value = rn.split("?", 1)[0]
+            return sanitize_api_path(value) if sanitize_routes else value
 
     return None
 
@@ -320,15 +350,19 @@ def main() -> int:
 
     kept = 0
     skipped_e2e = 0
+    skipped_e2e_routes: Dict[str, int] = {}
     skipped_missing_route = 0
     skipped_missing_ts = 0
 
     for span in spans:
+        route = extract_route(span, sanitize_routes=args.sanitize_routes)
+
         if skip_e2e and is_e2e_true(span):
             skipped_e2e += 1
+            if route:
+                skipped_e2e_routes[route] = skipped_e2e_routes.get(route, 0) + 1
             continue
 
-        route = extract_route(span, sanitize_routes=args.sanitize_routes)
         if not route:
             skipped_missing_route += 1
             continue
@@ -355,10 +389,16 @@ def main() -> int:
         w.writerow([route, cnt, fmt_ts(tmin), fmt_ts(tmax)])
 
     if args.verbose:
+        eprint(f"[ddog] query:        {query}")
+        eprint(f"[ddog] from/to:      {args.time_from} -> {args.time_to}")
         eprint(f"[ddog] spans fetched: {len(spans)}")
         eprint(f"[ddog] spans kept:   {kept}")
         if skip_e2e:
             eprint(f"[ddog] e2e skipped:  {skipped_e2e}")
+            if skipped_e2e_routes:
+                eprint("[ddog] top skipped e2e routes:")
+                for route, cnt in sorted(skipped_e2e_routes.items(), key=lambda x: (-x[1], x[0]))[:25]:
+                    eprint(f"[ddog]   {cnt:5d} {route}")
         eprint(f"[ddog] no-route:     {skipped_missing_route}")
         eprint(f"[ddog] no-ts:        {skipped_missing_ts}")
         eprint(f"[ddog] routes:       {len(stats)}")

utils/otel_dd/check_spans_in_ddog.sh

@@ -2,13 +2,12 @@
 set -euo pipefail
 
 # Example:
-# ./utils/otel_dd/check_spans_in_ddog.sh --env prod --skip-e2e \
-#   | jq -r '.data[].attributes.custom.http.route' | sort | uniq
+# ./utils/otel_dd/check_spans_in_ddog.sh --env prod --skip-e2e | jq -r '.data[].attributes.custom.http.route' | sort | uniq
 
 usage() {
   cat <<'EOF' >&2
 Usage:
-  check_spans_in_ddog.sh [--env <dev|prod|...>] [--stage <dev|prod|...>] [--skip-e2e|--no-skip-e2e]
+  check_spans_in_ddog.sh [--env <dev|prod|...>] [--stage <dev|prod|...>] [--from <time>] [--to <time>] [--skip-e2e|--no-skip-e2e]
 
 Env vars:
   DD_SITE, DD_API_KEY, DD_APP_KEY  (required)
@@ -24,6 +23,8 @@ EOF
 SKIP_E2E=0
 DD_ENV="${DD_ENV:-${ENV:-${STAGE:-dev}}}"
 DD_SERVICE="${DD_SERVICE:-easycla-backend}"
+TIME_FROM="now-60m"
+TIME_TO="now"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -34,6 +35,16 @@ while [[ $# -gt 0 ]]; do
       DD_ENV="$2"
       shift 2
       ;;
+    --from)
+      [[ $# -ge 2 ]] || { echo "ERROR: --from requires a value" >&2; usage; exit 2; }
+      TIME_FROM="$2"
+      shift 2
+      ;;
+    --to)
+      [[ $# -ge 2 ]] || { echo "ERROR: --to requires a value" >&2; usage; exit 2; }
+      TIME_TO="$2"
+      shift 2
+      ;;
     --service)
       [[ $# -ge 2 ]] || { echo "ERROR: --service requires a value" >&2; usage; exit 2; }
       DD_SERVICE="$2"
@@ -51,11 +62,11 @@ done
 QUERY="service:${DD_SERVICE} env:${DD_ENV}"
 
 # Build request JSON safely with jq (avoids quoting bugs)
-payload="$(jq -n --arg query "$QUERY" '{
+payload="$(TIME_FROM="$TIME_FROM" TIME_TO="$TIME_TO" jq -n --arg query "$QUERY" '{
   data: {
     type: "search_request",
     attributes: {
-      filter: { from: "now-60m", to: "now", query: $query },
+      filter: { from: env.TIME_FROM, to: env.TIME_TO, query: $query },
       sort: "timestamp",
       page: { limit: 5000 }
     }