Merge branch 'master' into milan.garnier/publish-ci-builds

# Conflicts:
#	.github/chainguard/gitlab-ci-publish-packages.sts.yaml
#	.gitlab/generate-package.php

Author: MilanGarnier

.claude/ci/appsec-gradle-integration.md

@@ -278,7 +278,7 @@ If you need to inspect sidecar/helper or PHP issues:
   (`pref -f dd-ipc-helper`) or to PHP (usually an apache or an FPM worker -- if
   you're investigating code run during processes it will not be the master
   process). sidecar requires as a first command `file /proc/<pid>/exe`).
-* See [gdb.md](../gdb.md) for more information on how to run gdb. Always read
+* See [gdb.md](../debugging/gdb.md) for more information on how to run gdb. Always read
   this file before attempting to use gdb.
 
 ## Gotchas

.claude/ci/benchmarks.md

@@ -12,7 +12,7 @@
 | `benchmarks-appsec` | same | Runs appsec microbenchmarks; produces `candidate.tar.gz` and `baseline.tar.gz` artifacts |
 | `benchmarks-profiler` | same | Runs profiler microbenchmarks |
 | `macrobenchmarks: [{PHP_VERSION}]` | `486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:php_laravel-realworld` | Runs a Laravel Realworld application under k6 load at three traffic levels; PHP 7.4 and 8.1 |
-| `check-big-regressions` | `registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest` | Post-benchmark gate: fails if `benchmarks-tracer` results contain a regression above threshold |
+| `check-big-regressions` | `registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu@sha256:b682acae509fa391ac559705b66df07cac876e4c46641d2457077972a1c0bbb8` | Post-benchmark gate: fails if `benchmarks-tracer` results contain a regression above threshold |
 | `check-slo-breaches` | (benchmarking-platform-tools template) | Post-macrobenchmark gate: evaluates SLO breaches |
 | `notify-slo-breaches` | (benchmarking-platform-tools template) | Posts SLO breach notifications to `#guild-dd-php` Slack channel |
 

.claude/ci/building-locally.md

@@ -69,6 +69,21 @@ This is needed for `build-appsec.sh` on centos-7 but not on bookworm
 the package pipeline's two-phase build). For local testing, always use
 `make` unless you specifically need the split build.
 
+### Linting the libdatadog submodule (fmt + clippy)
+
+To validate Rust changes in the `libdatadog/` submodule, run from inside it:
+
+```bash
+cd libdatadog
+cargo +nightly fmt --all --quiet && \
+  cargo clippy --workspace --all-targets --all-features -- -D warnings
+```
+
+A rustc ≥1.87 toolchain override is active in the repo, so plain `cargo clippy`
+picks up the correct toolchain — do NOT force `+stable` (the default stable is
+older) or a pinned `+1.87.0`, and do not lint per-crate with `-p`. Only `fmt`
+needs `+nightly`.
+
 ## Tracer Extension (ddtrace.so)
 
 ### For test jobs (bookworm, debug build)

.claude/ci/check-ci

@@ -46,9 +46,25 @@ def parse_args():
     parser.add_argument("--max-failures", type=int, default=50, help="Stop after this many failures (default: 50)")
     parser.add_argument("--timeout", type=int, default=7200, help="Overall timeout in seconds (default: 7200 = 2 hours)")
     parser.add_argument("--list-jobs", action="store_true", help="List all jobs grouped by pipeline and exit")
+    parser.add_argument("--jobs", type=str, default=None, help="Comma-separated substring patterns; monitor only matching jobs (case-insensitive)")
     return parser.parse_args()
 
 
+def parse_job_patterns(spec: str | None) -> list[str]:
+    """Split a comma-separated --jobs spec into lowercased substring patterns."""
+    if not spec:
+        return []
+    return [p.strip().lower() for p in spec.split(",") if p.strip()]
+
+
+def job_name_matches(name: str, patterns: list[str]) -> bool:
+    """Return True if name contains any pattern (case-insensitive substring). Empty patterns match all."""
+    if not patterns:
+        return True
+    lname = name.lower()
+    return any(p in lname for p in patterns)
+
+
 def resolve_sha(ref: str) -> str:
     result = subprocess.run(["git", "rev-parse", ref], capture_output=True, text=True)
     if result.returncode != 0:
@@ -383,8 +399,9 @@ async def gh_download_job_log(session: aiohttp.ClientSession, job_id: int, log_d
 # list-jobs mode
 # ---------------------------------------------------------------------------
 
-async def list_jobs(session: aiohttp.ClientSession, root_id: int):
-    """List all GitLab jobs grouped by pipeline, then exit."""
+async def list_jobs(session: aiohttp.ClientSession, root_id: int, patterns: list[str] | None = None):
+    """List all GitLab jobs grouped by pipeline, then exit. Filtered to matched jobs when patterns given."""
+    patterns = patterns or []
     pipeline_ids, pipeline_names, jobs_by_pipeline, pipeline_statuses = await discover_pipelines_and_jobs(session, root_id)
 
     for pid in pipeline_ids:
@@ -394,6 +411,10 @@ async def list_jobs(session: aiohttp.ClientSession, root_id: int):
         info = pipeline_statuses.get(pid, {"status": "unknown", "name": str(pid)})
         display_name = pipeline_names.get(pid, info["name"])
         jobs = jobs_by_pipeline.get(pid, [])
+        if patterns:
+            jobs = [j for j in jobs if job_name_matches(j.get("name", ""), patterns)]
+            if not jobs:
+                continue
         print(f"\nPipeline {pid} '{display_name}' (status: {info['status']}):")
         for job in jobs:
             job_status = job.get("status", "unknown")
@@ -402,8 +423,9 @@ async def list_jobs(session: aiohttp.ClientSession, root_id: int):
             print(f"  {job_status:<10}({job_id}) {job_name}")
 
 
-async def list_github_jobs(session: aiohttp.ClientSession, sha: str):
-    """List GitHub Actions workflow runs and their jobs for a SHA."""
+async def list_github_jobs(session: aiohttp.ClientSession, sha: str, patterns: list[str] | None = None):
+    """List GitHub Actions workflow runs and their jobs for a SHA. Filtered to matched jobs when patterns given."""
+    patterns = patterns or []
     try:
         runs = await gh_discover_runs(session, sha, timeout=15)
     except GitHubAuthError:
@@ -420,6 +442,10 @@ async def list_github_jobs(session: aiohttp.ClientSession, sha: str):
         run_conclusion = run.get("conclusion") or ""
         status_str = run_status if not run_conclusion else f"{run_status}, conclusion: {run_conclusion}"
         jobs = await gh_get_run_jobs(session, run_id)
+        if patterns:
+            jobs = [j for j in jobs if job_name_matches(j.get("name", ""), patterns)]
+            if not jobs:
+                continue
         print(f"\nGitHub Actions run {run_id} '{run_name}' (status: {status_str}):")
         for job in sorted(jobs, key=lambda j: j.get("name", "")):
             job_status = job.get("status", "unknown")
@@ -493,10 +519,14 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_
     else:
         root_id = await discover_pipeline(gl_session, sha, args.discovery_timeout)
 
+    patterns = parse_job_patterns(args.jobs)
+    if patterns:
+        print(f"Filtering to jobs matching (any of): {patterns}")
+
     if args.list_jobs:
-        await list_jobs(gl_session, root_id)
+        await list_jobs(gl_session, root_id, patterns)
         if gh_session and sha:
-            await list_github_jobs(gh_session, sha)
+            await list_github_jobs(gh_session, sha, patterns)
         return
 
     # Set up working directory
@@ -557,6 +587,8 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_
 
         pipeline_ids, _, jobs_by_pipeline, pipeline_statuses = gl_result
         all_jobs = [job for jobs in jobs_by_pipeline.values() for job in jobs]
+        if patterns:
+            all_jobs = [job for job in all_jobs if job_name_matches(job.get("name", ""), patterns)]
 
         # Phase 2: fetch GH run-jobs while we can (no GL dependency); process GL jobs locally.
         if gh_monitoring:
@@ -609,6 +641,7 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_
         gh_passed = 0
         gh_failed_count = 0
         gh_all_done = True
+        gh_matched_total = 0
         gh_new_failures: list[tuple[dict, dict]] = []
         gh_fail_log_dir = wdir / "gh_fail_logs"
 
@@ -618,6 +651,9 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_
             if run.get("status", "unknown") != "completed":
                 gh_all_done = False
             for job in run_jobs:
+                if patterns and not job_name_matches(job.get("name", ""), patterns):
+                    continue
+                gh_matched_total += 1
                 jid = job["id"]
                 job_status = job.get("status", "unknown")
                 conclusion = job.get("conclusion") or ""
@@ -695,7 +731,15 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_
             print(f"[{now}] pipelines={len(pipeline_ids)}  jobs={len(all_jobs)}  running={gl_running_count}  passed={passed}  failed={failed}  allow_fail={allow_fail}")
 
         # Check completion
-        all_done = all_gl_done and (gl_running_count == 0) and (not gh_monitoring or gh_all_done)
+        if patterns:
+            # Finish as soon as the MATCHED subset is terminal (no matched job still
+            # running), regardless of the rest of the pipeline. Require at least one
+            # matched job to exist so we keep polling while jobs are still being created.
+            matched_total = len(all_jobs) + gh_matched_total
+            matched_running = gl_running_count + gh_running
+            all_done = matched_total > 0 and matched_running == 0
+        else:
+            all_done = all_gl_done and (gl_running_count == 0) and (not gh_monitoring or gh_all_done)
 
         if all_done:
             total_failures = failure_count

.claude/ci/diagnosing-failures.md

@@ -0,0 +1,89 @@
+# Diagnosing CI Failures & Flakiness
+
+Transferable triage techniques to apply before proposing a fix for a failing or
+flaky CI job. For reproducing a specific job locally, see the group docs linked
+from [index.md](index.md).
+
+## Determine the failure mode first: hang vs test failure
+
+Before blaming any FAIL/diff line in the log, check the job **duration**. A
+duration equal to the job timeout (e.g. `windows test_c` dies at ~3601s = the
+1 h timeout) means the job died from a **hang/timeout**, not from the assertion
+failures printed earlier. Those FAIL lines may be real, but are separate,
+non-fatal issues.
+
+- Grep the tail for a hang signature (`execution took longer than`,
+  `Failed to terminate process`) vs an assertion `FAIL`/diff (a test bug).
+- **`run-tests.php -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP` suppresses PASS
+  output.** So the last log line is NOT necessarily the hang point — it is just
+  the last *reported* (failed/skipped) test before a run of silently-passing
+  ones. To find the real hang point, add `-v` (sets `$DETAILED`, printing
+  `TEST <path>` before each test): the last `TEST` line that never gets a
+  result is the culprit. Confirm determinism against a second matrix variant.
+- Timed-out jobs skip `after_script`, so they upload only `job.log` (no
+  `sidecar.log`/dumps). A passing sibling version is both the comparison
+  baseline and the source of the artifacts the failing job never emitted.
+
+## All-versions deterministic flip with no source change = mutable image
+
+If a test flips green→red **deterministically across every PHP version/SAPI**
+with no corresponding source/commit change, suspect a **mutable CI image tag
+being republished** under the same tag (the `bookworm-{N}` images are rebuilt
+in place with `docker buildx bake --no-cache --pull`), not a code regression.
+Verify the mechanism empirically before fixing:
+
+- Check that pipelines on the *same* sha previously passed — if so, the trigger
+  is the image republish, not the commit.
+- Diff green-vs-red job **artifacts** (`[ddtrace]` logs, sidecar data) rather
+  than theorizing about log levels / worker recycling.
+- Watch for env vars already forced by the Makefile test target (e.g.
+  `run_tests_debug` forces `DD_TRACE_DEBUG=1`), so a "config" fix isn't a silent
+  no-op.
+
+A slower rebuilt image commonly loses a pre-existing **retrieval-window race**:
+e.g. a `/telemetry` poll that stops as soon as it sees `spans_created` can miss
+the later batch carrying `logs_created`. Fix the race (accumulate the awaited
+metrics across requests); don't relax the assertion.
+
+## Retry-trait cascade masks the real failure
+
+Tests using the persistent-server retry trait
+(`tests/Common/RetryTraitVersionGeneric.php`) re-run the whole test against an
+already-warmed server on failure, and **only the last attempt's exception is
+reported**. The reported failure can therefore be a downstream symptom of the
+first attempt's real break:
+
+- One-shot-per-server-lifecycle metrics (e.g. `logs_created`, emitted only on
+  the first telemetry lifecycle of a server process; zero-valued metrics aren't
+  sent) fail on every *warmed* retry — so a reported first-window `logs_created`
+  failure is often really a later-window break cascading through the retry.
+- To see the true cause, temporarily `fwrite(STDERR, ...)` the swallowed
+  exception in the trait's `catch` block.
+- Hard-coded dependency-version assertions on `^x.y` fixtures break silently
+  when a fresh `composer update` resolves a new upstream patch release; verify
+  what actually resolves in the container with `composer show <pkg>`.
+
+## Root-cause sidecar error logs — don't suppress them
+
+Recurring sidecar error logs (`Failed synchronously flushing traces:
+Kind(TimedOut)`, `connection reset by peer (os error 104)`, `The sidecar
+transport is closed. Reconnecting...`) are **symptoms** of a real flush-timeout
+/ transport-reset problem. Do NOT green a flaky test by silencing them
+(`DD_TRACE_LOG_LEVEL=off`, per-target `datadog_ipc=off,datadog_sidecar=off`, or
+grep-ignoring the line). Investigate why the flush times out / the IPC transport
+resets and fix that root cause; silencing leaves the defect to resurface
+elsewhere.
+
+## Merge gate: no `allow_failure` for known-flaky jobs
+
+Known-flaky jobs are NOT hidden with `allow_failure`. A single `merge-gate` job
+(last stage, `when: always`, no skip rules) runs `.gitlab/merge-gate.sh`, which
+collects every failed job across the parent + child pipelines via the GitLab API
+(`/pipelines/:id/bridges` → downstream ids → `/pipelines/:id/jobs?scope[]=failed`)
+and matches each against glob patterns in `.gitlab/flaky-jobs.txt`. A failure
+matching no pattern fails the gate; matches are treated as known-flaky and
+ignored. Branch protection requires only the single `merge-gate` check.
+
+To mark a job flaky, add its glob to `.gitlab/flaky-jobs.txt` (matrix jobs reduce
+to `base:*`, so any failing version marks all versions flaky; non-matrix jobs
+stay exact). Never reach for `allow_failure`.

.claude/ci/index.md

@@ -229,6 +229,17 @@ tooling/bin/download-artifacts --job-id 98765 -o /tmp/artifacts
 
 ---
 
+## Diagnosing failures & flakiness
+
+→ **[diagnosing-failures.md](diagnosing-failures.md)**
+Triage techniques to apply before proposing a fix: distinguishing a hang from a
+test failure (duration + `-v`), spotting a mutable-image republish behind an
+all-versions flip, seeing through retry-trait cascades, root-causing sidecar
+error logs instead of silencing them, and the `merge-gate` / `flaky-jobs.txt`
+mechanism (no `allow_failure`).
+
+---
+
 ## Building artifacts locally
 
 → **[building-locally.md](building-locally.md)**

.claude/ci/system-tests.md

@@ -300,7 +300,7 @@ cd /path/to/system-tests
   architecture in the filename. CI always sets this variable.
   Alternatively, build an arm64 `.so` with `make` (not `make static`)
   and use the `.so` override path — see
-  [../debugging-system-tests.md](../debugging-system-tests.md).
+  [../debugging/system-tests.md](../debugging/system-tests.md).
 
 - Artifacts are collected from `system-tests/logs_parametric/` and
   `system-tests/logs/` -- these directories are always uploaded

.claude/ci/windows-tests.md

@@ -86,3 +86,14 @@ docker stop -t 5 $CONTAINER && docker rm -f $CONTAINER
 
 - **Windows `switch-php` is a PowerShell script**, not the Linux bash version —
   it configures `C:\php\` rather than symlinking into `/usr/local/bin/`.
+
+- **CGI-SAPI `.phpt` tests can hang the whole job on PHP 7.2/7.3.** CGI tests
+  (`--GET--`/`--POST--`) run their SKIPIF/CLEAN under `php-cgi`, not the CLI. On
+  PHP 7.2/7.3, a ddtrace log line emitted during a CGI request blocks forever on
+  a synchronous buffered stderr write, because the Windows CGI stderr pipe is
+  never drained ("On Windows cgi skips stderr output"). Only tests that enable
+  ddtrace logging (e.g. raising `datadog.trace.log_level` above the default
+  `error`) hit it; the job then dies at the full timeout instead of a FAIL.
+  7.4+ are unaffected. When `windows test_c` fails on 7.2/7.3, check duration
+  before the FAIL lines — see
+  [diagnosing-failures.md](diagnosing-failures.md#determine-the-failure-mode-first-hang-vs-test-failure).

.claude/debugging.md .claude/debugging/appsec-integration.md.claude/debugging.md renamed to .claude/debugging/appsec-integration.md

@@ -8,7 +8,7 @@ The appsec Gradle integration tests run inside Docker containers. Use
 attach gdb to processes inside the container (PHP-FPM workers, sidecar).
 
 See [gdb.md](gdb.md) for gdb-specific instructions.
-See [ci/appsec-gradle-integration.md](ci/appsec-gradle-integration.md) for
+See [ci/appsec-gradle-integration.md](../ci/appsec-gradle-integration.md) for
 Gradle test details.
 
 ## Workflow