fix(agent): stop scoped-session S3 hang in baseline build + reap hung suites + early ACK

agent/Dockerfile

@@ -44,6 +44,15 @@ RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
+# Enable Corepack so ``yarn`` / ``pnpm`` resolve out of the box. Many target
+# repos (including ABCA itself) drive installs with ``yarn install`` via their
+# build command; without this, ``yarn`` is "command not found" (exit 127) and
+# the build-verification GATE runs inert — a green build is reported for a repo
+# we never actually built (live-caught 2026-06-29 dogfooding ABCA-on-ABCA: the
+# agent had to hand-build a ``~/bin/yarn`` shim every run). Corepack ships with
+# Node 24; ``enable`` installs the yarn/pnpm shims onto PATH.
+RUN corepack enable && corepack prepare yarn@stable --activate 2>/dev/null || corepack enable
+
 # Install Claude Code CLI (the Python SDK requires this binary)
 # Then update known vulnerable transitive packages where fixed versions exist.
 # Pinned 2.1.191 to match the CLI bundled by claude-agent-sdk 0.2.110 (see
@@ -52,10 +61,19 @@ RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
 # returned creds are cached until 5 min before the JSON's `Expiration`, so an
 # 8 h task re-assumes the 1 h-capped SessionRole before expiry. Older builds
 # only refreshed hourly on a timer, racing the role-chaining cap.
+# claude-code@2.1.x ships an error-shim at bin/claude.exe that its postinstall
+# (install.cjs) replaces with the platform-native ELF binary. That postinstall
+# can silently fall back (leaving the shim in place) — the shim then dies at
+# runtime with "Exec format error: 'claude'" even though the native binary is
+# present as an optional dep, just never wired up. Re-run install.cjs explicitly
+# after the install so the native binary is placed at build time, and hard-verify
+# it exec's (``claude --version`` fails the build if the shim is still on PATH).
 RUN npm install -g npm@latest && \
     npm install -g @anthropic-ai/claude-code@2.1.191 && \
     CLAUDE_NPM_ROOT="$(npm root -g)/@anthropic-ai/claude-code" && \
-    npm --prefix "${CLAUDE_NPM_ROOT}" update tar minimatch glob cross-spawn picomatch
+    npm --prefix "${CLAUDE_NPM_ROOT}" update tar minimatch glob cross-spawn picomatch && \
+    node "${CLAUDE_NPM_ROOT}/install.cjs" && \
+    claude --version
 
 # Install uv (fast Python package manager) — pinned for reproducibility
 COPY --from=ghcr.io/astral-sh/uv:0.11.14 /uv /usr/local/bin/uv

agent/pyproject.toml

@@ -81,6 +81,7 @@ dev = [
     "pytest",
     "pygments==2.20.0",
     "pytest-cov==7.1.0",
+    "pytest-timeout==2.4.0",  # per-test wall-clock cap: a single hung test (network/subprocess/Bedrock without its own timeout) must fail LOUDLY with a traceback, not silently burn the whole build-verify budget (ABCA-684/686: one hang stalled the baseline build past its 3600s ceiling)
     "vulture==2.16",  # dead-code detection (#282): unused functions/classes ruff F can't see
 ]
 
@@ -122,6 +123,18 @@ ignore = [
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 pythonpath = ["src"]
+# pytest-timeout: hard wall-clock cap PER TEST. A unit test that blocks (an
+# unmocked network/subprocess/Bedrock call with no timeout of its own) otherwise
+# hangs the whole `mise run build` until the 3600s build-verify ceiling kills it,
+# turning one flaky test into an un-diagnosable 60-min stall (ABCA-684/686). With
+# this, the offending test fails with a dumped stack in 120s and the suite moves on.
+# method="signal" (SIGALRM) actually ABORTS the hung test — pytest runs
+# single-threaded in the main thread here, so the alarm interrupts even a blocked
+# C-level/socket call. method="thread" only PRINTS the stack and cannot interrupt a
+# syscall-blocked test (proven live on ABCA-688: the thread method dumped the stack
+# but the build kept hanging ~55 min until the 3600s ceiling). signal is the real fix.
+timeout = 120
+timeout_method = "signal"
 
 [tool.coverage.run]
 branch = true

agent/src/entrypoint.py

@@ -31,7 +31,7 @@
     TaskResult,
     TokenUsage,
 )
-from pipeline import main, run_task  # noqa: F401
+from pipeline import main, run_task, run_task_from_payload  # noqa: F401
 from post_hooks import (  # noqa: F401
     ensure_committed,
     ensure_pr,

agent/src/pipeline.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import hashlib
+import inspect
 import os
 import subprocess
 import sys
@@ -822,28 +823,22 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None:
             if prompt_version:
                 os.environ["PROMPT_VERSION"] = prompt_version
 
-            # Setup repo (deterministic pre-hooks)
-            with task_span("task.repo_setup") as setup_span:
-                setup = setup_repo(config, progress=progress)
-                setup_span.set_attribute("build.before", setup.build_before)
-            progress.write_agent_milestone(
-                "repo_setup_complete",
-                f"branch={setup.branch} build_before={setup.build_before}",
-            )
-
-            system_prompt = build_system_prompt(config, setup, hc, system_prompt_overrides)
-
-            # Channel-specific MCP wiring. Must happen before
-            # discover_project_config so the scan picks up the file we just
-            # wrote. Resolve the per-channel access token from Secrets
-            # Manager *before* writing .mcp.json so the child SDK process
-            # inherits the env var that the MCP server entry references
-            # (${LINEAR_API_TOKEN} / ${JIRA_API_TOKEN}).
+            # ── Early ACK ────────────────────────────────────────────────────
+            # Acknowledge the task is picked up BEFORE the (potentially long)
+            # pre-agent baseline build in setup_repo(). On a large repo that
+            # baseline is minutes (up to the build-verify ceiling); posting the
+            # 👀 only *after* it left the issue looking dead for the whole phase
+            # (no reaction, comment, or state change). None of these calls needs
+            # the cloned repo — they act on the channel issue via its API token +
+            # issue id from channel metadata — so they belong before the build.
+            #
+            # Resolve the per-channel access token from Secrets Manager first
+            # (react_task_started/comment_task_started read the env var it sets).
+            # configure_channel_mcp DOES need setup.repo_dir, so it stays below.
             if config.channel_source == "linear":
                 resolve_linear_api_token(config.channel_metadata)
             elif config.channel_source == "jira":
                 resolve_jira_oauth_token(config.channel_metadata)
-            configure_channel_mcp(setup.repo_dir, config.channel_source)
 
             # 👀 on the Linear issue — acknowledges the task is picked up.
             # No-op for non-Linear tasks. Best-effort; failures are logged
@@ -862,6 +857,29 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None:
                 config.channel_metadata,
             )
 
+            # Setup repo (deterministic pre-hooks). A failure/timeout/OOM in the
+            # pre-agent baseline build raises here; it needs no local handler —
+            # the outer ``except Exception`` at the bottom of this ``try`` writes
+            # the task FAILED, swaps the 👀 (posted above) to ❌, and posts the
+            # failure comment. Posting the 👀 earlier is what makes the outer
+            # handler's ❌-swap actually visible for setup failures.
+            with task_span("task.repo_setup") as setup_span:
+                setup = setup_repo(config, progress=progress)
+                setup_span.set_attribute("build.before", setup.build_before)
+            progress.write_agent_milestone(
+                "repo_setup_complete",
+                f"branch={setup.branch} build_before={setup.build_before}",
+            )
+
+            system_prompt = build_system_prompt(config, setup, hc, system_prompt_overrides)
+
+            # Channel-specific MCP wiring. Must happen before
+            # discover_project_config so the scan picks up the file we just
+            # wrote — and after the clone, since it writes .mcp.json into the
+            # repo dir. (Token resolution + the 👀/start ACK moved earlier so
+            # the user gets immediate feedback; see the Early ACK block above.)
+            configure_channel_mcp(setup.repo_dir, config.channel_source)
+
             # Download attachments from S3 (version-pinned, integrity-verified)
             prepared_attachments: list = []
             if config.attachments:
@@ -1255,6 +1273,64 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None:
             raise
 
 
+#: Orchestrator payload keys that map to a differently-named ``run_task`` kwarg.
+#: The orchestrator emits ``prompt``/``model_id``; ``run_task`` calls them
+#: ``task_description``/``anthropic_model``. Everything else is a 1:1 name match.
+_PAYLOAD_KEY_ALIASES = {
+    "prompt": "task_description",
+    "model_id": "anthropic_model",
+}
+
+#: ``run_task`` kwargs that must be coerced to ``str`` — the orchestrator may
+#: emit them as numbers (issue_number, pr_number) and ``run_task`` types them as
+#: strings. ``max_turns`` is coerced to int. Absent keys are left to the
+#: ``run_task`` defaults.
+_PAYLOAD_STR_KEYS = frozenset({"issue_number", "pr_number"})
+
+#: Parameter names ``run_task`` accepts — computed once at import from the REAL
+#: signature (not inside the function, so patching ``run_task`` in tests can't
+#: shadow it). Any payload key not in this set is ignored, never passed through.
+_RUN_TASK_PARAMS = frozenset(inspect.signature(run_task).parameters)
+
+
+def run_task_from_payload(payload: dict) -> dict:
+    """Invoke :func:`run_task` from a full orchestrator payload dict.
+
+    The ECS compute path (``ecs-strategy.ts``) hands the agent the *entire*
+    orchestrator payload (via the #502 S3 pointer). Previously the ECS boot
+    command hand-listed a subset of ``run_task`` kwargs and silently dropped the
+    rest — most visibly ``channel_source``/``channel_metadata`` (no Linear/Jira
+    reactions or channel MCP on ECS — ABCA-487), plus ``build_command``,
+    ``cedar_policies``, ``base_branch``/``merge_branches``, ``attachments``, etc.
+
+    This maps the payload to ``run_task``'s real signature so no field can be
+    silently dropped again: rename the aliased keys, filter to parameters
+    ``run_task`` actually accepts (unknown keys are ignored, not passed as
+    ``**kwargs`` which ``run_task`` doesn't accept), and coerce the str/int
+    fields the orchestrator may emit as numbers. ``aws_region`` falls back to the
+    ``AWS_REGION`` env var when the payload omits it (the boot command used to
+    supply this explicitly).
+
+    Single source of truth + unit-testable, replacing the untestable inline
+    Python string that already drifted once.
+    """
+    kwargs: dict = {}
+    for key, value in (payload or {}).items():
+        target = _PAYLOAD_KEY_ALIASES.get(key, key)
+        if target not in _RUN_TASK_PARAMS:
+            continue  # not a run_task parameter — ignore (e.g. github_token_secret_arn)
+        if value is None:
+            continue  # let run_task's default apply
+        if target in _PAYLOAD_STR_KEYS:
+            value = str(value)
+        elif target == "max_turns":
+            value = int(value)
+        kwargs[target] = value
+
+    kwargs.setdefault("aws_region", os.environ.get("AWS_REGION", ""))
+    return run_task(**kwargs)
+
+
 def main():
     config = get_config()
 

agent/src/shell.py

@@ -160,6 +160,83 @@ def _clean_env() -> dict[str, str]:
     return env
 
 
+# Substrings that mark a line as a real failure in build/test tool output — used
+# to pull the FAILING line out of the MIDDLE of a large interleaved parallel-DAG
+# log (a plain tail misses it). Lower-cased comparison; conservative so we don't
+# flood on benign "warning"/"0 errors" lines.
+_FAILURE_LINE_MARKERS = (
+    "fail ",  # jest "FAIL test/foo.test.ts", pytest "FAILED"
+    "failed",
+    "✕",
+    "✗",
+    "✖",
+    "●",  # jest failed-assertion bullet
+    "error ts",  # tsc "error TS2345:"
+    "error:",
+    "elifecycle",  # yarn/npm lifecycle failure
+    "npm err!",
+    "does not meet",  # jest coverage-threshold "global … does not meet threshold"
+    "coverage threshold",
+    'jest: "global"',
+    "not trusted",  # mise untrusted config
+    "no task ",  # mise "no task named"
+    "missing script",  # npm missing script
+    "assertionerror",
+    "traceback (most recent call last)",
+    "task failed",  # mise "[//pkg:task] ERROR task failed"
+)
+
+# Benign lines that CONTAIN a marker substring but are not failures — filtered so
+# the surfaced set stays signal. e.g. "0 errors", "--no-error-on-unmatched".
+_FAILURE_LINE_NOISE = (
+    "0 errors",
+    "0 failed",
+    "no error",
+    "--no-error",
+    "0 problems",
+    "may fail",  # advisory prose
+)
+
+# Cap surfaced failure lines so a genuinely huge red run (hundreds of failing
+# assertions) can't flood CloudWatch; the count + a tail still convey scale.
+_MAX_SURFACED_FAILURE_LINES = 40
+_FAILURE_TAIL_LINES = 15
+
+
+def _surface_failure_lines(stdout: str) -> list[str]:
+    """From a failed command's stdout, return the lines most likely to name the
+    cause: every failure-signature line (scanning the WHOLE output, not just the
+    tail — a parallel task DAG interleaves output so the red line is often in the
+    middle) followed by a trailing-context tail. Deduped, order-preserving,
+    capped. This is the fix for build-gate failures that a plain tail couldn't
+    explain (ABCA-662: the tail was a passing package's coverage table)."""
+    lines = stdout.strip().splitlines()
+    matched: list[str] = []
+    for ln in lines:
+        low = ln.lower()
+        if any(m in low for m in _FAILURE_LINE_MARKERS) and not any(
+            n in low for n in _FAILURE_LINE_NOISE
+        ):
+            matched.append(ln.rstrip())
+            if len(matched) >= _MAX_SURFACED_FAILURE_LINES:
+                matched.append("… (more failure lines truncated)")
+                break
+    tail = [ln.rstrip() for ln in lines[-_FAILURE_TAIL_LINES:]]
+    # Order: failure-signature lines first (the WHY), then the tail (context),
+    # dropping tail lines already surfaced as matches.
+    seen = set(matched)
+    out = list(matched)
+    if matched and tail:
+        out.append("--- (trailing output) ---")
+    for ln in tail:
+        if ln not in seen:
+            out.append(ln)
+            seen.add(ln)
+    # Fallback: nothing matched a failure marker (unknown tool) → just the tail,
+    # so we never log NOTHING on a failure.
+    return out or tail
+
+
 def run_cmd(
     cmd: list[str],
     label: str,
@@ -182,6 +259,24 @@ def run_cmd(
         if result.stderr:
             for line in result.stderr.strip().splitlines()[:20]:
                 log("CMD", f"  {line}")
+        # ALSO surface stdout on failure. Build/test tooling (jest, tsc, the mise
+        # task DAG) writes the ACTUAL failing-task error to STDOUT, not stderr —
+        # stderr often carries only the runner's plan echo. Logging stderr alone
+        # made build-gate failures undebuggable: a red ``mise run build`` showed
+        # every task STARTING but never WHICH one failed or why (ABCA-662).
+        #
+        # A plain tail is NOT enough for a PARALLEL task DAG: `mise run build`
+        # runs 4 packages concurrently and interleaves their output, so the
+        # failing task's error scrolls into the MIDDLE while the tail captures
+        # whatever finished LAST (e.g. a passing package's coverage table) —
+        # ABCA-662 follow-up: the tail showed a coverage table, not the red task.
+        # So FIRST scan the whole output for failure-signature lines and surface
+        # those (this is what names the failing sub-task), THEN a larger tail for
+        # trailing context. Redact — repo build output is untrusted.
+        if result.stdout:
+            surfaced = _surface_failure_lines(result.stdout)
+            for line in surfaced:
+                log("CMD", f"  {redact_secrets(line)}")
         if check:
             stderr_snippet = redact_secrets(result.stderr.strip()[:500]) if result.stderr else ""
             raise RuntimeError(f"{label} failed (exit {result.returncode}): {stderr_snippet}")