fix(agent): build/verify failure diagnosis — timeout classify, claude-binary Exec-format, failing-line surfacing

agent/Dockerfile

@@ -44,6 +44,15 @@ RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
+# Enable Corepack so ``yarn`` / ``pnpm`` resolve out of the box. Many target
+# repos (including ABCA itself) drive installs with ``yarn install`` via their
+# build command; without this, ``yarn`` is "command not found" (exit 127) and
+# the build-verification GATE runs inert — a green build is reported for a repo
+# we never actually built (live-caught 2026-06-29 dogfooding ABCA-on-ABCA: the
+# agent had to hand-build a ``~/bin/yarn`` shim every run). Corepack ships with
+# Node 24; ``enable`` installs the yarn/pnpm shims onto PATH.
+RUN corepack enable && corepack prepare yarn@stable --activate || corepack enable
+
 # Install Claude Code CLI (the Python SDK requires this binary)
 # Then update known vulnerable transitive packages where fixed versions exist.
 # Pinned 2.1.191 to match the CLI bundled by claude-agent-sdk 0.2.110 (see
@@ -52,10 +61,19 @@ RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
 # returned creds are cached until 5 min before the JSON's `Expiration`, so an
 # 8 h task re-assumes the 1 h-capped SessionRole before expiry. Older builds
 # only refreshed hourly on a timer, racing the role-chaining cap.
+# claude-code@2.1.x ships an error-shim at bin/claude.exe that its postinstall
+# (install.cjs) replaces with the platform-native ELF binary. That postinstall
+# can silently fall back (leaving the shim in place) — the shim then dies at
+# runtime with "Exec format error: 'claude'" even though the native binary is
+# present as an optional dep, just never wired up. Re-run install.cjs explicitly
+# after the install so the native binary is placed at build time, and hard-verify
+# it exec's (``claude --version`` fails the build if the shim is still on PATH).
 RUN npm install -g npm@latest && \
     npm install -g @anthropic-ai/claude-code@2.1.191 && \
     CLAUDE_NPM_ROOT="$(npm root -g)/@anthropic-ai/claude-code" && \
-    npm --prefix "${CLAUDE_NPM_ROOT}" update tar minimatch glob cross-spawn picomatch
+    npm --prefix "${CLAUDE_NPM_ROOT}" update tar minimatch glob cross-spawn picomatch && \
+    node "${CLAUDE_NPM_ROOT}/install.cjs" && \
+    claude --version
 
 # Install uv (fast Python package manager) — pinned for reproducibility
 COPY --from=ghcr.io/astral-sh/uv:0.11.14 /uv /usr/local/bin/uv

agent/src/entrypoint.py

@@ -31,7 +31,7 @@
     TaskResult,
     TokenUsage,
 )
-from pipeline import main, run_task  # noqa: F401
+from pipeline import main, run_task, run_task_from_payload  # noqa: F401
 from post_hooks import (  # noqa: F401
     ensure_committed,
     ensure_pr,

agent/src/pipeline.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import hashlib
+import inspect
 import os
 import sys
 import time
@@ -1287,6 +1288,129 @@ def _on_trace_truncated(max_bytes: int, first_dropped: int) -> None:
             raise
 
 
+#: Orchestrator payload keys that map to a differently-named ``run_task`` kwarg.
+#: The orchestrator emits ``prompt``/``model_id``; ``run_task`` calls them
+#: ``task_description``/``anthropic_model``. Everything else is a 1:1 name match.
+_PAYLOAD_KEY_ALIASES = {
+    "prompt": "task_description",
+    "model_id": "anthropic_model",
+}
+
+#: ``run_task`` kwargs that must be coerced to ``str`` — the orchestrator may
+#: emit them as numbers (issue_number, pr_number) and ``run_task`` types them as
+#: strings. ``max_turns`` is coerced to int. Absent keys are left to the
+#: ``run_task`` defaults.
+_PAYLOAD_STR_KEYS = frozenset({"issue_number", "pr_number"})
+
+#: Parameter names ``run_task`` accepts — computed once at import from the REAL
+#: signature (not inside the function, so patching ``run_task`` in tests can't
+#: shadow it). Any payload key not in this set is ignored, never passed through.
+_RUN_TASK_PARAMS = frozenset(inspect.signature(run_task).parameters)
+
+#: Orchestrator payload keys we KNOW about that ``run_task`` does not (yet)
+#: accept as a parameter. Dropping one of these is expected today (e.g.
+#: ``base_branch`` is consumed via ``hydrated_context``, not a run_task kwarg;
+#: ``github_token_secret_arn`` is resolved before this call), but a key that
+#: shows up here AND is silently dropped is exactly the "wired one side of an
+#: orchestrator→agent field, forgot the other" no-op that ABCA-487 was — so we
+#: WARN when we drop one, making a future contract gap visible instead of silent.
+#: Keys not in this set (genuinely foreign) are dropped quietly as before.
+_KNOWN_ORCHESTRATOR_KEYS = frozenset(
+    {
+        "build_command",
+        # ``lint_command``'s sibling: neither is a run_task param today (the build/
+        # lint commands are consumed via repo config, not passed through here), but
+        # listing both makes a future "wired build_command, forgot lint_command"
+        # contract gap WARN instead of drop silently (N4).
+        "lint_command",
+        "merge_branches",
+        "base_branch",
+        # NB: ``github_token_secret_arn`` is deliberately NOT listed (N3). It is
+        # ALWAYS in the payload and ALWAYS dropped here (resolved via the
+        # ``GITHUB_TOKEN_SECRET_ARN`` env in build_config, never a run_task
+        # param), so listing it would fire the known-key WARN on 100% of ECS
+        # boots — pure noise that dilutes the channel meant to surface genuine
+        # future "wired one side, forgot the other" gaps. It falls through as a
+        # quiet foreign-key drop instead.
+        # AgentCore's server.py exports task_started_at as TASK_STARTED_AT, which
+        # hooks._remaining_maxlifetime_s() uses to clip the Cedar HITL approval-gate
+        # maxLifetime. The ECS boot path bypasses server.py and does not (yet) set
+        # that env, so this key is dropped here — a silent AgentCore↔ECS HITL
+        # divergence (fail-open: the clip returns None, gate uses the task default).
+        # Listing it makes the drop WARN so the parity gap is visible until the ECS
+        # strategy sets TASK_STARTED_AT in containerEnv (tracked as a follow-up).
+        "task_started_at",
+    }
+)
+
+
+def run_task_from_payload(payload: dict) -> dict:
+    """Invoke :func:`run_task` from a full orchestrator payload dict.
+
+    The ECS compute path (``ecs-strategy.ts``) hands the agent the *entire*
+    orchestrator payload (via the #502 S3 pointer). Previously the ECS boot
+    command hand-listed a subset of ``run_task`` kwargs and silently dropped the
+    rest — most visibly ``channel_source``/``channel_metadata`` (no Linear/Jira
+    reactions or channel MCP on ECS — ABCA-487), plus ``build_command``,
+    ``cedar_policies``, ``base_branch``/``merge_branches``, ``attachments``, etc.
+
+    This maps the payload to ``run_task``'s real signature so no field can be
+    silently dropped again: rename the aliased keys, filter to parameters
+    ``run_task`` actually accepts (unknown keys are ignored, not passed as
+    ``**kwargs`` which ``run_task`` doesn't accept), and coerce the str/int
+    fields the orchestrator may emit as numbers. ``aws_region`` falls back to the
+    ``AWS_REGION`` env var when the payload omits it (the boot command used to
+    supply this explicitly).
+
+    Single source of truth + unit-testable, replacing the untestable inline
+    Python string that already drifted once.
+    """
+    kwargs: dict = {}
+    for key, value in (payload or {}).items():
+        target = _PAYLOAD_KEY_ALIASES.get(key, key)
+        if target not in _RUN_TASK_PARAMS:
+            # Not a run_task parameter — ignore. A KNOWN orchestrator key being
+            # dropped is expected today but worth a breadcrumb: if run_task ever
+            # grows a matching param, this WARN is where a "forgot to wire it
+            # through" no-op surfaces (N4 / ABCA-487 class). Foreign keys are
+            # dropped quietly.
+            if key in _KNOWN_ORCHESTRATOR_KEYS and value is not None:
+                log(
+                    "WARN",
+                    f"run_task_from_payload: dropping known orchestrator key '{key}' "
+                    f"(not a run_task parameter) — consumed elsewhere or not yet wired",
+                )
+            continue
+        if value is None:
+            continue  # let run_task's default apply
+        if target in _PAYLOAD_STR_KEYS:
+            value = str(value)
+        elif target == "max_turns":
+            # Defensive: a malformed max_turns must not crash the whole boot —
+            # drop it and let run_task's default apply (with a breadcrumb) rather
+            # than raise. Unlike the str keys above, this is the one field with a
+            # non-str coercion, so it also guards the surprising int() cases the
+            # orchestrator never emits but a hand-edited payload might: a bool
+            # (``int(True) == 1``) and a non-integral float (``int(3.9) == 3``)
+            # would both silently become a bogus turn count (N4).
+            if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+                log("WARN", f"run_task_from_payload: ignoring non-integer max_turns {value!r}")
+                continue
+            try:
+                coerced = int(value)
+            except (TypeError, ValueError):
+                log("WARN", f"run_task_from_payload: ignoring non-integer max_turns {value!r}")
+                continue
+            if isinstance(value, float) and coerced != value:
+                log("WARN", f"run_task_from_payload: ignoring non-integral max_turns {value!r}")
+                continue
+            value = coerced
+        kwargs[target] = value
+
+    kwargs.setdefault("aws_region", os.environ.get("AWS_REGION", ""))
+    return run_task(**kwargs)
+
+
 def main():
     config = get_config()
 

agent/src/shell.py

@@ -160,6 +160,83 @@ def _clean_env() -> dict[str, str]:
     return env
 
 
+# Substrings that mark a line as a real failure in build/test tool output — used
+# to pull the FAILING line out of the MIDDLE of a large interleaved parallel-DAG
+# log (a plain tail misses it). Lower-cased comparison; conservative so we don't
+# flood on benign "warning"/"0 errors" lines.
+_FAILURE_LINE_MARKERS = (
+    "fail ",  # jest "FAIL test/foo.test.ts", pytest "FAILED"
+    "failed",
+    "✕",
+    "✗",
+    "✖",
+    "●",  # jest failed-assertion bullet
+    "error ts",  # tsc "error TS2345:"
+    "error:",
+    "elifecycle",  # yarn/npm lifecycle failure
+    "npm err!",
+    "does not meet",  # jest coverage-threshold "global … does not meet threshold"
+    "coverage threshold",
+    'jest: "global"',
+    "not trusted",  # mise untrusted config
+    "no task ",  # mise "no task named"
+    "missing script",  # npm missing script
+    "assertionerror",
+    "traceback (most recent call last)",
+    "task failed",  # mise "[//pkg:task] ERROR task failed"
+)
+
+# Benign lines that CONTAIN a marker substring but are not failures — filtered so
+# the surfaced set stays signal. e.g. "0 errors", "--no-error-on-unmatched".
+_FAILURE_LINE_NOISE = (
+    "0 errors",
+    "0 failed",
+    "no error",
+    "--no-error",
+    "0 problems",
+    "may fail",  # advisory prose
+)
+
+# Cap surfaced failure lines so a genuinely huge red run (hundreds of failing
+# assertions) can't flood CloudWatch; the count + a tail still convey scale.
+_MAX_SURFACED_FAILURE_LINES = 40
+_FAILURE_TAIL_LINES = 15
+
+
+def _surface_failure_lines(stdout: str) -> list[str]:
+    """From a failed command's stdout, return the lines most likely to name the
+    cause: every failure-signature line (scanning the WHOLE output, not just the
+    tail — a parallel task DAG interleaves output so the red line is often in the
+    middle) followed by a trailing-context tail. Deduped, order-preserving,
+    capped. This is the fix for build-gate failures that a plain tail couldn't
+    explain (ABCA-662: the tail was a passing package's coverage table)."""
+    lines = stdout.strip().splitlines()
+    matched: list[str] = []
+    for ln in lines:
+        low = ln.lower()
+        if any(m in low for m in _FAILURE_LINE_MARKERS) and not any(
+            n in low for n in _FAILURE_LINE_NOISE
+        ):
+            matched.append(ln.rstrip())
+            if len(matched) >= _MAX_SURFACED_FAILURE_LINES:
+                matched.append("… (more failure lines truncated)")
+                break
+    tail = [ln.rstrip() for ln in lines[-_FAILURE_TAIL_LINES:]]
+    # Order: failure-signature lines first (the WHY), then the tail (context),
+    # dropping tail lines already surfaced as matches.
+    seen = set(matched)
+    out = list(matched)
+    if matched and tail:
+        out.append("--- (trailing output) ---")
+    for ln in tail:
+        if ln not in seen:
+            out.append(ln)
+            seen.add(ln)
+    # Fallback: nothing matched a failure marker (unknown tool) → just the tail,
+    # so we never log NOTHING on a failure.
+    return out or tail
+
+
 def run_cmd(
     cmd: list[str],
     label: str,
@@ -182,6 +259,24 @@ def run_cmd(
         if result.stderr:
             for line in result.stderr.strip().splitlines()[:20]:
                 log("CMD", f"  {line}")
+        # ALSO surface stdout on failure. Build/test tooling (jest, tsc, the mise
+        # task DAG) writes the ACTUAL failing-task error to STDOUT, not stderr —
+        # stderr often carries only the runner's plan echo. Logging stderr alone
+        # made build-gate failures undebuggable: a red ``mise run build`` showed
+        # every task STARTING but never WHICH one failed or why (ABCA-662).
+        #
+        # A plain tail is NOT enough for a PARALLEL task DAG: `mise run build`
+        # runs 4 packages concurrently and interleaves their output, so the
+        # failing task's error scrolls into the MIDDLE while the tail captures
+        # whatever finished LAST (e.g. a passing package's coverage table) —
+        # ABCA-662 follow-up: the tail showed a coverage table, not the red task.
+        # So FIRST scan the whole output for failure-signature lines and surface
+        # those (this is what names the failing sub-task), THEN a larger tail for
+        # trailing context. Redact — repo build output is untrusted.
+        if result.stdout:
+            surfaced = _surface_failure_lines(result.stdout)
+            for line in surfaced:
+                log("CMD", f"  {redact_secrets(line)}")
         if check:
             stderr_snippet = redact_secrets(result.stderr.strip()[:500]) if result.stderr else ""
             raise RuntimeError(f"{label} failed (exit {result.returncode}): {stderr_snippet}")