Skip to content

Commit 49c3db3

Browse files
TrecekTrecek
authored
Implementation Plan: T5-P6-A10-WP3 — Make NDJSON Vocabulary Drift Visible in Per-Session Diagnostics (#4146)
## Summary Wire the existing `seen_ndjson_unknown_event_count` and `seen_ndjson_unknown_item_count` counters — currently dead-ending on `ClaudeSessionResult` — through `SkillResult` (via a frozen `NdjsonDriftOutcome` bundle) into `flush_session_log`, so they appear in `summary.json`, `sessions.jsonl`, and (when non-zero) `anomalies.jsonl`. Add a doctor check (`codex_ndjson_drift`, Check 35) that scans `sessions.jsonl` for codex entries with non-zero drift counters and returns WARNING with the affected count. ## Implementation Plan Plan file: `/home/talon/projects/autoskillit-runs/impl-20260628-121003-915180/.autoskillit/temp/make-plan/t5_p6_a10_wp3_ndjson_drift_visibility_plan_2026-06-28_121500.md` Closes #4046 🤖 Generated with [Claude Code](https://claude.com/claude-code) via AutoSkillit <!-- autoskillit:pipeline-signature steps=prepare_pr,run_arch_lenses,compose_pr,annotate_pr_diff,review_pr --> ## Token Usage Summary | Step | Model | count | uncached | output | cache_read | peak_ctx | turns | cache_write | time | |------|-------|-------|----------|--------|------------|----------|-------|-------------|------| | plan* | opus[1m] | 2 | 6.6k | 30.7k | 2.3M | 135.4k | 81 | 151.9k | 17m 40s | | verify* | sonnet | 1 | 132 | 15.3k | 829.1k | 63.2k | 42 | 44.7k | 7m 23s | | implement* | MiniMax-M3 | 1 | 106.1k | 18.9k | 5.4M | 0 | 137 | 0 | 6m 7s | | fix* | sonnet | 1 | 430 | 43.8k | 5.7M | 147.0k | 152 | 128.8k | 22m 52s | | audit_impl* | sonnet | 1 | 817 | 18.6k | 392.4k | 71.0k | 23 | 54.5k | 7m 48s | | prepare_pr* | MiniMax-M3 | 1 | 46.0k | 3.4k | 150.1k | 0 | 12 | 0 | 58s | | compose_pr* | MiniMax-M3 | 1 | 36.6k | 1.3k | 140.7k | 0 | 11 | 0 | 36s | | **Total** | | | 196.6k | 131.9k | 14.9M | 147.0k | | 379.9k | 1h 3m | \* *Step used a non-Anthropic provider; caching behavior may differ.* ## Token Efficiency | Step | LoC Changed | cache_read/LoC | cache_write/LoC | output/LoC | |------|-------------|----------------|-----------------|------------| | plan | 0 | — | — | — | | verify | 0 | — | — | — | | implement | 306 | 17566.6 | 0.0 | 61.9 | | fix | 41 | 139541.6 | 3140.9 | 1067.1 | | audit_impl | 0 | — | — | — | | prepare_pr | 0 | — | — | — | | compose_pr | 0 | — | — | — | | **Total** | **347** | 42947.0 | 1094.9 | 380.1 | ## Model Usage Breakdown | Model | steps | uncached | output | cache_read | cache_write | time | |-------|-------|----------|--------|------------|-------------|------| | opus[1m] | 1 | 6.6k | 30.7k | 2.3M | 151.9k | 17m 40s | | sonnet | 3 | 1.4k | 77.6k | 6.9M | 228.0k | 38m 3s | | MiniMax-M3 | 3 | 188.7k | 23.6k | 5.7M | 0 | 7m 41s |
1 parent de19aa6 commit 49c3db3

19 files changed

Lines changed: 318 additions & 14 deletions

‎docs/installation.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ projects need.
6767

6868
autoskillit doctor
6969

70-
Doctor runs 33 checks (23 numbered + 5 lettered sub-checks: `2b`, `2c`, `2d`, `4b`, `7b` + 5 backend runtime probes, checks 30–34); up to 39 with the fleet feature enabled.
70+
Doctor runs 34 checks (23 numbered + 5 lettered sub-checks: `2b`, `2c`, `2d`, `4b`, `7b` + 6 backend runtime probes, checks 30–35); up to 40 with the fleet feature enabled.
7171
Enumerated by `run_doctor` in `src/autoskillit/cli/doctor/__init__.py`:
7272

7373
| # | Check | What it verifies |

‎src/autoskillit/cli/doctor/AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# doctor/
22

3-
Diagnostic health checks for the autoskillit installation (34 checks).
3+
Diagnostic health checks for the autoskillit installation (35 checks).
44

55
## Files
66

‎src/autoskillit/cli/doctor/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
_check_claude_process_state_breakdown,
6464
_check_cli_conformance_probes,
6565
_check_codex_graduation,
66+
_check_codex_ndjson_drift,
6667
_check_codex_version,
6768
_check_quota_cache_schema,
6869
_check_script_binary,
@@ -221,6 +222,9 @@ def run_doctor(*, output_json: bool = False) -> None:
221222
# Check 34: CLI config-acceptance probe
222223
results.append(_check_cli_conformance_probes(backend=_backend))
223224

225+
# Check 35: Codex NDJSON vocabulary drift
226+
results.append(_check_codex_ndjson_drift(log_dir=cfg.linux_tracing.log_dir, backend=_backend))
227+
224228
# Output
225229
if output_json:
226230
print(

‎src/autoskillit/cli/doctor/_doctor_runtime.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,3 +317,47 @@ def _check_cli_conformance_probes(*, backend: CodingAgentBackend | None = None)
317317
check_name,
318318
f"{cli_binary} rejected config probe (exit {result.returncode})",
319319
)
320+
321+
322+
def _check_codex_ndjson_drift(
323+
*,
324+
log_dir: str = "",
325+
backend: CodingAgentBackend | None = None,
326+
) -> DoctorResult:
327+
check_name = "codex_ndjson_drift"
328+
if backend is None:
329+
return DoctorResult(Severity.OK, check_name, "Skipped (no codex backend)")
330+
backend_name = backend.name
331+
log_root = Path(log_dir).expanduser() if log_dir else default_log_dir()
332+
sessions_path = log_root / "sessions.jsonl"
333+
if not sessions_path.exists():
334+
return DoctorResult(Severity.OK, check_name, "No sessions.jsonl found")
335+
336+
affected = 0
337+
try:
338+
for line in sessions_path.read_text().splitlines():
339+
line = line.strip()
340+
if not line:
341+
continue
342+
try:
343+
entry = json.loads(line)
344+
except (json.JSONDecodeError, ValueError):
345+
continue
346+
if entry.get("backend") != backend_name:
347+
continue
348+
if (
349+
entry.get("ndjson_unknown_event_count", 0) > 0
350+
or entry.get("ndjson_unknown_item_count", 0) > 0
351+
):
352+
affected += 1
353+
except OSError:
354+
return DoctorResult(Severity.OK, check_name, "Could not read sessions.jsonl")
355+
356+
if affected > 0:
357+
return DoctorResult(
358+
Severity.WARNING,
359+
check_name,
360+
f"{affected} codex session(s) contain unknown NDJSON event types"
361+
" — parser vocabulary may be stale",
362+
)
363+
return DoctorResult(Severity.OK, check_name, "No NDJSON vocabulary drift detected")

‎src/autoskillit/core/__init__.pyi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ from .types import ModelIdentity as ModelIdentity
296296
from .types import ModelTotalEntry as ModelTotalEntry
297297
from .types import ModelTranslation as ModelTranslation
298298
from .types import NamedResume as NamedResume
299+
from .types import NdjsonDriftOutcome as NdjsonDriftOutcome
299300
from .types import NoResume as NoResume
300301
from .types import OutputFormat as OutputFormat
301302
from .types import OutputPatternResolver as OutputPatternResolver

‎src/autoskillit/core/types/_type_results.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"ProviderOutcome",
3434
"InfraOutcome",
3535
"ApiRetryOutcome",
36+
"NdjsonDriftOutcome",
3637
"SkillResult",
3738
"CleanupResult",
3839
"CloneSuccessResult",
@@ -265,6 +266,21 @@ class ContaminationOutcome:
265266
subtype: str = ""
266267

267268

269+
@dataclass(frozen=True, slots=True)
270+
class NdjsonDriftOutcome:
271+
"""NDJSON parser vocabulary drift counters.
272+
273+
Aggregates unknown event and item counts emitted by Codex NDJSON parsing
274+
when the parser encounters event/item types not in its known vocabulary.
275+
Surfaced through ``SkillResult.ndjson_drift`` and propagated into
276+
``summary.json``, ``sessions.jsonl``, and ``anomalies.jsonl`` for
277+
diagnostics and the doctor ``codex_ndjson_drift`` check.
278+
"""
279+
280+
unknown_event_count: int = 0
281+
unknown_item_count: int = 0
282+
283+
268284
@dataclass(frozen=True, slots=True)
269285
class WriteEvidence:
270286
"""Bundled write evidence signals — either explicitly constructed or absent."""
@@ -334,6 +350,8 @@ class SkillResult:
334350
"""API retry event accumulation bundle."""
335351
contamination: ContaminationOutcome = field(default_factory=ContaminationOutcome)
336352
"""Pre-contamination context bundle — populated only when clone_guard fires."""
353+
ndjson_drift: NdjsonDriftOutcome = field(default_factory=NdjsonDriftOutcome)
354+
"""NDJSON parser vocabulary drift counters — populated by Codex sessions."""
337355
completion_required: bool = False
338356

339357
def to_json(self) -> str:
@@ -369,6 +387,8 @@ def to_json(self) -> str:
369387
"api_retry_exhausted": self.api_retry.exhausted,
370388
"pre_contamination_retry_reason": self.contamination.retry_reason,
371389
"pre_contamination_subtype": self.contamination.subtype,
390+
"ndjson_unknown_event_count": self.ndjson_drift.unknown_event_count,
391+
"ndjson_unknown_item_count": self.ndjson_drift.unknown_item_count,
372392
}
373393
if self.worktree_path is not None:
374394
data["worktree_path"] = self.worktree_path
@@ -606,4 +626,6 @@ class SessionIndexEntry(TypedDict):
606626
api_retry_exhausted: bool
607627
api_retry_last_error: str
608628
api_retry_last_status: int | None
629+
ndjson_unknown_event_count: int
630+
ndjson_unknown_item_count: int
609631
schema_version: int

‎src/autoskillit/execution/anomaly_detection.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class AnomalyKind(StrEnum):
2727
THINKING_ONLY_FINAL_TURN = "thinking_only_final_turn"
2828
API_RETRY_EXHAUSTION = "api_retry_exhaustion"
2929
MODEL_DRIFT = "model_drift"
30+
NDJSON_DRIFT = "ndjson_drift"
3031

3132

3233
class AnomalySeverity(StrEnum):

‎src/autoskillit/execution/headless/_headless_execute.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,8 @@ async def _execute_claude_headless(
538538
api_retry_last_error=skill_result.api_retry.last_error,
539539
api_retry_last_status=skill_result.api_retry.last_status,
540540
api_retry_exhausted=skill_result.api_retry.exhausted,
541+
ndjson_unknown_event_count=skill_result.ndjson_drift.unknown_event_count,
542+
ndjson_unknown_item_count=skill_result.ndjson_drift.unknown_item_count,
541543
write_path_warnings=skill_result.write_path_warnings,
542544
write_call_count=skill_result.evidence.write_call_count,
543545
fs_writes_detected=skill_result.evidence.fs_writes_detected,

‎src/autoskillit/execution/headless/_headless_result.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
InfraExitCategory,
1515
InfraOutcome,
1616
KillReason,
17+
NdjsonDriftOutcome,
1718
ProviderOutcome,
1819
RetryReason,
1920
SessionOutcome,
@@ -135,6 +136,10 @@ def _make_terminated_result(
135136
provider=ProviderOutcome(provider_used=provider_used, fallback_activated=False),
136137
infra=infra,
137138
api_retry=api_retry,
139+
ndjson_drift=NdjsonDriftOutcome(
140+
unknown_event_count=session.seen_ndjson_unknown_event_count,
141+
unknown_item_count=session.seen_ndjson_unknown_item_count,
142+
),
138143
)
139144

140145

@@ -727,6 +732,10 @@ def _build_skill_result(
727732
provider=ProviderOutcome(provider_used=provider_used, fallback_activated=False),
728733
infra=InfraOutcome(exit_category=infra_category.value),
729734
api_retry=api_retry,
735+
ndjson_drift=NdjsonDriftOutcome(
736+
unknown_event_count=session.seen_ndjson_unknown_event_count,
737+
unknown_item_count=session.seen_ndjson_unknown_item_count,
738+
),
730739
completion_required=completion_required,
731740
)
732741
if path_contamination:

‎src/autoskillit/execution/session_log.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,8 @@ def flush_session_log(
147147
api_retry_last_error: str = "",
148148
api_retry_last_status: int | None = None,
149149
api_retry_exhausted: bool = False,
150+
ndjson_unknown_event_count: int = 0,
151+
ndjson_unknown_item_count: int = 0,
150152
versions: dict[str, Any] | None = None,
151153
model_identity: ModelIdentity = ModelIdentity.unknown(),
152154
max_sessions: int | None = None,
@@ -326,6 +328,30 @@ def flush_session_log(
326328
}
327329
)
328330

331+
if ndjson_unknown_event_count > 0 or ndjson_unknown_item_count > 0:
332+
from autoskillit.execution.anomaly_detection import (
333+
OUTCOME_ANOMALY_PID_SENTINEL,
334+
OUTCOME_ANOMALY_SEQ_SENTINEL,
335+
AnomalyKind,
336+
AnomalySeverity,
337+
)
338+
339+
anomalies.append(
340+
{
341+
"ts": datetime.now(UTC).isoformat(),
342+
"seq": OUTCOME_ANOMALY_SEQ_SENTINEL,
343+
"event": "anomaly",
344+
"kind": str(AnomalyKind.NDJSON_DRIFT),
345+
"severity": str(AnomalySeverity.WARNING),
346+
"pid": OUTCOME_ANOMALY_PID_SENTINEL,
347+
"detail": {
348+
"ndjson_unknown_event_count": ndjson_unknown_event_count,
349+
"ndjson_unknown_item_count": ndjson_unknown_item_count,
350+
},
351+
"snapshot": {},
352+
}
353+
)
354+
329355
effective_model_id = model_identity.effective_model or _primary_model_identifier(token_usage)
330356
_observed = _primary_model_identifier(token_usage) if token_usage else ""
331357
anomalies.extend(
@@ -427,6 +453,8 @@ def flush_session_log(
427453
"api_retry_last_error": api_retry_last_error,
428454
"api_retry_last_status": api_retry_last_status,
429455
"api_retry_exhausted": api_retry_exhausted,
456+
"ndjson_unknown_event_count": ndjson_unknown_event_count,
457+
"ndjson_unknown_item_count": ndjson_unknown_item_count,
430458
}
431459
if versions is not None:
432460
summary["versions"] = {
@@ -555,6 +583,8 @@ def flush_session_log(
555583
"api_retry_exhausted": api_retry_exhausted,
556584
"api_retry_last_error": api_retry_last_error,
557585
"api_retry_last_status": api_retry_last_status,
586+
"ndjson_unknown_event_count": ndjson_unknown_event_count,
587+
"ndjson_unknown_item_count": ndjson_unknown_item_count,
558588
"model_identifier": effective_model_id,
559589
"configured_model": model_identity.configured_model,
560590
"profile_name": model_identity.profile_name,

0 commit comments

Comments
 (0)