T5-P6-A10-WP1 Unknown-NDJSON-Vocabulary Counters (#4120)

## Summary

Add `ndjson_unknown_event_count` and `ndjson_unknown_item_count` integer
counters to the Codex NDJSON parsing pipeline. The counters originate at
two parser layers (`_CodexParseAccumulator` for batch parsing,
`CodexStreamParser` for streaming), flow through
`AgentSessionResult.raw`, and land on `ClaudeSessionResult` via
`_adapt_agent_result`. This makes unknown-vocabulary occurrences
machine-readable for downstream WPs (P6-A10-WP2+) without log scraping.

## Implementation Plan

Plan file:
`/home/talon/projects/autoskillit-runs/impl-20260626-235325-737964/.autoskillit/temp/make-plan/t5_p6_a10_wp1_unknown_ndjson_counters_plan_2026-06-26_235500.md`

Closes #4045

🤖 Generated with [Claude Code](https://claude.com/claude-code) via
AutoSkillit
<!-- autoskillit:pipeline-signature
steps=prepare_pr,run_arch_lenses,compose_pr,annotate_pr_diff,review_pr
-->

## Token Usage Summary

| Step | Model | count | uncached | output | cache_read | peak_ctx |
turns | cache_write | time |

|------|-------|-------|----------|--------|------------|----------|-------|-------------|------|
| plan* | opus[1m] | 1 | 59 | 16.6k | 892.4k | 98.0k | 43 | 84.0k | 14m
40s |
| verify* | sonnet | 1 | 92 | 9.9k | 517.6k | 59.6k | 29 | 41.1k | 4m
21s |
| implement* | MiniMax-M3 | 1 | 97.8k | 11.5k | 2.5M | 0 | 97 | 0 | 4m
50s |
| audit_impl* | sonnet | 1 | 44 | 13.6k | 165.6k | 44.3k | 13 | 42.1k |
7m 33s |
| prepare_pr* | MiniMax-M3 | 1 | 42.6k | 1.9k | 112.4k | 43.9k | 11 | 0
| 37s |
| compose_pr* | MiniMax-M3 | 1 | 36.0k | 2.1k | 142.6k | 0 | 11 | 0 |
35s |
| **Total** | | | 176.6k | 55.6k | 4.3M | 98.0k | | 167.1k | 32m 37s |

\* *Step used a non-Anthropic provider; caching behavior may differ.*

## Token Efficiency

| Step | LoC Changed | cache_read/LoC | cache_write/LoC | output/LoC |
|------|-------------|----------------|-----------------|------------|
| plan | 0 | — | — | — |
| verify | 0 | — | — | — |
| implement | 129 | 19020.4 | 0.0 | 89.1 |
| audit_impl | 0 | — | — | — |
| prepare_pr | 0 | — | — | — |
| compose_pr | 0 | — | — | — |
| **Total** | **129** | 33211.0 | 1295.6 | 431.2 |

## Model Usage Breakdown

| Model | steps | uncached | output | cache_read | cache_write | time |
|-------|-------|----------|--------|------------|-------------|------|
| opus[1m] | 1 | 59 | 16.6k | 892.4k | 84.0k | 14m 40s |
| sonnet | 2 | 136 | 23.5k | 683.2k | 83.1k | 11m 54s |
| MiniMax-M3 | 3 | 176.4k | 15.5k | 2.7M | 0 | 6m 2s |

Author: Trecek

src/autoskillit/execution/backends/_codex_parse.py

@@ -37,6 +37,8 @@ class _CodexParseAccumulator:
     success: bool = False
     error_message: str = ""
     error_code: str = ""
+    ndjson_unknown_event_count: int = 0
+    ndjson_unknown_item_count: int = 0
 
 
 def _scan_codex_ndjson(stdout: str) -> _CodexParseAccumulator:
@@ -56,6 +58,7 @@ def _scan_codex_ndjson(stdout: str) -> _CodexParseAccumulator:
         event_type = CodexEventType.from_ndjson(obj.get("type", ""))
         if event_type == CodexEventType.UNKNOWN:
             logger.warning("codex_ndjson_unknown_event_type", type=obj.get("type", ""))
+            acc.ndjson_unknown_event_count += 1
             continue
         if event_type == CodexEventType.THREAD_STARTED:
             acc.session_id = obj.get("thread_id", "")
@@ -103,6 +106,7 @@ def _scan_codex_ndjson(stdout: str) -> _CodexParseAccumulator:
                 continue
             elif item_type == CodexItemType.UNKNOWN:
                 logger.warning("codex_ndjson_unknown_item_type", item_type=item.get("type", ""))
+                acc.ndjson_unknown_item_count += 1
                 continue
         elif event_type == CodexEventType.TURN_COMPLETED:
             usage = obj.get("usage")
@@ -186,6 +190,8 @@ def parse_stdout(self, stdout: str, *, exit_code: int = 0) -> AgentSessionResult
                 "mcp_tool_calls": acc.mcp_tool_calls,
                 "file_changes": acc.file_changes,
                 "error_code": acc.error_code,
+                "ndjson_unknown_event_count": acc.ndjson_unknown_event_count,
+                "ndjson_unknown_item_count": acc.ndjson_unknown_item_count,
             },
         )
 
@@ -200,6 +206,8 @@ class CodexStreamParser:
 
     completion_marker: str = ""
     _saw_marker: bool = field(default=False, init=False, repr=False)
+    ndjson_unknown_event_count: int = field(default=0, init=False, repr=False)
+    ndjson_unknown_item_count: int = field(default=0, init=False, repr=False)
 
     def _check_marker_text(self, text: str) -> None:
         if self.completion_marker and _marker_is_standalone(text, self.completion_marker):
@@ -308,6 +316,7 @@ def parse_line(self, line: str) -> SessionEvent | None:
                     has_marker=False,
                 )
 
+            self.ndjson_unknown_item_count += 1
             logger.warning("codex_ndjson_unknown_item_type", item_type=item.get("type", ""))
             return SessionEvent(
                 kind=BackendEventKind.IGNORED,
@@ -362,6 +371,7 @@ def parse_line(self, line: str) -> SessionEvent | None:
                 has_marker=False,
             )
 
+        self.ndjson_unknown_event_count += 1
         logger.warning("codex_ndjson_unknown_event_type", type=obj.get("type", ""))
         return SessionEvent(
             kind=BackendEventKind.IGNORED,

src/autoskillit/execution/headless/_headless_evidence.py

@@ -125,6 +125,9 @@ def _adapt_agent_result(agent_result: AgentSessionResult) -> ClaudeSessionResult
 
     assistant_messages: list[str] = raw.get("agent_messages", [])
 
+    seen_ndjson_unknown_event_count: int = raw.get("ndjson_unknown_event_count", 0)
+    seen_ndjson_unknown_item_count: int = raw.get("ndjson_unknown_item_count", 0)
+
     return ClaudeSessionResult(
         subtype=subtype,
         is_error=is_error,
@@ -139,6 +142,8 @@ def _adapt_agent_result(agent_result: AgentSessionResult) -> ClaudeSessionResult
         has_thinking_only_turn=False,
         seen_block_types=frozenset(),
         api_error_status=api_error_status,
+        seen_ndjson_unknown_event_count=seen_ndjson_unknown_event_count,
+        seen_ndjson_unknown_item_count=seen_ndjson_unknown_item_count,
     )
 
 

src/autoskillit/execution/session/_session_model.py

@@ -74,6 +74,8 @@ class ClaudeSessionResult:
     api_retry_last_status: int | None = None
     api_retry_exhausted: bool = False
     api_error_status: int | None = None
+    seen_ndjson_unknown_event_count: int = 0
+    seen_ndjson_unknown_item_count: int = 0
 
     def __post_init__(self) -> None:
         if not isinstance(self.result, str):

tests/execution/backends/test_codex_result_parser.py

@@ -101,6 +101,8 @@ def test_scan_empty_stdout_returns_empty_accumulator(self) -> None:
         assert acc.success is False
         assert acc.error_message == ""
         assert acc.error_code == ""
+        assert acc.ndjson_unknown_event_count == 0
+        assert acc.ndjson_unknown_item_count == 0
 
     def test_scan_thread_started_populates_session_id(self) -> None:
         acc = _scan_codex_ndjson(_thread_started_line("t1"))
@@ -182,6 +184,16 @@ def test_scan_turn_failed_after_completed_overrides_success(self) -> None:
         assert acc.success is False
         assert acc.error_message == "error after success"
 
+    def test_scan_codex_ndjson_accumulator_counters_increment_independently(self) -> None:
+        unknown_event_line = json.dumps({"type": "brand_new_event_type", "data": 1})
+        unknown_item_line = json.dumps(
+            {"type": "item.completed", "item": {"type": "brand_new_item_type", "data": 1}}
+        )
+        ndjson = unknown_event_line + "\n" + unknown_item_line
+        acc = _scan_codex_ndjson(ndjson)
+        assert acc.ndjson_unknown_event_count == 1
+        assert acc.ndjson_unknown_item_count == 1
+
 
 class TestCodexResultParserStdout:
     def test_parse_stdout_empty_returns_error(self) -> None:
@@ -289,6 +301,8 @@ def test_parse_stdout_raw_contains_all_fields(self) -> None:
         assert raw["mcp_tool_calls"][0]["tool_name"] == "mcp_tool"
         assert raw["file_changes"] == ["/path/file.py"]
         assert raw["error_code"] == ""
+        assert raw["ndjson_unknown_event_count"] == 0
+        assert raw["ndjson_unknown_item_count"] == 0
 
     def test_parse_result_completion_events_yield_success(self) -> None:
         parser = CodexResultParser()
@@ -759,6 +773,31 @@ def test_agent_message_populates_agent_messages(self) -> None:
         assert result.raw["agent_messages"] == ["hello from v0.136"]
         assert result.output == "hello from v0.136"
 
+    def test_ndjson_unknown_counters_in_raw_on_success(self) -> None:
+        """Normal success NDJSON stream → both counter keys present with value 0."""
+        ndjson = "\n".join(
+            [
+                _thread_started_line("s1"),
+                _item_completed_message_line("hello"),
+                _turn_completed_line({"input_tokens": 1, "output_tokens": 1}),
+            ]
+        )
+        result = CodexResultParser().parse_stdout(ndjson)
+        assert result.raw["ndjson_unknown_event_count"] == 0
+        assert result.raw["ndjson_unknown_item_count"] == 0
+
+    def test_ndjson_unknown_counters_in_raw_on_failure(self) -> None:
+        """Failure NDJSON stream → both counter keys present with value 0."""
+        ndjson = "\n".join(
+            [
+                _thread_started_line("s1"),
+                _turn_failed_line("something broke"),
+            ]
+        )
+        result = CodexResultParser().parse_stdout(ndjson)
+        assert result.raw["ndjson_unknown_event_count"] == 0
+        assert result.raw["ndjson_unknown_item_count"] == 0
+
     def test_command_execution_populates_command_executions(self) -> None:
         ndjson = "\n".join(
             [

tests/execution/backends/test_codex_stream_parser.py

@@ -605,3 +605,36 @@ def test_unknown_event_type_still_warns_after_item_updated_guard(self) -> None:
             log["event"] == "codex_ndjson_unknown_event_type" and log["log_level"] == "warning"
             for log in cap
         )
+
+
+class TestCodexUnknownCounters:
+    def test_stream_parser_event_counter_increments_on_unknown_event_type(self) -> None:
+        parser = CodexStreamParser()
+        line = json.dumps({"type": "brand_new_event_type", "data": 1})
+        parser.parse_line(line)
+        assert parser.ndjson_unknown_event_count == 1
+        assert parser.ndjson_unknown_item_count == 0
+
+    def test_stream_parser_item_counter_increments_on_unknown_item_type(self) -> None:
+        parser = CodexStreamParser()
+        line = json.dumps(
+            {"type": "item.completed", "item": {"type": "brand_new_item_type", "data": 1}}
+        )
+        parser.parse_line(line)
+        assert parser.ndjson_unknown_item_count == 1
+        assert parser.ndjson_unknown_event_count == 0
+
+    def test_counters_stay_zero_for_known_types(self) -> None:
+        parser = CodexStreamParser()
+        parser.parse_line(json.dumps({"type": "thread.started", "thread_id": "t1"}))
+        parser.parse_line(
+            json.dumps(
+                {
+                    "type": "item.completed",
+                    "item": {"type": "message", "content": [{"type": "text", "text": "hi"}]},
+                }
+            )
+        )
+        parser.parse_line(json.dumps({"type": "turn.completed", "usage": {}}))
+        assert parser.ndjson_unknown_event_count == 0
+        assert parser.ndjson_unknown_item_count == 0

tests/execution/test_adapt_agent_result.py

@@ -58,6 +58,8 @@ def test_hardcoded_thinking_defaults() -> None:
     result = _adapt_agent_result(_make_agent_result())
     assert result.has_thinking_only_turn is False
     assert result.seen_block_types == frozenset()
+    assert result.seen_ndjson_unknown_event_count == 0
+    assert result.seen_ndjson_unknown_item_count == 0
 
 
 def test_unknown_subtype_maps_to_unknown() -> None:
@@ -259,6 +261,8 @@ def test_unused_agent_fields_do_not_affect_output() -> None:
     assert base.stop_reasons == varied.stop_reasons
     assert base.has_thinking_only_turn == varied.has_thinking_only_turn
     assert base.seen_block_types == varied.seen_block_types
+    assert base.seen_ndjson_unknown_event_count == varied.seen_ndjson_unknown_event_count
+    assert base.seen_ndjson_unknown_item_count == varied.seen_ndjson_unknown_item_count
 
     assert not hasattr(base, "exit_code")
     assert not hasattr(base, "backend_name")
@@ -296,6 +300,8 @@ def test_full_round_trip_all_fields() -> None:
             "mcp_tool_calls": [{"name": "read", "path": "/tmp"}],
             "file_changes": ["main.py"],
             "agent_messages": ["I updated the file."],
+            "ndjson_unknown_event_count": 5,
+            "ndjson_unknown_item_count": 2,
         },
     )
     result = _adapt_agent_result(agent)
@@ -317,6 +323,8 @@ def test_full_round_trip_all_fields() -> None:
     assert result.has_thinking_only_turn is False
     assert result.seen_block_types == frozenset()
     assert result.api_error_status is None
+    assert result.seen_ndjson_unknown_event_count == 5
+    assert result.seen_ndjson_unknown_item_count == 2
 
 
 def test_context_exhaustion_from_code_field() -> None:
@@ -396,6 +404,24 @@ def test_classify_infra_exit_rate_limited_via_adapted_error_code() -> None:
     assert exit_category == InfraExitCategory.RATE_LIMITED
 
 
+def test_ndjson_unknown_event_count_round_trip() -> None:
+    result = _adapt_agent_result(_make_agent_result(raw={"ndjson_unknown_event_count": 3}))
+    assert result.seen_ndjson_unknown_event_count == 3
+
+
+def test_ndjson_unknown_item_count_round_trip() -> None:
+    result = _adapt_agent_result(_make_agent_result(raw={"ndjson_unknown_item_count": 7}))
+    assert result.seen_ndjson_unknown_item_count == 7
+
+
+def test_ndjson_unknown_both_counters_present() -> None:
+    result = _adapt_agent_result(
+        _make_agent_result(raw={"ndjson_unknown_event_count": 3, "ndjson_unknown_item_count": 7})
+    )
+    assert result.seen_ndjson_unknown_event_count == 3
+    assert result.seen_ndjson_unknown_item_count == 7
+
+
 class TestAdaptAgentResultFilePathKey:
     """Verify file_change entries use 'file_path' key for synthesis compatibility."""