feat: a-priori sign convention + operator-side withheld-narration logging (#81)

* feat(explain): add an a-priori sign convention to the narration prompt

Signed numeric fields (deltas, CI bounds) must be written WITH their sign; a bare
magnitude is disallowed even when a verb carries the direction. Stated once in
SYSTEM_PROMPT, before any generation -- defence in depth mirroring the guardrail's
sign-aware token matching. An honest magnitude ("0.3391 만큼 감소") and a genuine
sign error ("델타는 0.3391", which inverts the leakage conclusion) emit a
byte-identical token, so the gate cannot tell them apart; this steers the model
away from that ambiguity at the source.

This is NOT a retry after a violation: the verifier never becomes a training
signal, and no retry logic is added. The gate, the allowed-set derivation, and
_NUM_TOKEN are untouched. The instruction carries no numeric literal or worked
example (cf. b24737a: literals injected into prompt text came back in narrations
and tripped the number gate); a test pins that it tokenizes to [] under the
guardrail's own extractor and that it is present in the standing prompt.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(ui): log withheld-narration prose server-side for operator adjudication

A withheld narration's GuardrailViolation still holds the full offending prose
after the `--- offending output ---` marker; narrate_diagnosis split it off and
discarded it with no side effect, so a withheld run could never be adjudicated
afterwards. Emit a logging.WARNING on the "permea_ui.narration" logger carrying
the violation headline, the full offending prose, and the report sha256, before
discarding it.

The browser redaction is unchanged: narrate_diagnosis still returns
Narration(status="withheld", detail=headline), app.py serialization is untouched,
and the offending prose never reaches the HTTP response. A caplog test asserts
both that the log record contains the offending text and that the returned
Narration carries no prose.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Permea-lab-admin

permea_explain/prompt.py

@@ -37,7 +37,9 @@
    없습니다. 신뢰수준(예: "95% 신뢰구간"의 95)처럼 숫자 필드가 아니라 설명 문자열
    안에만 있는 값은 숫자로 다시 쓰지 마십시오. 아래의 레지스트리 설명(경고 코드의
    의미)도 숫자의 출처가 아닙니다. 그 안에 숫자·버전·논문 참조가 보이더라도 출력에
-   옮기지 마십시오.
+   옮기지 마십시오. 부호가 있는 숫자 필드(델타, 신뢰구간 경계 등)는 항상 그 부호를
+   붙여 그대로 쓰십시오. 방향을 동사로 나타내더라도 부호를 뗀 크기(절댓값)만 적는
+   것은 허용되지 않습니다. 음수 필드는 반드시 음의 부호와 함께 기재하십시오.
 
 2. fired 목록에 있는 경고만 서술하십시오. fired 에 없는 경고 코드(PERMEA-W###)를
    지어내지 마십시오.

permea_ui/runner.py

@@ -15,7 +15,9 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
+import logging
 import os
 import tempfile
 from dataclasses import dataclass, field
@@ -26,6 +28,11 @@
 from permea_core.diagnose.rules import DiagnosePolicy
 from permea_core.eval.run import run_from_dataset
 
+#: Operator-only sink for withheld narrations. The offending prose is logged HERE (server
+#: side) so a withheld run can be adjudicated after the fact; it is never returned to the
+#: browser -- the HTTP response still carries only the headline. See narrate_diagnosis.
+_log = logging.getLogger("permea_ui.narration")
+
 #: Demo-scale evaluation settings. The CLI defaults (seeds=5, k=5, n_boot=2000) take ~53s
 #: and saturate every core; these take ~9s on the example fixture and are the settings the
 #: W501 -> W101 flip was verified at. A UI knob may raise them -- it must not silently
@@ -201,7 +208,19 @@ def narrate_diagnosis(diagnosis: dict[str, Any], *, provider: Any = None) -> Nar
         except GuardrailViolation as exc:
             # Show that a check failed and how many -- never the offending text, which the
             # exception appends after a `--- offending output ---` marker.
-            headline = str(exc).split("\n--- offending output ---", 1)[0]
+            message = str(exc)
+            headline = message.split("\n--- offending output ---", 1)[0]
+            # OPERATOR-ONLY observability: the full exception still holds the offending prose
+            # after the marker. Log it here (server side) with the report hash so a withheld
+            # narration can be adjudicated later. This never reaches the browser -- the
+            # returned Narration below carries only the headline, and app.py serializes no
+            # more than that. `message` is logged whole: headline + the offending prose.
+            source_report_sha256 = hashlib.sha256(report_path.read_bytes()).hexdigest()
+            _log.warning(
+                "narration withheld [source_report_sha256=%s]: %s",
+                source_report_sha256,
+                message,
+            )
             return Narration(status="withheld", detail=headline)
         except Exception as exc:  # transport, timeout, malformed response
             return Narration(status="error", detail=f"{type(exc).__name__}: {exc}")

tests/test_explain_narrate.py

@@ -12,7 +12,7 @@
 
 from permea_core.diagnose import diagnose
 from permea_core.eval.run import EvalRun, MetricRow
-from permea_explain import GuardrailViolation, guardrails, narrate, source
+from permea_explain import GuardrailViolation, guardrails, narrate, prompt, source
 from permea_explain.extract import extract
 from permea_explain.providers.base import Provider, ProviderResponse
 from permea_explain.violations import NUMERIC_UNTRACED
@@ -373,6 +373,38 @@ def test_pure_digit_and_decimal_tokenization_unchanged():
     assert guardrails.numbers_in_narrative("PERMEA-W101 이 발화, 정확도 0.87") == ["0.87"]
 
 
+# --------------------------------------------------------------------------------------
+# A-priori sign convention (#0034)
+# --------------------------------------------------------------------------------------
+# SYSTEM_PROMPT carries a standing instruction to always write signed fields (deltas, CI
+# bounds) WITH their sign -- stated ONCE, before any generation, as defence in depth mirroring
+# the guardrail's sign-aware token matching. It is NOT a retry after a violation (the verifier
+# must never become a training signal). The instruction itself must carry no numeric literal:
+# commit b24737a exists because literals injected into prompt-visible text came back in
+# narrations and tripped the number gate, so the added sentence must tokenize to [] under the
+# SAME extractor the gate uses at runtime.
+_SIGN_CONVENTION = (
+    "부호가 있는 숫자 필드(델타, 신뢰구간 경계 등)는 항상 그 부호를 "
+    "붙여 그대로 쓰십시오. 방향을 동사로 나타내더라도 부호를 뗀 크기(절댓값)만 적는 "
+    "것은 허용되지 않습니다. 음수 필드는 반드시 음의 부호와 함께 기재하십시오."
+)
+
+
+def test_sign_convention_present_in_system_prompt():
+    """The convention is a-priori: it lives in the standing prompt, sent before generation.
+
+    Matched whitespace-insensitively because SYSTEM_PROMPT hard-wraps its rules across
+    indented lines, so the sentence spans several ``\\n   `` breaks in the source.
+    """
+    collapse = lambda s: re.sub(r"\s+", " ", s).strip()
+    assert collapse(_SIGN_CONVENTION) in collapse(prompt.SYSTEM_PROMPT)
+
+
+def test_sign_convention_carries_no_numeric_literal():
+    """A faithful copy of the instruction must not itself fail the provenance gate (b24737a)."""
+    assert guardrails._standalone_number_tokens(_SIGN_CONVENTION) == []
+
+
 def test_extract_is_available_alongside_narrate():
     """Both task layers share one Provider and one verification discipline (#0023)."""
     import inspect

tests/test_ui_drylab.py

@@ -10,6 +10,7 @@
 
 import ast
 import json
+import logging
 import sys
 from pathlib import Path
 
@@ -117,6 +118,29 @@ def test_narration_quoting_the_data_sha256_is_not_withheld(diagnosis):
     assert n.text == text
 
 
+def test_withheld_narration_logs_offending_prose_for_operator(diagnosis, caplog):
+    """Design 1 (#0034): a withheld narration's offending prose is logged SERVER-SIDE for
+    operator adjudication, while the returned Narration -- and therefore the HTTP response --
+    still carries none of it. The browser redaction is unchanged; only observability is added.
+    """
+    bad = "정확도는 99.7% 였고 PERMEA-W999 경고가 발생했습니다."  # fabricated number + unfired code
+    with caplog.at_level(logging.WARNING, logger="permea_ui.narration"):
+        n = runner.narrate_diagnosis(diagnosis, provider=FakeProvider(bad))
+
+    # What the browser gets: withheld, no prose, no offending text leaked into the detail.
+    assert n.status == "withheld"
+    assert n.text is None, "withheld narration must not carry renderable prose"
+    assert "offending output" not in (n.detail or "")
+    assert bad not in (n.detail or ""), "offending prose must not reach the returned Narration"
+
+    # What the operator gets: the full offending prose, on the dedicated server-side logger.
+    records = [r for r in caplog.records if r.name == "permea_ui.narration"]
+    assert records, "expected a WARNING on the 'permea_ui.narration' logger"
+    logged = "\n".join(r.getMessage() for r in records)
+    assert bad in logged, "offending prose must be present in the operator log"
+    assert "source_report_sha256=" in logged, "log must carry the report hash for adjudication"
+
+
 def test_missing_credentials_yields_unavailable(diagnosis, monkeypatch):
     for var in ("PERMEA_LLM_TOKEN", "PERMEA_LLM_ENDPOINT_ID", "PERMEA_LLM_BASE_URL"):
         monkeypatch.delenv(var, raising=False)