Skip to content

Commit 3da58d2

Browse files
committed
Blind-spot fix #2 (generator — case + special tags): the net can now DISCOVER a case-asymmetry bug
The mixed-case raw-text bug was invisible to the generative net for TWO reasons, both now fixed in the input generator (test-only; no grammar change): - The materializer only ever emitted the grammar's literals VERBATIM (lowercase) and generic placeholder tag names (`a`) — it never varied case and never used the SPECIFIC special-tag literals. So it could not produce `<SCRIPT>` (a raw-text tag, uppercased), the exact shape the bug needs. - Fix: (a) `markupRawText()` — a directed emission that builds raw-text/void elements from the SPECIFIC `markup.rawText.tags` / `voidTags` (`<script><b</script>`, `<br>`), so the case-insensitive raw-text / void behaviours are exercised instead of a generic `<a>`; (b) `caseVaryTags()` in `push` — for every markup input, a CASE-VARIED copy that UPPERCASEs the tag names (derived from `markup.tagOpen`/ `closeMarker`; tag-name matching is case-insensitive, length-preserving so offsets stay valid). Proven by revert-test: with the #2 grammar fix disabled, the improved generator now makes the scope≡role net report a gated inconsistency on `<SCRIPT>…` — the class the old generator missed entirely. With the fix in place it is green. 26/26 gates; grammars untouched (the change is in the test generator only).
1 parent fe4afc0 commit 3da58d2

1 file changed

Lines changed: 50 additions & 0 deletions

File tree

test/grammar-gen.ts

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,30 @@ class Walker {
799799
];
800800
}
801801

802+
// DIRECTED RAW-TEXT / VOID element using the SPECIFIC declared tag literals (`grammar.markup.rawText.tags`
803+
// / `voidTags`), NOT a generic identifier sample. The un-biased enumeration + tokenCover only ever
804+
// materialise a generic placeholder tag name (`a`), so the SPECIAL-tag behaviours — a case-insensitive
805+
// raw-text region, a void element — are NEVER exercised by generation. This forces them: `<script><b</script>`
806+
// (a raw-text body carrying a would-be `<b` tag the region must keep as content) and `<br>`. Combined with
807+
// the tag-name CASE variation in `push`, this is what produces `<SCRIPT><b</SCRIPT>` — the witness for a
808+
// case-sensitive raw-text region. Returns [] when the grammar declares no rawText/void tags.
809+
markupRawText(): Emission[][] {
810+
const mk = this.grammar.markup;
811+
const nameTok = this.grammar.tokens.find((t) => t.identifier);
812+
if (!mk || !nameTok) return [];
813+
const open = mk.tagOpen, closeTag = mk.tagClose, closeMk = mk.closeMarker ?? '';
814+
const out: Emission[][] = [];
815+
for (const tag of mk.rawText?.tags ?? []) out.push([
816+
{ t: 'lit', value: open }, { t: 'tok', name: nameTok.name, text: tag }, { t: 'lit', value: closeTag },
817+
{ t: 'lit', value: open + 'b' }, // raw-text body: a would-be `<b` tag
818+
{ t: 'lit', value: open + closeMk }, { t: 'tok', name: nameTok.name, text: tag }, { t: 'lit', value: closeTag },
819+
]);
820+
for (const tag of (mk.voidTags ?? []).slice(0, 1)) out.push([
821+
{ t: 'lit', value: open }, { t: 'tok', name: nameTok.name, text: tag }, { t: 'lit', value: closeTag },
822+
]);
823+
return out;
824+
}
825+
802826
// The leading literal of an alt arm's seq/group spine (the indicator a `? …`/`- …` arm starts with).
803827
private armLeadLiteral(e: RuleExpr): string | null {
804828
if (e.type === 'literal') return e.value;
@@ -1137,6 +1161,17 @@ function compactify(ems: Emission[], compactLits: Set<string>): Emission[] {
11371161
}
11381162

11391163
// ─── TOP LEVEL ────────────────────────────────────────────────────────────────────
1164+
// UPPERCASE the tag NAMES in a markup string — the letters right after `tagOpen` (`<`) or
1165+
// `tagOpen`+`closeMarker` (`</`). Tag names are case-insensitive in markup (the lexer folds case), so this
1166+
// is an equally-legal variant; `<!`-led forms (comments / doctype) start with a non-letter and are untouched.
1167+
// Derived from `grammar.markup` (tagOpen / closeMarker) — no hardcoded `<`.
1168+
function caseVaryTags(grammar: CstGrammar, text: string): string {
1169+
const m = grammar.markup; if (!m) return text;
1170+
const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
1171+
const re = new RegExp(`(${esc(m.tagOpen)}${m.closeMarker ? `${esc(m.closeMarker)}?` : ''})([A-Za-z][A-Za-z0-9]*)`, 'g');
1172+
return text.replace(re, (_full, open: string, name: string) => open + name.toUpperCase());
1173+
}
1174+
11401175
export function generateInputs(grammar: CstGrammar, opts: GenOptions = {}): GenInput[] {
11411176
const depth = opts.depth ?? 5;
11421177
const cap = opts.cap ?? 6;
@@ -1186,6 +1221,18 @@ export function generateInputs(grammar: CstGrammar, opts: GenOptions = {}): GenI
11861221
if (!text.trim() || text.length > 2000 || seen.has(text)) continue; // skip blank / over-long / duplicate
11871222
seen.add(text);
11881223
out.push({ text, tokens, strategy: job.strat, rule });
1224+
// markup: a CASE-VARIED copy (UPPERCASE the tag NAMES). Tag-name matching is case-INSENSITIVE (the
1225+
// lexer folds case), so this is an equally-legal shape — but the materializer only ever emits the
1226+
// grammar's lowercase literals, so without this the generator NEVER produces `<SCRIPT>` and is blind
1227+
// to a parser↔highlighter case asymmetry (a case-sensitive highlighter region the lexer folds past —
1228+
// exactly the mixed-case raw-text bug). UPPERCASE preserves length, so the token offsets stay valid.
1229+
if (mode === 'markup') {
1230+
const up = caseVaryTags(grammar, text);
1231+
if (up !== text && !seen.has(up)) {
1232+
seen.add(up);
1233+
out.push({ text: up, tokens: tokens.map((t) => ({ ...t, text: up.slice(t.start, t.end) })), strategy: `case:${job.strat}`, rule });
1234+
}
1235+
}
11891236
}
11901237
};
11911238

@@ -1268,6 +1315,9 @@ export function generateInputs(grammar: CstGrammar, opts: GenOptions = {}): GenI
12681315
if (mode === 'markup') {
12691316
const sc = w.markupSelfCloseAttr();
12701317
if (sc.length) push(sc, 'fuzz:markupSelfClose', entry.name);
1318+
// raw-text / void elements with the SPECIFIC tag literals (+ the case-varied UPPERCASE copy from push)
1319+
// → exercises the case-insensitive raw-text region the generic-placeholder enumeration never reached.
1320+
for (const ems of w.markupRawText()) push(ems, 'markupRawText', entry.name);
12711321
}
12721322

12731323
// 7) DIRECTED INDENT EXPLICIT-KEY-WITH-BRACKET-SCALAR (indent grammars) — `? k [y :\n - p\n - q`. The

0 commit comments

Comments
 (0)