-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathemit-parser.ts
More file actions
2083 lines (1999 loc) · 95.4 KB
/
Copy pathemit-parser.ts
File metadata and controls
2083 lines (1999 loc) · 95.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// ── emit-parser ──
//
// An EMITTER (vs the createParser RUNTIME INTERPRETER in gen-parser.ts). It takes
// the same CstGrammar, re-derives the SAME static analysis createParser does
// (precedence/binding-power tables, Pratt NUD/LED classification, FIRST sets,
// nullability, mixfix/access-tail detection), and emits SELF-CONTAINED JavaScript
// for a parser that bakes those tables in as constants and emits the PER-RULE +
// PER-ARM matching as SPECIALIZED straight-line code — replacing the interpreter's
// matchExpr/matchSeq/matchQuantifier/matchSep tree-walk dispatch.
//
// The algorithmic CONTROL cores (the Pratt precedence loop, the left-recursion
// loop, the longest-match non-recursive loop, the mixfix operand re-bind, the
// packrat memo, save/restore backtracking) stay as a generic runtime that the
// emitted code CALLS — copied VERBATIM from gen-parser.ts so their longest-match /
// tail-closing / suppress / mixfix semantics are byte-identical to the oracle.
// What those loops used to dispatch per arm (matchExpr(alt) / matchSeq(items)) is
// now a GENERATED specialized matcher per NUD/LED/alt arm, so the interpretive
// dispatch — the ~third of parse time the CPU profile attributes to matchExpr +
// per-arm re-classification — is removed.
//
// The LEXER is identical in both worlds and is OUTSIDE this optimization, so the
// emitted code imports the fixed createLexer RUNTIME and feeds it the grammar's
// baked token+config DATA (a "grammar-lite"); it never imports the grammar
// DEFINITION object. createParser is the correctness oracle — the emitted parser
// must reproduce its CST byte-for-byte.
import type { CstGrammar, RuleExpr, RuleDecl, PrecLevel } from './types.ts';
import { isKeywordLiteral, collectLiterals } from './grammar-utils.ts';
import { emitLexer } from './emit-lexer.ts';
// ── Static analysis (re-derived; mirrors gen-parser.ts exactly) ──
interface OpInfo {
lbp: number;
rbp: number;
assoc: 'left' | 'right' | 'none';
position: 'infix' | 'prefix' | 'postfix';
}
type FirstTok = { lit: string } | { tok: string } | null;
type MixfixInfo = { openLit: string; sepLit: string };
function hasMarker(expr: RuleExpr): boolean {
if (expr.type === 'op' || expr.type === 'prefix' || expr.type === 'postfix') return true;
if (expr.type === 'seq' || expr.type === 'alt') return expr.items.some(hasMarker);
if (expr.type === 'quantifier' || expr.type === 'group') return hasMarker(expr.body);
if (expr.type === 'sep') return hasMarker(expr.element);
return false;
}
function findEntryRule(grammar: CstGrammar): string {
return grammar.rules[grammar.rules.length - 1].name;
}
/** Build the full static analysis createParser performs, returned as plain data. */
function analyze(grammar: CstGrammar) {
const tokenNames = new Set(grammar.tokens.map(t => t.name));
// Precedence table — identical to gen-parser.ts.
const opTable = new Map<string, OpInfo>();
const prefixOps = new Map<string, OpInfo>();
const noUnaryLhsOps = new Set<string>();
const postfixOpValues = new Set<string>();
for (let i = 0; i < grammar.precs.length; i++) {
const level = grammar.precs[i];
const bp = (i + 1) * 2;
for (const op of level.operators) {
if (op.position === 'prefix') {
prefixOps.set(op.value, { lbp: 0, rbp: level.assoc === 'right' ? bp - 1 : bp, assoc: level.assoc, position: 'prefix' });
} else if (op.position === 'postfix') {
postfixOpValues.add(op.value);
opTable.set(op.value, { lbp: bp, rbp: 0, assoc: level.assoc, position: 'postfix' });
} else {
const lbp = bp;
const rbp = level.assoc === 'right' ? bp - 1 : bp;
opTable.set(op.value, { lbp, rbp, assoc: level.assoc, position: 'infix' });
if (op.noUnaryLhs) noUnaryLhsOps.add(op.value);
}
}
}
// Alternative-form LED binding powers (mirrors gen-parser.ts — the two engines must
// resolve IDENTICAL lbp numbers or their CSTs diverge).
const ledPrecByConnector = new Map<string, { lbp: number; rhsBp: number | null }>();
for (const lp of grammar.ledPrecs ?? []) {
const anchorOp = lp.sameAs ?? lp.below;
if (!anchorOp) throw new Error(`ledPrec ${lp.connector}: needs sameAs or below`);
const op = opTable.get(anchorOp);
if (!op) throw new Error(`ledPrec ${lp.connector}: anchor ${JSON.stringify(anchorOp)} is not a ladder operator`);
const lbp = lp.sameAs !== undefined ? op.lbp : op.lbp - 1;
ledPrecByConnector.set(lp.connector, { lbp, rhsBp: lp.chainRhs ? lbp : null });
}
// Pratt rules.
const prattRules = new Set<string>();
for (const rule of grammar.rules) if (hasMarker(rule.body)) prattRules.add(rule.name);
function classifyAlts(rule: RuleDecl) {
const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
const nuds: RuleExpr[] = [];
const leds: { expr: RuleExpr; items: RuleExpr[] }[] = [];
for (const alt of alts) {
const items = alt.type === 'seq' ? alt.items : [alt];
if (items[0]?.type === 'ref' && items[0].name === rule.name) leds.push({ expr: alt, items: items.slice(1) });
else nuds.push(alt);
}
return { nuds, leds };
}
function classifyLeftRec(rule: RuleDecl) {
const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
const atoms: RuleExpr[] = [];
const continuations: RuleExpr[][] = [];
for (const alt of alts) {
const items = alt.type === 'seq' ? alt.items : [alt];
if (items[0]?.type === 'ref' && items[0].name === rule.name) continuations.push(items.slice(1));
else atoms.push(alt);
}
return { atoms, continuations };
}
function isLeftRecursive(rule: RuleDecl): boolean {
const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
return alts.some(alt => {
const items = alt.type === 'seq' ? alt.items : [alt];
return items[0]?.type === 'ref' && items[0].name === rule.name;
});
}
const maxBp = (grammar.precs.length + 1) * 2;
const ruleByName = new Map<string, RuleDecl>(grammar.rules.map(r => [r.name, r]));
const leftRecSet = new Set<string>(grammar.rules.filter(isLeftRecursive).map(r => r.name));
const prattClassified = new Map<string, ReturnType<typeof classifyAlts>>();
const leftRecClassified = new Map<string, ReturnType<typeof classifyLeftRec>>();
for (const rule of grammar.rules) {
if (prattRules.has(rule.name)) prattClassified.set(rule.name, classifyAlts(rule));
else if (leftRecSet.has(rule.name)) leftRecClassified.set(rule.name, classifyLeftRec(rule));
}
const templateTokenName = grammar.tokens.find(t => t.template)?.name;
const templateTokenNames = new Set<string>(grammar.tokens.filter(t => t.template).map(t => t.name));
// First-token dispatch.
function firstTokenOf(alt: RuleExpr): FirstTok {
const items = alt.type === 'seq' ? alt.items : [alt];
const first = items[0];
if (!first) return null;
if (first.type === 'literal') return { lit: first.value };
if (first.type === 'ref' && tokenNames.has(first.name)) return { tok: first.name };
return null;
}
// Mixfix operand re-bind shape: `<lit L1> $self <lit L2> …`.
function selfRefName(e: RuleExpr | undefined, ruleName: string): boolean {
return !!e && e.type === 'ref' && e.name === ruleName;
}
function mixfixOf(items: RuleExpr[], ruleName: string): MixfixInfo | null {
if (items.length >= 4 && items[0]?.type === 'literal' && selfRefName(items[1], ruleName) && items[2]?.type === 'literal') {
return { openLit: items[0].value, sepLit: items[2].value };
}
return null;
}
// Access-tail + tail-closing LED classification (Pratt).
// Returns, per Pratt rule, parallel arrays of flags aligned to the leds array.
const ledMeta = new Map<string, { accessTail: boolean[]; tailClosing: boolean[]; mixfix: (MixfixInfo | null)[]; first: FirstTok[]; prec: ({ lbp: number; rhsBp: number | null } | null)[] }>();
for (const [ruleName, { leds }] of prattClassified.entries()) {
const accessTail: boolean[] = [];
const tailClosing: boolean[] = [];
const mixfix: (MixfixInfo | null)[] = [];
const first: FirstTok[] = [];
const prec: ({ lbp: number; rhsBp: number | null } | null)[] = [];
for (const led of leds) {
const it = led.items;
let isAccessTail = false, isTailClosing = false;
if (it.length > 0 && it[0].type !== 'op' && it[0].type !== 'postfix') {
const last = it[it.length - 1];
const lastIsOperand = selfRefName(last, ruleName);
const wordConnector = it[0].type === 'literal' && /^[A-Za-z]/.test(it[0].value);
if (!lastIsOperand && !wordConnector) isAccessTail = true;
if (last.type === 'not') isTailClosing = true;
}
accessTail.push(isAccessTail);
tailClosing.push(isTailClosing);
mixfix.push(mixfixOf(led.items, ruleName));
first.push(firstTokenOf({ type: 'seq', items: led.items } as RuleExpr));
const firstItem = led.items[0];
const lp = firstItem?.type === 'literal' ? ledPrecByConnector.get(firstItem.value) ?? null : null;
if (lp !== null && lp.rhsBp !== null) {
const last = led.items[led.items.length - 1];
if (!(last?.type === 'ref' && last.name === ruleName)) {
throw new Error(`ledPrec ${firstItem.type === 'literal' ? firstItem.value : '?'}: chainRhs requires a trailing self-operand`);
}
}
prec.push(lp);
}
ledMeta.set(ruleName, { accessTail, tailClosing, mixfix, first, prec });
}
// Left-rec continuation mixfix.
const contMeta = new Map<string, (MixfixInfo | null)[]>();
for (const [ruleName, { continuations }] of leftRecClassified.entries()) {
contMeta.set(ruleName, continuations.map(c => mixfixOf(c, ruleName)));
}
// Nullability.
const nullableRules = new Set<string>();
function exprNullable(e: RuleExpr): boolean {
switch (e.type) {
case 'literal': return false;
case 'ref': return tokenNames.has(e.name) ? false : nullableRules.has(e.name);
case 'seq': return e.items.every(exprNullable);
case 'alt': return e.items.some(exprNullable);
case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true;
case 'group': return exprNullable(e.body);
case 'not': return true;
case 'sep': return true;
default: return true;
}
}
for (let changed = true; changed; ) {
changed = false;
for (const rule of grammar.rules) {
if (!nullableRules.has(rule.name) && exprNullable(rule.body)) { nullableRules.add(rule.name); changed = true; }
}
}
// FIRST sets.
//
// Reserved-aware keys: a `not(alt('if', 'var', …))` guard immediately before the
// first consuming element proves those keyword texts can never be this position's
// token. A token-name key gathered under such a guard becomes a QUALIFIED key
// ('\0Q:'-prefixed, registered in qualKeys) and is emitted as TM[0] plus every
// non-excluded keyword t-bit instead of the blanket k-bit — so a reserved-keyword
// lookahead no longer admits identifier-led alternatives it provably cannot start
// (the dominant longest-match waste: expr-stmt/labeled-stmt arms on 'var'/'if'/…).
// Sound: every pruned (alt, token) pair fails its not-guard before consuming.
const qualKeys = new Map<string, { tok: string; except: Set<string> }>();
function qualKey(tok: string, except: Set<string>): string {
const sorted = [...except].sort();
const key = '\u0000Q:' + tok + ':' + sorted.join(',');
if (!qualKeys.has(key)) qualKeys.set(key, { tok, except: new Set(sorted) });
return key;
}
// null = the key is guarded out entirely (a keyword literal inside its own not-class).
function excludeKey(k: string, pending: Set<string>): string | null {
const q = qualKeys.get(k);
if (q) return qualKey(q.tok, new Set([...q.except, ...pending]));
if (tokenNames.has(k)) return qualKey(k, pending);
if (pending.has(k)) return null;
return k;
}
// A not() whose body is purely keyword literals (the reserved-word guard shape).
function notKeywordClass(body: RuleExpr): Set<string> | null {
const items = body.type === 'alt' ? body.items : [body];
const out = new Set<string>();
for (const it of items) {
if (it.type !== 'literal' || !isKeywordLiteral(it.value)) return null;
out.add(it.value);
}
return out.size > 0 ? out : null;
}
const firstSets = new Map<string, Set<string> | null>();
function exprFirst(e: RuleExpr): Set<string> | null {
switch (e.type) {
case 'literal': return new Set([e.value]);
case 'ref': {
if (tokenNames.has(e.name)) return new Set([e.name]);
return firstSets.has(e.name) ? firstSets.get(e.name)! : new Set();
}
case 'seq': {
const acc = new Set<string>();
let pending: Set<string> | null = null;
for (const item of e.items) {
if (item.type === 'prefix') {
// A pratt prefix form ([prefix, operand]): its first token is one of the
// prefix-operator literals — a real set, not unknown. Keeps FIRST(Expr)
// from collapsing to null/always-admit.
for (const op of prefixOps.keys()) {
const ek = pending ? excludeKey(op, pending) : op;
if (ek !== null) acc.add(ek);
}
return acc;
}
if (item.type === 'not') {
const kws = notKeywordClass(item.body);
if (kws) pending = pending ? new Set([...pending, ...kws]) : kws;
continue;
}
if (item.type === 'op' || item.type === 'postfix' || item.type === 'sameLine' || item.type === 'noCommentBefore' || item.type === 'noMultilineFlowBefore') continue;
const f = exprFirst(item);
if (f === null) return null;
for (const k of f) {
const ek = pending ? excludeKey(k, pending) : k;
if (ek !== null) acc.add(ek);
}
if (!exprNullable(item)) return acc;
}
return acc;
}
case 'alt': {
const acc = new Set<string>();
for (const item of e.items) {
const f = exprFirst(item);
if (f === null) return null;
for (const k of f) acc.add(k);
}
return acc;
}
case 'quantifier': case 'group': return exprFirst(e.body);
case 'not': case 'sameLine': case 'noCommentBefore': case 'noMultilineFlowBefore': return new Set();
case 'sep': return exprFirst(e.element);
default: return null;
}
}
for (let changed = true; changed; ) {
changed = false;
for (const rule of grammar.rules) {
const prev = firstSets.get(rule.name);
if (prev === null) continue;
const next = exprFirst(rule.body);
if (next === null) { firstSets.set(rule.name, null); changed = true; continue; }
const merged = prev ? new Set(prev) : new Set<string>();
let grew = false;
for (const k of next) if (!merged.has(k)) { merged.add(k); grew = true; }
if (grew || prev === undefined) { firstSets.set(rule.name, merged); changed = true; }
}
}
// Deep per-alternative FIRST set + nullability for the longest-match dispatch — the
// emitted mirror of gen-parser.ts's altMightStart. An alternative whose FIRST element
// is a rule ref (`Decl …`, `Expr …`) is pruned when the lookahead can't begin that
// rule (resolved through the transitive firstSets), not only when it begins with a
// known literal/token. Sound: exprFirst over-approximates (never omits a startable
// token) and a nullable alt is always tried (its empty match never wins longest-match).
const altDeepFirst = new Map<RuleExpr, Set<string> | null>();
const altNullable = new Map<RuleExpr, boolean>();
for (const rule of grammar.rules) {
const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
for (const alt of alts) { altDeepFirst.set(alt, exprFirst(alt)); altNullable.set(alt, exprNullable(alt)); }
}
// SECOND sets: the keys admissible as a match's SECOND token, plus whether a
// one-token match exists (len1). Refines the longest-match dispatch: an admitted
// alternative whose SECOND set excludes the actual second token — and that cannot
// end after one token — provably fails, so its arm can be skipped. Over-approximated
// everywhere (unknown shapes → TOP, no guard exclusions applied at depth 2), and
// op/prefix/postfix pratt items are one-op-token consumers with known literal sets.
type Sec = { s: Set<string> | null; len1: boolean };
const SEC_TOP: Sec = { s: null, len1: true };
const ruleSecond = new Map<string, Sec>();
const opKeys = new Set<string>([...opTable.keys(), ...postfixOpValues]);
// SECOND inputs use PLAIN FIRST semantics (no reserved-qualified keys, prefix → top),
// an exact mirror of gen-parser's exprFirst: the interpreter computes the same SECOND
// sets, and the prune decisions must be ENGINE-IDENTICAL — an arm skipped by only one
// engine would consume a token in the other and skew the farthest-position error state
// (the emit-reject-messages gate caught exactly this).
const firstSetsPlain = new Map<string, Set<string> | null>();
function exprFirstPlain(e: RuleExpr): Set<string> | null {
switch (e.type) {
case 'literal': return new Set([e.value]);
case 'ref': {
if (tokenNames.has(e.name)) return new Set([e.name]);
return firstSetsPlain.has(e.name) ? firstSetsPlain.get(e.name)! : new Set();
}
case 'seq': {
const acc = new Set<string>();
for (const item of e.items) {
if (item.type === 'prefix') return null;
if (item.type === 'op' || item.type === 'postfix' || item.type === 'not' || item.type === 'sameLine' || item.type === 'noCommentBefore' || item.type === 'noMultilineFlowBefore') continue;
const f = exprFirstPlain(item);
if (f === null) return null;
for (const k of f) acc.add(k);
if (!exprNullable(item)) return acc;
}
return acc;
}
case 'alt': {
const acc = new Set<string>();
for (const item of e.items) {
const f = exprFirstPlain(item);
if (f === null) return null;
for (const k of f) acc.add(k);
}
return acc;
}
case 'quantifier': case 'group': return exprFirstPlain(e.body);
case 'not': case 'sameLine': case 'noCommentBefore': case 'noMultilineFlowBefore': return new Set();
case 'sep': return exprFirstPlain(e.element);
default: return null;
}
}
for (let changed = true; changed; ) {
changed = false;
for (const rule of grammar.rules) {
const prev = firstSetsPlain.get(rule.name);
if (prev === null) continue;
const next = exprFirstPlain(rule.body);
if (next === null) { firstSetsPlain.set(rule.name, null); changed = true; continue; }
const merged = prev ? new Set(prev) : new Set<string>();
let grew = false;
for (const k of next) if (!merged.has(k)) { merged.add(k); grew = true; }
if (grew || prev === undefined) { firstSetsPlain.set(rule.name, merged); changed = true; }
}
}
// FIRST of a seq suffix for second-token purposes (op items consume an op literal;
// zero-width skipped; nullable items scanned through), and its nullability.
function suffixFirst(items: RuleExpr[], j: number): Set<string> | null {
const acc = new Set<string>();
for (let i = j; i < items.length; i++) {
const item = items[i];
if (item.type === 'not' || item.type === 'sameLine' || item.type === 'noCommentBefore' || item.type === 'noMultilineFlowBefore') continue;
if (item.type === 'op' || item.type === 'postfix') { for (const k of opKeys) acc.add(k); return acc; }
if (item.type === 'prefix') { for (const k of prefixOps.keys()) acc.add(k); return acc; }
const f = exprFirstPlain(item);
if (f === null) return null;
for (const k of f) acc.add(k);
if (!exprNullable(item)) return acc;
}
return acc;
}
function suffixNullable(items: RuleExpr[], j: number): boolean {
for (let i = j; i < items.length; i++) {
const item = items[i];
if (item.type === 'not' || item.type === 'sameLine' || item.type === 'noCommentBefore' || item.type === 'noMultilineFlowBefore') continue;
if (item.type === 'op' || item.type === 'prefix' || item.type === 'postfix') return false;
if (!exprNullable(item)) return false;
}
return true;
}
function exprSecond(e: RuleExpr): Sec {
switch (e.type) {
case 'literal': return { s: new Set(), len1: true };
case 'ref':
if (tokenNames.has(e.name)) return { s: new Set(), len1: true };
return ruleSecond.get(e.name) ?? { s: new Set(), len1: false };
case 'seq': {
const acc = new Set<string>();
let len1 = false;
const items = e.items;
for (let i = 0; i < items.length; i++) {
const item = items[i];
if (item.type === 'not' || item.type === 'sameLine' || item.type === 'noCommentBefore' || item.type === 'noMultilineFlowBefore') continue;
let isec: Sec;
let itemNullable: boolean;
if (item.type === 'op' || item.type === 'postfix' || item.type === 'prefix') {
isec = { s: new Set(), len1: true };
itemNullable = false;
} else {
isec = exprSecond(item);
itemNullable = exprNullable(item);
}
if (isec.s === null) return SEC_TOP;
for (const k of isec.s) acc.add(k);
if (isec.len1) {
const rf = suffixFirst(items, i + 1);
if (rf === null) return SEC_TOP;
for (const k of rf) acc.add(k);
if (suffixNullable(items, i + 1)) len1 = true;
}
if (!itemNullable) return { s: acc, len1 };
}
return { s: acc, len1 };
}
case 'alt': {
const acc = new Set<string>();
let len1 = false;
for (const item of e.items) {
const sec = exprSecond(item);
if (sec.s === null) return SEC_TOP;
for (const k of sec.s) acc.add(k);
len1 ||= sec.len1;
}
return { s: acc, len1 };
}
case 'quantifier': {
const sec = exprSecond(e.body);
if (sec.s === null) return SEC_TOP;
const acc = new Set(sec.s);
if (e.kind !== '?' && sec.len1) {
const bf = exprFirstPlain(e.body);
if (bf === null) return SEC_TOP;
for (const k of bf) acc.add(k);
}
return { s: acc, len1: sec.len1 };
}
case 'group': return exprSecond(e.body);
case 'sep': {
const sec = exprSecond(e.element);
if (sec.s === null) return SEC_TOP;
const acc = new Set(sec.s);
if (sec.len1) acc.add(e.delimiter);
return { s: acc, len1: sec.len1 };
}
case 'not': case 'sameLine': case 'noCommentBefore': case 'noMultilineFlowBefore':
return { s: new Set(), len1: false };
case 'op': case 'prefix': case 'postfix':
return { s: new Set(), len1: true };
default: return SEC_TOP;
}
}
for (let changed = true; changed; ) {
changed = false;
for (const rule of grammar.rules) {
const prev = ruleSecond.get(rule.name);
if (prev && prev.s === null && prev.len1) continue;
const next = exprSecond(rule.body);
let nv: Sec;
if (!prev) nv = next;
else if (next.s === null || prev.s === null) nv = { s: null, len1: prev.len1 || next.len1 };
else nv = { s: new Set([...prev.s, ...next.s]), len1: prev.len1 || next.len1 };
const grew = !prev || (nv.s === null) !== (prev.s === null) || nv.len1 !== prev.len1
|| (nv.s !== null && prev.s !== null && nv.s.size > prev.s.size);
if (grew) { ruleSecond.set(rule.name, nv); changed = true; }
}
}
const altSecond = new Map<RuleExpr, Sec>();
for (const rule of grammar.rules) {
const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
for (const alt of alts) altSecond.set(alt, exprSecond(alt));
}
// ── Lever 1: integer token kinds ──
// Replace the per-call string dispatch in literal/token matching and FIRST gating
// (matchLiteral / matchToken / the membershipFn byte tables) with integer compares.
// Two int fields are interned onto each token at parse time (see the emitted tokenize
// wiring):
// tok.k = TYPE kind — an int for tok.type ('' punctuation → PUNCT sentinel;
// each declared token name → its own int; the three
// template-token-kinds $templateHead/$templateMiddle/
// $templateTail get their own ints below PUNCT-named).
// tok.t = LITERAL kind — if tok.text is a known keyword literal (for a named,
// non-'' token) OR a known punctuation literal (for a ''
// token), that literal's int; else 0 (NONE). Keyword ints
// and punct ints occupy DISJOINT ranges, so tok.t alone
// disambiguates keyword-vs-punct.
// A keyword token (an identifier-type token whose text is e.g. "if") therefore has
// tok.k = <Ident kind> AND tok.t = <KW_if>, so it matches BOTH the identifier
// token-name key and the keyword key — exactly the dual-match keyMatchesTok did.
//
// Kind ranges (so a single >= test answers "is a declared token name"):
// PUNCT = 1; $templateHead = 2, $templateMiddle = 3, $templateTail = 4;
// declared token names: 5, 6, … (NAMED_MIN = 5).
const KIND_PUNCT = 1;
const KIND_TEMPLATE_HEAD = 2;
const KIND_TEMPLATE_MIDDLE = 3;
const KIND_TEMPLATE_TAIL = 4;
const KIND_NAMED_MIN = 5;
const typeKind = new Map<string, number>();
typeKind.set('', KIND_PUNCT);
typeKind.set('$templateHead', KIND_TEMPLATE_HEAD);
typeKind.set('$templateMiddle', KIND_TEMPLATE_MIDDLE);
typeKind.set('$templateTail', KIND_TEMPLATE_TAIL);
let nextKind = KIND_NAMED_MIN;
for (const name of tokenNames) if (!typeKind.has(name)) typeKind.set(name, nextKind++);
// Literal ints: keyword literals in [1 .. kwCount], punct literals after, disjoint.
// The vocabulary must be a SUPERSET of every string classifyKey()/matchLiteralCall()
// is ever called with — otherwise an unlisted literal gets int 0 (NONE) and a t===0
// compare would false-match every plain token. classifyKey/matchLiteral see:
// • every `literal` node value reachable in a rule body — INCLUDING inside `not`
// (the reserved-word negative-lookahead), which shared collectLiterals
// deliberately does NOT descend into; so we walk the full tree here.
// • every operator value (prefix/infix/postfix) — keyword-shaped ops like `delete`
// are matched as literals inside the reserved-word `not`, yet live only in `precs`.
// • every FIRST-set member (defensive; these are literals or token names).
const allLiterals = new Set<string>();
function collectAllLiterals(e: RuleExpr): void {
switch (e.type) {
case 'literal': allLiterals.add(e.value); return;
case 'seq': case 'alt': e.items.forEach(collectAllLiterals); return;
case 'quantifier': case 'group': case 'not': collectAllLiterals(e.body); return;
case 'sep': collectAllLiterals(e.element); allLiterals.add(e.delimiter); return;
default: return;
}
}
for (const rule of grammar.rules) collectAllLiterals(rule.body);
for (const level of grammar.precs) for (const op of level.operators) allLiterals.add(op.value);
for (const fs of firstSets.values()) if (fs) for (const k of fs) if (!tokenNames.has(k) && !qualKeys.has(k)) allLiterals.add(k);
const kwLitKind = new Map<string, number>();
const puLitKind = new Map<string, number>();
let nextLit = 1;
for (const lit of allLiterals) if (isKeywordLiteral(lit) && !kwLitKind.has(lit)) kwLitKind.set(lit, nextLit++);
for (const lit of allLiterals) if (!isKeywordLiteral(lit) && !puLitKind.has(lit)) puLitKind.set(lit, nextLit++);
// Pre-classify a FIRST-set key the SAME way keyMatchesTok does, into an int
// descriptor the emitted checks consume. Order MUST match keyMatchesTok:
// tokenNames.has → token-name; else isKeywordLiteral → keyword; else punct.
type KeyDesc =
| { kind: 'tok'; k: number; template: boolean }
| { kind: 'kw'; t: number }
| { kind: 'punct'; t: number; v: string };
function classifyKey(key: string): KeyDesc {
if (tokenNames.has(key)) {
return { kind: 'tok', k: typeKind.get(key)!, template: templateTokenNames.has(key) };
}
if (isKeywordLiteral(key)) {
// A keyword key whose literal int we know (it is a literal in some rule body).
// If somehow not in the vocabulary (defensive), fall back to a never-matching 0.
return { kind: 'kw', t: kwLitKind.get(key) ?? 0 };
}
return { kind: 'punct', t: puLitKind.get(key) ?? 0, v: key };
}
// A sentinel kind for any non-'' token type the lexer did not declare (unreachable
// for this closed lexer, but kept faithful): one past the max declared kind, so it
// is >= NAMED_MIN (behaves as "a named token" for the keyword-by-text branch) yet
// collides with NO real token-name kind (so matchToken(name) never false-matches it).
const KIND_NAMED_FALLBACK = nextKind;
const symtab = {
KIND_PUNCT, KIND_TEMPLATE_HEAD, KIND_NAMED_MIN, KIND_NAMED_FALLBACK,
typeKind, kwLitKind, puLitKind, classifyKey,
};
return {
grammar, tokenNames, opTable, prefixOps, noUnaryLhsOps, postfixOpValues,
prattRules, leftRecSet, ruleByName, prattClassified, leftRecClassified,
maxBp, templateTokenName, templateTokenNames, firstTokenOf, altDeepFirst, altNullable,
altSecond, ledMeta, contMeta, nullableRules, firstSets, symtab, qualKeys,
};
}
// ── Code-emission helpers ──
const J = (v: unknown) => JSON.stringify(v);
function sanitize(name: string): string {
return name.replace(/[^A-Za-z0-9_]/g, '_');
}
/**
* Emit a specialized matcher BODY for a RuleExpr — straight-line code that mirrors
* matchExpr/matchSeq/matchQuantifier/matchSep exactly, but with each step inlined as
* a direct call (matchLiteral('x'), the ref's parse function, matchToken, an unrolled
* quantifier loop, the zero-width assertions, sep). Pushes matched leaves/nodes into a
* local array `out` (declared by the caller). On failure it `pos = <saveVar>; return null`.
* Returns a string of statements. `saveVar` is the position to restore to on failure.
*
* The semantics are a 1:1 transcription of the interpreter's matchExpr switch, so the
* emitted code accepts/rejects and shapes the CST identically.
*/
class Emitter {
private buf: string[] = [];
private tmp = 0;
// Generated compound-matcher helpers, deduplicated by their structural key so an
// expr shared across rules emits one helper. Each is a fn returning children[]|null
// (the matchExpr contract) — control flow is `return`, never `break`, so inlined
// loops (quantifier/sep) never clash with an enclosing construct's loop.
private helpers = new Map<string, string>(); // structural key → fn name
private helperDefs: string[] = [];
// Deduped FIRST-set membership tables + their per-set test fns, hoisted to module
// level. The longest-match alt guards and the rule-ref guards sit on the hottest
// dispatch paths; each set bakes to two Uint8Array byte tables (indexed by tok.k /
// tok.t) so the test is two loads + an or — no per-desc loop. Spliced at the same
// `//${HELPERS}` sentinel (top-level, above the rule fns).
private u8Consts = new Map<string, string>(); // `${size}:${ones}` → const name
private memberFns = new Map<string, string>(); // `${kConst}|${tConst}` → fn name
private u32Consts = new Map<string, string>(); // mask-table values → const name
private u8Emitted = false;
readonly a: ReturnType<typeof analyze>;
constructor(a: ReturnType<typeof analyze>) { this.a = a; }
// Token-text materialization: a source-span slice for an emitted (SoA) lexer, the
// converter-filled text column for createLexer-fallback grammars (synthetic tokens
// there — indent/dedent etc. — have text that is NOT a source span). Chosen at emit
// time; the runtime has a single form.
soa = false;
textAt(idx: string): string {
return this.soa ? `src.slice(tkOff[${idx}], tkEnd[${idx}])` : `tkText[${idx}]`;
}
private id() { return `_t${this.tmp++}`; }
emit(line = '') { this.buf.push(line); }
// The compound-matcher helpers are function declarations (hoisted), but they must sit
// BELOW the module's import statements. The header emits a `${HELPERS}` sentinel right
// after the imports/tables; we splice the collected helpers there.
toString() {
// MEMO_RULES is known only after every memoized rule allocated its slot.
this.helperDefs.push(`const MEMO_RULES = ${this.memoIdx.size};`);
const src = this.buf.join('\n');
return src.replace('//${HELPERS}', this.helperDefs.join('\n\n'));
}
// Per-memoized-rule slot in the parse-wide memo array (replaces the string-keyed
// outer Map: parseRuleEntry runs per rule entry, so the name hash was on a hot path).
private memoIdx = new Map<string, number>();
memoIndex(name: string): number {
let i = this.memoIdx.get(name);
if (i === undefined) { i = this.memoIdx.size; this.memoIdx.set(name, i); }
return i;
}
// Reference to a rule's parse function (token refs are inlined where used).
private ruleFn(name: string) { return `R_${sanitize(name)}`; }
/**
* Emit (once) a helper fn for a compound `expr` and return its name. The helper
* has the matchExpr contract: returns the matched children array or null, with pos
* restored on failure. Using a function (not inlined statements) means failure is a
* `return null` and loops use `return`, so nested compounds never have a `break`
* that escapes to the wrong loop — the one real hazard of inlining the tree-walk.
*/
private matchFn(expr: RuleExpr): string {
const key = JSON.stringify(expr);
const existing = this.helpers.get(key);
if (existing) return existing;
const name = `m${this.helpers.size}`;
this.helpers.set(key, name);
const body: string[] = [`function ${name}() {`];
const single = this.singleLeafBody(expr);
if (single) {
// Matcher produces exactly one child: the matcher call already pushes it —
// no save needed (a failing single matcher consumes nothing).
body.push(single);
} else {
body.push(` const _save = pos; const _sn = scn;`);
body.push(this.matchInto(expr, `pos = _save; scn = _sn; return false;`));
body.push(` return true;`);
}
body.push(`}`);
this.helperDefs.push(body.join('\n'));
return name;
}
// Public wrapper so free-function emitters (emitArmNamed) can reuse the single-leaf
// specialization.
singleLeafBodyPublic(expr: RuleExpr): string | null { return this.singleLeafBody(expr); }
// If `expr` matches exactly one child (a single literal, a single non-template token
// ref, or a single rule ref — possibly wrapped in a transparent group / one-item seq),
// return a body that just delegates to the (already pushing) matcher call — skipping
// the save/restore (a failing single matcher consumes nothing and pushes nothing).
// Else null. Excludes template-token refs (two-branch) and suppress-groups.
private singleLeafBody(expr: RuleExpr): string | null {
const a = this.a;
// Unwrap transparent wrappers that don't themselves emit/consume.
while (true) {
if (expr.type === 'group' && !(expr.suppress && expr.suppress.length)) { expr = expr.body; continue; }
if (expr.type === 'seq') {
const real = expr.items.filter(it => it.type !== 'op' && it.type !== 'prefix' && it.type !== 'postfix');
if (real.length === 1) { expr = real[0]; continue; }
}
break;
}
if (expr.type === 'literal') {
return ` return ${this.matchLiteralCall(expr.value)};`;
}
if (expr.type === 'ref') {
if (a.tokenNames.has(expr.name)) {
if (a.templateTokenNames.has(expr.name)) return null; // two-branch (template) — not single-leaf
return ` return ${this.matchTokenCall(expr.name)};`;
}
// Rule ref: keep the FIRST-set guard, then the call.
const guard = this.firstGuard(expr.name);
const guardLine = guard ? ` if (${guard}) { return false; }\n` : '';
return `${guardLine} return ${this.ruleFn(expr.name)}();`;
}
return null;
}
/**
* Generate statements that match `expr`, PUSHING children onto the scratch stack
* (the arena protocol — every matcher/rule call pushes its own result), and on any
* failure execute `onFail` (which restores pos + scn and returns). Simple/flat
* shapes are INLINED here (specialized straight-line code, no matchExpr switch);
* compound shapes (alt / quantifier / sep / nested seq with a contained loop)
* delegate to a generated helper fn via matchFn — keeping control flow `return`-based.
*/
matchInto(expr: RuleExpr, onFail: string): string {
const a = this.a;
switch (expr.type) {
case 'literal': {
return `if (!${this.matchLiteralCall(expr.value)}) { ${onFail} }`;
}
case 'ref': {
if (a.tokenNames.has(expr.name)) {
// Template tokens: route to parseTemplateExpr first (interpolated templates).
if (a.templateTokenNames.has(expr.name)) {
return `if (!parseTemplateExpr() && !${this.matchTokenCall(expr.name)}) { ${onFail} }`;
}
return `if (!${this.matchTokenCall(expr.name)}) { ${onFail} }`;
}
// Rule ref: FIRST-set guard (ruleMightStart) baked as a direct check, then call.
const guard = this.firstGuard(expr.name);
return [
guard ? `if (${guard}) { ${onFail} }` : ``,
`if (!${this.ruleFn(expr.name)}()) { ${onFail} }`,
].filter(Boolean).join('\n');
}
case 'seq': {
// Inline each item in order; share the caller's onFail (which restores to the
// seq's save point). matchSeq skips op/prefix/postfix markers. A nested seq is
// flattened inline too — its failure restores to the SAME save point (the whole
// matcher fn's _save), exactly like matchSeq's single saved/restore.
const parts: string[] = [];
for (const item of expr.items) {
if (item.type === 'op' || item.type === 'prefix' || item.type === 'postfix') continue;
parts.push(this.matchInto(item, onFail));
}
return parts.join('\n');
}
case 'alt': {
// matchExpr 'alt': try each arm (a helper fn) from a shared save; first
// success wins — its children are already on scratch.
const save = this.id(), sn = this.id(), r = this.id();
const lines: string[] = [`const ${save} = pos; const ${sn} = scn;`, `let ${r} = false;`];
for (const item of expr.items) {
const fn = this.matchFn(item);
lines.push(`if (!${r}) { pos = ${save}; scn = ${sn}; ${r} = ${fn}(); }`);
}
lines.push(`if (!${r}) { pos = ${save}; scn = ${sn}; ${onFail} }`);
return lines.join('\n');
}
case 'quantifier':
return this.matchQuantifierInto(expr.body, expr.kind, onFail);
case 'group': {
// A suppress-carrying group stages the LED-connector exclusion for the next
// parseRule, then matches its body (same as matchExpr 'group').
const pre = (expr.suppress && expr.suppress.length)
? `suppressNext = new Set(${J(expr.suppress)});`
: ``;
return [pre, this.matchInto(expr.body, onFail)].filter(Boolean).join('\n');
}
case 'not': {
// Zero-width negative lookahead: succeed (no children) iff body does NOT match.
const kinds = this.notKwKinds(expr.body);
if (kinds) {
// Fast: one keyword-kind membership test (no body matcher, nothing pushed).
const cond = kinds.map(k => `_tt === ${k}`).join(' || ');
return `if (pos < cap && tkK[pos] >= K_NAMED_MIN) { const _tt = tkT[pos]; if (${cond}) { ${onFail} } }`;
}
const save = this.id(), sn = this.id(), fn = this.matchFn(expr.body), m = this.id();
return [
`{ const ${save} = pos; const ${sn} = scn; const ${m} = ${fn}(); pos = ${save}; scn = ${sn};`,
` if (${m}) { ${onFail} } }`,
].join('\n');
}
case 'sameLine':
return `if (!(pos < cap && (tkFl[pos] & 1) === 0)) { ${onFail} }`;
case 'noCommentBefore':
return `if (!(pos < cap && (tkFl[pos] & 2) === 0)) { ${onFail} }`;
case 'noMultilineFlowBefore':
return `if (!(pos < cap && (tkFl[pos] & 4) === 0)) { ${onFail} }`;
case 'sep':
return this.matchSepInto(expr.element, expr.delimiter, onFail);
default:
// op/prefix/postfix — handled by Pratt; in matchExpr these return null.
return `{ ${onFail} }`;
}
}
// Quantifier: body is matched via a helper fn (pushes + boolean), so the loop here
// uses `return`/`break` only against ITS OWN while — no nested-loop hazard.
private matchQuantifierInto(body: RuleExpr, kind: '*' | '+' | '?', onFail: string): string {
const fn = this.matchFn(body);
if (kind === '?') {
// Try once; on failure the helper restored pos/scn itself.
return `${fn}();`;
}
if (kind === '*') {
const before = this.id(), bsn = this.id();
return [
`while (true) {`,
` const ${before} = pos; const ${bsn} = scn;`,
` if (!${fn}()) break;`,
` if (pos === ${before} && scn === ${bsn}) break;`,
`}`,
].join('\n');
}
// '+': first mandatory, then the same loop.
const before = this.id(), bsn = this.id();
return [
`if (!${fn}()) { ${onFail} }`,
`while (true) {`,
` const ${before} = pos; const ${bsn} = scn;`,
` if (!${fn}()) break;`,
` if (pos === ${before} && scn === ${bsn}) break;`,
`}`,
].join('\n');
}
// sep = (element (delimiter element)*)? — never fails (matches zero elements).
// element matched via helper fn. A delimiter with no following element stays kept
// (trailing-delimiter semantics) — it was already pushed by its matcher.
private matchSepInto(element: RuleExpr, delimiter: string, _onFail: string): string {
const fn = this.matchFn(element);
return [
`if (${fn}()) {`,
` while (true) {`,
` const _ds = pos; if (!${this.matchLiteralCall(delimiter)}) { pos = _ds; break; }`,
` if (!${fn}()) break;`,
` }`,
`}`,
].join('\n');
}
// Baked FIRST-set guard for a rule ref: the NEGATED ruleMightStart condition (so
// the caller writes `if (<guard>) { onFail }`). Returns '' when the rule can always
// start (nullable / null FIRST set) → no guard (matches ruleMightStart returning true).
firstGuard(name: string): string {
const a = this.a;
if (a.nullableRules.has(name)) return '';
const fs = a.firstSets.get(name);
if (!fs || fs.size === 0) return '';
// ruleMightStart: true iff some key in fs matches peek(); guard = NOT that. The set
// is baked as a per-set membership fn over two byte tables (see membershipFn).
return `!${this.membershipFn(fs)}(pos)`;
}
// Deep per-alternative dispatch condition (mirrors gen-parser.ts altMightStart): the
// POSITIVE "this alt might start at startTok" test for the longest-match loops. `true`
// when the alt is nullable or its FIRST set is unknown/empty (always tried — an empty
// match never wins longest-match); else a membership test over the alt's transitive
// FIRST set, baked as a hoisted int-descriptor array (same encoding firstGuard uses).
altGuard(alt: RuleExpr): string {
const a = this.a;
if (a.altNullable.get(alt)) return 'true';
const fs = a.altDeepFirst.get(alt);
if (!fs || fs.size === 0) return 'true';
return `${this.membershipFn(fs)}(saved)`;
}
// A `not(...)` over a literal / alternation of KEYWORD literals → the int keyword-kinds,
// else null. Lets the not be one membership test instead of matching each keyword arm
// (mirrors gen-parser.ts notKwSet; emits the same check matchKwLit uses, so byte-identical).
notKwKinds(body: RuleExpr): number[] | null {
const kinds: number[] = [];
const collect = (e: RuleExpr): boolean =>
e.type === 'literal' ? (isKeywordLiteral(e.value) ? (kinds.push(this.a.symtab.kwLitKind.get(e.value)!), true) : false)
: e.type === 'alt' ? e.items.every(collect)
: false;
return collect(body) && kinds.length > 0 ? kinds : null;
}
// Register (deduped) a FIRST-set's membership test as a module-level fn over two
// byte tables and return the fn's NAME. Test: `!tok || (KT[tok.k] | TT[tok.t])` —
// faithful to the old per-desc loop because the kw int range and the punct int range
// are DISJOINT (so the loop's k!==K_PUNCT / k===K_PUNCT guards were redundant), and
// a punct desc's text.startsWith(v) arm is enumerated over the closed punct
// vocabulary at emit time. The one narrowing: a K_PUNCT token whose text is OUTSIDE
// the vocabulary (t=0) can't startsWith-match here — such a token is unreachable
// (the lexer scans only vocabulary puncts; `>`-split rests stay in-vocabulary),
// and the full-corpus byte-identical gate covers it empirically.
membershipFn(fs: Set<string>): string {
const { kArr, tArr } = this.membershipTables(fs);
const fnKey = `${kArr}|${tArr}`;
let nm = this.memberFns.get(fnKey);
if (!nm) {
nm = `_q${this.memberFns.size}`;
this.memberFns.set(fnKey, nm);
this.helperDefs.push(`function ${nm}(i) { return i >= cap || (${kArr}[tkK[i]] | ${tArr}[tkT[i]]) !== 0; }`);
}
return nm;
}
// The first-token gate for an alt/LED whose tok is already known non-null: the same
// two-table membership as membershipFn, open-coded (no call). null FirstTok → no gate.
ftCond(ft: FirstTok, idxVar: string): string | null {
if (!ft) return null;
const key = 'tok' in ft ? ft.tok : ft.lit;
const { kArr, tArr } = this.membershipTables(new Set([key]));
return `(${kArr}[tkK[${idxVar}]] | ${tArr}[tkT[${idxVar}]]) !== 0`;
}
// A FIRST set's admitted (tok.k, tok.t) index sets — the shared classification behind
// the byte tables and the alt-dispatch masks (same split keyMatchesTok used).
private firstSetOnes(fs: Set<string>): { kOnes: Set<number>; tOnes: Set<number> } {
const st = this.a.symtab;
const kOnes = new Set<number>(), tOnes = new Set<number>();
for (const key of [...fs].sort()) {
const q = this.a.qualKeys.get(key);
if (q) {
// Reserved-qualified token key: admit by t instead of the blanket k-bit —
// t=0 (covers plain members of the token class; over-admits other t=0 kinds
// harmlessly) plus every keyword t outside the guard class.
tOnes.add(0);
for (const [text, id] of st.kwLitKind) if (!q.except.has(text)) tOnes.add(id);
continue;
}
const d = st.classifyKey(key);
if (d.kind === 'tok') {
kOnes.add(d.k);
if (d.template) kOnes.add(st.KIND_TEMPLATE_HEAD);
} else if (d.kind === 'kw') {
if (d.t === 0) throw new Error(`emit: FIRST key ${J(key)} missing from the literal vocabulary`);
tOnes.add(d.t);
} else {
if (d.t === 0) throw new Error(`emit: FIRST key ${J(key)} missing from the literal vocabulary`);
for (const [text, id] of st.puLitKind) if (text.startsWith(d.v)) tOnes.add(id);
}
}
return { kOnes, tOnes };
}
private kSize(): number { return this.a.symtab.KIND_NAMED_FALLBACK + 1; }
private tSize(): number {
const st = this.a.symtab;
let n = 1;
for (const v of st.kwLitKind.values()) n = Math.max(n, v + 1);
for (const v of st.puLitKind.values()) n = Math.max(n, v + 1);
return n;
}
// Build (deduped) the two byte tables for a FIRST set's membership test.