Fix JS line-terminator conformance across all four lexers (CR / LS / PS)

The lexers counted only LF as a line terminator, but ECMAScript also defines CR
(U+000D), LS (U+2028), and PS (U+2029) — the set that drives ASI and the
"no LineTerminator here" restrictions. So `return\r1` was parsed `return 1`
where a conforming JS parser applies ASI (bare `return`, then `1`).

Fixed consistently in all four lexer implementations so they stay in lockstep:
- gen-lexer.ts (interpreter, the oracle): LF/CR in the ASCII path, LS/PS via the
  \s-run regex, and the comment-span check.
- emit-lexer.ts (emitted SoA/JS lexer): the same, in its codegen.
- target-ts.ts (portable, UTF-16): LF/CR/LS/PS.
- target-go.ts / target-rust.ts (portable, byte-based): LF/CR (LS/PS are multi-byte
  and fall under the documented non-ASCII offset boundary).

CRLF is unchanged (the LF already set newline-before), so the existing corpus is
unaffected — the change only reaches lone-CR and LS/PS inputs. This supersedes the
earlier direction (which had made the portable lexers match the LF-only interpreter);
now the interpreter is conforming and all four agree on the full set.

sljs gate extended: `return\r1;` / `return\r\n1;` / `return /*\r*/ 1;` reject,
`return\t1;` accepts (tab is whitespace, not a terminator) — checked across ts/go/rust.
emit-parser-verify byte-identical, portable-targets 16 grammars ×3, full suite 42/42.

Author: johnsoncodehk

src/emit-lexer.ts

@@ -43,7 +43,7 @@ const NON_ASCII_WS_FN =
 // The non-ASCII whitespace fallback, emitted at the two sites that need it (after an ASCII run,
 // and as the lead char). `cont` appends the `continue` the lead-char site needs.
 const nonAsciiWsConsume = (v: string, cont: boolean, indent: string): string =>
-  `${indent}if (${v} > 127 && lxNonAsciiWs(${v})) { LX_WS.lastIndex = pos; const m = LX_WS.exec(source); if (m !== null) { if (m[0].includes('\\n')) pendingNl = true; pos += m[0].length;${cont ? ' continue;' : ''} } }`;
+  `${indent}if (${v} > 127 && lxNonAsciiWs(${v})) { LX_WS.lastIndex = pos; const m = LX_WS.exec(source); if (m !== null) { if (/[\\n\\r\\u2028\\u2029]/.test(m[0])) pendingNl = true; pos += m[0].length;${cont ? ' continue;' : ''} } }`;
 
 export function emitSoaLexer(grammar: CstGrammar, st: LexerSymtab): string | null {
   // Out of scope: the markup / indentation / newline state machines.
@@ -390,7 +390,7 @@ export function emitSoaLexer(grammar: CstGrammar, st: LexerSymtab): string | nul
   emit(`    if (cc === 32 || (cc >= 9 && cc <= 13)) {`);
   emit(`      let wc = cc;`);
   emit(`      do {`);
-  emit(`        if (wc === 10) pendingNl = true;`);
+  emit(`        if (wc === 10 || wc === 13) pendingNl = true;`);   // JS line terminators LF/CR (LS/PS via the \\s regex below)
   emit(`        pos++;`);
   emit(`        wc = source.charCodeAt(pos);`);
   emit(`      } while (wc === 32 || (wc >= 9 && wc <= 13));`);
@@ -476,7 +476,7 @@ export function emitSoaLexer(grammar: CstGrammar, st: LexerSymtab): string | nul
       emit(`${ind}    }`);
     }
     if (m.skip) {
-      emit(`${ind}    if (m[0].includes('\\n')) pendingNl = true;`);
+      emit(`${ind}    if (/[\\n\\r\\u2028\\u2029]/.test(m[0])) pendingNl = true;`);
       emit(`${ind}    pos += m[0].length;`);
     } else {
       emit(`${ind}    const _e = pos + m[0].length;`);

src/gen-lexer.ts

@@ -842,21 +842,23 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
         let wc = source.charCodeAt(pos);
         if (wc === 32 || (wc >= 9 && wc <= 13)) {
           do {
-            if (wc === 10) pendingNl = true;
+            // JS line terminators: LF, CR, LS, PS (the ECMAScript set, driving ASI / "no
+            // LineTerminator here"). LF/CR are ASCII (here); LS/PS arrive via the \s regex below.
+            if (wc === 10 || wc === 13) pendingNl = true;
             pos++;
             wc = source.charCodeAt(pos);
           } while (wc === 32 || (wc >= 9 && wc <= 13));
           if (wc > 127) {   // a Unicode space may continue the run — absorb it like the old regex did
             wsReY.lastIndex = pos;
             const wsMatch = wsReY.exec(source);
-            if (wsMatch) { if (wsMatch[0].includes('\n')) pendingNl = true; pos += wsMatch[0].length; }
+            if (wsMatch) { if (/[\n\r\u2028\u2029]/.test(wsMatch[0])) pendingNl = true; pos += wsMatch[0].length; }
           }
           continue;
         }
         if (wc > 127) {
           wsReY.lastIndex = pos;
           const wsMatch = wsReY.exec(source);
-          if (wsMatch) { if (wsMatch[0].includes('\n')) pendingNl = true; pos += wsMatch[0].length; continue; }
+          if (wsMatch) { if (/[\n\r\u2028\u2029]/.test(wsMatch[0])) pendingNl = true; pos += wsMatch[0].length; continue; }
         }
       }
 
@@ -1178,7 +1180,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
           if (!tm.skip) {
             push(mkNamed(tm.name, m[0], pos, tm.k));
           } else {
-            if (m[0].includes('\n')) pendingNl = true;   // a skipped comment spanning a newline still terminates the previous line
+            if (/[\n\r\u2028\u2029]/.test(m[0])) pendingNl = true;   // a skipped comment spanning a line terminator still terminates the previous line
             // An inline comment (indentation grammars) ENDS a plain scalar — flag the next token so a
             // multi-line fold won't reabsorb a post-comment line (yaml-test-suite 8XDJ / BF9H).
             if (indent?.comment && m[0].startsWith(indent.comment)) pendingComment = true;

src/target-go.ts

@@ -50,7 +50,7 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st
   const name = (t as { name: string }).name;
   const stateful = rxTok !== undefined || tplTok !== undefined;
   if (tplTok !== undefined && name === tplTok) return '';   // template token scanned by the state machine
-  const push = (endE: string) => (t.skip ? `if strings.Contains(src[pos:${endE}], "\\n") { pendingNl = true }; ` : stateful ? `emit(${J(name)}, src[pos:${endE}], pos, ${endE}); ` : `pushTok(${J(name)}, src[pos:${endE}], pos, ${endE}); `);
+  const push = (endE: string) => (t.skip ? `if strings.ContainsAny(src[pos:${endE}], "\\n\\r") { pendingNl = true }; ` : stateful ? `emit(${J(name)}, src[pos:${endE}], pos, ${endE}); ` : `pushTok(${J(name)}, src[pos:${endE}], pos, ${endE}); `);
   const gate = rxTok !== undefined && name === rxTok ? '!prevIsValue() && ' : '';
   if (t.kind === 'run') return `\t\tif ${gate}${rangeCond('c', t.first)} {
 \t\t\te := pos + 1
@@ -156,8 +156,8 @@ ${emitHooks}
 \t_ = pendingNl
 ${rxState}${tplState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\n' : ''}\tfor pos < n {
 \t\tc := int(src[pos])
-\t\tif c == 10 { pendingNl = true; pos++; continue }   // only LF (10) is newline-before (matches the interpreter); CR is plain whitespace
-\t\tif c == 13 || c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue }
+\t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue }   // JS line terminators LF/CR (matches the interpreter; LS/PS are multi-byte: non-ASCII boundary)
+\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue }
 ${tplDispatch}${toks}
 ${puncts}
 \t\tpanic(fmt.Sprintf("lex error at %d", pos))

src/target-rust.ts

@@ -54,7 +54,7 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st
   const stateful = rxTok !== undefined || tplTok !== undefined;
   if (tplTok !== undefined && name === tplTok) return '';   // template token scanned by the state machine
   const nlVar = stateful ? 'st.pending_nl' : 'pending_nl';
-  const push = (endE: string) => (t.skip ? `if src[pos..${endE}].contains('\\n') { ${nlVar} = true; } ` : stateful ? `st.emit(${J(name)}, &src[pos..${endE}], pos, ${endE}); ` : `toks.push(Tok { kind: ${J(name)}, text: &src[pos..${endE}], off: pos, end: ${endE}, nl: pending_nl }); pending_nl = false; `);
+  const push = (endE: string) => (t.skip ? `if src[pos..${endE}].bytes().any(|c| c == 10 || c == 13) { ${nlVar} = true; } ` : stateful ? `st.emit(${J(name)}, &src[pos..${endE}], pos, ${endE}); ` : `toks.push(Tok { kind: ${J(name)}, text: &src[pos..${endE}], off: pos, end: ${endE}, nl: pending_nl }); pending_nl = false; `);
   const gate = rxTok !== undefined && name === rxTok ? '!st.prev_is_value() && ' : '';
   if (t.kind === 'run') return `        if ${gate}${rangeCond('c', t.first)} {
             let mut e = pos + 1;
@@ -163,8 +163,8 @@ ${open}
     let mut pos = 0usize;
     while pos < n {
         let c = b[pos] as u32;
-        if c == 32 || c == 9 || c == 13 { pos += 1; continue; }   // CR is plain whitespace, NOT newline-before
-        if c == 10 { ${nlVar} = true; pos += 1; continue; }   // only LF (10) is newline-before (matches the interpreter)
+        if c == 32 || c == 9 { pos += 1; continue; }
+        if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; }   // JS line terminators LF/CR (matches the interpreter; LS/PS multi-byte: non-ASCII boundary)
 ${tplDispatch}${toks}
 ${puncts}
         panic!("lex error at {}", pos);

src/target-ts.ts

@@ -50,7 +50,7 @@ function scanTok(t: LexTok, defs: string[], rxTok?: string, tplTok?: string): st
   if (tplTok !== undefined && name === tplTok) return '';   // template token is scanned by the state machine
   // `emit(...)` threads the lexer state in stateful mode; a plain push otherwise. A skipped
   // token (comment) still records a newline it spans, so `sameLine` sees it.
-  const push = (endExpr: string) => (t.skip ? `if (src.slice(pos, ${endExpr}).indexOf('\\n') >= 0) pendingNl = true; ` : `${stateful ? 'emit' : 'push'}(${J(name)}, src.slice(pos, ${endExpr}), pos, ${endExpr}); `);
+  const push = (endExpr: string) => (t.skip ? `if (/[\\n\\r\\u2028\\u2029]/.test(src.slice(pos, ${endExpr}))) pendingNl = true; ` : `${stateful ? 'emit' : 'push'}(${J(name)}, src.slice(pos, ${endExpr}), pos, ${endExpr}); `);
   const gate = rxTok !== undefined && name === rxTok ? '!prevIsValue() && ' : '';
   if (t.kind === 'run') return `    if (${gate}${rangeCond('c', t.first)}) {
       let e = pos + 1;
@@ -147,10 +147,9 @@ ${emitHooks}
   let pendingNl = false;
 ${defs.length ? '  _s = src;\n' : ''}${rxState}${tplState}${stateful ? emitFn : '  const push = (kind: string, text: string, off: number, end: number) => { toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; };\n'}  while (pos < n) {
     const c = src.charCodeAt(pos);
-    // Only LF (char 10) sets newline-before, matching the interpreter (gen-lexer.ts: only wc === 10).
-    // CR/LS/PS are whitespace but NOT newline-before there, so a lone CR must not flip sameLine.
-    if (c === 10) { pendingNl = true; pos++; continue; }
-    if (c === 13 || c === 8232 || c === 8233 || c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; }
+    // JS line terminators LF/CR/LS/PS set newline-before, matching the interpreter (gen-lexer.ts).
+    if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; }
+    if (c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; }
 ${tplDispatch}${toks}
 ${puncts}
     throw new Error('lex error at ' + pos + ': ' + JSON.stringify(src[pos]));

test/portable-targets.ts

@@ -135,12 +135,12 @@ const CASES: Case[] = [
     grammar: 'sljs', path: './fixtures/sljs.ts',
     accept: [
       'return 1;', 'return;', 'return 1 + 2;', '1 + 2;', 'return /* c */ 1;',
-      '(a);', 'return (1);',
-      // Only `\n` is newline-before — a lone `\r` is plain whitespace, so `return` still takes its
-      // same-line operand (matches the interpreter; was a portable-lexer bug). CRLF still has the `\n`.
-      'return\r1;',
+      '(a);', 'return (1);', 'return\t1;',
     ],
-    reject: ['return\n1;', 'return\nx;', 'return /*\n*/ 1;', 'return // c\n 1;', 'return\r\n1;'],
+    // `\r`, LS, PS are JS line terminators just like `\n` (ASI / "no LineTerminator here"), so a
+    // `return` followed by any of them takes no operand — across all four lexers (interpreter,
+    // emitted JS, portable ts/go/rust). A `\t` (tab) is whitespace but NOT a terminator → accepted above.
+    reject: ['return\n1;', 'return\nx;', 'return /*\n*/ 1;', 'return // c\n 1;', 'return\r1;', 'return\r\n1;', 'return /*\r*/ 1;'],
   },
   {
     // capBelow (assignment-level) arrow functions: a NUD parsed only when minBp < the