Skip to content

Commit e48584e

Browse files
hyperpolymathclaude
hyperpolymath
and
claude
committed
fix(codegen-js): emit \uXXXX/\u{X} for non-ASCII (closes #460)
OCaml's `String.escaped` emits non-ASCII bytes as `\NNN` *decimal* sequences. JavaScript parses `\NNN` as *octal* escapes which strict-mode ESM rejects outright (`SyntaxError: Octal escape sequences are not allowed in strict mode`), and which would decode to wrong characters even outside strict mode. Adds `Js_codegen.js_string_lit` that walks the UTF-8 byte sequence, decodes code points, and emits `\uXXXX` (BMP) or `\u{XXXXX}` (non-BMP) Unicode escapes. ASCII printable bytes pass through unchanged; `\\` `\"` `\n` `\r` `\t` use conventional escapes; ASCII control bytes use `\xHH`. Wired into both `js_codegen.ml` (Node target) and `codegen_deno.ml` (Deno-ESM target) LitString/LitChar emit sites. Regression fixture `tests/codegen-deno/non_ascii.affine` + harness exercise BMP emoji (❌ ✓), CJK (你好), Latin accents (café résumé), non-BMP code points (😭 = U+1F62D), mixed strings, and the existing-escape regression path (\\ and \"). Pre-fix: harness `import` itself fails with SyntaxError. Post-fix: 8/8 assertions pass. Verified: full `tools/run_codegen_deno_tests.sh` (13/13 harnesses green); full `dune test` suite (352/352 green). Closes #460 Refs hyperpolymath/standards#284 (the seam-analyst PR that surfaced this) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 4ae169d commit e48584e

4 files changed

Lines changed: 105 additions & 4 deletions

File tree

‎lib/codegen_deno.ml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -721,8 +721,8 @@ and gen_literal (lit : literal) : string =
721721
if String.length s > 0 && s.[String.length s - 1] = '.' then s ^ "0" else s
722722
| LitBool (true, _) -> "true"
723723
| LitBool (false, _) -> "false"
724-
| LitString (s, _) -> "\"" ^ String.escaped s ^ "\""
725-
| LitChar (c, _) -> "\"" ^ Char.escaped c ^ "\""
724+
| LitString (s, _) -> Js_codegen.js_string_lit s
725+
| LitChar (c, _) -> Js_codegen.js_string_lit (String.make 1 c)
726726
| LitUnit _ -> "Unit"
727727

728728
and gen_pattern ctx (pat : pattern) : string =

‎lib/js_codegen.ml‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,62 @@ let mangle (name : string) : string =
101101
if List.mem name js_reserved then name ^ "_"
102102
else name
103103

104+
(** Lower a UTF-8 byte string to a JS double-quoted literal that is
105+
safe under strict-mode ESM.
106+
107+
OCaml's [String.escaped] emits non-ASCII bytes as [\NNN] *decimal*
108+
sequences; JavaScript parses [\NNN] as *octal* escapes which strict
109+
mode rejects ([SyntaxError: Octal escape sequences are not allowed
110+
in strict mode]) and which would decode to wrong characters even
111+
outside strict mode. This helper instead decodes the UTF-8 byte
112+
sequence to code points and emits [\uXXXX] (BMP) or [\u{XXXXX}]
113+
(non-BMP) Unicode escapes — accepted everywhere, no parser-mode
114+
surprises, and preserves the original character. Closes #460. *)
115+
let js_string_lit (s : string) : string =
116+
let buf = Buffer.create (String.length s + 8) in
117+
Buffer.add_char buf '"';
118+
let n = String.length s in
119+
let i = ref 0 in
120+
while !i < n do
121+
let b0 = Char.code s.[!i] in
122+
if b0 < 0x80 then begin
123+
(match Char.chr b0 with
124+
| '\\' -> Buffer.add_string buf "\\\\"
125+
| '"' -> Buffer.add_string buf "\\\""
126+
| '\n' -> Buffer.add_string buf "\\n"
127+
| '\r' -> Buffer.add_string buf "\\r"
128+
| '\t' -> Buffer.add_string buf "\\t"
129+
| c when b0 >= 0x20 && b0 <= 0x7E -> Buffer.add_char buf c
130+
| _ -> Buffer.add_string buf (Printf.sprintf "\\x%02X" b0));
131+
incr i
132+
end else begin
133+
let cp, len =
134+
if b0 < 0xC0 then (b0, 1)
135+
else if b0 < 0xE0 && !i + 1 < n then
136+
let b1 = Char.code s.[!i + 1] in
137+
(((b0 land 0x1F) lsl 6) lor (b1 land 0x3F), 2)
138+
else if b0 < 0xF0 && !i + 2 < n then
139+
let b1 = Char.code s.[!i + 1] in
140+
let b2 = Char.code s.[!i + 2] in
141+
(((b0 land 0x0F) lsl 12) lor ((b1 land 0x3F) lsl 6) lor (b2 land 0x3F), 3)
142+
else if !i + 3 < n then
143+
let b1 = Char.code s.[!i + 1] in
144+
let b2 = Char.code s.[!i + 2] in
145+
let b3 = Char.code s.[!i + 3] in
146+
(((b0 land 0x07) lsl 18) lor ((b1 land 0x3F) lsl 12)
147+
lor ((b2 land 0x3F) lsl 6) lor (b3 land 0x3F), 4)
148+
else (b0, 1)
149+
in
150+
if cp <= 0xFFFF then
151+
Buffer.add_string buf (Printf.sprintf "\\u%04X" cp)
152+
else
153+
Buffer.add_string buf (Printf.sprintf "\\u{%X}" cp);
154+
i := !i + len
155+
end
156+
done;
157+
Buffer.add_char buf '"';
158+
Buffer.contents buf
159+
104160
(* ============================================================================
105161
Expression Code Generation
106162
============================================================================ *)
@@ -230,8 +286,8 @@ and gen_literal (lit : literal) : string =
230286
if String.length s > 0 && s.[String.length s - 1] = '.' then s ^ "0" else s
231287
| LitBool (true, _) -> "true"
232288
| LitBool (false, _) -> "false"
233-
| LitString (s, _) -> "\"" ^ String.escaped s ^ "\""
234-
| LitChar (c, _) -> "\"" ^ Char.escaped c ^ "\""
289+
| LitString (s, _) -> js_string_lit s
290+
| LitChar (c, _) -> js_string_lit (String.make 1 c)
235291
| LitUnit _ -> "Unit"
236292

237293
and gen_pattern ctx (pat : pattern) : string =

‎tests/codegen-deno/non_ascii.affine‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// SPDX-License-Identifier: MPL-2.0
2+
// issue #460 — non-ASCII string literals must round-trip under
3+
// strict-mode ESM. Pre-fix, the JS codegen used OCaml `String.escaped`
4+
// which emitted `\NNN` decimal sequences; the JS parser reads `\NNN`
5+
// as OCTAL escapes, which strict-mode ESM rejects with
6+
// `SyntaxError: Octal escape sequences are not allowed in strict mode`.
7+
// Post-fix, non-ASCII bytes lower to `\uXXXX` / `\u{XXXXX}` Unicode
8+
// escapes which all JS parser modes accept.
9+
10+
pub fn emoji_cross() -> String { return "❌"; }
11+
pub fn emoji_check() -> String { return "✓"; }
12+
pub fn cjk_hello() -> String { return "你好"; }
13+
pub fn latin_accent() -> String { return "café résumé"; }
14+
pub fn non_bmp_sob() -> String { return "😭"; }
15+
pub fn mixed() -> String { return "[OK] café 你好 ❌"; }
16+
pub fn ascii_only() -> String { return "plain ASCII"; }
17+
pub fn quotes_and_backslash() -> String { return "\"escaped\" and \\back"; }

‎tests/codegen-deno/non_ascii.harness.mjs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// SPDX-License-Identifier: MPL-2.0
2+
// issue #460 — round-trip non-ASCII string literals through the
3+
// Deno-ESM backend under strict-mode ESM. The `import` itself is the
4+
// strictest test: if the emitted `.deno.js` contains octal escapes,
5+
// the module fails to parse and the import throws SyntaxError before
6+
// any assertion can run.
7+
import assert from "node:assert/strict";
8+
import {
9+
emoji_cross,
10+
emoji_check,
11+
cjk_hello,
12+
latin_accent,
13+
non_bmp_sob,
14+
mixed,
15+
ascii_only,
16+
quotes_and_backslash,
17+
} from "./non_ascii.deno.js";
18+
19+
assert.equal(emoji_cross(), "❌", "BMP emoji ❌ round-trips");
20+
assert.equal(emoji_check(), "✓", "BMP check mark ✓ round-trips");
21+
assert.equal(cjk_hello(), "你好", "CJK 'nihao' round-trips");
22+
assert.equal(latin_accent(), "café résumé", "Latin accented round-trips");
23+
assert.equal(non_bmp_sob(), "\u{1F62D}", "non-BMP code point round-trips");
24+
assert.equal(mixed(), "[OK] café 你好 ❌", "mixed ASCII+non-ASCII round-trips");
25+
assert.equal(ascii_only(), "plain ASCII", "ASCII-only unchanged");
26+
assert.equal(quotes_and_backslash(), "\"escaped\" and \\back", "quote+backslash escapes preserved");
27+
28+
console.log("non_ascii.harness.mjs OK");

0 commit comments

Comments
 (0)