fix(native-preview): preserve template surrogate escapes

Author: TorinAsakura

_packages/native-preview/test/async/api.test.ts

@@ -234,6 +234,38 @@ test("unicode escapes", async () => {
     }
 });
 
+test("template unicode escapes", async () => {
+    const api = spawnAPI({
+        "/tsconfig.json": "{}",
+        "/src/index.ts": "`\\ud800${0}\\udc00`",
+    });
+    try {
+        const snapshot = await api.updateSnapshot({ openProject: "/tsconfig.json" });
+        const project = snapshot.getProject("/tsconfig.json")!;
+        const sourceFile = await project.program.getSourceFile("/src/index.ts");
+        assert.ok(sourceFile);
+
+        let sawHead = false;
+        let sawTail = false;
+        sourceFile.forEachChild(function visit(node) {
+            if (isTemplateHead(node)) {
+                assert.equal(node.text, "\ud800");
+                sawHead = true;
+            }
+            else if (isTemplateTail(node)) {
+                assert.equal(node.text, "\udc00");
+                sawTail = true;
+            }
+            node.forEachChild(visit);
+        });
+        assert.ok(sawHead);
+        assert.ok(sawTail);
+    }
+    finally {
+        await api.close();
+    }
+});
+
 test("Object equality", async () => {
     const api = spawnAPI();
     try {

_packages/native-preview/test/sync/api.test.ts

@@ -242,6 +242,38 @@ test("unicode escapes", () => {
     }
 });
 
+test("template unicode escapes", () => {
+    const api = spawnAPI({
+        "/tsconfig.json": "{}",
+        "/src/index.ts": "`\\ud800${0}\\udc00`",
+    });
+    try {
+        const snapshot = api.updateSnapshot({ openProject: "/tsconfig.json" });
+        const project = snapshot.getProject("/tsconfig.json")!;
+        const sourceFile = project.program.getSourceFile("/src/index.ts");
+        assert.ok(sourceFile);
+
+        let sawHead = false;
+        let sawTail = false;
+        sourceFile.forEachChild(function visit(node) {
+            if (isTemplateHead(node)) {
+                assert.equal(node.text, "\ud800");
+                sawHead = true;
+            }
+            else if (isTemplateTail(node)) {
+                assert.equal(node.text, "\udc00");
+                sawTail = true;
+            }
+            node.forEachChild(visit);
+        });
+        assert.ok(sawHead);
+        assert.ok(sawTail);
+    }
+    finally {
+        api.close();
+    }
+});
+
 test("Object equality", () => {
     const api = spawnAPI();
     try {

internal/api/encoder/encoder.go

@@ -547,21 +547,21 @@ func recordExtendedData_SourceFile(node *ast.Node, strs *stringTable, positionMa
 
 func recordExtendedData_TemplateHead(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
 	n := node.AsTemplateHead()
-	textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
+	textIndex := strs.add(encodeTemplateTextForJS(n.Text, n.RawText), node.Kind, node.Pos(), node.End())
 	rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End())
 	*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))
 }
 
 func recordExtendedData_TemplateMiddle(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
 	n := node.AsTemplateMiddle()
-	textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
+	textIndex := strs.add(encodeTemplateTextForJS(n.Text, n.RawText), node.Kind, node.Pos(), node.End())
 	rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End())
 	*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))
 }
 
 func recordExtendedData_TemplateTail(node *ast.Node, strs *stringTable, positionMap *ast.PositionMap, extendedData *[]byte, structuredData *[]byte) {
 	n := node.AsTemplateTail()
-	textIndex := strs.add(n.Text, node.Kind, node.Pos(), node.End())
+	textIndex := strs.add(encodeTemplateTextForJS(n.Text, n.RawText), node.Kind, node.Pos(), node.End())
 	rawTextIndex := strs.add(n.RawText, node.Kind, node.Pos(), node.End())
 	*extendedData = appendUint32s(*extendedData, textIndex, rawTextIndex, uint32(n.TemplateFlags))
 }

internal/api/encoder/encoder_test.go

@@ -75,6 +75,25 @@ func TestEncodeSourceFilePreservesSurrogateEscapes(t *testing.T) {
 	})
 }
 
+func TestEncodeSourceFilePreservesTemplateSurrogateEscapes(t *testing.T) {
+	t.Parallel()
+	sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{
+		FileName: "/test.ts",
+		Path:     "/test.ts",
+	}, "let s = `\\uD800${1}\\uDC00`;", core.ScriptKindTS)
+
+	buf, err := encoder.EncodeSourceFile(sourceFile)
+	assert.NilError(t, err)
+
+	headText, ok := findExtendedNodeText(buf, ast.KindTemplateHead)
+	assert.Assert(t, ok)
+	assert.DeepEqual(t, headText, []byte{0xed, 0xa0, 0x80})
+
+	tailText, ok := findExtendedNodeText(buf, ast.KindTemplateTail)
+	assert.Assert(t, ok)
+	assert.DeepEqual(t, tailText, []byte{0xed, 0xb0, 0x80})
+}
+
 func TestEncodeSourceFileFallsBackForUnterminatedSurrogateEscape(t *testing.T) {
 	t.Parallel()
 	sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{

internal/api/encoder/literal_text.go

@@ -50,20 +50,40 @@ func decodeQuotedLiteralText(raw string) (text string, hasSurrogate bool, ok boo
 	if len(raw) < 2 {
 		return "", false, false
 	}
+	return decodeEscapedLiteralText(raw[1:len(raw)-1], false)
+}
+
+func encodeTemplateTextForJS(text string, rawText string) string {
+	decoded, hasSurrogate, ok := decodeEscapedLiteralText(rawText, true)
+	if !ok || !hasSurrogate {
+		return text
+	}
+	return decoded
+}
+
+func decodeEscapedLiteralText(raw string, normalizeTemplateLineEndings bool) (text string, hasSurrogate bool, ok bool) {
 	var out strings.Builder
-	for i := 1; i < len(raw)-1; {
+	for i := 0; i < len(raw); {
 		if raw[i] != '\\' {
+			if normalizeTemplateLineEndings && raw[i] == '\r' {
+				out.WriteByte('\n')
+				i++
+				if i < len(raw) && raw[i] == '\n' {
+					i++
+				}
+				continue
+			}
 			out.WriteByte(raw[i])
 			i++
 			continue
 		}
-		ch, next, ok := decodeEscape(raw, i, len(raw)-1)
+		ch, next, ok := decodeEscape(raw, i, len(raw))
 		if !ok {
 			return "", false, false
 		}
 		if codePointIsHighSurrogate(ch) {
 			hasSurrogate = true
-			if nextCh, nextNext, ok := decodeUnicodeEscape(raw, next, len(raw)-1); ok && codePointIsLowSurrogate(nextCh) {
+			if nextCh, nextNext, ok := decodeUnicodeEscape(raw, next, len(raw)); ok && codePointIsLowSurrogate(nextCh) {
 				out.WriteRune(surrogatePairToCodepoint(ch, nextCh))
 				i = nextNext
 				continue