Add t-strings support to native parser (#21007)

This is mypy counterpart of
mypyc/ast_serialize#34

Author: ilevkivskyi

.github/workflows/test.yml

@@ -220,7 +220,7 @@ jobs:
     # To speed-up process until ast_serialize is on PyPI.
     - name: Install pinned ast-serialize
       if: ${{ matrix.dev_ast_serialize }}
-      run: pip install ast-serialize@git+https://github.com/mypyc/ast_serialize.git@da5a16cf268dbec63ed6b2e6b715470576e2d1a6
+      run: pip install ast-serialize@git+https://github.com/mypyc/ast_serialize.git@d277690a078c7784667a640ed1045e725bc42c00
 
     - name: Setup tox environment
       run: |

mypy/nativeparse.py

@@ -111,6 +111,7 @@
     Statement,
     StrExpr,
     SuperExpr,
+    TemplateStrExpr,
     TempNode,
     TryStmt,
     TupleExpr,
@@ -168,6 +169,7 @@ def __init__(self, options: Options) -> None:
         self.options = options
         self.errors: list[dict[str, Any]] = []
         self.num_funcs = 0
+        self.uses_template_strings = False
 
     def add_error(
         self,
@@ -233,6 +235,7 @@ def native_parse(
     node = MypyFile(defs, imports)
     node.path = filename
     node.is_partial_stub_package = is_partial_package
+    node.uses_template_strings = state.uses_template_strings
     # Merge deserialization errors with parsing errors
     all_errors = errors + state.errors
     return node, all_errors, ignores
@@ -243,7 +246,7 @@ def expect_end_tag(data: ReadBuffer) -> None:
 
 
 def expect_tag(data: ReadBuffer, tag: Tag) -> None:
-    assert read_tag(data) == tag
+    assert (actual := read_tag(data)) == tag, actual
 
 
 def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]:
@@ -261,9 +264,9 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]:
 def parse_to_binary_ast(
     filename: str, options: Options, skip_function_bodies: bool = False
 ) -> tuple[bytes, list[dict[str, Any]], TypeIgnores, bytes, bool]:
-    ast_bytes, errors, ignores, import_bytes, is_partial_package = ast_serialize.parse(
+    ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse(
         filename,
-        skip_function_bodies,
+        skip_function_bodies=skip_function_bodies,
         python_version=options.python_version,
         platform=options.platform,
         always_true=options.always_true,
@@ -274,7 +277,7 @@ def parse_to_binary_ast(
         cast("list[dict[str, Any]]", errors),
         ignores,
         import_bytes,
-        is_partial_package,
+        ast_data["is_partial_package"],
     )
 
 
@@ -1524,6 +1527,32 @@ def read_expression(state: State, data: ReadBuffer) -> Expression:
         expr = build_fstring_join(state, data, fitems)
         expect_end_tag(data)
         return expr
+    elif tag == nodes.TSTRING_EXPR:
+        state.uses_template_strings = True
+        nparts = read_int(data)
+        titems: list[Expression | tuple[Expression, str, str | None, Expression | None]] = []
+        for _ in range(nparts):
+            if read_bool(data):
+                e = read_expression(state, data)
+                s = read_str(data)
+                if read_bool(data):
+                    conv = read_str(data)
+                else:
+                    conv = None
+                if read_bool(data):
+                    # Parse format spec as a JoinedStr, this matches the old parser behavior.
+                    format_spec = read_fstring_items(state, data)
+                else:
+                    format_spec = None
+                titems.append((e, s, conv, format_spec))
+            else:
+                s = StrExpr(read_str(data))
+                read_loc(data, s)
+                titems.append(s)
+        expr = TemplateStrExpr(titems)
+        read_loc(data, expr)
+        expect_end_tag(data)
+        return expr
     elif tag == nodes.LAMBDA_EXPR:
         arguments, has_ann = read_parameters(state, data)
         body = read_block(state, data)

mypy/nodes.py

@@ -5242,6 +5242,7 @@ def local_definitions(
 IMPORT_METADATA: Final[Tag] = 226
 IMPORTFROM_METADATA: Final[Tag] = 227
 IMPORTALL_METADATA: Final[Tag] = 228
+TSTRING_EXPR: Final[Tag] = 229
 
 
 def read_symbol(data: ReadBuffer) -> SymbolNode:

test-data/unit/check-python314.test

@@ -1,4 +1,4 @@
-[case testTemplateStringBasics_no_native_parse]
+[case testTemplateStringBasics]
 reveal_type(t"foobar")  # N: Revealed type is "string.templatelib.Template"
 t"{'foobar'}"
 t"foo{'bar'}"
@@ -14,32 +14,32 @@ a = t"foobar"
 a = t"{'foobar'}"
 [builtins fixtures/f_string.pyi]
 
-[case testTemplateStringWithoutExplicitImport_no_native_parse]
+[case testTemplateStringWithoutExplicitImport]
 reveal_type(t"implicit import works")  # N: Revealed type is "string.templatelib.Template"
 [builtins fixtures/f_string.pyi]
 
-[case testTemplateStringExpressionsOk_no_native_parse]
+[case testTemplateStringExpressionsOk]
 t".{1 + 1}."
 t".{1 + 1}.{'foo' + 'bar'}"
 [builtins fixtures/f_string.pyi]
 
-[case testTemplateStringExpressionsErrors_no_native_parse]
+[case testTemplateStringExpressionsErrors]
 t"{1 + ''}"   # E: Unsupported operand types for + ("int" and "str")
 t".{1 + ''}"  # E: Unsupported operand types for + ("int" and "str")
 [builtins fixtures/f_string.pyi]
 
-[case testTemplateStringParseFormatOptions_no_native_parse]
+[case testTemplateStringParseFormatOptions]
 value = 10.5142
 width = 10
 precision = 4
 t"result: {value:{width}.{precision}}"
 [builtins fixtures/f_string.pyi]
 
-[case testTemplateStringNestedExpressionsTypeChecked_no_native_parse]
+[case testTemplateStringNestedExpressionsTypeChecked]
 t"{2:{3 + ''}}"  # E: Unsupported operand types for + ("int" and "str")
 [builtins fixtures/f_string.pyi]
 
-[case testIncrementalTemplateStringImplicitDependency_no_native_parse]
+[case testIncrementalTemplateStringImplicitDependency]
 import m
 reveal_type(m.x)
 [file m.py]

test-data/unit/native-parser.test

@@ -2552,7 +2552,7 @@ MypyFile:1(
 x = [1 2]
 y = 2
 [out]
-1:8: error: Expected ',', found int
+1:8: error: Expected `,`, found int
 MypyFile:1(
   AssignmentStmt:1(
     NameExpr(x)
@@ -2603,7 +2603,7 @@ MypyFile:1(
 from m
 1
 [out]
-1:7: error: Expected 'import', found newline
+1:7: error: Expected `import`, found newline
 MypyFile:1(
   ImportFrom:1(m, [])
   ExpressionStmt:2(
@@ -2613,7 +2613,7 @@ MypyFile:1(
 from m
 1
 [out]
-1:7: error: Expected 'import', found newline
+1:7: error: Expected `import`, found newline
 MypyFile:1(
   ImportFrom:1(m, [])
   ExpressionStmt:2(
@@ -2626,7 +2626,7 @@ def f():
 
 def g(): ...
 [out]
-3:8: error: Expected ')', found newline
+3:8: error: Expected `)`, found newline
 MypyFile:1(
   FuncDef:1(
     f
@@ -2651,7 +2651,7 @@ def f(
 
 def g(): ...
 [out]
-3:6: error: Expected ')', found newline
+3:6: error: Expected `)`, found newline
 5:1: error: Expected an indented block after function definition
 MypyFile:1(
   FuncDef:1(
@@ -2671,7 +2671,7 @@ class A
 
 def f(): pass
 [out]
-1:8: error: Expected ':', found newline
+1:8: error: Expected `:`, found newline
 3:1: error: Expected an indented block after `class` definition
 MypyFile:1(
   ClassDef:1(