Guard builtin formatter tokenization

src/datamodel_code_generator/_builtin_formatter.py

@@ -196,6 +196,8 @@ def _has_inline_comment(lines: list[str], node: ast.AST) -> bool:
 
 
 def _has_comment_token(line: str) -> bool:
+    if "#" not in line:
+        return False
     try:
         tokens = tokenize.generate_tokens(StringIO(line).readline)
         return any(token.type == tokenize.COMMENT for token in tokens)
@@ -1715,9 +1717,11 @@ def _ensure_post_class_annotation_assignment_spacing(
 
 def _normalize_top_level_blank_lines(code: str) -> str:
     string_lines: set[int] = set()
-    for token in tokenize.generate_tokens(StringIO(code).readline):
-        if token.type == tokenize.STRING and token.start[0] != token.end[0]:
-            string_lines.update(range(token.start[0], token.end[0] + 1))
+    # Multi-line STRING tokens can only come from triple quotes or backslash-continuation.
+    if ('"""' in code) or ("'''" in code) or ("\\\n" in code) or ("\\\r\n" in code):
+        for token in tokenize.generate_tokens(StringIO(code).readline):
+            if token.type == tokenize.STRING and token.start[0] != token.end[0]:
+                string_lines.update(range(token.start[0], token.end[0] + 1))
 
     lines = code.splitlines()
     formatted_lines: list[str] = []

tests/main/test_performance.py

@@ -125,6 +125,38 @@ def test_perf_large_models_pydantic_v2(tmp_path: Path) -> None:
     assert content.count("class Model") >= 500
 
 
+@pytest.mark.perf
+@pytest.mark.benchmark
+def test_perf_large_models_pydantic_v2_builtin(tmp_path: Path) -> None:
+    """Performance test: Generate 500 Pydantic v2 models with the built-in formatter."""
+    output_file = tmp_path / "output.py"
+    generate(
+        input_=PERFORMANCE_DATA_PATH / "large_models.json",
+        input_file_type=InputFileType.JsonSchema,
+        output=output_file,
+        output_model_type=DataModelType.PydanticV2BaseModel,
+        formatters=[Formatter.BUILTIN],
+    )
+    content = output_file.read_text()
+    assert content.count("class Model") >= 500
+
+
+@pytest.mark.perf
+@pytest.mark.benchmark
+def test_perf_large_models_pydantic_v2_noformat(tmp_path: Path) -> None:
+    """Performance test: Generate 500 Pydantic v2 models without formatting."""
+    output_file = tmp_path / "output.py"
+    generate(
+        input_=PERFORMANCE_DATA_PATH / "large_models.json",
+        input_file_type=InputFileType.JsonSchema,
+        output=output_file,
+        output_model_type=DataModelType.PydanticV2BaseModel,
+        formatters=[],
+    )
+    content = output_file.read_text()
+    assert content.count("class Model") >= 500
+
+
 @pytest.mark.perf
 @pytest.mark.parametrize(
     ("formatter_case", "formatters"),

tests/test_format.py

@@ -931,6 +931,81 @@ def test_apply_builtin_formatter_normalizes_blank_lines_without_imports() -> Non
     assert apply_builtin_formatter(code) == "Alias = str\n\n\nOtherAlias = Alias\n"
 
 
+def test_builtin_formatter_comment_token_guard_skips_lines_without_hash(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Test comment-token detection avoids tokenization when a line cannot contain comments."""
+
+    def fail_generate_tokens(*_args: object, **_kwargs: object) -> None:
+        pytest.fail("comment-token guard should skip tokenization")  # pragma: no cover
+
+    monkeypatch.setattr(builtin_formatter.tokenize, "generate_tokens", fail_generate_tokens)
+
+    assert not builtin_formatter._has_comment_token("value = 1")
+
+
+@pytest.mark.parametrize(
+    ("line", "expected"),
+    [
+        ("value = '# not a comment'", False),
+        ("value = 1  # comment", True),
+    ],
+)
+def test_builtin_formatter_comment_token_guard_tokenizes_hash_lines(line: str, expected: bool) -> None:
+    """Test comment-token detection still tokenizes lines that contain a hash."""
+    assert builtin_formatter._has_comment_token(line) is expected
+
+
+def test_builtin_formatter_blank_line_guard_skips_tokenize_without_multiline_strings(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Test blank-line normalization avoids tokenization when multi-line strings are impossible."""
+
+    def fail_generate_tokens(*_args: object, **_kwargs: object) -> None:
+        pytest.fail("blank-line guard should skip tokenization")  # pragma: no cover
+
+    monkeypatch.setattr(builtin_formatter.tokenize, "generate_tokens", fail_generate_tokens)
+
+    code = "class Model:\n    pass\n\n\n\nclass OtherModel:\n    pass"
+
+    assert builtin_formatter._normalize_top_level_blank_lines(code) == (
+        "class Model:\n    pass\n\n\nclass OtherModel:\n    pass"
+    )
+
+
+def test_builtin_formatter_blank_line_guard_keeps_docstring_body_lines() -> None:
+    """Test blank-line normalization tokenizes code that may contain multi-line strings."""
+    code = '"""doc\nclass NotCode:\n    pass\n"""\n\n\n\nclass Model:\n    pass'
+
+    assert builtin_formatter._normalize_top_level_blank_lines(code) == (
+        '"""doc\nclass NotCode:\n    pass\n"""\n\n\nclass Model:\n    pass'
+    )
+
+
+@pytest.mark.parametrize("line_separator", ["\n", "\r\n"])
+def test_builtin_formatter_blank_line_guard_keeps_backslash_continuation_body_lines(line_separator: str) -> None:
+    """Test backslash-continued string lines are not treated as top-level code."""
+    code = line_separator.join([
+        "value = 'first\\",
+        "class NotCode:'",
+        "",
+        "",
+        "",
+        "class Model:",
+        "    pass",
+    ])
+    expected_lines = [
+        "value = 'first\\",
+        "class NotCode:'",
+        "",
+        "",
+        "class Model:",
+        "    pass",
+    ]
+
+    assert builtin_formatter._normalize_top_level_blank_lines(code) == "\n".join(expected_lines)
+
+
 @pytest.mark.skipif(sys.version_info < (3, 12), reason="type statements require Python 3.12")
 def test_apply_builtin_formatter_normalizes_type_alias_blank_lines_without_imports() -> None:
     """Test built-in formatter normalizes top-level blanks between type statements."""