diff --git a/src/datamodel_code_generator/_builtin_formatter.py b/src/datamodel_code_generator/_builtin_formatter.py index c0fe2fad8..9910321f2 100644 --- a/src/datamodel_code_generator/_builtin_formatter.py +++ b/src/datamodel_code_generator/_builtin_formatter.py @@ -196,6 +196,8 @@ def _has_inline_comment(lines: list[str], node: ast.AST) -> bool: def _has_comment_token(line: str) -> bool: + if "#" not in line: + return False try: tokens = tokenize.generate_tokens(StringIO(line).readline) return any(token.type == tokenize.COMMENT for token in tokens) @@ -1715,9 +1717,11 @@ def _ensure_post_class_annotation_assignment_spacing( def _normalize_top_level_blank_lines(code: str) -> str: string_lines: set[int] = set() - for token in tokenize.generate_tokens(StringIO(code).readline): - if token.type == tokenize.STRING and token.start[0] != token.end[0]: - string_lines.update(range(token.start[0], token.end[0] + 1)) + # Multi-line STRING tokens can only come from triple quotes or backslash-continuation. + if ('"""' in code) or ("'''" in code) or ("\\\n" in code) or ("\\\r\n" in code): + for token in tokenize.generate_tokens(StringIO(code).readline): + if token.type == tokenize.STRING and token.start[0] != token.end[0]: + string_lines.update(range(token.start[0], token.end[0] + 1)) lines = code.splitlines() formatted_lines: list[str] = [] diff --git a/tests/main/test_performance.py b/tests/main/test_performance.py index 4d38f6a58..c9bae5df2 100644 --- a/tests/main/test_performance.py +++ b/tests/main/test_performance.py @@ -125,6 +125,38 @@ def test_perf_large_models_pydantic_v2(tmp_path: Path) -> None: assert content.count("class Model") >= 500 +@pytest.mark.perf +@pytest.mark.benchmark +def test_perf_large_models_pydantic_v2_builtin(tmp_path: Path) -> None: + """Performance test: Generate 500 Pydantic v2 models with the built-in formatter.""" + output_file = tmp_path / "output.py" + generate( + input_=PERFORMANCE_DATA_PATH / "large_models.json", + input_file_type=InputFileType.JsonSchema, + output=output_file, + output_model_type=DataModelType.PydanticV2BaseModel, + formatters=[Formatter.BUILTIN], + ) + content = output_file.read_text() + assert content.count("class Model") >= 500 + + +@pytest.mark.perf +@pytest.mark.benchmark +def test_perf_large_models_pydantic_v2_noformat(tmp_path: Path) -> None: + """Performance test: Generate 500 Pydantic v2 models without formatting.""" + output_file = tmp_path / "output.py" + generate( + input_=PERFORMANCE_DATA_PATH / "large_models.json", + input_file_type=InputFileType.JsonSchema, + output=output_file, + output_model_type=DataModelType.PydanticV2BaseModel, + formatters=[], + ) + content = output_file.read_text() + assert content.count("class Model") >= 500 + + @pytest.mark.perf @pytest.mark.parametrize( ("formatter_case", "formatters"), diff --git a/tests/test_format.py b/tests/test_format.py index 7f9b653bb..dc154f11e 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -931,6 +931,81 @@ def test_apply_builtin_formatter_normalizes_blank_lines_without_imports() -> Non assert apply_builtin_formatter(code) == "Alias = str\n\n\nOtherAlias = Alias\n" +def test_builtin_formatter_comment_token_guard_skips_lines_without_hash( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test comment-token detection avoids tokenization when a line cannot contain comments.""" + + def fail_generate_tokens(*_args: object, **_kwargs: object) -> None: + pytest.fail("comment-token guard should skip tokenization") # pragma: no cover + + monkeypatch.setattr(builtin_formatter.tokenize, "generate_tokens", fail_generate_tokens) + + assert not builtin_formatter._has_comment_token("value = 1") + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + ("value = '# not a comment'", False), + ("value = 1 # comment", True), + ], +) +def test_builtin_formatter_comment_token_guard_tokenizes_hash_lines(line: str, expected: bool) -> None: + """Test comment-token detection still tokenizes lines that contain a hash.""" + assert builtin_formatter._has_comment_token(line) is expected + + +def test_builtin_formatter_blank_line_guard_skips_tokenize_without_multiline_strings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test blank-line normalization avoids tokenization when multi-line strings are impossible.""" + + def fail_generate_tokens(*_args: object, **_kwargs: object) -> None: + pytest.fail("blank-line guard should skip tokenization") # pragma: no cover + + monkeypatch.setattr(builtin_formatter.tokenize, "generate_tokens", fail_generate_tokens) + + code = "class Model:\n pass\n\n\n\nclass OtherModel:\n pass" + + assert builtin_formatter._normalize_top_level_blank_lines(code) == ( + "class Model:\n pass\n\n\nclass OtherModel:\n pass" + ) + + +def test_builtin_formatter_blank_line_guard_keeps_docstring_body_lines() -> None: + """Test blank-line normalization tokenizes code that may contain multi-line strings.""" + code = '"""doc\nclass NotCode:\n pass\n"""\n\n\n\nclass Model:\n pass' + + assert builtin_formatter._normalize_top_level_blank_lines(code) == ( + '"""doc\nclass NotCode:\n pass\n"""\n\n\nclass Model:\n pass' + ) + + +@pytest.mark.parametrize("line_separator", ["\n", "\r\n"]) +def test_builtin_formatter_blank_line_guard_keeps_backslash_continuation_body_lines(line_separator: str) -> None: + """Test backslash-continued string lines are not treated as top-level code.""" + code = line_separator.join([ + "value = 'first\\", + "class NotCode:'", + "", + "", + "", + "class Model:", + " pass", + ]) + expected_lines = [ + "value = 'first\\", + "class NotCode:'", + "", + "", + "class Model:", + " pass", + ] + + assert builtin_formatter._normalize_top_level_blank_lines(code) == "\n".join(expected_lines) + + @pytest.mark.skipif(sys.version_info < (3, 12), reason="type statements require Python 3.12") def test_apply_builtin_formatter_normalizes_type_alias_blank_lines_without_imports() -> None: """Test built-in formatter normalizes top-level blanks between type statements."""