PYTHON-5890 Automate sync file generation and staging (#2887)

Author: blink1073

.github/workflows/test-python.yml

@@ -46,6 +46,9 @@ jobs:
       - name: Run typecheck
         run: |
           just typing
+      - name: Run synchro smoke test
+        run: |
+          uv run --extra test --with unasync pytest tools/test_synchro.py -v --override-ini="addopts="
       - run: |
           sudo apt-get install -y cppcheck
       - run: |

.gitignore

@@ -30,6 +30,7 @@ expansion.yml
 .evergreen/scripts/test-env.sh
 specifications/
 results.json
+.synchro-modified
 .evergreen/atlas_x509_dev_client_certificate.pem
 
 # Lambda temp files

.pre-commit-config.yaml

@@ -18,25 +18,36 @@ repos:
     exclude: .patch
     exclude_types: [json]
 
-- repo: https://github.com/astral-sh/ruff-pre-commit
-  # Ruff version.
-  rev: v0.15.16
+- repo: local
   hooks:
     - id: ruff
-      args: ["--fix", "--show-fixes"]
+      name: ruff
+      entry: uv run --group lint ruff check --fix --show-fixes
+      language: system
+      types_or: [python, pyi]
+      require_serial: true
     - id: ruff-format
+      name: ruff-format
+      entry: uv run --group lint ruff format
+      language: system
+      types_or: [python, pyi]
+      require_serial: true
 
 - repo: local
   hooks:
     - id: synchro
       name: synchro
-      entry: bash ./tools/synchro.sh
-      language: python
+      entry: uv run --group unasync ./tools/synchro.py
+      language: system
+      require_serial: true
+      always_run: true
+    - id: synchro-stage-files
+      name: synchro-stage-files
+      entry: bash -c 'if [ -s .synchro-modified ]; then xargs git add < .synchro-modified || true; fi'
+      language: system
       require_serial: true
-      fail_fast: true
-      additional_dependencies:
-        - ruff==0.15.16
-        - unasync
+      always_run: true
+      pass_filenames: false
 
 - repo: https://github.com/adamchainz/blacken-docs
   rev: "1.16.0"

justfile

@@ -16,6 +16,10 @@ default:
 resync:
  @uv sync --quiet
 
+[private]
+run-synchro:
+    uv run --group unasync ./tools/synchro.py
+
 # Set up the development environment
 install:
    bash .evergreen/scripts/setup-dev-env.sh
@@ -66,7 +70,7 @@ lint-manual *args="": && resync
 
 # Run pytest (e.g. just test test/test_uri_parser.py)
 [group('test')]
-test *args="-v --durations=5 --maxfail=10": && resync
+test *args="-v --durations=5 --maxfail=10": run-synchro && resync
     #!/usr/bin/env bash
     set -euo pipefail
     uv run ${USE_ACTIVE_VENV:+--active} --extra test python -m pytest {{args}}
@@ -79,7 +83,7 @@ test-numpy *args="": && resync
 
 # Run tests via the Evergreen test runner script
 [group('test')]
-run-tests *args: && resync
+run-tests *args: run-synchro && resync
     bash ./.evergreen/run-tests.sh {{args}}
 
 # Set up the test environment (auth, TLS, etc.)

pyproject.toml

@@ -48,6 +48,13 @@ Tracker = "https://jira.mongodb.org/projects/PYTHON/issues"
 
 [dependency-groups]
 dev = []
+lint = [
+    "ruff==0.15.16",
+]
+unasync = [
+    {include-group = "lint"},
+    "unasync",
+]
 pip = ["pip>=20.2"]
 gevent = ["gevent>=21.12"]
 coverage = [
@@ -141,6 +148,10 @@ pretty = true
 disable_error_code = ["no-any-return"]
 disallow_any_generics = true
 
+[[tool.mypy.overrides]]
+module = ["synchro"]
+warn_unused_ignores = false
+
 [[tool.mypy.overrides]]
 module = ["test.*"]
 disable_error_code = ["type-arg", "no-untyped-def", "no-untyped-call"]
@@ -234,6 +245,7 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?)|dummy.*)$"
     "RET", "ARG", "F405", "B028", "PGH001", "B018", "F403", "RUF015", "E731", "B007",
     "UP031", "F401", "B023", "F811"]
 "tools/*.py" = ["T201"]
+"tools/test_*.py" = ["E402", "S"]
 "hatch_build.py" = ["S"]
 "_setup.py" = ["SIM112"]
 

tools/synchro.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 """Synchronization of asynchronous modules.
 
 Used as part of our build system to generate synchronous code.
@@ -21,10 +22,13 @@
 
 import os
 import re
+import subprocess
 import sys
 from pathlib import Path
 
-from unasync import Rule, unasync_files  # type: ignore[import-not-found]
+from unasync import Rule, unasync_files  # type: ignore[import-untyped,import-not-found]
+
+MANIFEST = ".synchro-modified"
 
 replacements = {
     "AsyncCollection": "Collection",
@@ -424,6 +428,7 @@ def unasync_directory(files: list[str], src: str, dest: str, replacements: dict[
 
 
 def main() -> None:
+    is_ci = bool(os.environ.get("CI"))
     modified_files = [f"./{f}" for f in sys.argv[1:]]
     errored = False
     for fname in async_files + gridfs_files + test_files:
@@ -438,29 +443,77 @@ def main() -> None:
             print(f"Refusing to overwrite {test_sync_name}")
             errored = True
     if errored:
-        raise ValueError("Aborting synchro due to errors")
-
-    unasync_directory(async_files, _pymongo_base, _pymongo_dest_base, replacements)
-    unasync_directory(gridfs_files, _gridfs_base, _gridfs_dest_base, replacements)
-    unasync_directory(test_files, _test_base, _test_dest_base, replacements)
-
-    sync_files = [
-        _pymongo_dest_base + f.name for f in Path(_pymongo_dest_base).iterdir() if f.is_file()
-    ]
+        sys.exit(1)
+
+    # When called with specific files, only process those; otherwise process everything.
+    modified_set = set(modified_files)
+    filtered_async = [f for f in async_files if not modified_set or f in modified_set]
+    filtered_gridfs = [f for f in gridfs_files if not modified_set or f in modified_set]
+    filtered_tests = [f for f in test_files if not modified_set or f in modified_set]
+
+    ruff_extra = [] if is_ci else ["--silent"]
+
+    # Check async source files for problems before generating sync output.
+    async_sources = filtered_async + filtered_gridfs + filtered_tests
+    if async_sources:
+        subprocess.run(  # noqa: S603
+            [sys.executable, "-m", "ruff", "check", *async_sources, *ruff_extra],
+            check=True,
+        )
 
-    sync_gridfs_files = [
-        _gridfs_dest_base + f.name for f in Path(_gridfs_dest_base).iterdir() if f.is_file()
-    ]
-    sync_test_files = [
-        _test_dest_base + f for f in converted_tests if (Path(_test_dest_base) / f).is_file()
+    unasync_directory(filtered_async, _pymongo_base, _pymongo_dest_base, replacements)
+    unasync_directory(filtered_gridfs, _gridfs_base, _gridfs_dest_base, replacements)
+    unasync_directory(filtered_tests, _test_base, _test_dest_base, replacements)
+
+    # Derive generated output paths directly from filtered source paths.
+    converted_tests_set = set(converted_tests)
+    generated_pymongo = [_pymongo_dest_base + Path(f).name for f in filtered_async]
+    generated_gridfs = [_gridfs_dest_base + Path(f).name for f in filtered_gridfs]
+    generated_tests = [
+        _test_dest_base + Path(f).name
+        for f in filtered_tests
+        if Path(f).name in converted_tests_set and (Path(_test_dest_base) / Path(f).name).is_file()
     ]
 
-    docstring_translate_files = sync_files + sync_gridfs_files + sync_test_files
+    docstring_translate_files = generated_pymongo + generated_gridfs + generated_tests
 
     process_files(
-        sync_files + sync_gridfs_files + sync_test_files, docstring_translate_files, sync_test_files
+        generated_pymongo + generated_gridfs + generated_tests,
+        docstring_translate_files,
+        generated_tests,
     )
 
+    generated_files = generated_pymongo + generated_gridfs + generated_tests
+
+    if is_ci and generated_files:
+        print(f"Synchro generated {len(generated_files)} file(s):")
+        for f in generated_files:
+            print(f"  {f}")
+
+    subprocess.run(  # noqa: S603
+        [sys.executable, "-m", "ruff", "check", *generated_files, "--fix", *ruff_extra],
+        check=is_ci,
+    )
+    subprocess.run(  # noqa: S603
+        [sys.executable, "-m", "ruff", "format", *generated_files, *ruff_extra],
+        check=is_ci,
+    )
+
+    if is_ci and generated_files:
+        result = subprocess.run(  # noqa: S603
+            ["git", "diff", "--name-only", "--", *generated_files],  # noqa: S607
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        if result.stdout.strip():
+            print("Sync files are out of date. Run `just lint --all-files synchro` to regenerate:")
+            for f in result.stdout.strip().splitlines():
+                print(f"  {f}")
+            sys.exit(1)
+
+    Path(MANIFEST).write_text("\n".join(generated_files) + ("\n" if generated_files else ""))
+
 
 if __name__ == "__main__":
     main()

tools/synchro.sh

@@ -0,0 +1,40 @@
+"""Smoke test for tools/synchro.py."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+
+def test_synchro(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Synchro processes only the async files passed as arguments.
+
+    collection.py is a valid async source that would be processed when
+    synchro runs with no arguments. When mongo_client.py is passed instead,
+    collection.py must not reach unasync_directory.
+    """
+    monkeypatch.setattr("sys.argv", ["synchro.py", "pymongo/asynchronous/mongo_client.py"])
+    monkeypatch.delenv("CI", raising=False)
+
+    processed: list[str] = []
+
+    def capture(files: list[str], *args: object, **kwargs: object) -> None:
+        processed.extend(files)
+
+    with (
+        patch("synchro.unasync_directory", side_effect=capture),
+        patch("synchro.process_files"),
+        patch("subprocess.run"),
+        patch("pathlib.Path.write_text"),
+    ):
+        from synchro import main
+
+        main()
+
+    assert "./pymongo/asynchronous/mongo_client.py" in processed
+    assert "./pymongo/asynchronous/collection.py" not in processed