Preserve snapshots and ignore pycache in sync

Author: Miskler

README.md

@@ -114,7 +114,7 @@ The repository ships the sync workflow template used by generated package reposi
 
 The generated `source-sync` workflow stays thin: it checks out this repository as the generator logic, reads the consumer repo's `.msra` source tree, generates the artifact into a staging tree, syncs the generated tree into `main`, validates the result, and pushes the commit so the target repo's `publish.yml` can run on `push` to `main`.
 
-If a project defines `[app.sync].preserved_target_paths` in its MSRA source, the generated sync step preserves those repo-specific runtime artifacts. For the FixPrice layout that means `tests/__snapshots__` stays in `main` and is not rewritten by the generator.
+If a project defines `[app.sync]` in its MSRA source, the generated sync step can preserve repo-specific runtime artifacts and ignore generated noise. For the FixPrice layout that means `tests/__snapshots__` stays in `main`, while `**/__pycache__` and `**/*.pyc` are removed before commit.
 
 Before validation, the generated workflow installs the target project's `requirements-dev.txt` so the validation step runs with the generated dependencies available.
 

docs/msra-repo-b-sync.md

@@ -8,7 +8,7 @@
 - этот workflow запускается вручную через `workflow_dispatch`
 - внутри job он сам checkout-ит repo с логикой генератора
 - затем читает `repo B/source`, генерирует артефакт и пушит его в `repo B/main`
-- при синке сохраняет repo-specific runtime artifacts, перечисленные в `[app.sync].preserved_target_paths` source-MSTRA, например `tests/__snapshots__`
+- при синке сохраняет repo-specific runtime artifacts, перечисленные в `[app.sync].preserved_target_paths`, и удаляет generated noise из `[app.sync].ignored_generated_patterns`
 - после `push` в `main` автоматически стартует `publish.yml`
 
 ## Почему workflow должен лежать в `main`
@@ -109,13 +109,31 @@ jobs:
       - name: Replace target tree contents
         run: |
           python - <<'PY'
+          from fnmatch import fnmatch
           from pathlib import Path
           import shutil
 
           target = Path("target")
           generated = Path("generated")
           preserved_root = Path("preserved")
           preserve_paths = ["tests/__snapshots__"]
+          ignored_patterns = ["**/__pycache__", "**/*.pyc"]
+
+          def is_ignored(relative_path: Path) -> bool:
+              relative_text = relative_path.as_posix()
+              return any(fnmatch(relative_text, pattern) for pattern in ignored_patterns)
+
+          def remove_ignored(root: Path) -> None:
+              ignored_paths = sorted(
+                  [path for path in root.rglob("*") if is_ignored(path.relative_to(root))],
+                  key=lambda path: len(path.parts),
+                  reverse=True,
+              )
+              for path in ignored_paths:
+                  if path.is_dir():
+                      shutil.rmtree(path)
+                  else:
+                      path.unlink()
 
           for relative_path in preserve_paths:
               source_path = target / relative_path
@@ -151,6 +169,9 @@ jobs:
               else:
                   destination_path.parent.mkdir(parents=True, exist_ok=True)
                   shutil.copy2(source_path, destination_path)
+
+          remove_ignored(target)
+          shutil.rmtree(preserved_root, ignore_errors=True)
           PY
 
       - name: Validate generated project

msra_codegen/github_workflows.py

@@ -24,6 +24,9 @@ def build_github_workflows_context(project: dict[str, Any], package_name: str) -
     preserved_target_paths = app_sync_config.get("preserved_target_paths", [])
     if not isinstance(preserved_target_paths, list):
         raise RuntimeError("app.sync.preserved_target_paths must be a list.")
+    ignored_generated_patterns = app_sync_config.get("ignored_generated_patterns", [])
+    if not isinstance(ignored_generated_patterns, list):
+        raise RuntimeError("app.sync.ignored_generated_patterns must be a list.")
 
     return {
         "package_name": package_name,
@@ -87,6 +90,7 @@ def build_github_workflows_context(project: dict[str, Any], package_name: str) -
             "commit_user_email": str(source_sync_config["commit_user_email"]),
             "source_msra_path": source_msra_path,
             "preserved_target_paths": [str(item) for item in preserved_target_paths],
+            "ignored_generated_patterns": [str(item) for item in ignored_generated_patterns],
         },
         "makefile": {
             "package_name": package_name,

msra_codegen/project_model.py

@@ -184,7 +184,22 @@ def get_assignment(table: dict[str, Any] | None, key: str, default: Any = None)
             text = item.strip()
             if text:
                 preserved_target_paths.append(text)
-        app["sync"] = {"preserved_target_paths": preserved_target_paths}
+        ignored_generated_patterns_value = get_plain_value(
+            get_assignment(sync_table, "ignored_generated_patterns", [])
+        )
+        if not isinstance(ignored_generated_patterns_value, list):
+            raise TypeError("app.sync.ignored_generated_patterns must be a list of strings.")
+        ignored_generated_patterns: list[str] = []
+        for item in ignored_generated_patterns_value:
+            if not isinstance(item, str):
+                raise TypeError("app.sync.ignored_generated_patterns entries must be strings.")
+            text = item.strip()
+            if text:
+                ignored_generated_patterns.append(text)
+        app["sync"] = {
+            "preserved_target_paths": preserved_target_paths,
+            "ignored_generated_patterns": ignored_generated_patterns,
+        }
 
     variable_tables = [
         table

msra_codegen/templates/github/workflows/source-sync.yml.tpl

@@ -62,13 +62,44 @@ jobs:
       - name: Replace target tree contents
         run: |
           python - <<'PY'
+          from fnmatch import fnmatch
           from pathlib import Path
           import shutil
 
           target = Path("target")
           generated = Path("generated")
           preserved_root = Path("preserved")
           preserve_paths = {{ source_sync.preserved_target_paths | tojson }}
+          ignored_patterns = {{ source_sync.ignored_generated_patterns | tojson }}
+
+          def is_ignored(relative_path: Path) -> bool:
+              relative_text = relative_path.as_posix()
+              return any(fnmatch(relative_text, pattern) for pattern in ignored_patterns)
+
+          def copy_tree(source_root: Path, destination_root: Path) -> None:
+              for item in source_root.iterdir():
+                  relative_path = item.relative_to(source_root)
+                  if is_ignored(relative_path):
+                      continue
+                  destination_path = destination_root / relative_path
+                  if item.is_dir():
+                      destination_path.mkdir(parents=True, exist_ok=True)
+                      copy_tree(item, destination_path)
+                  else:
+                      destination_path.parent.mkdir(parents=True, exist_ok=True)
+                      shutil.copy2(item, destination_path)
+
+          def remove_ignored(root: Path) -> None:
+              ignored_paths = sorted(
+                  [path for path in root.rglob("*") if is_ignored(path.relative_to(root))],
+                  key=lambda path: len(path.parts),
+                  reverse=True,
+              )
+              for path in ignored_paths:
+                  if path.is_dir():
+                      shutil.rmtree(path)
+                  else:
+                      path.unlink()
 
           for relative_path in preserve_paths:
               source_path = target / relative_path
@@ -89,12 +120,7 @@ jobs:
               else:
                   child.unlink()
 
-          for item in generated.iterdir():
-              destination = target / item.name
-              if item.is_dir():
-                  shutil.copytree(item, destination, dirs_exist_ok=True)
-              else:
-                  shutil.copy2(item, destination)
+          copy_tree(generated, target)
 
           for relative_path in preserve_paths:
               source_path = preserved_root / relative_path
@@ -105,6 +131,7 @@ jobs:
                   destination_path.parent.mkdir(parents=True, exist_ok=True)
                   shutil.copy2(source_path, destination_path)
 
+          remove_ignored(target)
           shutil.rmtree(preserved_root, ignore_errors=True)
           PY
 

tests/codegen.test.js

@@ -855,7 +855,7 @@ test("source sync workflow preserves target artifacts from app.sync", () => {
     .replace('package_name="ozon_api"', 'package_name="sync_api"')
     .replace(
       /\n\[app\.warmup\]\n/,
-      '\n[app.sync]\npreserved_target_paths=["tests/__snapshots__"]\n\n[app.warmup]\n',
+      '\n[app.sync]\npreserved_target_paths=["tests/__snapshots__"]\nignored_generated_patterns=["**/__pycache__", "**/*.pyc"]\n\n[app.warmup]\n',
     );
 
   try {
@@ -876,6 +876,7 @@ test("source sync workflow preserves target artifacts from app.sync", () => {
       readFileSync(path.join(outputDir, ".github", "workflows", "source-sync.yml"), "utf8"),
     );
     assert.match(sourceSyncWorkflowText, /preserve_paths = \["tests\/__snapshots__"\]/);
+    assert.match(sourceSyncWorkflowText, /ignored_patterns = \["\*\*\/__pycache__", "\*\*\/\*\.pyc"\]/);
   } finally {
     rmSync(workDir, { recursive: true, force: true });
   }

vscode-extension/lsp/assignment-schema.js

@@ -679,6 +679,7 @@ const TABLE_SCHEMAS = [
   }),
   makeFixedSchema(exactPath(["app", "sync"]), {
     preserved_target_paths: arrayOf(STRING),
+    ignored_generated_patterns: arrayOf(STRING),
   }),
   makeFixedSchema(exactPath(["app", "variables", "*"]), {
     types: arrayOf(VARIABLE_TYPE_ITEM_SPEC),