Generate snapshots in parallel (#56184)

Summary:
Pull Request resolved: #56184

Changelog: [Internal]

Updates the snapshot generator script to run generation steps for all snapshots in parallel instead of in sequence, significantly improving the generation time.

Reviewed By: cortinico

Differential Revision: D97714334

fbshipit-source-id: b643dd7bb2fa4f3f9db1495bc7a44c0069a25274

Author: j-piasecki

packages/react-native/.doxygen.config.template

@@ -74,7 +74,7 @@ PROJECT_ICON           =
 # entered, it will be relative to the location where Doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = api
+OUTPUT_DIRECTORY       = ${OUTPUT_DIR}
 
 # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096
 # sub-directories (in 2 levels) under the output directory of each output format

scripts/cxx-api/manual_test/.doxygen.config.template

@@ -74,7 +74,7 @@ PROJECT_ICON           =
 # entered, it will be relative to the location where Doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = api
+OUTPUT_DIRECTORY       = ${OUTPUT_DIR}
 
 # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096
 # sub-directories (in 2 levels) under the output directory of each output format

scripts/cxx-api/parser/__main__.py

@@ -12,11 +12,12 @@
 """
 
 import argparse
+import concurrent.futures
 import os
-import shutil
 import subprocess
 import sys
 import tempfile
+import traceback
 
 from .config import ApiViewSnapshotConfig, parse_config_file
 from .doxygen import get_doxygen_bin, run_doxygen
@@ -36,13 +37,20 @@ def run_command(
     if result.returncode != 0:
         if verbose:
             print(f"{label} finished with error: {result.stderr}")
-        sys.exit(1)
+        raise RuntimeError(
+            f"{label} finished with error (exit code {result.returncode})"
+        )
     elif verbose:
         print(f"{label} finished successfully")
     return result
 
 
-def build_codegen(platform: str, verbose: bool = False) -> str:
+def build_codegen(
+    platform: str,
+    verbose: bool = False,
+    output_path: str = "./api/codegen",
+    label: str = "",
+) -> str:
     react_native_dir = os.path.join(get_react_native_dir(), "packages", "react-native")
 
     run_command(
@@ -52,17 +60,21 @@ def build_codegen(platform: str, verbose: bool = False) -> str:
             "--path",
             "./",
             "--outputPath",
-            "./api/codegen",
+            output_path,
             "--targetPlatform",
             platform,
             "--forceOutputPath",
         ],
-        label="Codegen",
+        label=f"[{label}] Codegen" if label else "Codegen",
         verbose=verbose,
         cwd=react_native_dir,
+        capture_output=True,
+        text=True,
     )
 
-    return os.path.join(react_native_dir, "api", "codegen")
+    if os.path.isabs(output_path):
+        return output_path
+    return os.path.join(react_native_dir, output_path)
 
 
 def build_snapshot_for_view(
@@ -75,21 +87,29 @@ def build_snapshot_for_view(
     codegen_platform: str | None = None,
     verbose: bool = True,
     input_filter: str = None,
+    work_dir: str | None = None,
 ) -> str:
     if verbose:
-        print(f"Generating API view: {api_view}")
+        print(f"[{api_view}] Generating API view")
 
-    api_dir = os.path.join(react_native_dir, "api")
-    if os.path.exists(api_dir):
-        if verbose:
-            print("  Deleting existing output directory")
-        shutil.rmtree(api_dir)
+    include_directories = list(include_directories)
+
+    if work_dir is None:
+        work_dir = os.path.join(react_native_dir, "api")
 
     if codegen_platform is not None:
-        codegen_dir = build_codegen(codegen_platform, verbose=verbose)
+        codegen_output = os.path.join(work_dir, "codegen")
+        codegen_dir = build_codegen(
+            codegen_platform,
+            verbose=verbose,
+            output_path=codegen_output,
+            label=api_view,
+        )
         include_directories.append(codegen_dir)
     elif verbose:
-        print("  Skipping codegen")
+        print(f"[{api_view}] Skipping codegen")
+
+    config_file = f".doxygen.config.{api_view}.generated"
 
     run_doxygen(
         working_dir=react_native_dir,
@@ -98,12 +118,15 @@ def build_snapshot_for_view(
         definitions=definitions,
         input_filter=input_filter,
         verbose=verbose,
+        output_dir=work_dir,
+        config_file=config_file,
+        label=api_view,
     )
 
     if verbose:
-        print("  Building snapshot")
+        print(f"[{api_view}] Building snapshot")
 
-    snapshot = build_snapshot(os.path.join(api_dir, "xml"))
+    snapshot = build_snapshot(os.path.join(work_dir, "xml"))
     snapshot_string = snapshot.to_string()
 
     output_file = os.path.join(output_dir, f"{api_view}Cxx.api")
@@ -126,36 +149,66 @@ def build_snapshots(
     is_test: bool = False,
 ) -> None:
     if not is_test:
-        for config in snapshot_configs:
-            if view_filter and config.snapshot_name != view_filter:
-                continue
-
-            build_snapshot_for_view(
-                api_view=config.snapshot_name,
+        configs_to_build = [
+            config
+            for config in snapshot_configs
+            if not view_filter or config.snapshot_name == view_filter
+        ]
+
+        with tempfile.TemporaryDirectory(prefix="cxx-api-") as parent_tmp:
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                futures = {}
+                for config in configs_to_build:
+                    work_dir = os.path.join(parent_tmp, config.snapshot_name)
+                    os.makedirs(work_dir, exist_ok=True)
+                    future = executor.submit(
+                        build_snapshot_for_view,
+                        api_view=config.snapshot_name,
+                        react_native_dir=react_native_dir,
+                        include_directories=config.inputs,
+                        exclude_patterns=config.exclude_patterns,
+                        definitions=config.definitions,
+                        output_dir=output_dir,
+                        codegen_platform=config.codegen_platform,
+                        verbose=verbose,
+                        input_filter=input_filter if config.input_filter else None,
+                        work_dir=work_dir,
+                    )
+                    futures[future] = config.snapshot_name
+
+                errors = []
+                for future in concurrent.futures.as_completed(futures):
+                    view_name = futures[future]
+                    try:
+                        future.result()
+                    except Exception as e:
+                        errors.append((view_name, e))
+                        if verbose:
+                            print(
+                                f"[{view_name}] Error generating:\n"
+                                f"{traceback.format_exc()}"
+                            )
+
+                if errors:
+                    failed_views = ", ".join(name for name, _ in errors)
+                    raise RuntimeError(f"Failed to generate snapshots: {failed_views}")
+    else:
+        with tempfile.TemporaryDirectory(prefix="cxx-api-test-") as work_dir:
+            snapshot = build_snapshot_for_view(
+                api_view="Test",
                 react_native_dir=react_native_dir,
-                include_directories=config.inputs,
-                exclude_patterns=config.exclude_patterns,
-                definitions=config.definitions,
+                include_directories=[],
+                exclude_patterns=[],
+                definitions={},
                 output_dir=output_dir,
-                codegen_platform=config.codegen_platform,
+                codegen_platform=None,
                 verbose=verbose,
-                input_filter=input_filter if config.input_filter else None,
+                input_filter=input_filter,
+                work_dir=work_dir,
             )
-    else:
-        snapshot = build_snapshot_for_view(
-            api_view="Test",
-            react_native_dir=react_native_dir,
-            include_directories=[],
-            exclude_patterns=[],
-            definitions={},
-            output_dir=output_dir,
-            codegen_platform=None,
-            verbose=verbose,
-            input_filter=input_filter,
-        )
 
-        if verbose:
-            print(snapshot)
+            if verbose:
+                print(snapshot)
 
 
 def get_default_snapshot_dir() -> str:

scripts/cxx-api/parser/doxygen.py

@@ -9,7 +9,6 @@
 
 import os
 import subprocess
-import sys
 
 _DOXYGEN_CONFIG_FILE = ".doxygen.config.generated"
 
@@ -24,6 +23,8 @@ def build_doxygen_config(
     exclude_patterns: list[str] = None,
     definitions: dict[str, str | int] = None,
     input_filter: str = None,
+    output_dir: str = "api",
+    config_file: str = _DOXYGEN_CONFIG_FILE,
 ) -> None:
     if include_directories is None:
         include_directories = []
@@ -54,9 +55,10 @@ def build_doxygen_config(
         .replace("${EXCLUDE_PATTERNS}", exclude_patterns_str)
         .replace("${PREDEFINED}", definitions_str)
         .replace("${DOXYGEN_INPUT_FILTER}", input_filter_str)
+        .replace("${OUTPUT_DIR}", output_dir)
     )
 
-    with open(os.path.join(directory, _DOXYGEN_CONFIG_FILE), "w") as f:
+    with open(os.path.join(directory, config_file), "w") as f:
         f.write(config)
 
 
@@ -67,40 +69,46 @@ def run_doxygen(
     definitions: dict[str, str | int],
     input_filter: str = None,
     verbose: bool = True,
+    output_dir: str = "api",
+    config_file: str = _DOXYGEN_CONFIG_FILE,
+    label: str = "",
 ) -> None:
     """Generate Doxygen config, run Doxygen, and clean up the config file."""
+    prefix = f"[{label}] " if label else ""
     if verbose:
-        print("  Generating Doxygen config file")
+        print(f"{prefix}Generating Doxygen config file")
 
     build_doxygen_config(
         working_dir,
         include_directories=include_directories,
         exclude_patterns=exclude_patterns,
         definitions=definitions,
         input_filter=input_filter,
+        output_dir=output_dir,
+        config_file=config_file,
     )
 
     if verbose:
-        print("  Running Doxygen")
+        print(f"{prefix}Running Doxygen")
         if input_filter:
-            print(f"    Using input filter: {input_filter}")
+            print(f"{prefix}Using input filter: {input_filter}")
 
     doxygen_bin = get_doxygen_bin()
 
     result = subprocess.run(
-        [doxygen_bin, _DOXYGEN_CONFIG_FILE],
+        [doxygen_bin, config_file],
         cwd=working_dir,
         capture_output=True,
         text=True,
     )
 
     if result.returncode != 0:
         if verbose:
-            print(f"  Doxygen finished with error: {result.stderr}")
-        sys.exit(1)
+            print(f"{prefix}Doxygen finished with error: {result.stderr}")
+        raise RuntimeError(f"Doxygen finished with error: {result.stderr}")
     elif verbose:
-        print("  Doxygen finished successfully")
+        print(f"{prefix}Doxygen finished successfully")
 
     if verbose:
-        print("  Deleting Doxygen config file")
-    os.remove(os.path.join(working_dir, _DOXYGEN_CONFIG_FILE))
+        print(f"{prefix}Deleting Doxygen config file")
+    os.remove(os.path.join(working_dir, config_file))