Merge pull request #516 from coding-kitties/fix/497-report-html-scaling

feat(backtesting): single-bundle binary persistence format (#487)

Author: MDUYN

README.md

@@ -86,6 +86,7 @@ This framework is built around the full loop: **create strategies → vector bac
 - 🎯 **Return Scenario Projections** — Good, average, bad & very bad year projections from backtest data
 - 📉 **Benchmark Comparison** — Beat-rate analysis vs Buy & Hold, DCA, risk-free & custom benchmarks
 - 📄 **One-Click HTML Report** — Self-contained file, no server, dark & light theme, shareable
+- 📦 **Custom `.iafbt` Backtest Bundle Format** — An explicit, versioned, compressed, language-portable container (zstd + msgpack with magic-byte header) plus a separate parquet index for fast filtering without loading. ~21× smaller and ~27× fewer files than standard filebased directory layouts, with parallel I/O for fast load/save of large amounts of backtests.
 - 🌐 **Load External Data** — Fetch CSV, JSON, or Parquet from any URL with caching and auto-refresh
 - � **[Record Custom Variables](https://coding-kitties.github.io/investing-algorithm-framework/Advanced%20Concepts/recording-variables)** — Track any indicator or metric during backtests with `context.record()`
 - �🚀 **Build → Backtest → Deploy** — Local dev, cloud deploy (AWS / Azure), or monetize on Finterion

docusaurus/docs/Getting Started/backtest-reports.md

@@ -60,7 +60,17 @@ report = BacktestReport.open(directory_path="./my_backtests")
 report.show()
 ```
 
-The `open()` method recursively finds all valid backtest directories (containing `algorithm_id.json` and a `runs/` folder) and loads them into a single report.
+The `open()` method recursively finds all valid backtest directories (containing `algorithm_id.json` and a `runs/` folder) **and** any `.iafbt` bundle files, and loads them into a single report.
+
+:::tip Optimized `.iafbt` bundle format
+Backtests are saved by default in the framework's custom **`.iafbt` bundle format** — a single binary file per backtest combining zstd compression and MessagePack encoding. It is purpose-built for backtest reports: ~21× smaller and ~27× fewer files than the legacy directory format, and `BacktestReport.open()` loads it ~3× faster. The legacy directory format is still fully supported for backwards compatibility, and you can mix both in the same folder.
+
+For very large batches, opt into parallel loading:
+
+```python
+report = BacktestReport.open(directory_path="./my_backtests", workers=4)
+```
+:::
 
 You can also combine disk and in-memory backtests:
 

docusaurus/docs/Getting Started/backtesting.md

@@ -85,6 +85,49 @@ from investing_algorithm_framework import load_backtests_from_directory
 backtests = load_backtests_from_directory("./my_backtests")
 ```
 
+:::info `.iafbt` bundle format
+Backtests are persisted in the framework's optimized **`.iafbt` bundle format** — a single binary file per backtest using zstd compression + MessagePack encoding. Compared to the legacy directory layout it is ~21× smaller, ~27× fewer files, and ~3× faster to load. Both `save_backtests_to_directory` and `load_backtests_from_directory` support parallel I/O via `workers=N`. Existing legacy directories keep working transparently; use `iaf migrate-backtests --src ... --dst ...` to convert them.
+:::
+
+### Migrating Existing Backtests
+
+Convert a directory of legacy backtest folders to the new bundle
+format with `migrate_backtests`. The migration is **streamed**
+(load+save fused per worker), so memory usage stays roughly
+constant regardless of how many backtests you migrate, and an
+interrupted run can simply be re-invoked to resume — destination
+bundles that already exist are skipped by default.
+
+```python
+from investing_algorithm_framework import migrate_backtests
+
+n = migrate_backtests(
+    src_dir="./old_backtests",       # legacy folders and/or .iafbt
+    dst_dir="./bundled_backtests",   # destination
+    workers=8,                        # parallel; None = min(8, cpu)
+    show_progress=True,
+    write_index=True,                 # also build index.parquet
+    include_ohlcv=False,
+    skip_existing=True,               # resume-safe (default)
+    delete_source=False,              # set True to remove originals
+)
+print(f"migrated {n} backtests")
+```
+
+Or from the CLI:
+
+```bash
+iaf migrate-backtests \
+    --src ./old_backtests \
+    --dst ./bundled_backtests \
+    --workers 8
+```
+
+Pass `--delete-source` (or `delete_source=True` in Python) to remove
+each legacy directory/bundle after its destination has been written
+successfully. Sources are deleted per-backtest, so an interrupted
+run leaves only the not-yet-migrated ones behind.
+
 ## Reporting
 
 Use [Backtest Reports](/docs/Getting%20Started/backtest-reports) to turn

investing_algorithm_framework/app/reporting/backtest_report.py

@@ -12,7 +12,7 @@
 from jinja2 import Environment, FileSystemLoader
 
 from investing_algorithm_framework.domain import (
-    Backtest, OperationalException, tqdm
+    Backtest, OperationalException, tqdm, BUNDLE_EXT
 )
 
 logger = logging.getLogger("investing_algorithm_framework")
@@ -217,9 +217,15 @@ def save(self, path):
 
     @staticmethod
     def _is_backtest(backtest_path):
+        if not os.path.exists(backtest_path):
+            return False
+        # Bundle file (.iafbt)
+        if os.path.isfile(backtest_path) and \
+                backtest_path.endswith(BUNDLE_EXT):
+            return True
+        # Legacy directory layout
         return (
-            os.path.exists(backtest_path)
-            and os.path.isdir(backtest_path)
+            os.path.isdir(backtest_path)
             and os.path.isfile(
                 os.path.join(backtest_path, "algorithm_id.json")
             )
@@ -231,6 +237,7 @@ def open(
         backtests: List[Backtest] = None,
         directory_path: Union[str, List[str], None] = None,
         show_progress: bool = False,
+        workers: Union[int, None] = None,
     ) -> "BacktestReport":
         loaded = []
         source_tags = []
@@ -254,7 +261,7 @@ def open(
             if BacktestReport._is_backtest(dp):
                 backtest_paths.append((dp, tag))
             else:
-                for root, dirs, _ in os.walk(dp):
+                for root, dirs, files in os.walk(dp):
                     for dir_name in dirs:
                         subdir = os.path.join(
                             root, dir_name
@@ -265,6 +272,11 @@ def open(
                             backtest_paths.append(
                                 (subdir, tag)
                             )
+                    for file_name in files:
+                        if file_name.endswith(BUNDLE_EXT):
+                            backtest_paths.append(
+                                (os.path.join(root, file_name), tag)
+                            )
 
         iterator = backtest_paths
         if show_progress:
@@ -273,9 +285,43 @@ def open(
                 desc="Loading backtests",
             )
 
-        for path, tag in iterator:
-            loaded.append(Backtest.open(path))
-            source_tags.append(tag)
+        # Parallel load is opt-in (workers > 1). ProcessPoolExecutor
+        # startup costs typically dwarf the per-backtest decode for
+        # batches < ~30, so keep default behaviour serial. Pass
+        # ``workers=N`` (or ``-1`` for cpu_count) to load large
+        # batches in parallel.
+        from investing_algorithm_framework.domain.backtesting.\
+backtest_utils import (
+            _load_one_dispatch, _resolve_workers,
+        )
+
+        n = len(backtest_paths)
+        resolved_workers = (
+            _resolve_workers(workers) if workers is not None else 1
+        )
+        if resolved_workers > 1 and n >= 4:
+            from concurrent.futures import ProcessPoolExecutor
+
+            items = [
+                (path, "bundle" if path.endswith(BUNDLE_EXT) else "dir")
+                for path, _ in backtest_paths
+            ]
+            with ProcessPoolExecutor(max_workers=resolved_workers) as ex:
+                results = list(
+                    tqdm(
+                        ex.map(_load_one_dispatch, items),
+                        total=n,
+                        desc="Loading backtests",
+                        disable=not show_progress,
+                    )
+                )
+            for bt, (_, tag) in zip(results, backtest_paths):
+                loaded.append(bt)
+                source_tags.append(tag)
+        else:
+            for path, tag in iterator:
+                loaded.append(Backtest.open(path))
+                source_tags.append(tag)
 
         for bt in backtests:
             if not isinstance(bt, Backtest):

investing_algorithm_framework/cli/cli.py

@@ -250,3 +250,73 @@ def mcp(directory):
 
 
 cli.add_command(mcp)
+
+
+@click.command(name="migrate-backtests")
+@click.option(
+    "--src", "-s",
+    required=True,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    help="Source directory containing legacy backtest sub-directories.",
+)
+@click.option(
+    "--dst", "-d",
+    required=True,
+    type=click.Path(file_okay=False, dir_okay=True),
+    help="Destination directory for the new ``.iafbt`` bundle files.",
+)
+@click.option(
+    "--workers", "-w", type=int, default=None,
+    help="Number of parallel workers (default: min(8, CPU count)).",
+)
+@click.option(
+    "--no-index", is_flag=True, default=False,
+    help="Skip writing index.parquet at the destination.",
+)
+@click.option(
+    "--include-ohlcv", is_flag=True, default=False,
+    help="Include OHLCV data in the destination bundles.",
+)
+@click.option(
+    "--no-skip-existing", is_flag=True, default=False,
+    help="Re-migrate even if the destination bundle already exists.",
+)
+@click.option(
+    "--delete-source", is_flag=True, default=False,
+    help=(
+        "Delete each source directory/bundle after its destination "
+        "has been written successfully. Use with care."
+    ),
+)
+def migrate_backtests_cmd(
+    src, dst, workers, no_index, include_ohlcv, no_skip_existing,
+    delete_source,
+):
+    """Convert a directory of legacy backtest folders into the bundled
+    binary format introduced in issue #487.
+
+    The new ``.iafbt`` format is a single zstd-compressed MessagePack
+    file per backtest. Loading bundled directories is dramatically
+    faster than the legacy multi-file layout for large batches.
+
+    Migration is streamed (load+save fused per worker) so memory
+    usage stays roughly constant regardless of source size, and
+    interrupted runs can be resumed (existing destination bundles
+    are skipped by default).
+    """
+    from investing_algorithm_framework.domain import migrate_backtests
+
+    n = migrate_backtests(
+        src,
+        dst,
+        workers=workers,
+        show_progress=True,
+        write_index=not no_index,
+        include_ohlcv=include_ohlcv,
+        skip_existing=not no_skip_existing,
+        delete_source=delete_source,
+    )
+    click.echo(f"Migrated {n} backtest(s) from {src} to {dst}")
+
+
+cli.add_command(migrate_backtests_cmd)

investing_algorithm_framework/domain/__init__.py

@@ -44,7 +44,10 @@
     BacktestDateRange, Backtest, BacktestMetrics, combine_backtests, \
     BacktestPermutationTest, BacktestEvaluationFocus, \
     generate_backtest_summary_metrics, load_backtests_from_directory, \
-    save_backtests_to_directory, retag_backtests
+    save_backtests_to_directory, retag_backtests, migrate_backtests, \
+    BacktestIndex, save_bundle, open_bundle, BUNDLE_EXT, \
+    BUNDLE_FORMAT_VERSION
+from .backtesting.backtest_utils import resolve_backtest_path
 from .algorithm_id import generate_algorithm_id
 from .pipeline import (
     Pipeline,
@@ -173,6 +176,13 @@
     "load_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
+    "migrate_backtests",
+    "resolve_backtest_path",
+    "BacktestIndex",
+    "save_bundle",
+    "open_bundle",
+    "BUNDLE_EXT",
+    "BUNDLE_FORMAT_VERSION",
     "generate_algorithm_id",
     # Pipeline API (Phase 1, see docs/design/pipeline-api.md)
     "Pipeline",

investing_algorithm_framework/domain/backtesting/__init__.py

@@ -7,8 +7,19 @@
 from .backtest_evaluation_focuss import BacktestEvaluationFocus
 from .combine_backtests import combine_backtests, \
     generate_backtest_summary_metrics
-from .backtest_utils import load_backtests_from_directory, \
-    save_backtests_to_directory, retag_backtests
+from .backtest_utils import (
+    load_backtests_from_directory,
+    save_backtests_to_directory,
+    retag_backtests,
+    migrate_backtests,
+    BacktestIndex,
+)
+from .bundle import (
+    save_bundle,
+    open_bundle,
+    BUNDLE_EXT,
+    BUNDLE_FORMAT_VERSION,
+)
 
 __all__ = [
     "Backtest",
@@ -18,9 +29,15 @@
     "BacktestRun",
     "BacktestPermutationTest",
     "BacktestEvaluationFocus",
+    "BacktestIndex",
     "combine_backtests",
     "generate_backtest_summary_metrics",
     "load_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
+    "migrate_backtests",
+    "save_bundle",
+    "open_bundle",
+    "BUNDLE_EXT",
+    "BUNDLE_FORMAT_VERSION",
 ]