feat(backtesting): single-bundle binary persistence format (#487)

Introduce .iafbt bundle format (zstd + msgpack) and parallel I/O for
backtest persistence, addressing the linear-scaling pain points in
issue #487.

Changes:
- New domain/backtesting/bundle.py with save_bundle/open_bundle and
  IAFB magic + format-version header.
- from_dict() classmethods on Backtest, BacktestRun, BacktestMetrics,
  BacktestSummaryMetrics, BacktestPermutationTest. open() delegates.
- Backtest.save() / open() auto-detect .iafbt files (and sibling
  bundles) for full backward compatibility.
- save_backtests_to_directory() / load_backtests_from_directory()
  now default to bundle format with ProcessPoolExecutor parallelism.
- New BacktestIndex helper backed by index.parquet for filtering
  large batches without opening any bundles.
- migrate_backtests() API + 'iaf migrate-backtests' CLI command.
- Optional parallel workers= flag on recalculate_backtests().
- 9 new tests in tests/domain/backtests/test_bundle.py.

Benchmarked on a real 6-strategy batch (11 runs each):
  size  : bundle 21x smaller than legacy directory format
  files : 27x fewer files per backtest
  save  : 3.6x faster at N=200
  load  : 2.4x faster at N=200

Full suite: 1642 tests pass (1633 existing + 9 new).

Author: MDUYN

investing_algorithm_framework/cli/cli.py

@@ -250,3 +250,47 @@ def mcp(directory):
 
 
 cli.add_command(mcp)
+
+
+@click.command(name="migrate-backtests")
+@click.option(
+    "--src", "-s",
+    required=True,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    help="Source directory containing legacy backtest sub-directories.",
+)
+@click.option(
+    "--dst", "-d",
+    required=True,
+    type=click.Path(file_okay=False, dir_okay=True),
+    help="Destination directory for the new ``.iafbt`` bundle files.",
+)
+@click.option(
+    "--workers", "-w", type=int, default=None,
+    help="Number of parallel workers (default: min(8, CPU count)).",
+)
+@click.option(
+    "--no-index", is_flag=True, default=False,
+    help="Skip writing index.parquet at the destination.",
+)
+def migrate_backtests_cmd(src, dst, workers, no_index):
+    """Convert a directory of legacy backtest folders into the bundled
+    binary format introduced in issue #487.
+
+    The new ``.iafbt`` format is a single zstd-compressed MessagePack
+    file per backtest. Loading bundled directories is dramatically
+    faster than the legacy multi-file layout for large batches.
+    """
+    from investing_algorithm_framework.domain import migrate_backtests
+
+    n = migrate_backtests(
+        src,
+        dst,
+        workers=workers,
+        show_progress=True,
+        write_index=not no_index,
+    )
+    click.echo(f"Migrated {n} backtest(s) from {src} to {dst}")
+
+
+cli.add_command(migrate_backtests_cmd)

investing_algorithm_framework/domain/__init__.py

@@ -44,7 +44,10 @@
     BacktestDateRange, Backtest, BacktestMetrics, combine_backtests, \
     BacktestPermutationTest, BacktestEvaluationFocus, \
     generate_backtest_summary_metrics, load_backtests_from_directory, \
-    save_backtests_to_directory, retag_backtests
+    save_backtests_to_directory, retag_backtests, migrate_backtests, \
+    BacktestIndex, save_bundle, open_bundle, BUNDLE_EXT, \
+    BUNDLE_FORMAT_VERSION
+from .backtesting.backtest_utils import resolve_backtest_path
 from .algorithm_id import generate_algorithm_id
 from .pipeline import (
     Pipeline,
@@ -173,6 +176,13 @@
     "load_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
+    "migrate_backtests",
+    "resolve_backtest_path",
+    "BacktestIndex",
+    "save_bundle",
+    "open_bundle",
+    "BUNDLE_EXT",
+    "BUNDLE_FORMAT_VERSION",
     "generate_algorithm_id",
     # Pipeline API (Phase 1, see docs/design/pipeline-api.md)
     "Pipeline",

investing_algorithm_framework/domain/backtesting/__init__.py

@@ -7,8 +7,19 @@
 from .backtest_evaluation_focuss import BacktestEvaluationFocus
 from .combine_backtests import combine_backtests, \
     generate_backtest_summary_metrics
-from .backtest_utils import load_backtests_from_directory, \
-    save_backtests_to_directory, retag_backtests
+from .backtest_utils import (
+    load_backtests_from_directory,
+    save_backtests_to_directory,
+    retag_backtests,
+    migrate_backtests,
+    BacktestIndex,
+)
+from .bundle import (
+    save_bundle,
+    open_bundle,
+    BUNDLE_EXT,
+    BUNDLE_FORMAT_VERSION,
+)
 
 __all__ = [
     "Backtest",
@@ -18,9 +29,15 @@
     "BacktestRun",
     "BacktestPermutationTest",
     "BacktestEvaluationFocus",
+    "BacktestIndex",
     "combine_backtests",
     "generate_backtest_summary_metrics",
     "load_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
+    "migrate_backtests",
+    "save_bundle",
+    "open_bundle",
+    "BUNDLE_EXT",
+    "BUNDLE_FORMAT_VERSION",
 ]

investing_algorithm_framework/domain/backtesting/backtest.py

@@ -57,6 +57,11 @@ class Backtest:
     strategy_ids: List[int] = field(default_factory=list)
     parameters: Dict = field(default_factory=dict)
     tag: str = None
+    # OHLCV payload optionally attached for save_bundle(include_ohlcv=True).
+    # Keys are conventionally "<symbol>@<timeframe>" (e.g.
+    # "BTC/EUR@1h"), values are pandas DataFrames or any object that
+    # has ``to_pandas()``. See issue #487.
+    ohlcv: Dict[str, object] = field(default_factory=dict, repr=False)
 
     def get_all_backtest_runs(
         self, backtest_date_ranges=None
@@ -194,17 +199,58 @@ def to_dict(self) -> dict:
             "tag": self.tag,
         }
 
+    @classmethod
+    def from_dict(cls, data: dict) -> 'Backtest':
+        """
+        Reconstruct a ``Backtest`` from a plain dict produced by
+        :py:meth:`to_dict`. Used by the binary bundle loader (#487).
+
+        Permutation-test entries that were serialized via
+        :py:meth:`BacktestPermutationTest.to_dict` are accepted, but the
+        ``permutated_dataframes`` field of those tests cannot round-trip
+        through this path and is left empty (the directory loader has
+        the same limitation).
+        """
+        if data is None:
+            return None
+
+        runs = []
+        for r in data.get("backtest_runs") or []:
+            runs.append(BacktestRun.from_dict(r))
+
+        summary_dict = data.get("backtest_summary")
+        summary = (
+            BacktestSummaryMetrics.from_dict(summary_dict)
+            if summary_dict else None
+        )
+
+        perm_tests = []
+        for pt in data.get("backtest_permutation_tests") or []:
+            perm_tests.append(BacktestPermutationTest.from_dict(pt))
+
+        return cls(
+            algorithm_id=data.get("algorithm_id"),
+            backtest_runs=runs,
+            backtest_summary=summary,
+            backtest_permutation_tests=perm_tests,
+            metadata=data.get("metadata") or {},
+            risk_free_rate=data.get("risk_free_rate"),
+            strategy_ids=data.get("strategy_ids") or [],
+            parameters=data.get("parameters") or {},
+            tag=data.get("tag"),
+        )
+
     @staticmethod
     def open(
         directory_path: Union[str, Path],
         backtest_date_ranges: List[BacktestDateRange] = None,
     ) -> 'Backtest':
         """
-        Open a backtest report from a directory and return a Backtest instance.
+        Open a backtest report from a directory **or** a ``.iafbt``
+        bundle file (issue #487) and return a :class:`Backtest`.
 
         Args:
-            directory_path (str): The path to the directory containing the
-                backtest report files.
+            directory_path: Path to the backtest directory or bundle file.
             backtest_date_ranges (List[BacktestDateRange], optional): A list of
                 date ranges to filter the backtest runs. If provided, only
                 backtest runs matching these date ranges will be loaded.
@@ -217,6 +263,34 @@ def open(
             OperationalException: If the directory does not exist or if
             there is an error loading the files.
         """
+        path_str = str(directory_path)
+
+        # If the path is a bundle file (or a regular file ending in the
+        # bundle extension), load via the bundle reader.
+        if os.path.isfile(path_str):
+            from .bundle import BUNDLE_EXT, is_bundle_file, open_bundle
+            if path_str.endswith(BUNDLE_EXT) or is_bundle_file(path_str):
+                bt = open_bundle(path_str)
+                if backtest_date_ranges is not None:
+                    bt.backtest_runs = bt.get_all_backtest_runs(
+                        backtest_date_ranges
+                    )
+                return bt
+
+        # Fallback: caller passed a path without extension but a sibling
+        # bundle file exists (e.g. session_cache stores
+        # "<storage>/<algorithm_id>" while the new default save format
+        # writes "<storage>/<algorithm_id>.iafbt").
+        if not os.path.exists(path_str):
+            from .bundle import BUNDLE_EXT, open_bundle
+            candidate = path_str + BUNDLE_EXT
+            if os.path.isfile(candidate):
+                bt = open_bundle(candidate)
+                if backtest_date_ranges is not None:
+                    bt.backtest_runs = bt.get_all_backtest_runs(
+                        backtest_date_ranges
+                    )
+                return bt
         algorithm_id = None
         backtest_runs = []
         backtest_summary_metrics = None
@@ -399,6 +473,23 @@ def save(
             None: This method does not return anything, it saves the
             metrics to a file.
         """
+        # Bundle-format dispatch (issue #487):
+        #   * If the caller passed a path ending in ``.iafbt``, save as
+        #     a bundle file.
+        #   * If the caller passed a base path (no extension) and a
+        #     sibling ``<path>.iafbt`` exists, replace it in place.
+        #   * Otherwise fall through to the legacy directory format
+        #     below (preserved for backward compatibility).
+        from .bundle import BUNDLE_EXT, save_bundle as _save_bundle
+        path_str = str(directory_path)
+        if path_str.endswith(BUNDLE_EXT):
+            _save_bundle(self, path_str)
+            return
+        sibling = path_str + BUNDLE_EXT
+        if os.path.isfile(sibling):
+            _save_bundle(self, sibling)
+            return
+
         if not os.path.exists(directory_path):
             os.makedirs(directory_path)
 
@@ -523,6 +614,37 @@ def __repr__(self):
             self.to_dict(), indent=4, sort_keys=True, default=str
         )
 
+    def save_bundle(
+        self,
+        path: Union[str, Path],
+        *,
+        include_ohlcv: bool = False,
+        ohlcv_store: Union[str, Path, None] = None,
+    ) -> Path:
+        """Persist this backtest as a single ``.iafbt`` bundle.
+
+        See :py:func:`investing_algorithm_framework.domain.backtesting.
+        bundle.save_bundle` for details. This is a thin convenience
+        wrapper.
+        """
+        from .bundle import save_bundle as _save_bundle
+        return _save_bundle(
+            self,
+            path,
+            include_ohlcv=include_ohlcv,
+            ohlcv_store=ohlcv_store,
+        )
+
+    @staticmethod
+    def open_bundle(
+        path: Union[str, Path],
+        *,
+        ohlcv_store: Union[str, Path, None] = None,
+    ) -> 'Backtest':
+        """Load a :class:`Backtest` from a ``.iafbt`` bundle file."""
+        from .bundle import open_bundle as _open_bundle
+        return _open_bundle(path, ohlcv_store=ohlcv_store)
+
     def merge(self, other: 'Backtest') -> 'Backtest':
         """
         Function to merge another Backtest instance into this one.

investing_algorithm_framework/domain/backtesting/backtest_metrics.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from logging import getLogger
 from typing import Tuple, List, Dict
 from datetime import datetime, date
@@ -448,6 +448,21 @@ def open(file_path: str | Path) -> 'BacktestMetrics':
         with open(file_path, 'r') as file:
             data = json.load(file)
 
+        return BacktestMetrics.from_dict(data)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'BacktestMetrics':
+        """
+        Reconstruct a ``BacktestMetrics`` from a plain dict (the inverse
+        of :py:meth:`to_dict`). Used by the JSON loader and by the
+        binary bundle loader (issue #487).
+        """
+        if data is None:
+            return None
+
+        # Work on a shallow copy to avoid mutating the caller's dict
+        data = dict(data)
+
         # Parse datetime fields
         data['backtest_start_date'] = datetime.fromisoformat(
             data['backtest_start_date']
@@ -457,37 +472,50 @@ def open(file_path: str | Path) -> 'BacktestMetrics':
         )
 
         # Parse tuple lists with datetime
-        data['equity_curve'] = BacktestMetrics._parse_tuple_list_datetime(
+        data['equity_curve'] = cls._parse_tuple_list_datetime(
             data.get('equity_curve', [])
         )
-        data['rolling_sharpe_ratio'] = BacktestMetrics\
-            ._parse_tuple_list_datetime(data.get('rolling_sharpe_ratio', []))
-        data['monthly_returns'] = BacktestMetrics\
-            ._parse_tuple_list_datetime(data.get('monthly_returns', []))
-        data['drawdown_series'] = BacktestMetrics\
-            ._parse_tuple_list_datetime(data.get('drawdown_series', []))
+        data['rolling_sharpe_ratio'] = cls._parse_tuple_list_datetime(
+            data.get('rolling_sharpe_ratio', [])
+        )
+        data['monthly_returns'] = cls._parse_tuple_list_datetime(
+            data.get('monthly_returns', [])
+        )
+        data['drawdown_series'] = cls._parse_tuple_list_datetime(
+            data.get('drawdown_series', [])
+        )
+        data['cumulative_return_series'] = cls._parse_tuple_list_datetime(
+            data.get('cumulative_return_series', [])
+        )
 
         # Parse tuple lists with date
-        data['yearly_returns'] = BacktestMetrics\
-            ._parse_tuple_list_date(data.get('yearly_returns', []))
+        data['yearly_returns'] = cls._parse_tuple_list_date(
+            data.get('yearly_returns', [])
+        )
 
         # Parse single tuples
-        data['best_month'] = BacktestMetrics\
-            ._parse_tuple_datetime(data.get('best_month'))
-        data['worst_month'] = BacktestMetrics\
-            ._parse_tuple_datetime(data.get('worst_month'))
-        data['best_year'] = BacktestMetrics\
-            ._parse_tuple_date(data.get('best_year'))
-        data['worst_year'] = BacktestMetrics\
-            ._parse_tuple_date(data.get('worst_year'))
+        data['best_month'] = cls._parse_tuple_datetime(data.get('best_month'))
+        data['worst_month'] = cls._parse_tuple_datetime(
+            data.get('worst_month')
+        )
+        data['best_year'] = cls._parse_tuple_date(data.get('best_year'))
+        data['worst_year'] = cls._parse_tuple_date(data.get('worst_year'))
 
         # Parse Trade objects if they exist
         if data.get('best_trade'):
             data['best_trade'] = Trade.from_dict(data['best_trade'])
         if data.get('worst_trade'):
             data['worst_trade'] = Trade.from_dict(data['worst_trade'])
 
-        return BacktestMetrics(**data)
+        # Drop fields computed in __post_init__ to avoid duplicate kwargs
+        data.pop('total_number_of_days', None)
+
+        # Drop any unknown keys (forward-compat with new fields written
+        # by a newer version)
+        valid = {f.name for f in fields(cls)}
+        data = {k: v for k, v in data.items() if k in valid}
+
+        return cls(**data)
 
     def __repr__(self):
         """