perf(backtesting): 3x faster Backtest.open + dashboard bundle support

- Add domain/datetime_parsing.parse_datetime fast helper using
  datetime.fromisoformat with dateutil fallback. The dateutil parser
  was 88% of Backtest.open time per profiling.
- Use the fast helper in PortfolioSnapshot, Order, Trade, TradeStopLoss,
  TradeTakeProfit hot from_dict paths. Backtest.open: 1025ms -> 334ms
  on the real example batch (3x faster).
- BacktestReport.open() now discovers .iafbt bundle files in addition
  to legacy directory layouts, and accepts an optional workers= kwarg
  for parallel loading of large batches.
- Document the .iafbt bundle format in README and Getting Started docs
  as the framework's custom optimized backtest persistence format
  (~21x smaller, ~27x fewer files, ~3x faster to load).

Author: MDUYN

README.md

@@ -86,6 +86,7 @@ This framework is built around the full loop: **create strategies → vector bac
 - 🎯 **Return Scenario Projections** — Good, average, bad & very bad year projections from backtest data
 - 📉 **Benchmark Comparison** — Beat-rate analysis vs Buy & Hold, DCA, risk-free & custom benchmarks
 - 📄 **One-Click HTML Report** — Self-contained file, no server, dark & light theme, shareable
+- 📦 **Custom `.iafbt` Bundle Format** — Optimized binary persistence (zstd + MessagePack) for backtest results: ~21× smaller and ~27× fewer files than the legacy directory layout, with parallel I/O for fast load/save of large batches
 - 🌐 **Load External Data** — Fetch CSV, JSON, or Parquet from any URL with caching and auto-refresh
 - � **[Record Custom Variables](https://coding-kitties.github.io/investing-algorithm-framework/Advanced%20Concepts/recording-variables)** — Track any indicator or metric during backtests with `context.record()`
 - �🚀 **Build → Backtest → Deploy** — Local dev, cloud deploy (AWS / Azure), or monetize on Finterion

docusaurus/docs/Getting Started/backtest-reports.md

@@ -60,7 +60,17 @@ report = BacktestReport.open(directory_path="./my_backtests")
 report.show()
 ```
 
-The `open()` method recursively finds all valid backtest directories (containing `algorithm_id.json` and a `runs/` folder) and loads them into a single report.
+The `open()` method recursively finds all valid backtest directories (containing `algorithm_id.json` and a `runs/` folder) **and** any `.iafbt` bundle files, and loads them into a single report.
+
+:::tip Optimized `.iafbt` bundle format
+Backtests are saved by default in the framework's custom **`.iafbt` bundle format** — a single binary file per backtest combining zstd compression and MessagePack encoding. It is purpose-built for backtest reports: ~21× smaller and ~27× fewer files than the legacy directory format, and `BacktestReport.open()` loads it ~3× faster. The legacy directory format is still fully supported for backwards compatibility, and you can mix both in the same folder.
+
+For very large batches, opt into parallel loading:
+
+```python
+report = BacktestReport.open(directory_path="./my_backtests", workers=4)
+```
+:::
 
 You can also combine disk and in-memory backtests:
 

docusaurus/docs/Getting Started/backtesting.md

@@ -85,6 +85,10 @@ from investing_algorithm_framework import load_backtests_from_directory
 backtests = load_backtests_from_directory("./my_backtests")
 ```
 
+:::info `.iafbt` bundle format
+Backtests are persisted in the framework's optimized **`.iafbt` bundle format** — a single binary file per backtest using zstd compression + MessagePack encoding. Compared to the legacy directory layout it is ~21× smaller, ~27× fewer files, and ~3× faster to load. Both `save_backtests_to_directory` and `load_backtests_from_directory` support parallel I/O via `workers=N`. Existing legacy directories keep working transparently; use `iaf migrate-backtests --src ... --dst ...` to convert them.
+:::
+
 ## Reporting
 
 Use [Backtest Reports](/docs/Getting%20Started/backtest-reports) to turn

investing_algorithm_framework/app/reporting/backtest_report.py

@@ -12,7 +12,7 @@
 from jinja2 import Environment, FileSystemLoader
 
 from investing_algorithm_framework.domain import (
-    Backtest, OperationalException, tqdm
+    Backtest, OperationalException, tqdm, BUNDLE_EXT
 )
 
 logger = logging.getLogger("investing_algorithm_framework")
@@ -217,9 +217,15 @@ def save(self, path):
 
     @staticmethod
     def _is_backtest(backtest_path):
+        if not os.path.exists(backtest_path):
+            return False
+        # Bundle file (.iafbt)
+        if os.path.isfile(backtest_path) and \
+                backtest_path.endswith(BUNDLE_EXT):
+            return True
+        # Legacy directory layout
         return (
-            os.path.exists(backtest_path)
-            and os.path.isdir(backtest_path)
+            os.path.isdir(backtest_path)
             and os.path.isfile(
                 os.path.join(backtest_path, "algorithm_id.json")
             )
@@ -231,6 +237,7 @@ def open(
         backtests: List[Backtest] = None,
         directory_path: Union[str, List[str], None] = None,
         show_progress: bool = False,
+        workers: Union[int, None] = None,
     ) -> "BacktestReport":
         loaded = []
         source_tags = []
@@ -254,7 +261,7 @@ def open(
             if BacktestReport._is_backtest(dp):
                 backtest_paths.append((dp, tag))
             else:
-                for root, dirs, _ in os.walk(dp):
+                for root, dirs, files in os.walk(dp):
                     for dir_name in dirs:
                         subdir = os.path.join(
                             root, dir_name
@@ -265,6 +272,11 @@ def open(
                             backtest_paths.append(
                                 (subdir, tag)
                             )
+                    for file_name in files:
+                        if file_name.endswith(BUNDLE_EXT):
+                            backtest_paths.append(
+                                (os.path.join(root, file_name), tag)
+                            )
 
         iterator = backtest_paths
         if show_progress:
@@ -273,9 +285,43 @@ def open(
                 desc="Loading backtests",
             )
 
-        for path, tag in iterator:
-            loaded.append(Backtest.open(path))
-            source_tags.append(tag)
+        # Parallel load is opt-in (workers > 1). ProcessPoolExecutor
+        # startup costs typically dwarf the per-backtest decode for
+        # batches < ~30, so keep default behaviour serial. Pass
+        # ``workers=N`` (or ``-1`` for cpu_count) to load large
+        # batches in parallel.
+        from investing_algorithm_framework.domain.backtesting.\
+backtest_utils import (
+            _load_one_dispatch, _resolve_workers,
+        )
+
+        n = len(backtest_paths)
+        resolved_workers = (
+            _resolve_workers(workers) if workers is not None else 1
+        )
+        if resolved_workers > 1 and n >= 4:
+            from concurrent.futures import ProcessPoolExecutor
+
+            items = [
+                (path, "bundle" if path.endswith(BUNDLE_EXT) else "dir")
+                for path, _ in backtest_paths
+            ]
+            with ProcessPoolExecutor(max_workers=resolved_workers) as ex:
+                results = list(
+                    tqdm(
+                        ex.map(_load_one_dispatch, items),
+                        total=n,
+                        desc="Loading backtests",
+                        disable=not show_progress,
+                    )
+                )
+            for bt, (_, tag) in zip(results, backtest_paths):
+                loaded.append(bt)
+                source_tags.append(tag)
+        else:
+            for path, tag in iterator:
+                loaded.append(Backtest.open(path))
+                source_tags.append(tag)
 
         for bt in backtests:
             if not isinstance(bt, Backtest):

investing_algorithm_framework/domain/datetime_parsing.py

@@ -0,0 +1,28 @@
+"""Fast ISO-8601 datetime parsing helper.
+
+``dateutil.parser.parse`` is the bottleneck in :class:`Backtest` loading
+(see issue #487 profiling notes). The strings emitted by :py:meth:`Backtest.to_dict`
+are always produced by :py:meth:`datetime.isoformat`, so the standard-library
+:py:meth:`datetime.fromisoformat` parser handles them ~50x faster.
+
+This helper:
+
+- Returns ``None`` for falsy / ``None`` values.
+- Passes already-parsed :class:`datetime.datetime` objects through unchanged.
+- Tries :py:meth:`datetime.fromisoformat` first (fast path).
+- Falls back to ``dateutil.parser.parse`` for anything exotic.
+"""
+from datetime import datetime
+
+
+def parse_datetime(value):
+    if value is None or value == "":
+        return None
+    if isinstance(value, datetime):
+        return value
+    try:
+        return datetime.fromisoformat(value)
+    except (TypeError, ValueError):
+        pass
+    from dateutil.parser import parse as _du_parse
+    return _du_parse(value)

investing_algorithm_framework/domain/models/order/order.py

@@ -2,6 +2,9 @@
 from datetime import datetime, timezone
 
 from dateutil.parser import parse
+from investing_algorithm_framework.domain.datetime_parsing import (
+    parse_datetime as _parse_dt,
+)
 
 from investing_algorithm_framework.domain.exceptions import \
     OperationalException
@@ -280,10 +283,10 @@ def from_dict(data: dict):
             trading_symbol = data.get("trading_symbol", None)
 
         if created_at is not None:
-            created_at = parse(created_at)
+            created_at = _parse_dt(created_at)
 
         if updated_at is not None:
-            updated_at = parse(updated_at)
+            updated_at = _parse_dt(updated_at)
 
         order = Order(
             id=data.get("id", None),

investing_algorithm_framework/domain/models/portfolio/portfolio_snapshot.py

@@ -3,6 +3,9 @@
 from dateutil import parser
 
 from investing_algorithm_framework.domain.models.base_model import BaseModel
+from investing_algorithm_framework.domain.datetime_parsing import (
+    parse_datetime as _parse_dt,
+)
 
 
 class PortfolioSnapshot(BaseModel):
@@ -36,7 +39,7 @@ def __init__(
         self.metadata = metadata if metadata is not None else {}
 
         if created_at is not None and isinstance(created_at, str):
-            self.created_at = parser.parse(created_at)
+            self.created_at = _parse_dt(created_at)
         else:
             self.created_at = created_at
 
@@ -185,7 +188,7 @@ def from_dict(data):
             PortfolioSnapshot: An instance of PortfolioSnapshot.
         """
         created_at_str = data.get("created_at")
-        created_at = parser.parse(created_at_str)
+        created_at = _parse_dt(created_at_str)
 
         # Ensure created_at is timezone aware
         created_at = created_at.replace(tzinfo=timezone.utc)

investing_algorithm_framework/domain/models/trade/trade.py

@@ -1,4 +1,7 @@
 from dateutil.parser import parse
+from investing_algorithm_framework.domain.datetime_parsing import (
+    parse_datetime as _parse_dt,
+)
 from datetime import timezone
 
 from investing_algorithm_framework.domain.models.base_model import BaseModel
@@ -318,13 +321,13 @@ def from_dict(data):
         orders = None
 
         if "opened_at" in data and data["opened_at"] is not None:
-            opened_at = parse(data["opened_at"])
+            opened_at = _parse_dt(data["opened_at"])
 
         if "closed_at" in data and data["closed_at"] is not None:
-            closed_at = parse(data["closed_at"])
+            closed_at = _parse_dt(data["closed_at"])
 
         if "updated_at" in data and data["updated_at"] is not None:
-            updated_at = parse(data["updated_at"])
+            updated_at = _parse_dt(data["updated_at"])
 
         if "stop_losses" in data and data["stop_losses"] is not None:
             stop_losses = [

investing_algorithm_framework/domain/models/trade/trade_stop_loss.py

@@ -1,4 +1,7 @@
 from dateutil.parser import parse
+from investing_algorithm_framework.domain.datetime_parsing import (
+    parse_datetime as _parse_dt,
+)
 from datetime import timezone
 from datetime import datetime
 
@@ -273,13 +276,13 @@ def ensure_iso(value):
 
     @staticmethod
     def from_dict(data: dict):
-        created_at = parse(data["created_at"]) \
+        created_at = _parse_dt(data["created_at"]) \
             if data.get("created_at") is not None else None
-        updated_at = parse(data["updated_at"]) \
+        updated_at = _parse_dt(data["updated_at"]) \
             if data.get("updated_at") is not None else None
-        triggered_at = parse(data["triggered_at"]) \
+        triggered_at = _parse_dt(data["triggered_at"]) \
             if data.get("triggered_at") is not None else None
-        high_water_mark_date = parse(data.get("high_water_mark_date")) \
+        high_water_mark_date = _parse_dt(data.get("high_water_mark_date")) \
             if data.get("high_water_mark_date") is not None else None
 
         # Make sure all the dates are timezone utc aware

investing_algorithm_framework/domain/models/trade/trade_take_profit.py

@@ -1,5 +1,8 @@
 from datetime import timezone, datetime
 from dateutil.parser import parse
+from investing_algorithm_framework.domain.datetime_parsing import (
+    parse_datetime as _parse_dt,
+)
 
 from investing_algorithm_framework.domain.models.base_model import BaseModel
 
@@ -307,13 +310,13 @@ def ensure_iso(value):
 
     @staticmethod
     def from_dict(data: dict):
-        created_at = parse(data["created_at"]) \
+        created_at = _parse_dt(data["created_at"]) \
             if data.get("created_at") is not None else None
-        updated_at = parse(data["updated_at"]) \
+        updated_at = _parse_dt(data["updated_at"]) \
             if data.get("updated_at") is not None else None
-        triggered_at = parse(data["triggered_at"]) \
+        triggered_at = _parse_dt(data["triggered_at"]) \
             if data.get("triggered_at") is not None else None
-        high_water_mark_date = parse(data.get("high_water_mark_date")) \
+        high_water_mark_date = _parse_dt(data.get("high_water_mark_date")) \
             if data.get("high_water_mark_date") is not None else None
 
         # Make sure all the dates are timezone utc aware