Merge pull request #2 from HanSun103/live-forward-test-2026-07-17

Update project implementation

Author: HanSun103

README.md

@@ -196,20 +196,74 @@ python -m src.cli evaluate-rank-model \
   --top-k 40
 ```
 
-Create a frozen live-forward paper snapshot from the latest yfinance close:
+Install the optional tuning dependency, then create a frozen live-forward paper snapshot from the
+latest yfinance close:
 
 ```bash
-python -m src.cli run-live-forward-snapshot --as-of 2026-07-17
+pip install -e ".[tuning]"
+
+python -m src.cli run-live-forward-snapshot \
+  --as-of 2026-07-20 \
+  --optuna-trials 25 \
+  --tuning-folds 3 \
+  --validation-days 42 \
+  --purge-days 1 \
+  --optuna-timeout-seconds 600
 ```
 
-By default this rebuilds a rolling daily-proxy training panel from recent yfinance prices, trains
-candidate Ridge and XGBoost rankers on all known one-session forward residual-rank labels, selects
-the winner on a trailing temporal validation window, refits the selected model, scores the active
-membership list using the latest close, and saves a local paper portfolio plus
-`model_selection.csv` under `reports/live_forward/YYYY-MM-DD/`. Do not edit a saved snapshot
-before measuring realized next-session returns; otherwise the forward test stops being clean. This
-daily proxy is designed to collect evidence faster than waiting ten calendar weeks for ten weekly
-rebalance observations.
+This rebuilds a rolling daily-proxy training panel from recent yfinance prices, builds only labels
+whose future one-session returns are already known, and uses Optuna to choose between Ridge and
+XGBoost rankers on purged rolling temporal validation folds. After selection, the winning model is
+refit on all data available before the decision date, the active membership list is scored using
+the latest close, and a frozen paper portfolio plus `model_selection.csv` are saved under
+`reports/live_forward/YYYY-MM-DD/`. Do not edit a saved snapshot before measuring realized
+next-session returns; otherwise the forward test stops being clean. This daily proxy is designed to
+collect evidence faster than waiting ten calendar weeks for ten weekly rebalance observations.
+
+The July 20, 2026 production-style run requested 25 Optuna trials with a 600-second timeout. Ten
+trials completed, using three 42-session validation folds separated from training by a one-session
+purge. The selected model was Ridge with `alpha=6156.9973`, aggregate validation Rank IC of
+approximately `0.0121`, and average top-40 raw excess return of approximately `0.067%` per
+validation session. Treat that as model-selection evidence only; the true forward result comes
+from evaluating the frozen snapshot after the next session is realized.
+
+Generate an hourly live-monitoring chart versus SPY for a saved snapshot:
+
+```bash
+python -m src.cli report-live-intraday \
+  --as-of 2026-07-17 \
+  --monitor-date 2026-07-20 \
+  --benchmark SPY \
+  --interval 60m \
+  --baseline previous-close
+```
+
+This writes `intraday_vs_benchmark.csv`, `intraday_vs_benchmark_summary.json`, and a lightweight
+SVG chart under the snapshot folder. The default baseline is the monitored day's previous close,
+matching common one-day market charts. It is a monitoring view, not a replacement for the frozen
+next-session evaluation.
+
+Evaluate a snapshot at stock and portfolio level for a named horizon:
+
+```bash
+python -m src.cli evaluate-live-forward \
+  --as-of 2026-07-17 \
+  --eval-date 2026-07-20 \
+  --benchmark SPY \
+  --interval 60m \
+  --baseline previous-close \
+  --horizon-label monday_intraday
+```
+
+Build the static monitoring dashboard:
+
+```bash
+python -m src.cli build-live-dashboard
+```
+
+The dashboard reads every saved snapshot under `reports/live_forward/`, supports changing the
+snapshot and evaluation horizon, and shows portfolio return, benchmark return, active return,
+stock-level winners/losers versus SPY, model-selection evidence, and experiment history.
 
 Build the earlier conservative final-reference package:
 

pyproject.toml

@@ -15,6 +15,7 @@ portfolio = "src.cli:main"
 
 [project.optional-dependencies]
 wrds = ["wrds>=3.2"]
+tuning = ["optuna>=4.9"]
 
 [tool.setuptools.packages.find]
 where = ["."]
@@ -30,6 +31,8 @@ addopts = "-v --tb=short"
 [tool.ruff]
 line-length = 100
 target-version = "py311"
+
+[tool.ruff.lint]
 select = ["E", "F", "I", "N", "W"]
 
 [tool.mypy]

src/cli.py

@@ -53,7 +53,7 @@ def main(log_level: str, log_file: str | None) -> None:
 @click.option("--seed",  default=42, show_default=True, type=int)
 def generate_demo(output: str, start: str, end: str, seed: int) -> None:
     """Generate synthetic demo prices and NLP data (no API keys needed)."""
-    from .utils.demo_data import save_demo_prices, save_demo_nlp
+    from .utils.demo_data import save_demo_nlp, save_demo_prices
     save_demo_prices(output, start=start, end=end, seed=seed)
     save_demo_nlp(output, start=start, end=end, seed=seed)
     click.echo(f"Demo data written to {output}")
@@ -79,9 +79,9 @@ def build_dataset(
     validate_experiment_config(cfg)
     logger.info("Building dataset (demo=%s)", use_demo)
 
-    from .data.market_data import MarketDataLoader
-    from .data.macro_data import MacroDataLoader
     from .data.feature_store import FeatureStore
+    from .data.macro_data import MacroDataLoader
+    from .data.market_data import MarketDataLoader
 
     mdl = MarketDataLoader(cfg, use_demo=use_demo)
     prices = mdl.load()
@@ -121,13 +121,11 @@ def build_nlp_features(
     if experiment_config:
         paths.append(experiment_config)
     cfg = load_config(*paths)
-    nlp_cfg = cfg.extra.get("sources", {})
-
-    from .nlp.source_connectors import ConnectorRegistry
+    from .features.nlp_features import NLPFeatureBuilder
     from .nlp.entity_linking import EntityLinker
-    from .nlp.sentiment_model import SentimentModel
     from .nlp.event_detector import EventDetector
-    from .features.nlp_features import NLPFeatureBuilder
+    from .nlp.sentiment_model import SentimentModel
+    from .nlp.source_connectors import ConnectorRegistry
 
     registry = ConnectorRegistry(cfg, use_demo=use_demo)
     posts = registry.fetch_all()
@@ -169,6 +167,7 @@ def build_nlp_features(
     feature_store_path = Path(cfg.paths.data_processed) / "features.parquet"
     if feature_store_path.exists() and not features.empty:
         import pandas as pd
+
         from .data.feature_store import FeatureStore
 
         feature_store = pd.read_parquet(feature_store_path)
@@ -265,7 +264,11 @@ def collect_news(config: str, base_config: str) -> None:
 @main.command("import-fnspid")
 @click.option("--config", default="configs/base.yaml", show_default=True)
 @click.option("--nlp-config", default="configs/nlp.yaml", show_default=True)
-@click.option("--tickers", default=None, help="Comma-separated symbols; defaults to configured universe")
+@click.option(
+    "--tickers",
+    default=None,
+    help="Comma-separated symbols; defaults to configured universe",
+)
 @click.option("--start", default=None, help="Inclusive date; defaults to configured start")
 @click.option("--end", default=None, help="Exclusive date; defaults to configured end")
 @click.option(
@@ -426,8 +429,8 @@ def train_model(
     from .experiments.forecast_evaluation import (
         development_slice,
         persist_forecast_evaluation,
-        validate_feature_store_metadata,
         validate_experiment_config,
+        validate_feature_store_metadata,
     )
     validate_experiment_config(cfg)
     model_name = model or cfg.extra.get("active_model", "xgboost")
@@ -440,18 +443,19 @@ def train_model(
     validate_feature_store_metadata(features_path, cfg)
 
     import pandas as pd
-    from .models.ensemble import build_model
+
     from .backtest.walk_forward import WalkForwardSplitter
+    from .models.ensemble import build_model
 
-    X = development_slice(pd.read_parquet(features_path), cfg, unlock_lockbox)
+    features = development_slice(pd.read_parquet(features_path), cfg, unlock_lockbox)
     splitter = WalkForwardSplitter(cfg)
 
     results = []
-    for fold, (train_idx, val_idx, test_idx) in enumerate(splitter.split(X)):
+    for fold, (train_idx, val_idx, test_idx) in enumerate(splitter.split(features)):
         mdl = build_model(model_name, cfg)
-        train = X.iloc[train_idx]
-        val = X.iloc[val_idx]
-        test = X.iloc[test_idx]
+        train = features.iloc[train_idx]
+        val = features.iloc[val_idx]
+        test = features.iloc[test_idx]
         mdl.fit(train, train["target"], val, val["target"])
         preds = mdl.predict(test)
         fold_result = test[["target"]].copy()
@@ -617,7 +621,9 @@ def prepare_fnspid_v2_inputs(
     coverage = len(covered_stocks) / len(stock_tickers)
     covered_membership = relevant.loc[relevant["ticker"].isin(covered_stocks)].copy()
     covered_membership.to_csv(out / "membership_intervals_covered.csv", index=False)
-    pd.DataFrame({"ticker": missing_stocks}).to_csv(out / "missing_membership_prices.csv", index=False)
+    pd.DataFrame({"ticker": missing_stocks}).to_csv(
+        out / "missing_membership_prices.csv", index=False
+    )
     prices.to_parquet(out / "prices_with_sector_etfs.parquet")
     repair_audit.to_csv(out / "price_adjustment_repairs.csv", index=False)
     sectors = infer_sector_etf_history(prices, covered_stocks)
@@ -879,6 +885,10 @@ def run_exit_prediction_v8_command(
     help="Train fixed config parameters only; intended for debugging, not evidence runs.",
 )
 @click.option("--validation-days", default=42, show_default=True, type=int)
+@click.option("--tuning-folds", default=3, show_default=True, type=int)
+@click.option("--purge-days", default=None, type=int)
+@click.option("--optuna-trials", default=50, show_default=True, type=int)
+@click.option("--optuna-timeout-seconds", default=None, type=int)
 @click.option("--selection-metric", default="top40_raw_excess", show_default=True)
 def run_live_forward_snapshot_command(
     base_config: str,
@@ -895,6 +905,10 @@ def run_live_forward_snapshot_command(
     use_static_training_dataset: bool,
     skip_hyperparameter_tuning: bool,
     validation_days: int,
+    tuning_folds: int,
+    purge_days: int | None,
+    optuna_trials: int,
+    optuna_timeout_seconds: int | None,
     selection_metric: str,
 ) -> None:
     """Create a frozen paper-trading snapshot from latest yfinance data."""
@@ -917,6 +931,10 @@ def run_live_forward_snapshot_command(
         use_static_training_dataset=use_static_training_dataset,
         tune_hyperparameters=not skip_hyperparameter_tuning,
         validation_days=validation_days,
+        tuning_folds=tuning_folds,
+        purge_days=purge_days,
+        optuna_trials=optuna_trials,
+        optuna_timeout_seconds=optuna_timeout_seconds,
         selection_metric=selection_metric,
     )
     click.echo(json.dumps({
@@ -934,6 +952,110 @@ def run_live_forward_snapshot_command(
     }, indent=2))
 
 
+@main.command("report-live-intraday")
+@click.option(
+    "--snapshot-dir",
+    default=None,
+    help="Live snapshot directory; defaults to reports/live_forward/<as-of>.",
+)
+@click.option("--as-of", default=None, help="Snapshot date used when snapshot-dir is omitted.")
+@click.option("--benchmark", default="SPY", show_default=True)
+@click.option("--interval", default="60m", show_default=True)
+@click.option("--monitor-date", default=None, help="Intraday date to plot; defaults to today.")
+@click.option(
+    "--baseline",
+    default="previous-close",
+    show_default=True,
+    type=click.Choice(["previous-close", "first-bar"]),
+)
+def report_live_intraday_command(
+    snapshot_dir: str | None,
+    as_of: str | None,
+    benchmark: str,
+    interval: str,
+    monitor_date: str | None,
+    baseline: str,
+) -> None:
+    """Generate intraday portfolio-vs-benchmark CSV and SVG for a live snapshot."""
+    import json
+
+    import pandas as pd
+
+    from .experiments.live_forward_test import generate_intraday_benchmark_report
+
+    if snapshot_dir is None:
+        date = as_of or pd.Timestamp.today().strftime("%Y-%m-%d")
+        snapshot_dir = str(Path("reports/live_forward") / date)
+    result = generate_intraday_benchmark_report(
+        snapshot_dir,
+        benchmark=benchmark,
+        interval=interval,
+        monitor_date=monitor_date,
+        baseline=baseline,
+    )
+    click.echo(json.dumps(result, indent=2))
+
+
+@main.command("evaluate-live-forward")
+@click.option(
+    "--snapshot-dir",
+    default=None,
+    help="Live snapshot directory; defaults to reports/live_forward/<as-of>.",
+)
+@click.option("--as-of", default=None, help="Snapshot date used when snapshot-dir is omitted.")
+@click.option("--eval-date", required=True, help="Evaluation date.")
+@click.option("--benchmark", default="SPY", show_default=True)
+@click.option("--interval", default="60m", show_default=True)
+@click.option(
+    "--baseline",
+    default="previous-close",
+    show_default=True,
+    type=click.Choice(["previous-close", "first-bar"]),
+)
+@click.option("--horizon-label", default=None, help="Display/storage label for this horizon.")
+def evaluate_live_forward_command(
+    snapshot_dir: str | None,
+    as_of: str | None,
+    eval_date: str,
+    benchmark: str,
+    interval: str,
+    baseline: str,
+    horizon_label: str | None,
+) -> None:
+    """Evaluate a frozen live-forward snapshot versus a benchmark."""
+    import json
+
+    import pandas as pd
+
+    from .experiments.live_forward_test import evaluate_live_forward_snapshot
+
+    if snapshot_dir is None:
+        date = as_of or pd.Timestamp.today().strftime("%Y-%m-%d")
+        snapshot_dir = str(Path("reports/live_forward") / date)
+    result = evaluate_live_forward_snapshot(
+        snapshot_dir,
+        eval_date=eval_date,
+        benchmark=benchmark,
+        interval=interval,
+        baseline=baseline,
+        horizon_label=horizon_label,
+    )
+    click.echo(json.dumps(result, indent=2))
+
+
+@main.command("build-live-dashboard")
+@click.option("--live-root", default="reports/live_forward", show_default=True)
+@click.option("--output-file", default="dashboard.html", show_default=True)
+def build_live_dashboard_command(live_root: str, output_file: str) -> None:
+    """Build a static dashboard for live-forward snapshots and evaluations."""
+    import json
+
+    from .experiments.live_forward_test import build_live_dashboard
+
+    result = build_live_dashboard(live_root, output_file=output_file)
+    click.echo(json.dumps(result, indent=2))
+
+
 @main.command("run-sparse-sentiment-ablation")
 @click.option(
     "--v2-base-config", default="configs/experiments/residual_rank_v2.yaml",
@@ -1193,6 +1315,7 @@ def select_pairs(
     cfg = load_config(*paths)
 
     import pandas as pd
+
     from .data.market_data import MarketDataLoader
     from .statarb.pair_selection import PairSelector
 
@@ -1274,7 +1397,6 @@ def run_ablation(config: str, base_config: str, output: str) -> None:
         return
 
     from .backtest.engine import BacktestEngine
-    from .utils.config import _deep_merge
 
     out = Path(output)
     out.mkdir(parents=True, exist_ok=True)