phi_forest_V3.py

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, GradientBoostingRegressor, VotingRegressor
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error
from sklearn.exceptions import NotFittedError
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from datetime import date
import os
import joblib
import math
import sys
import json
import time
import argparse
import matplotlib.pyplot as plt
import yfinance as yf

# === Configuration defaults ===
# Change these at the top of the file or pass via CLI to control symbol and download ranges
DEFAULT_ASSET_SYMBOL = "GBPJPY=X"
DEFAULT_DOWNLOAD_PERIOD = "5D"   # e.g., '2y', '1y', '6mo'
DEFAULT_DOWNLOAD_INTERVAL = "1H"  # e.g., '1wk', '1d', '1h'

# Feature/dataset persistence settings
FEATURE_VERSION = "1"  # bump if feature engineering changes
WINDOW_SIZE = 10        # rolling window length used for features

# Intraday default settings used when running in intraday mode
INTRADAY_DEFAULTS = {
    '1H': {'interval': '60m', 'period': '90d'},   # More data for better models
    '30M': {'interval': '30m', 'period': '60d'},  # Increased data range
    '5M': {'interval': '5m', 'period': '30d'},    # Much more 5M data
    '1M': {'interval': '1m', 'period': '7d'},     # Add 1-minute resolution
}
DEFAULT_MAX_TRAIN_SAMPLES = 8000       # Begin decay after ~8000 samples (rolling cap)
DEFAULT_RECENCY_HALF_LIFE = 250
DEFAULT_USE_RECENCY_WEIGHTING = True

# Fibonacci tier configuration
DEFAULT_MAX_FIB_MULTIPLIER = 100.0          # Explore tiers up to +/- this multiple
DEFAULT_INCLUDE_NEGATIVE_FIB_TIERS = True   # Include tiers above the high (negative ratios)
DEFAULT_MAX_FIB_LEVELS = 1000               # Safety cap on total tier count - increased to 1000 for thorough exploration

def save_config(symbol, period, interval, path='config.json'):
    cfg = {'symbol': symbol, 'period': period, 'interval': interval}
    with open(path, 'w') as f:
        json.dump(cfg, f, indent=2)
    print(f"Saved config to {path}")

def load_config(path='config.json'):
    if os.path.exists(path):
        try:
            with open(path, 'r') as f:
                return json.load(f)
        except Exception:
            return None
    return None

class PhiForestEngineWithVolumeTiers:
    def __init__(self, symbol=None):
        self.asset_symbol = (symbol or DEFAULT_ASSET_SYMBOL).upper()
        print(f"[INIT] Using asset: {self.asset_symbol}")
        self.model_version = "v1.0"
        self.hyperparam_history = []

        # Start with small, manageable hyperparameters - grow slowly with evolution
        self.hyperparams = {
            'n_estimators': 100,            # Start small, grow with evolution
            'random_state': 42,
            'n_jobs': -1,                   # Use all CPU cores
            'oob_score': True,
            'min_samples_leaf': 1,          # More granular splits (higher compute)
            'max_depth': 10,                # Start shallow, grow with evolution
            'max_features': 'sqrt',         # More feature evaluation
            'min_impurity_decrease': 1e-9,  # More sensitive splits
            'bootstrap': True,              # Enable bootstrap sampling
            'max_samples': None,            # Use all samples for each tree
        }

        # Meta flags (not passed to sklearn)
        self.actual_price = True
        self.prediction_price = True

        self.setup_directories()
        # load model and hyperparams if present
        self.model, loaded_version = self.load_model()
        # If hyperparams file contained meta flags, they will be in self.hyperparams; extract them
        self.actual_price = self.hyperparams.pop('actual_price', self.actual_price)
        self.prediction_price = self.hyperparams.pop('prediction_price', self.prediction_price)

        if not self.model:
            # Filter hyperparams to valid RandomForest parameters
            allowed = RandomForestRegressor().get_params().keys()
            model_kwargs = {k: v for k, v in self.hyperparams.items() if k in allowed}
            self.model = RandomForestRegressor(**model_kwargs)
            print("Created new model with initial hyperparameters")
        else:
            self.model_version = loaded_version
            print(f"Loaded model version {self.model_version}")
            
            # Check if loaded model is compatible with current feature set
            # We'll do a quick feature compatibility test during first use
            self._needs_retraining = False

        self.history = []
        self.metrics_log = []
        # Small rolling history for EMC values to form adaptive thresholds
        self._emc_history = []
        # Whether to use phi-aware hyperparameter evolution (can be toggled later)
        self.phi_evolve_enabled = True
        self.X_train = []
        self.y_train = []
        # Track whether we've achieved perfect accuracy in the most recent cycle
        self.perfect_accuracy_achieved = False
        # Extensive Fibonacci tier controls
        self.max_fib_multiplier = DEFAULT_MAX_FIB_MULTIPLIER
        self.include_negative_fib_tiers = DEFAULT_INCLUDE_NEGATIVE_FIB_TIERS
        self.max_fib_levels = DEFAULT_MAX_FIB_LEVELS
        self._tiers_info_logged = False
        
        # Load accumulated training samples if they exist
        self.load_training_samples()
        
        # Clean samples to ensure shape consistency
        self.clean_training_samples()
        
        self.save_fingerprint()
        # Optional runtime context for dataset caching (set from CLI)
        self.data_interval = None
        self.data_period = None
        self.use_training_cache = True
        self.rebuild_training_cache = False
        # Disable adaptive scaling by default - let trees grow freely
        self.use_adaptive_scaling = False

    def setup_directories(self):
        if self.asset_symbol != "SPY":
            old_model = f"models/SPY_phi_forest_model.pkl"
            if os.path.exists(old_model):
                os.remove(old_model)
                print(f"[CLEANUP] Removed old model: {old_model}")

        os.makedirs(f"models/{self.asset_symbol}", exist_ok=True)
        os.makedirs(f"training_data/{self.asset_symbol}", exist_ok=True)
        os.makedirs(f"metrics_log/{self.asset_symbol}", exist_ok=True)
        os.makedirs(f"forecasts/{self.asset_symbol}", exist_ok=True)
        os.makedirs(f"hyperparams/{self.asset_symbol}", exist_ok=True)
        os.makedirs(f"actual_price", exist_ok=True)
        os.makedirs(f"prediction_price", exist_ok=True)
        os.makedirs(f"actual_price/{self.asset_symbol}", exist_ok=True)
        os.makedirs(f"prediction_price/{self.asset_symbol}", exist_ok=True)

    def save_fingerprint(self):
        os.makedirs("symbols_used", exist_ok=True)
        with open(f"symbols_used/{self.asset_symbol}_fingerprint.txt", "w") as f:
            f.write(f"Symbol: {self.asset_symbol}\nModel Version: {self.model_version}")

    def load_model(self):
        path = f"models/{self.asset_symbol}/phi_forest_model.pkl"
        hyper_path = f"hyperparams/{self.asset_symbol}/current_hyperparams.json"
        if os.path.exists(path):
            try:
                if os.path.getsize(path) < 1024:
                    raise ValueError("Model file too small, likely corrupted")
                model = joblib.load(path)
                print(f"Successfully loaded model from {path}")

                if os.path.exists(hyper_path):
                    try:
                        with open(hyper_path, 'r') as f:
                            self.hyperparams = json.load(f)
                        print("Loaded associated hyperparameters")
                    except Exception as e:
                        print(f"Error loading hyperparameters: {e}. Using current hyperparameters.")

                version_path = f"models/{self.asset_symbol}/model_version.txt"
                version = "unknown"
                if os.path.exists(version_path):
                    try:
                        with open(version_path, 'r') as f:
                            version = f.read().strip()
                    except:
                        print("Could not read version file")

                return model, version
            except Exception as e:
                print(f"Error loading model: {e}")
                try:
                    os.remove(path)
                    print(f"Removed corrupted model: {path}")
                except:
                    pass
        return None, ""

    def check_feature_compatibility_and_retrain_if_needed(self, df):
        """Check if current model is compatible with current feature set and retrain if needed."""
        if not hasattr(self, 'model') or self.model is None:
            return False
        
        try:
            # Test feature generation with current dataset
            test_window = df.iloc[-WINDOW_SIZE:] if len(df) >= WINDOW_SIZE else df
            result = self.phi_fold_features_with_tiers(test_window)
            if result is None:
                return False
            
            feat, _ = result
            expected_features = getattr(self.model, 'n_features_in_', None)
            actual_features = feat.shape[1]
            
            if expected_features is not None and expected_features != actual_features:
                print(f"[COMPATIBILITY] Model expects {expected_features} features, current code generates {actual_features}")
                print("[COMPATIBILITY] Retraining model with current feature set...")
                
                # Try to use cached data first
                cached_result = self.load_cached_training_dataset(df)
                if cached_result is not None:
                    print("[COMPATIBILITY] Using cached dataset for retraining")
                    success = self.train_from_cached_dataset()
                    if success:
                        print("[COMPATIBILITY] Successfully retrained from cache")
                        return True
                
                # Fall back to training from current data
                print("[COMPATIBILITY] No compatible cache found, training from current data")
                self._needs_retraining = True
                return False
                
        except Exception as e:
            print(f"[COMPATIBILITY] Error during compatibility check: {e}")
            self._needs_retraining = True
            return False
            
        return True

    # ===== Training dataset caching helpers =====
    def _cache_dir(self):
        d = f"training_data/{self.asset_symbol}"
        os.makedirs(d, exist_ok=True)
        return d

    def _sanitize(self, s: str) -> str:
        if s is None:
            return ""
        return ''.join(c if c.isalnum() or c in ('-', '_') else '-' for c in str(s))

    def _dataset_key(self, df: pd.DataFrame) -> str:
        try:
            start = pd.to_datetime(df.index[0]).strftime('%Y%m%d%H%M%S')
            end = pd.to_datetime(df.index[-1]).strftime('%Y%m%d%H%M%S')
        except Exception:
            start = 'start'
            end = 'end'
        interval = self._sanitize(self.data_interval or 'unknown')
        period = self._sanitize(self.data_period or 'unknown')
        nrows = len(df)
        key = f"feat_v{FEATURE_VERSION}_{interval}_{period}_{start}_{end}_n{nrows}"
        return key

    def _dataset_paths(self, df: pd.DataFrame):
        key = self._dataset_key(df)
        base = os.path.join(self._cache_dir(), key)
        return f"{base}.npz", f"{base}.json"

    def clear_training_cache(self):
        """Delete cached training datasets (.npz/.json) for this symbol.
        Removes files like feat_v*.npz/.json and the latest.json pointer.
        """
        cache_dir = self._cache_dir()
        removed = []
        kept = []
        try:
            for fname in os.listdir(cache_dir):
                fpath = os.path.join(cache_dir, fname)
                if not os.path.isfile(fpath):
                    continue
                if fname == 'latest.json' or (fname.startswith(f"feat_v{FEATURE_VERSION}_") and (fname.endswith('.npz') or fname.endswith('.json'))):
                    try:
                        os.remove(fpath)
                        removed.append(fname)
                    except Exception as e:
                        print(f"[CACHE] Failed to remove {fname}: {e}")
                else:
                    kept.append(fname)
            print(f"[CACHE] Removed {len(removed)} cache files from {cache_dir}")
            if removed:
                for r in removed[:10]:
                    print(f"  - {r}")
                if len(removed) > 10:
                    print(f"  ... and {len(removed)-10} more")
            if kept:
                print(f"[CACHE] Kept {len(kept)} non-cache files")
            return True
        except Exception as e:
            print(f"[CACHE] Error clearing cache: {e}")
            return False

    def orbital_cannon_reset(self):
        """ORBITAL CANNON RESET: Nuke both caches for this symbol.
        - Clears training_data/<symbol> feature caches (.npz/.json + latest.json)
        - Clears models/<symbol>/accumulated_training_samples.npz and resets memory
        """
        print("\n===== ORBITAL CANNON RESET ENGAGED =====")
        ok_cache = self.clear_training_cache()
        ok_samples = self.clear_samples_cache()
        overall = bool(ok_cache and ok_samples)
        if overall:
            print("[ORBITAL CANNON] All caches purged for symbol:", self.asset_symbol)
        else:
            print("[ORBITAL CANNON] Completed with warnings for symbol:", self.asset_symbol)
        print("===== ORBITAL CANNON RESET COMPLETE =====\n")
        return overall

    def prepare_and_save_training_dataset(self, df: pd.DataFrame):
        """Build full training pairs (X,y) for df and persist to training_data/<symbol>/.
        Returns (npz_path, meta_path)."""
        if df is None or len(df) < WINDOW_SIZE + 2:
            print(f"[CACHE] Not enough rows to prepare dataset (got {len(df) if df is not None else 0})")
            return None, None
        npz_path, meta_path = self._dataset_paths(df)
        if os.path.exists(npz_path) and not self.rebuild_training_cache and self.use_training_cache:
            print(f"[CACHE] Training dataset already exists: {npz_path}")
            return npz_path, meta_path
        X = []
        y = []
        emc_total_energy = 0.0
        for i in range(WINDOW_SIZE, len(df) - 1):
            window = df.iloc[i-(WINDOW_SIZE-1):i+1]
            result = self.phi_fold_features_with_tiers(window)
            if result is not None:
                feat, emc_pi = result
                X.append(feat.flatten())
                y.append(df['Close'].iloc[i + 1].item())
                try:
                    emc_total_energy += float(emc_pi)
                except Exception:
                    pass
        if not X:
            print("[CACHE] No features constructed; skipping save.")
            return None, None
        X = np.array(X)
        y = np.array(y)
        # enforce rolling cap consistent with training
        max_samples = getattr(self, 'max_train_samples', DEFAULT_MAX_TRAIN_SAMPLES)
        if len(X) > max_samples:
            X = X[-max_samples:]
            y = y[-max_samples:]
        # Save compressed arrays
        try:
            np.savez_compressed(npz_path, X=X, y=y)
            meta = {
                'symbol': self.asset_symbol,
                'interval': self.data_interval,
                'period': self.data_period,
                'feature_version': FEATURE_VERSION,
                'rows': len(df),
                'n_samples': int(len(X)),
                'window': WINDOW_SIZE,
                'start': str(df.index[0]),
                'end': str(df.index[-1]),
                'emc_energy': float(emc_total_energy),
                'created_at': pd.Timestamp.now().isoformat()
            }
            with open(meta_path, 'w') as f:
                json.dump(meta, f, indent=2)
            # also write/update a "latest" pointer file for convenience
            latest_ptr = os.path.join(self._cache_dir(), 'latest.json')
            with open(latest_ptr, 'w') as f:
                json.dump({'npz_path': npz_path, 'meta_path': meta_path, 'meta': meta}, f, indent=2)
            print(f"[CACHE] Saved training dataset: {npz_path}")
            return npz_path, meta_path
        except Exception as e:
            print(f"[CACHE] Failed to save training dataset: {e}")
            return None, None

    def load_cached_training_dataset(self, df: pd.DataFrame):
        """Load cached (X,y) arrays for the given df range if available."""
        if not self.use_training_cache:
            return None
        npz_path, meta_path = self._dataset_paths(df)
        if not os.path.exists(npz_path):
            return None
        try:
            data = np.load(npz_path)
            X = data['X']
            y = data['y']
            print(f"[CACHE] Loaded training dataset from {npz_path} (n={len(X)})")
            return X, y
        except Exception as e:
            print(f"[CACHE] Failed to load dataset: {e}")
            return None

    def train_from_cached_dataset(self, npz_path: str = None):
        """Train and save the main model using a previously cached dataset.
        If npz_path is None, attempts to use latest.json pointer."""
        try:
            if npz_path is None:
                latest_ptr = os.path.join(self._cache_dir(), 'latest.json')
                if not os.path.exists(latest_ptr):
                    print("[CACHE] No latest cached dataset found.")
                    return False
                with open(latest_ptr, 'r') as f:
                    info = json.load(f)
                npz_path = info.get('npz_path')
            if not os.path.exists(npz_path):
                print(f"[CACHE] Cached dataset not found: {npz_path}")
                return False
            data = np.load(npz_path)
            X = data['X']
            y = data['y']
            if len(X) == 0:
                print("[CACHE] Empty cached dataset.")
                return False
                
            # Scale hyperparameters based on cached dataset size
            scaled = self.scale_hyperparams_for_data_size(len(X))
            if scaled:
                print(f"[CACHE-ADAPTIVE] Hyperparameters automatically scaled for cached dataset size ({len(X)} samples)")
                
            # validation split (time-ordered)
            val_size = max(1, int(len(X) * 0.1))
            if val_size >= len(X):
                val_size = 1
            train_size = len(X) - val_size
            X_tr, y_tr = X[:train_size], y[:train_size]
            X_val, y_val = X[train_size:], y[train_size:]
            sample_weight_tr = None
            try:
                if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                    half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                    if half_life and half_life > 0 and len(X_tr) > 0:
                        idx = np.arange(len(X_tr))
                        sample_weight_tr = 0.5 ** ((len(X_tr) - 1 - idx) / float(half_life))
            except Exception:
                sample_weight_tr = None
            if sample_weight_tr is not None:
                self.model.fit(X_tr, y_tr, sample_weight=sample_weight_tr)
            else:
                self.model.fit(X_tr, y_tr)
            # Evaluate and then train on full
            try:
                y_val_pred = self.model.predict(X_val)
                r2 = r2_score(y_val, y_val_pred) if len(y_val) >= 2 else float('nan')
                mae = mean_absolute_error(y_val, y_val_pred)
                mse = mean_squared_error(y_val, y_val_pred)
                acc = np.mean(np.isclose(y_val, y_val_pred, rtol=0.00001))
                print(f"[CACHE-TRAIN] R²: {r2:.6f}, MAE: {mae:.6f}, MSE: {mse:.6f}, ACC: {acc*100:.4f}%")
            except Exception:
                pass
            # final fit on full data
            final_sw = None
            try:
                if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                    half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                    if half_life and half_life > 0 and len(X) > 0:
                        idx = np.arange(len(X))
                        final_sw = 0.5 ** ((len(X) - 1 - idx) / float(half_life))
            except Exception:
                final_sw = None
            if final_sw is not None:
                self.model.fit(X, y, sample_weight=final_sw)
            else:
                self.model.fit(X, y)
            # accumulate X_train/y_train samples (don't replace, extend)
            try:
                # Add new samples to existing ones
                new_X = [row for row in X]
                new_y = [val for val in y]
                self.X_train.extend(new_X)
                self.y_train.extend(new_y)
                # Apply rolling decay to accumulated samples
                self.apply_sample_decay()

                print(f"[CACHE-TRAIN] Total accumulated samples: {len(self.X_train)}")
            except Exception:
                pass
            self.save_model()
            print("[CACHE-TRAIN] Model trained and saved from cached dataset.")
            return True
        except Exception as e:
            print(f"[CACHE-TRAIN] Failed to train from cache: {e}")
            return False

    def save_model(self):
        path = f"models/{self.asset_symbol}/phi_forest_model.pkl"
        hyper_path = f"hyperparams/{self.asset_symbol}/current_hyperparams.json"
        version_path = f"models/{self.asset_symbol}/model_version.txt"
        joblib.dump(self.model, path)
        with open(hyper_path, 'w') as f:
            json.dump(self.hyperparams, f)
        with open(version_path, 'w') as f:
            f.write(self.model_version)
        
        # Save accumulated training samples
        self.save_training_samples()
        
        # save sample_count metadata for main model
        try:
            sample_count = len(getattr(self, 'X_train', [])) if hasattr(self, 'X_train') else 0
            # Also save the expected feature count for compatibility checking
            expected_features = getattr(self.model, 'n_features_in_', None)
            meta = {
                'sample_count': sample_count, 
                'model_path': path, 
                'timestamp': pd.Timestamp.now().isoformat(),
                'expected_features': expected_features,
                'feature_version': FEATURE_VERSION
            }
            meta_path = f"models/{self.asset_symbol}/phi_forest_model_meta.json"
            with open(meta_path, 'w') as mf:
                json.dump(meta, mf, indent=2)
        except Exception:
            pass

    def save_training_samples(self):
        """Save accumulated training samples to disk for persistence across sessions."""
        try:
            if not hasattr(self, 'X_train') or not hasattr(self, 'y_train'):
                return
            if not self.X_train or not self.y_train:
                return
                
            samples_dir = f"models/{self.asset_symbol}"
            os.makedirs(samples_dir, exist_ok=True)
            samples_path = f"{samples_dir}/accumulated_training_samples.npz"
            
            X = np.array(self.X_train)
            y = np.array(self.y_train)
            
            # Apply rolling window cap before saving
            max_samples = getattr(self, 'max_train_samples', DEFAULT_MAX_TRAIN_SAMPLES)
            if len(X) > max_samples:
                X = X[-max_samples:]
                y = y[-max_samples:]
                # Update the in-memory arrays too
                self.X_train = [row for row in X]
                self.y_train = [val for val in y]
            
            np.savez_compressed(samples_path, X=X, y=y)
            print(f"[PERSIST] Saved {len(X)} accumulated training samples to {samples_path}")
        except Exception as e:
            print(f"[PERSIST] Failed to save training samples: {e}")

    def apply_sample_decay(self):
        """Apply rolling sample decay by keeping only the most recent max_train_samples.
        Drops the oldest samples first once the cap is exceeded.
        """
        try:
            max_samples = getattr(self, 'max_train_samples', DEFAULT_MAX_TRAIN_SAMPLES)
            cur_n = len(self.X_train)
            if cur_n > max_samples:
                self.X_train = self.X_train[-max_samples:]
                self.y_train = self.y_train[-max_samples:]
                print(f"[DECAY] Applied sample decay, kept {len(self.X_train)} most recent samples")
        except Exception as e:
            print(f"[DECAY] Error applying sample decay: {e}")

    def accuracy_rounded_to_dimes(self, y_true, y_pred):
        """Compute accuracy after rounding both y_true and y_pred to the nearest $0.10.
        Returns fraction in [0,1]. If arrays are empty, returns 0.0.
        """
        try:
            yt = np.asarray(y_true, dtype=float)
            yp = np.asarray(y_pred, dtype=float)
            if yt.size == 0 or yp.size == 0:
                return 0.0

            # Only consider finite pairs
            finite_mask = np.isfinite(yt) & np.isfinite(yp)
            if not np.any(finite_mask):
                return 0.0
            yt = yt[finite_mask]
            yp = yp[finite_mask]

            # Round both to nearest dime
            yt_q = np.round(yt * 10.0) / 10.0
            yp_q = np.round(yp * 10.0) / 10.0

            # Overshoot penalty: any prediction greater than actual counts as zero accuracy
            overshoot = yp > yt

            # Accurate only if rounded values match AND it's not an overshoot
            correct_and_not_overshoot = (yt_q == yp_q) & (~overshoot)
            return float(np.mean(correct_and_not_overshoot))
        except Exception:
            return 0.0

    def clear_samples_cache(self):
        """Delete accumulated training samples file and reset in-memory samples."""
        try:
            samples_path = f"models/{self.asset_symbol}/accumulated_training_samples.npz"
            removed = False
            if os.path.exists(samples_path):
                try:
                    os.remove(samples_path)
                    removed = True
                except Exception as e:
                    print(f"[PERSIST] Failed to remove samples file: {e}")
            # Reset in-memory buffers
            self.X_train = []
            self.y_train = []
            if removed:
                print(f"[PERSIST] Removed accumulated samples file: {samples_path}")
            else:
                print(f"[PERSIST] No accumulated samples file to remove at: {samples_path}")
            print("[PERSIST] In-memory training samples reset to 0")
            return True
        except Exception as e:
            print(f"[PERSIST] Error clearing samples cache: {e}")
            return False

    def get_expected_feature_count(self):
        """Get the expected number of features from current feature engineering."""
        # Create a small test dataframe to get feature count
        try:
            test_data = {
                'Close': [100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0],
                'High': [101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0],
                'Low': [99.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0],
                'Volume': [1000.0] * 11
            }
            test_df = pd.DataFrame(test_data)
            result = self.phi_fold_features_with_tiers(test_df.iloc[-10:])
            if result is not None:
                feat, _ = result
                return len(feat.flatten())
        except Exception:
            pass
        return 25  # fallback to expected count

    def load_training_samples(self):
        """Load previously accumulated training samples from disk."""
        try:
            samples_path = f"models/{self.asset_symbol}/accumulated_training_samples.npz"
            if not os.path.exists(samples_path):
                print(f"[PERSIST] No accumulated training samples found at {samples_path}")
                return
                
            data = np.load(samples_path)
            X = data['X']
            y = data['y']
            
            # Validate feature shape consistency
            expected_features = self.get_expected_feature_count()
            valid_samples = []
            valid_targets = []
            
            for i, (sample, target) in enumerate(zip(X, y)):
                if len(sample) == expected_features:
                    valid_samples.append(sample)
                    valid_targets.append(target)
                else:
                    print(f"[PERSIST] Skipping sample {i}: has {len(sample)} features, expected {expected_features}")
            
            self.X_train = valid_samples
            self.y_train = valid_targets
            
            if len(valid_samples) < len(X):
                print(f"[PERSIST] Filtered {len(X)} -> {len(valid_samples)} samples due to shape mismatch")
            
            print(f"[PERSIST] Loaded {len(valid_samples)} accumulated training samples from {samples_path}")
        except Exception as e:
            print(f"[PERSIST] Failed to load training samples: {e}")
            # Initialize empty if loading fails
            self.X_train = []
            self.y_train = []

    def evolve_hyperparams(self, accuracy, r2, predicted_price=None, phi_signals=None, actual_price=None):
        """Evolve hyperparameters under a strict policy:
        - If accuracy < 100%, only INCREASE model complexity (monotonic increases)
        - Once 100% is reached, hold complexity and only fine-tune stability knobs
        - If it drops below 100% again, resume increasing until back to 100%
        - PARENT VOTE: If parent had higher accuracy, keep parent's hyperparameters
        """
        prev_params = self.hyperparams.copy()
        
        # Track parent's accuracy and last prediction/actual for comparison
        parent_accuracy = getattr(self, '_last_accuracy', 0.0)
        parent_pred = getattr(self, '_last_predicted_price', None)
        parent_actl = getattr(self, '_last_actual_price', None)

        # Overshoot-aware selection policy takes precedence over parent-vote by accuracy
        selection_result = 'none'  # 'prefer_parent' | 'prefer_child' | 'none'
        current_overshoot = None
        parent_overshoot = None
        try:
            if predicted_price is not None and actual_price is not None:
                current_overshoot = float(predicted_price) > float(actual_price)
            if parent_pred is not None and parent_actl is not None:
                parent_overshoot = float(parent_pred) > float(parent_actl)
        except Exception:
            pass

        # If we can compare overshoot statuses, enforce the user's policy
        if current_overshoot is not None and parent_overshoot is not None:
            print(f"[SELECTION] Overshoot statuses → parent: {parent_overshoot}, child: {current_overshoot}")
            if current_overshoot and not parent_overshoot:
                selection_result = 'prefer_parent'
                print("[SELECTION] Child overshot while parent did not — keeping parent (no evolution)")
            elif (not current_overshoot) and parent_overshoot:
                selection_result = 'prefer_child'
                print("[SELECTION] Parent overshot while child did not — evolving child (bypass parent accuracy vote)")
            elif current_overshoot and parent_overshoot:
                # Both overshot — TRIGGER TIER SELECTION to find a non-overshooting Fibonacci tier
                print("[SELECTION] ⚠ Both parent and child overshot — triggering tier selection")
                self._trigger_tier_selection = True  # Flag for update() to perform tier selection
                
                # DO NOT prefer child or parent - tier selection will handle it
                # Skip normal evolution logic - tier selection will retrain and re-evaluate
                selection_result = 'tier_selection'
                print("[SELECTION] Will use tier-based training instead of parent/child selection")

        # Apply selection result or fall back to parent-vote by accuracy
        if selection_result == 'prefer_parent':
            # Keep parent; do not evolve
            self._last_accuracy = accuracy
            # Persist last prediction/actual for next cycle
            try:
                self._last_predicted_price = float(predicted_price) if predicted_price is not None else None
                self._last_actual_price = float(actual_price) if actual_price is not None else None
            except Exception:
                self._last_predicted_price = predicted_price
                self._last_actual_price = actual_price
            return False
        elif selection_result == 'tier_selection':
            # Both overshot - tier selection will handle retraining in update()
            # Don't evolve hyperparams, just return False and let tier selection take over
            print("[SELECTION] Skipping normal evolution - tier selection will retrain the model")
            self._last_accuracy = accuracy
            try:
                self._last_predicted_price = float(predicted_price) if predicted_price is not None else None
                self._last_actual_price = float(actual_price) if actual_price is not None else None
            except Exception:
                self._last_predicted_price = predicted_price
                self._last_actual_price = actual_price
            return False
        elif selection_result == 'prefer_child':
            # Proceed to evolve; bypass parent accuracy vote
            self._last_accuracy = accuracy
        else:
            # PARENT VOTE: If parent had higher accuracy, revert to parent and skip evolution
            if parent_accuracy > accuracy:
                print(f"[PARENT-VOTE] Parent had higher accuracy ({parent_accuracy*100:.2f}% vs {accuracy*100:.2f}%) - keeping parent hyperparameters")
                self._last_accuracy = accuracy
                try:
                    self._last_predicted_price = float(predicted_price) if predicted_price is not None else None
                    self._last_actual_price = float(actual_price) if actual_price is not None else None
                except Exception:
                    self._last_predicted_price = predicted_price
                    self._last_actual_price = actual_price
                return False
            # Store current accuracy as the new parent for next generation
            self._last_accuracy = accuracy

        # Helper to safely get/set and bound values
        def _inc(name, step, max_val=None):
            v = self.hyperparams.get(name)
            if isinstance(v, (int, float)):
                nv = v + step
                if max_val is not None:
                    nv = min(max_val, nv)
                self.hyperparams[name] = int(nv) if isinstance(v, int) else float(nv)
            return self.hyperparams.get(name)

        def _dec(name, step, min_val=None):
            v = self.hyperparams.get(name)
            if isinstance(v, (int, float)):
                nv = v - step
                if min_val is not None:
                    nv = max(min_val, nv)
                self.hyperparams[name] = int(nv) if isinstance(v, int) else float(nv)
            return self.hyperparams.get(name)

        # Determine if we are at perfect accuracy for this evaluation
        is_perfect = (accuracy >= 1.0)
        if is_perfect:
            self.perfect_accuracy_achieved = True
        else:
            # If we dropped below perfect, resume growth loop
            if getattr(self, 'perfect_accuracy_achieved', False):
                print("[EVOLVE] Dropped below 100% — resuming complexity growth loop")
            self.perfect_accuracy_achieved = False

        # Baseline evolution driven by the policy
        if not is_perfect:
            # Only increase complexity knobs until we hit 100% - VERY SLOW 1% growth
            before = self.hyperparams.copy()
            _inc('n_estimators', step= max(1, int(self.hyperparams.get('n_estimators', 100) * 0.01)), max_val=None)
            _inc('max_depth', step=1, max_val=None)
            # Smaller leaves allow more complex trees
            if isinstance(self.hyperparams.get('min_samples_leaf', 1), (int, float)):
                self.hyperparams['min_samples_leaf'] = max(1, int(self.hyperparams.get('min_samples_leaf', 2) - 1))
            # Encourage considering more features (if numeric)
            mf = self.hyperparams.get('max_features')
            if isinstance(mf, (int, float)) and mf is not None:
                self.hyperparams['max_features'] = min(1.0, float(mf) + 0.01)  # Reduced from 0.05 to 0.01
            # Reduce impurity threshold to allow finer splits
            mid = self.hyperparams.get('min_impurity_decrease', 1e-7)
            try:
                self.hyperparams['min_impurity_decrease'] = max(1e-12, float(mid) * 0.5)
            except Exception:
                self.hyperparams['min_impurity_decrease'] = 1e-9
            print("[EVOLVE] Very slowly increasing complexity to reach 100% accuracy (1% growth)")
        else:
            # At perfect accuracy: avoid increasing complexity; fine-tune stability only
            # Nudge impurity up a bit to prefer simpler, more stable splits while preserving accuracy
            mid = self.hyperparams.get('min_impurity_decrease', 1e-9)
            try:
                self.hyperparams['min_impurity_decrease'] = min(1e-5, float(mid) * 1.1)
            except Exception:
                pass
            # Optionally nudge max_features slightly toward sqrt if numeric and > 0.8
            mf = self.hyperparams.get('max_features')
            if isinstance(mf, (int, float)) and mf is not None and mf > 0.8:
                self.hyperparams['max_features'] = max(0.7, float(mf) - 0.02)
            print("[EVOLVE] Perfect accuracy achieved — holding complexity; light stability tuning only")

        # If actual price provided, compute error and use it to influence evolution
        pred_error = None
        if predicted_price is not None and actual_price is not None:
            try:
                pred_error = float(predicted_price) - float(actual_price)
                abs_err = abs(pred_error)
                rel_err = abs_err / float(actual_price) if float(actual_price) != 0 else None
                print(f"[EVAL] pred={predicted_price}, actual={actual_price}, abs_err={abs_err}, rel_err={rel_err}")
                # If relative error unusually high, increase model complexity conservatively
                if rel_err is not None and rel_err > 0.02:
                    # Very slow 1% increase
                    self.hyperparams['n_estimators'] = self.hyperparams.get('n_estimators', 100) + max(1, int(self.hyperparams.get('n_estimators', 100) * 0.01))
                    self.hyperparams['max_depth'] = self.hyperparams.get('max_depth', 10) + 1
                    print(f"[EVAL] High relative error ({rel_err:.4f}) detected; very slowly increasing complexity (1%).")
            except Exception:
                pred_error = None

        # phi-aware adjustments (respect monotonic growth policy)
        if self.phi_evolve_enabled:
            if phi_signals is None:
                phi_signals = getattr(self, '_last_phi_signals', None)
            if phi_signals:
                try:
                    phi_pressure = abs(float(phi_signals.get('phi_pressure', 0.0)))
                    emc_pi = float(phi_signals.get('EMC_Pi', 0.0))
                except Exception:
                    phi_pressure = 0.0
                    emc_pi = 0.0

                median_emc = None
                try:
                    if hasattr(self, '_emc_history') and len(self._emc_history) > 0:
                        median_emc = float(np.median(self._emc_history))
                except Exception:
                    median_emc = None

                volatile = False
                if phi_pressure > 0.6:
                    volatile = True
                elif median_emc and median_emc > 0 and emc_pi > median_emc * 2.5:
                    volatile = True

                if volatile:
                    if not is_perfect:
                        # Only allow increases while below 100% - very slow 1% growth
                        self.hyperparams['n_estimators'] = self.hyperparams.get('n_estimators', 100) + max(1, int(self.hyperparams.get('n_estimators', 100) * 0.01))
                        self.hyperparams['max_depth'] = self.hyperparams.get('max_depth', 10) + 1
                        # Don't increase min_samples_leaf beyond current (avoid decreasing complexity)
                        print(f"[PHI-EVOLVE] Volatile regime (phi_pressure={phi_pressure:.3f}, emc={emc_pi:.3f}) — very slowly increasing (1%)")
                    else:
                        # At perfect, keep complexity stable
                        print(f"[PHI-EVOLVE] Volatile regime but perfect accuracy — holding complexity")
                else:
                    if not is_perfect:
                        # Stable and below perfect — very slow 1% growth
                        self.hyperparams['max_depth'] = self.hyperparams.get('max_depth', 10) + 1
                        self.hyperparams['min_samples_leaf'] = max(1, self.hyperparams.get('min_samples_leaf', 2) - 1)
                        current_max_features = self.hyperparams.get('max_features', 0.8)
                        if isinstance(current_max_features, (int, float)) and current_max_features is not None:
                            self.hyperparams['max_features'] = min(1.0, current_max_features + 0.01)  # Reduced from 0.05 to 0.01
                        print(f"[PHI-EVOLVE] Stable regime — very slowly increasing complexity towards 100% (1%)")
                    else:
                        # At perfect — very light, stability-oriented nudges handled above
                        pass

        if prev_params != self.hyperparams:
            self.model_version = f"v{self.model_version[1:].split('.')[0]}.{int(time.time()) % 10000}"
            allowed = RandomForestRegressor().get_params().keys()
            model_kwargs = {k: v for k, v in self.hyperparams.items() if k in allowed}
            self.model = RandomForestRegressor(**model_kwargs)
            # record evolution event including predicted price and phi_signals
            evo_record = {
                'timestamp': pd.Timestamp.now().isoformat(),
                'from': prev_params,
                'to': self.hyperparams.copy(),
                'accuracy': accuracy,
                'r2': r2,
                'predicted_price': float(predicted_price) if predicted_price is not None else None,
                'actual_price': float(actual_price) if actual_price is not None else None,
                'prediction_error': float(pred_error) if pred_error is not None else None,
                'phi_signals': phi_signals,
                'model_version': self.model_version
            }
            try:
                self.hyperparam_history.append(evo_record)
            except Exception:
                pass
            # persist to disk
            try:
                hp_dir = f"hyperparams/{self.asset_symbol}"
                os.makedirs(hp_dir, exist_ok=True)
                hist_path = f"{hp_dir}/evolution_history.json"
                existing = []
                if os.path.exists(hist_path):
                    try:
                        with open(hist_path, 'r') as hf:
                            existing = json.load(hf)
                    except Exception:
                        existing = []
                existing.append(evo_record)
                with open(hist_path, 'w') as hf:
                    json.dump(existing, hf, indent=2)
            except Exception as e:
                print(f"Could not persist hyperparam evolution: {e}")
            print(f"Evolved hyperparameters to version {self.model_version} | predicted_price: {evo_record['predicted_price']}")
            # Persist last prediction/actual used for selection into 'parent' for next cycle
            try:
                self._last_predicted_price = float(predicted_price) if predicted_price is not None else None
                self._last_actual_price = float(actual_price) if actual_price is not None else None
            except Exception:
                self._last_predicted_price = predicted_price
                self._last_actual_price = actual_price
            return True
        # No evolution; still persist last prediction/actual for next cycle
        try:
            self._last_predicted_price = float(predicted_price) if predicted_price is not None else None
            self._last_actual_price = float(actual_price) if actual_price is not None else None
        except Exception:
            self._last_predicted_price = predicted_price
            self._last_actual_price = actual_price
        return False

    def phi_fold_features_with_tiers(self, df, force_actual_price_tier=False, actual_price=None, exclude_tiers=None):
        """
        Generate phi-folded features with Fibonacci tier analysis.
        
        Args:
            df: DataFrame with price/volume data
            force_actual_price_tier: If True, override volume-based tier selection with closest tier to actual_price
            actual_price: The actual price to use for forced tier selection (required if force_actual_price_tier=True)
            exclude_tiers: Set of tier indices to exclude from selection (for overshoot elimination loop)
        """
        phi = 1.61803398875
        if len(df) < 10:
            return None

        # Normalize DataFrame to handle MultiIndex issues
        try:
            # If df has MultiIndex columns, flatten them
            if hasattr(df.columns, 'nlevels') and df.columns.nlevels > 1:
                df = df.copy()
                df.columns = df.columns.get_level_values(0) if len(df.columns.levels) > 0 else df.columns
            
            # Ensure we have the required columns
            required_cols = ['Close', 'High', 'Low', 'Volume']
            for col in required_cols:
                if col not in df.columns:
                    print(f"[ERROR] Missing required column: {col}")
                    return None
        except Exception as e:
            print(f"[ERROR] DataFrame structure issue: {e}")
            return None

        anchor = df.iloc[-2]['Close'].item()
        latest = df.iloc[-1]
        high = latest['High'].item()
        low = latest['Low'].item()
        volume = latest['Volume'].item()

        delta_high = high - anchor
        delta_low = anchor - low

        folded_high = delta_high / phi
        folded_low = delta_low / phi
        non_euclid_high = anchor + folded_high
        non_euclid_low = anchor - folded_low
        fib_range = non_euclid_high - non_euclid_low

        # Dynamically expanded Fibonacci-style ratios across positive and negative ranges
        max_mult = float(getattr(self, 'max_fib_multiplier', DEFAULT_MAX_FIB_MULTIPLIER))
        include_neg = bool(getattr(self, 'include_negative_fib_tiers', DEFAULT_INCLUDE_NEGATIVE_FIB_TIERS))
        max_levels = int(getattr(self, 'max_fib_levels', DEFAULT_MAX_FIB_LEVELS))

        # Extended Fibonacci-based ratios using golden ratio (phi) relationships
        # FOCUS: High granularity between 0-1 for near-price accuracy, then coarser for extremes
        phi = 1.61803398875
        base = [0.0]
        
        # FINE GRANULARITY 0.00 to 1.00 - Every 0.01 for precise near-price tiers
        fib_classics = []
        
        # Ultra-fine: 0.00 to 1.00 in 0.01 increments (100 levels)
        for i in range(101):
            fib_classics.append(round(i * 0.01, 2))
        
        # Classic Fibonacci retracements/extensions embedded in the range
        # (already covered by 0.01 increments, but add exact values for mathematical purity)
        fib_classics.extend([
            0.236, 0.382, 0.5, 0.618, 0.786,  # Classic retracements
            1.0, 1.236, 1.382, 1.5, 1.618, 1.786,  # Extensions
            2.0, 2.236, 2.382, 2.5, 2.618,  # Higher extensions
            3.0, 3.236, 3.382, 3.618,
            4.0, 4.236, 4.618,
            5.0, 5.618,
            6.0, 6.854102,
            8.0, 8.618,
            10.0, 11.090170,
            13.0, 13.618,
            17.944271, 21.0,
            29.034441, 34.0,
            46.978713, 55.0,
            76.013155, 89.0,
            122.991869, 144.0,
            199.005034, 233.0
        ])
        
        # Remove duplicates and sort
        fib_classics = sorted(set(fib_classics))
        
        base.extend(fib_classics)
        
        # Simplified additional ratios - focus on Fibonacci sequence convergence
        fib_nums = [1, 1]
        for _ in range(15):
            fib_nums.append(fib_nums[-1] + fib_nums[-2])
        for i in range(1, min(12, len(fib_nums))):
            if fib_nums[i] > 0:
                ratio = fib_nums[i] / fib_nums[i-1] if i > 0 else 1.0
                base.append(round(float(ratio), 6))
        
        # Remove duplicates and sort
        base = sorted(set(base))
        
        ratios_set = set()
        try:
            m_max = int(math.floor(max_mult))
        except Exception:
            m_max = 100

        # Positive side: integer offsets plus base decimals
        for m in range(m_max + 1):
            for b in base:
                r = m + b
                if r <= max_mult + 1e-9:
                    ratios_set.add(round(float(r), 6))
        # Ensure exact integers up to max_mult are included
        for k in range(0, m_max + 1):
            ratios_set.add(float(k))

        # Negative side (levels above non_euclid_high)
        if include_neg:
            negs = set()
            for r in list(ratios_set):
                if r > 0:
                    negs.add(round(float(-r), 6))
            ratios_set |= negs

        fib_ratios = sorted(ratios_set)
        # Safety cap to avoid runaway counts
        if len(fib_ratios) > max_levels:
            idx = np.linspace(0, len(fib_ratios) - 1, max_levels, dtype=int)
            fib_ratios = [fib_ratios[i] for i in idx]

        if not getattr(self, '_tiers_info_logged', False):
            try:
                print(f"[PHI] Fibonacci tiers: {len(fib_ratios)} levels (max_mult={max_mult}, include_neg={include_neg}, cap={max_levels})")
            except Exception:
                pass
            self._tiers_info_logged = True

        levels = [non_euclid_high - r * fib_range for r in fib_ratios]
        
        # Store total tiers available for overshoot loop safety check
        self._total_tiers_available = len(levels)
        # Store the levels array so we can look up tier prices later
        self._fibonacci_levels = levels

        # More computationally intensive volume analysis with multiple tolerance bands
        vol_bands = []
        vol_bands_tight = []
        vol_bands_wide = []
        
        try:
            for lvl in levels:
                try:
                    # Standard band
                    mask_std = (df['Close'] >= lvl * 0.998) & (df['Close'] <= lvl * 1.002)
                    vol_near_lvl = df.loc[mask_std, 'Volume'].sum() if mask_std.any() else 0.0
                    vol_bands.append(float(vol_near_lvl))
                    
                    # Tighter band for precision  
                    mask_tight = (df['Close'] >= lvl * 0.9995) & (df['Close'] <= lvl * 1.0005)
                    vol_tight = df.loc[mask_tight, 'Volume'].sum() if mask_tight.any() else 0.0
                    vol_bands_tight.append(float(vol_tight))
                    
                    # Wider band for broader context
                    mask_wide = (df['Close'] >= lvl * 0.995) & (df['Close'] <= lvl * 1.005)
                    vol_wide = df.loc[mask_wide, 'Volume'].sum() if mask_wide.any() else 0.0
                    vol_bands_wide.append(float(vol_wide))
                except Exception as e:
                    print(f"[WARNING] Volume analysis error for level {lvl}: {e}")
                    vol_bands.append(0.0)
                    vol_bands_tight.append(0.0)
                    vol_bands_wide.append(0.0)
        except Exception as e:
            print(f"[ERROR] Volume analysis failed: {e}")
            # Fallback to simple analysis
            vol_bands = [0.0] * len(levels)
            vol_bands_tight = [0.0] * len(levels)
            vol_bands_wide = [0.0] * len(levels)

        vol_bands = np.array(vol_bands)
        vol_bands_tight = np.array(vol_bands_tight)
        vol_bands_wide = np.array(vol_bands_wide)
        
        # Tier selection: ALWAYS use closest tier to actual price in post-evolution mode
        # OR exclude specific tiers during overshoot elimination loop
        if force_actual_price_tier and actual_price is not None:
            # Find the Fibonacci tier closest to the actual price
            distances = [abs(lvl - actual_price) for lvl in levels]
            tier = int(np.argmin(distances))
            tier_tight = tier  # Use same tier for all bands when forcing
            tier_wide = tier
            print(f"[EVOLUTION-TIER] FORCED tier selection: tier={tier} at level {levels[tier]:.4f} (closest to actual_price={actual_price:.4f}, distance={distances[tier]:.4f})")
        elif exclude_tiers is not None and len(exclude_tiers) > 0:
            # Overshoot elimination mode: select highest-volume tier that hasn't been used yet
            # Create a mask to zero out volumes for excluded tiers
            vol_bands_masked = vol_bands.copy()
            vol_bands_tight_masked = vol_bands_tight.copy()
            vol_bands_wide_masked = vol_bands_wide.copy()
            
            for excluded_tier in exclude_tiers:
                if 0 <= excluded_tier < len(vol_bands_masked):
                    vol_bands_masked[excluded_tier] = -1.0  # Use negative value to ensure it's never selected
                    vol_bands_tight_masked[excluded_tier] = -1.0
                    vol_bands_wide_masked[excluded_tier] = -1.0
            
            # Debug: check how many tiers have positive volume after masking
            positive_volume_tiers = np.sum(vol_bands_masked > 0)
            print(f"[TIER-EXCLUSION] After masking {len(exclude_tiers)} tiers: {positive_volume_tiers}/{len(levels)} tiers have positive volume")
            
            # Select the highest-volume tier from remaining options
            # If all tiers are excluded or volumes are zero, cycle through tiers systematically
            if np.max(vol_bands_masked) <= 0:
                # All tiers excluded or no volume - use systematic exploration
                # Pick the next tier after the highest excluded tier
                max_excluded = max(exclude_tiers) if exclude_tiers else -1
                tier = (max_excluded + 1) % len(levels)
                # Skip forward until we find a non-excluded tier
                attempts = 0
                while tier in exclude_tiers and attempts < len(levels):
                    tier = (tier + 1) % len(levels)
                    attempts += 1
                
                # Double-check we didn't cycle back to an excluded tier
                if tier in exclude_tiers:
                    print(f"[TIER-EXCLUSION] ⚠ ERROR: All {len(levels)} tiers exhausted or logic error!")
                    tier = 0  # Fallback to tier 0
                
                tier_tight = tier
                tier_wide = tier
                print(f"[TIER-EXCLUSION] Systematic exploration: tier {tier} at level {levels[tier]:.4f} (all volumes excluded, tried {attempts} positions)")
            else:
                tier = int(np.argmax(vol_bands_masked))
                tier_tight = int(np.argmax(vol_bands_tight_masked)) if np.max(vol_bands_tight_masked) > 0 else tier
                tier_wide = int(np.argmax(vol_bands_wide_masked)) if np.max(vol_bands_wide_masked) > 0 else tier
                
                # Safety check: if selected tier is somehow still in excluded set, force systematic selection
                if tier in exclude_tiers:
                    max_excluded = max(exclude_tiers)
                    tier = (max_excluded + 1) % len(levels)
                    attempts = 0
                    while tier in exclude_tiers and attempts < len(levels):
                        tier = (tier + 1) % len(levels)
                        attempts += 1
                    tier_tight = tier
                    tier_wide = tier
                    print(f"[TIER-EXCLUSION] Safety override: forced tier {tier} at level {levels[tier]:.4f}")
            
            # Store the selected tier for tracking
            self._last_selected_tier = tier
            print(f"[TIER-EXCLUSION] Selected tier {tier} at level {levels[tier]:.4f} (excluded {len(exclude_tiers)} tiers: {sorted(list(exclude_tiers))})")
        else:
            # Normal volume-based tier selection
            tier = int(np.argmax(vol_bands)) if len(vol_bands) > 0 and np.sum(vol_bands) > 0 else 0
            tier_tight = int(np.argmax(vol_bands_tight)) if len(vol_bands_tight) > 0 and np.sum(vol_bands_tight) > 0 else 0
            tier_wide = int(np.argmax(vol_bands_wide)) if len(vol_bands_wide) > 0 and np.sum(vol_bands_wide) > 0 else 0
            # Store the selected tier
            self._last_selected_tier = tier

        spread = high - low
        last_close = latest['Close'].item()
        center_band = (non_euclid_high + non_euclid_low) / 2
        focus_range = fib_range * 0.15
        in_focus_band = int(abs(last_close - center_band) < focus_range)
        phi_pressure = float(np.clip((anchor - last_close) / anchor, -1.0, 1.0))

        deltaP = high - low
        EMC_Pi = volume * math.pow(abs(deltaP), math.pi)
        
        # Additional computationally intensive features (with safe scalar conversion)
        try:
            price_velocity = float((last_close - df.iloc[-3]['Close'].item()) / 2) if len(df) >= 3 else 0.0
        except:
            price_velocity = 0.0
            
        try:
            if len(df) >= 5:
                vol_mean_5_series = df['Volume'].rolling(5).mean().iloc[-1:]
                vol_mean_5 = float(vol_mean_5_series.iloc[0]) if len(vol_mean_5_series) > 0 else 1.0
                volume_momentum = float(volume / (vol_mean_5 + 1e-8)) if not pd.isna(vol_mean_5) else 1.0
            else:
                volume_momentum = 1.0
        except:
            volume_momentum = 1.0
            
        phi_oscillator = math.sin(phi_pressure * math.pi) * math.cos(phi_pressure * math.pi * phi)
        fractal_dimension = abs(math.log(abs(deltaP) + 1e-8)) / math.log(phi)
        volume_phi_interaction = volume * math.pow(phi, abs(phi_pressure))
        
        # Multi-timeframe EMC calculations (more compute intensive)
        EMC_Pi_short = volume * math.pow(abs(deltaP), math.pi * 0.5)
        EMC_Pi_long = volume * math.pow(abs(deltaP), math.pi * 1.5)
        EMC_combined = (EMC_Pi + EMC_Pi_short + EMC_Pi_long) / 3

        # Safely compute statistical features with proper scalar conversion
        try:
            close_std = float(df['Close'].std()) if not pd.isna(df['Close'].std()) else 0.0
        except:
            close_std = 0.0
            
        try:
            volume_std = float(df['Volume'].std()) if not pd.isna(df['Volume'].std()) else 0.0
        except:
            volume_std = 0.0
            
        try:
            price_range = float(df['High'].max() - df['Low'].min())
        except:
            price_range = 0.0
            
        try:
            if len(df) >= 3:
                close_ma3_series = df['Close'].rolling(3).mean().iloc[-1:]
                close_ma3 = float(close_ma3_series.iloc[0]) if len(close_ma3_series) > 0 else float(last_close)
            else:
                close_ma3 = float(last_close)
        except:
            close_ma3 = float(last_close)
            
        try:
            if len(df) >= 3:
                volume_ma3_series = df['Volume'].rolling(3).mean().iloc[-1:]
                volume_ma3 = float(volume_ma3_series.iloc[0]) if len(volume_ma3_series) > 0 else float(volume)
            else:
                volume_ma3 = float(volume)
        except:
            volume_ma3 = float(volume)

        features = [
            float(anchor),
            float(folded_high),
            float(folded_low),
            float(spread),
            float(volume),
            int(tier),
            int(tier_tight),
            int(tier_wide),
            int(in_focus_band),
            float(phi_pressure),
            float(EMC_Pi),
            float(EMC_Pi_short),
            float(EMC_Pi_long),
            float(EMC_combined),
            float(price_velocity),
            float(volume_momentum),
            float(phi_oscillator),
            float(fractal_dimension),
            float(volume_phi_interaction),
            # Statistical features (safely converted to scalars)
            close_std,
            volume_std,
            price_range,
            close_ma3,
            volume_ma3
        ]
        
        # Final safety check - ensure all features are numeric scalars
        safe_features = []
        for i, feature in enumerate(features):
            try:
                if hasattr(feature, 'item'):  # pandas scalar
                    safe_features.append(float(feature.item()))
                elif np.isscalar(feature):
                    safe_features.append(float(feature))
                else:
                    print(f"[WARNING] Feature {i} is not scalar: {type(feature)}, using 0.0")
                    safe_features.append(0.0)
            except Exception as e:
                print(f"[WARNING] Error processing feature {i}: {e}, using 0.0")
                safe_features.append(0.0)
        
        features = safe_features
        # Proper recording in phi_fold_features_with_tiers before return
        try:
            self._last_phi_signals = {
                'tier': int(tier),
                'in_focus_band': int(in_focus_band),
                'phi_pressure': float(phi_pressure),
                'EMC_Pi': float(EMC_Pi),
                'anchor': float(anchor),
                'last_close': float(last_close),
                'spread': float(spread)
            }
        except Exception:
            self._last_phi_signals = None

        return np.array(features).reshape(1, -1), EMC_Pi

    def scale_hyperparams_for_data_size(self, n_samples):
        """Intelligently scale hyperparameters based on available training data."""
        if not getattr(self, 'use_adaptive_scaling', True):
            return False  # Adaptive scaling disabled
            
        if n_samples < 10:
            print(f"[ADAPTIVE] Too few samples ({n_samples}) for meaningful training")
            return False
            
        original_estimators = self.hyperparams.get('n_estimators', 1000)
        original_depth = self.hyperparams.get('max_depth', 10)
        
        # Scale down aggressively for small datasets to prevent overfitting and hanging
        if n_samples < 50:
            # Very small dataset - minimal complexity
            scale_factor = 0.01  # 1% of original complexity
            max_estimators = 50
            max_depth = 5
            print(f"[ADAPTIVE] Very small dataset ({n_samples} samples) - using minimal complexity")
        elif n_samples < 200:
            # Small dataset - low complexity
            scale_factor = 0.05  # 5% of original complexity
            max_estimators = 200
            max_depth = 8
            print(f"[ADAPTIVE] Small dataset ({n_samples} samples) - using low complexity")
        elif n_samples < 1000:
            # Medium dataset - moderate complexity
            scale_factor = 0.2   # 20% of original complexity
            max_estimators = 2000
            max_depth = 15
            print(f"[ADAPTIVE] Medium dataset ({n_samples} samples) - using moderate complexity")
        elif n_samples < 5000:
            # Large dataset - high complexity
            scale_factor = 0.5   # 50% of original complexity
            max_estimators = 10000
            max_depth = 25
            print(f"[ADAPTIVE] Large dataset ({n_samples} samples) - using high complexity")
        else:
            # Very large dataset - full complexity
            scale_factor = 1.0   # Full complexity
            max_estimators = None  # No cap
            max_depth = None  # No cap
            print(f"[ADAPTIVE] Very large dataset ({n_samples} samples) - using maximum complexity")
        
        # Apply scaling
        scaled_estimators = min(max_estimators, max(10, int(original_estimators * scale_factor))) if max_estimators else max(10, int(original_estimators * scale_factor))
        scaled_depth = min(max_depth, max(3, int(original_depth * scale_factor))) if max_depth else max(3, int(original_depth * scale_factor))
        
        # Update hyperparameters
        old_estimators = self.hyperparams.get('n_estimators')
        old_depth = self.hyperparams.get('max_depth')
        
        self.hyperparams['n_estimators'] = scaled_estimators
        self.hyperparams['max_depth'] = scaled_depth
        
        # For very small datasets, also adjust other parameters
        if n_samples < 100:
            self.hyperparams['min_samples_leaf'] = max(2, min(n_samples // 10, 5))
            self.hyperparams['min_samples_split'] = max(2, min(n_samples // 5, 10))
            if n_samples < 20:
                self.hyperparams['oob_score'] = False  # Disable OOB for tiny datasets
        
        if old_estimators != scaled_estimators or old_depth != scaled_depth:
            print(f"[ADAPTIVE] Scaled: n_estimators {old_estimators} -> {scaled_estimators}, max_depth {old_depth} -> {scaled_depth}")
            
            # Recreate model with scaled hyperparameters
            allowed = RandomForestRegressor().get_params().keys()
            model_kwargs = {k: v for k, v in self.hyperparams.items() if k in allowed}
            self.model = RandomForestRegressor(**model_kwargs)
            return True
            
        return False

    def update(self, df, current_date=None):
        if len(df) < 12:
            print(f"Skipping update - only {len(df)} rows available (need at least 12)")
            return

        # Log current hyperparameters at start of update
        print(f"[UPDATE-START] Current hyperparams: n_estimators={self.hyperparams.get('n_estimators')}, max_depth={self.hyperparams.get('max_depth')}")

        emc_total_energy = 0
        for i in range(10, len(df) - 1):
            window = df.iloc[i-9:i+1]
            result = self.phi_fold_features_with_tiers(window)
            if result is not None:
                feat, emc_pi = result
                flattened_feat = feat.flatten()
                
                # Debug: Check feature array shape consistency
                if len(self.X_train) > 0:
                    expected_shape = len(self.X_train[0])
                    if len(flattened_feat) != expected_shape:
                        print(f"[WARNING] Feature shape mismatch! Expected {expected_shape}, got {len(flattened_feat)}")
                        print(f"[WARNING] Skipping this sample to maintain consistency")
                        continue
                
                self.X_train.append(flattened_feat)
                self.y_train.append(df['Close'].iloc[i + 1].item())
                emc_total_energy += emc_pi

        if self.X_train:
            # Final safety check before numpy array creation
            try:
                X = np.array(self.X_train)
                y = np.array(self.y_train)
            except ValueError as e:
                print(f"[ERROR] Failed to create training arrays: {e}")
                print(f"[ERROR] Cleaning samples and retrying...")
                self.clean_training_samples()
                if not self.X_train:
                    print("[ERROR] No valid samples remaining after cleaning")
                    return
                try:
                    X = np.array(self.X_train)
                    y = np.array(self.y_train)
                except Exception as e2:
                    print(f"[ERROR] Still failed after cleaning: {e2}")
                    print("[ERROR] Resetting training samples")
                    self.X_train = []
                    self.y_train = []
                    return
            # enforce rolling window cap via decay on in-memory buffers
            self.apply_sample_decay()
            # Recreate arrays from decayed lists to ensure consistency
            X = np.array(self.X_train)
            y = np.array(self.y_train)
            print(f"[DECAY] Applied sample decay, kept most recent {len(X)} samples")
            print(f"\n[TRAINING] Samples accumulated: {len(X)} (total ever trained: {len(self.X_train)})")
            
            # Check if we have enough samples for meaningful training
            if len(X) < 10:
                print(f"[WARNING] Only {len(X)} samples available - skipping training (need at least 10)")
                return
            
            # Adaptive scaling disabled - let trees grow freely without data-size constraints
            # scaled = self.scale_hyperparams_for_data_size(len(X))
            # if scaled:
            #     print(f"[ADAPTIVE] Hyperparameters automatically scaled for dataset size")
            
            print(f"Current Hyperparameters: {self.hyperparams}")

            # Use a time-series holdout (recent samples) for validation to avoid data leakage
            val_size = max(1, int(len(X) * 0.1))
            if val_size >= len(X):
                val_size = 1
            train_size = len(X) - val_size
            X_tr, y_tr = X[:train_size], y[:train_size]
            X_val, y_val = X[train_size:], y[train_size:]

            # compute recency weights only on training portion
            sample_weight_tr = None
            try:
                if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                    half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                    if half_life and half_life > 0 and len(X_tr) > 0:
                        idx = np.arange(len(X_tr))
                        sample_weight_tr = 0.5 ** ((len(X_tr) - 1 - idx) / float(half_life))
            except Exception:
                sample_weight_tr = None
            
            # OVERSHOOT PENALTY WEIGHTING: Teach trees to avoid overshoot
            # Add synthetic "negative examples" with slightly lower targets to discourage overshoot
            # This helps trees learn the asymmetric cost: overshooting is worse than undershooting
            X_tr_augmented = X_tr.copy()
            y_tr_augmented = y_tr.copy()
            sample_weight_augmented = sample_weight_tr.copy() if sample_weight_tr is not None else np.ones(len(X_tr))
            
            try:
                # For each training sample, add a "penalty sample" with target slightly below actual
                # This teaches: "if you predict this high, you should actually predict lower"
                n_penalty_samples = min(len(X_tr), 50)  # Don't add too many
                penalty_indices = np.random.choice(len(X_tr), n_penalty_samples, replace=False)
                
                for idx in penalty_indices:
                    # Add a sample with same features but target reduced by 0.5% (conservative adjustment)
                    penalty_target = y_tr[idx] * 0.995  # 0.5% below actual
                    X_tr_augmented = np.vstack([X_tr_augmented, X_tr[idx]])
                    y_tr_augmented = np.append(y_tr_augmented, penalty_target)
                    # Give penalty samples HIGH weight so trees learn from them
                    penalty_weight = sample_weight_augmented[idx] * 5.0  # 5x weight
                    sample_weight_augmented = np.append(sample_weight_augmented, penalty_weight)
                
                print(f"[OVERSHOOT-LEARN] Added {n_penalty_samples} penalty samples (0.5% below actual, 5x weight) to teach conservative prediction")
            except Exception as e:
                print(f"[OVERSHOOT-LEARN] Could not add penalty samples: {e}")
                # Fall back to original data
                X_tr_augmented = X_tr
                y_tr_augmented = y_tr
                sample_weight_augmented = sample_weight_tr

            # Train on augmented data with overshoot penalty samples
            if sample_weight_augmented is not None:
                self.model.fit(X_tr_augmented, y_tr_augmented, sample_weight=sample_weight_augmented)
            else:
                self.model.fit(X_tr_augmented, y_tr_augmented)

            # Evaluate out-of-sample on the validation holdout
            try:
                y_val_pred = self.model.predict(X_val)
                if len(y_val) >= 2:
                    r2 = r2_score(y_val, y_val_pred)
                else:
                    r2 = float('nan')
                mae = mean_absolute_error(y_val, y_val_pred)
                mse = mean_squared_error(y_val, y_val_pred)
                acc = self.accuracy_rounded_to_dimes(y_val, y_val_pred)
            except Exception:
                # fallback to training-set metrics if val evaluation fails
                y_tr_pred = self.model.predict(X_tr)
                r2 = r2_score(y_tr, y_tr_pred) if len(y_tr) >= 2 else float('nan')
                mae = mean_absolute_error(y_tr, y_tr_pred)
                mse = mean_squared_error(y_tr, y_tr_pred)
                acc = self.accuracy_rounded_to_dimes(y_tr, y_tr_pred)

            print(f"[VALIDATION] R² (val): {r2:.8f}")
            print(f"[VALIDATION] MAE (val): {mae:.8f}")
            print(f"[VALIDATION] MSE (val): {mse:.8f}")
            try:
                overs_val = int(np.sum(np.asarray(y_val_pred) > np.asarray(y_val)))
                total_val = int(len(y_val)) if y_val is not None else 0
                rate_val = (overs_val / total_val) if total_val > 0 else 0.0
                print(f"[VALIDATION] Overshoots (pred > actual): {overs_val}/{total_val} ({rate_val*100:.2f}%)")
            except Exception:
                pass
            print(f"[VALIDATION] Accuracy (val, rounded to $0.10): {acc*100:.6f}%")
            print(f"[VALIDATION] Rule: Overshoot penalty ENABLED (predictions > actual count as 0 accuracy)")
            # Print a numeric sample of actual vs predicted (last training sample)
            try:
                # prefer the last sample of the validation set for reporting
                if 'y_val' in locals() and len(y_val) > 0:
                    last_actual = float(y_val[-1])
                    last_predicted = float(y_val_pred[-1]) if 'y_val_pred' in locals() and len(y_val_pred) > 0 else float('nan')
                else:
                    last_actual = float(y[-1]) if len(y) > 0 else float('nan')
                    # fallback to training prediction if validation pred not available
                    last_predicted = float(y_tr_pred[-1]) if 'y_tr_pred' in locals() and len(y_tr_pred) > 0 else float('nan')
                # Round predicted price to nearest $0.10 for display
                last_predicted_rounded = round(last_predicted * 10.0) / 10.0 if not np.isnan(last_predicted) else float('nan')
                print(f"[VALIDATION] sample actual: {last_actual:.4f}, sample predicted: {last_predicted:.4f} (rounded to $0.10: ${last_predicted_rounded:.2f})")
            except Exception as e:
                print(f"[VALIDATION] Could not print sample actual/predicted: {e}")
            # Also show whether the flags to log/display prices are enabled
            print(f"[VALIDATION] actual_price_enabled: {self.actual_price}, prediction_price_enabled: {self.prediction_price}")
            

            # get a representative predicted price (last predicted on validation set)
            # AND its corresponding actual value from the validation set for consistent comparison
            try:
                sample_pred_price = float(y_val_pred[-1]) if len(y_val_pred) > 0 else None
                # Use the actual from the SAME validation sample, not y[-1]
                sample_actual_price = float(y_val[-1]) if len(y_val) > 0 else None
                # Round predicted to nearest $0.10
                if sample_pred_price is not None:
                    sample_pred_price = round(sample_pred_price * 10.0) / 10.0
            except Exception:
                sample_pred_price = None
                sample_actual_price = None
            
            # TIER SELECTION: Only runs when BOTH parent and child overshoot
            # Find the highest Fibonacci tier at or below actual price
            # This is a ONE-TIME selection - once we find the right tier, we use it and move on
            selected_tier = None
            tier_target_price = None
            
            # Check if tier selection was triggered by evolve_hyperparams (both parent and child overshot)
            trigger_tier_selection = getattr(self, '_trigger_tier_selection', False)
            
            if trigger_tier_selection:
                print("\n[TIER-SELECT] Both parent and child overshot — finding optimal Fibonacci tier")
                self._trigger_tier_selection = False  # Reset flag
                
                try:
                    fib_levels = getattr(self, '_fibonacci_levels', None)
                    if fib_levels is not None and sample_actual_price is not None:
                        previous_tier = None
                        previous_price = None
                        
                        # Loop through tiers sequentially to find the last one at or below actual
                        for tier_idx in range(len(fib_levels)):
                            tier_price = fib_levels[tier_idx]
                            
                            # Check if this tier is ABOVE the actual price
                            if tier_price > sample_actual_price:
                                # Found first tier above actual - use the PREVIOUS tier
                                if previous_tier is not None:
                                    selected_tier = previous_tier
                                    tier_target_price = previous_price
                                    print(f"[TIER-SELECT] Tier {tier_idx} exceeds actual ({tier_price:.4f} > {sample_actual_price:.4f})")
                                    print(f"[TIER-SELECT] → Using tier {selected_tier}: price={tier_target_price:.4f}")
                                else:
                                    # Edge case: first tier is already above actual - use it anyway
                                    selected_tier = tier_idx
                                    tier_target_price = tier_price
                                    print(f"[TIER-SELECT] First tier {tier_idx} already above actual - using it: price={tier_target_price:.4f}")
                                break
                            else:
                                # This tier is at or below actual - remember it
                                previous_tier = tier_idx
                                previous_price = tier_price
                        
                        # If we didn't break (all tiers below actual), use the last one
                        if selected_tier is None and previous_tier is not None:
                            selected_tier = previous_tier
                            tier_target_price = previous_price
                            print(f"[TIER-SELECT] All tiers below actual - using last tier {selected_tier}: price={tier_target_price:.4f}")
                        
                        if selected_tier is not None:
                            print(f"[TIER-SELECT] ✓ Final tier {selected_tier} → Fibonacci price: {tier_target_price:.4f} (actual: {sample_actual_price:.4f}, delta: {tier_target_price - sample_actual_price:+.4f})")
                            
                            # Generate features for this specific tier and augment training
                            if len(df) >= WINDOW_SIZE + 1:
                                val_window_idx = len(df) - 1
                                window_start = val_window_idx - (WINDOW_SIZE - 1)
                                if window_start >= 0:
                                    val_window = df.iloc[window_start:val_window_idx + 1].copy()
                                    
                                    # FORCE this specific tier by using the tier's price
                                    result = self.phi_fold_features_with_tiers(
                                        val_window,
                                        force_actual_price_tier=True,
                                        actual_price=tier_target_price
                                    )
                                    if result is not None:
                                        new_feat, _ = result
                                        
                                        # AUGMENT the training set: teach model these features → tier's Fibonacci price
                                        X_tr_augmented = np.vstack([X_tr, new_feat.flatten()])
                                        y_tr_augmented = np.append(y_tr, tier_target_price)
                                        
                                        print(f"[TIER-SELECT] Training augmented: {len(X_tr)} → {len(X_tr_augmented)} samples, target: {tier_target_price:.4f}")
                                        
                                        # Retrain with augmented data
                                        allowed = RandomForestRegressor().get_params().keys()
                                        model_kwargs = {k: v for k, v in self.hyperparams.items() if k in allowed}
                                        self.model = RandomForestRegressor(**model_kwargs)
                                        
                                        if sample_weight_tr is not None:
                                            # Give the tier sample high weight
                                            augmented_weights = np.append(sample_weight_tr, 10.0)
                                            self.model.fit(X_tr_augmented, y_tr_augmented, sample_weight=augmented_weights)
                                        else:
                                            self.model.fit(X_tr_augmented, y_tr_augmented)
                                        
                                        # Re-evaluate on validation
                                        y_val_pred = self.model.predict(X_val)
                                        acc = self.accuracy_rounded_to_dimes(y_val, y_val_pred)
                                        sample_pred_price = float(y_val_pred[-1]) if len(y_val_pred) > 0 else None
                                        if sample_pred_price is not None:
                                            sample_pred_price = round(sample_pred_price * 10.0) / 10.0
                                        
                                        print(f"[TIER-SELECT] After tier-based training: pred={sample_pred_price:.4f}, actual={sample_actual_price:.4f}, acc={acc*100:.6f}%")
                        else:
                            print(f"[TIER-SELECT] ⚠ Could not find suitable tier")
                except Exception as e:
                    print(f"[TIER-SELECT] Error during tier selection: {e}")
                    import traceback
                    traceback.print_exc()
            
            evolved = self.evolve_hyperparams(acc, r2, predicted_price=sample_pred_price, phi_signals=getattr(self, '_last_phi_signals', None), actual_price=sample_actual_price)
            if evolved:
                print("[EVOLUTION] Hyperparameters evolved. Retraining with new model and FEEDING actual price...")
                
                # BOOST model complexity for post-evolution to ensure it can learn forced-tier samples
                # Store original params to restore later
                original_n_estimators = self.hyperparams.get('n_estimators', 100)
                original_max_depth = self.hyperparams.get('max_depth', 10)
                
                # Grow complexity VERY SLOWLY - 1% incremental growth to build strong foundations
                # Slow, steady growth allows the model to solidify patterns at each level
                increment_estimators = max(1, int(original_n_estimators * 0.01))  # 1% growth or 1, whichever is larger
                increment_depth = max(1, int(original_max_depth * 0.01))  # 1% growth or 1, whichever is larger
                
                # Always increment for forced-tier learning (no caps)
                self.hyperparams['n_estimators'] = original_n_estimators + increment_estimators
                self.hyperparams['max_depth'] = original_max_depth + increment_depth
                
                print(f"[EVOLUTION] Very slow growth: n_estimators {original_n_estimators} -> {self.hyperparams['n_estimators']} (+{increment_estimators})")
                print(f"[EVOLUTION] Very slow growth: max_depth {original_max_depth} -> {self.hyperparams['max_depth']} (+{increment_depth})")
                
                # Recreate model with boosted params
                allowed = RandomForestRegressor().get_params().keys()
                model_kwargs = {k: v for k, v in self.hyperparams.items() if k in allowed}
                self.model = RandomForestRegressor(**model_kwargs)
                
                # Save the boosted hyperparams immediately to ensure they persist
                try:
                    hp_dir = f"hyperparams/{self.asset_symbol}"
                    os.makedirs(hp_dir, exist_ok=True)
                    hyper_path = f"{hp_dir}/current_hyperparams.json"
                    with open(hyper_path, 'w') as f:
                        json.dump(self.hyperparams, f)
                    print(f"[EVOLUTION] Saved boosted hyperparams to {hyper_path}")
                    print(f"[EVOLUTION] Persisted: n_estimators={self.hyperparams['n_estimators']}, max_depth={self.hyperparams['max_depth']}")
                    # Verify the save worked by reading it back
                    with open(hyper_path, 'r') as f:
                        verified = json.load(f)
                    print(f"[EVOLUTION] Verified saved values: n_estimators={verified.get('n_estimators')}, max_depth={verified.get('max_depth')}")
                except Exception as e:
                    print(f"[EVOLUTION] ERROR: Could not save boosted hyperparams: {e}")
                
                # FEED ACTUAL PRICE: Regenerate features using Fibonacci tier logic with actual price
                # This ensures the evolution model stays synced with the main model's feature engineering
                evolved_feat_for_eval = None
                if sample_actual_price is not None and len(df) >= WINDOW_SIZE + 1:
                    # Get the window of data up to the point where we have the actual price
                    # We need to reconstruct features using the same phi_fold_features_with_tiers logic
                    latest_window_idx = len(df) - 1
                    window_start = latest_window_idx - (WINDOW_SIZE - 1)
                    
                    if window_start >= 0:
                        # Get the window for feature generation
                        evolution_window = df.iloc[window_start:latest_window_idx + 1].copy()
                        
                        # Temporarily set the last close to the actual price to regenerate features
                        # This simulates having known the actual price when building features
                        evolution_window.loc[evolution_window.index[-1], 'Close'] = sample_actual_price
                        
                        # Regenerate features using Fibonacci tier logic with FORCED tier selection
                        # ALWAYS pick the Fibonacci tier closest to the actual price for proper training
                        result = self.phi_fold_features_with_tiers(
                            evolution_window, 
                            force_actual_price_tier=True, 
                            actual_price=sample_actual_price
                        )
                        
                        if result is not None:
                            evolved_feat, evolved_emc = result
                            evolved_feat_flat = evolved_feat.flatten()
                            evolved_feat_for_eval = evolved_feat  # keep for post-evolution prediction on trained sample
                            
                            # Create augmented training set with regenerated features using actual price
                            X_tr_augmented = np.vstack([X_tr, evolved_feat_flat.reshape(1, -1)])
                            y_tr_augmented = np.append(y_tr, sample_actual_price)
                            
                            print(f"[EVOLUTION] Fed actual price {sample_actual_price:.4f} with FORCED closest Fibonacci tier selection (EMC: {evolved_emc:.2f})")
                            print(f"[EVOLUTION] Model trained on EXACT tier closest to actual price for maximum accuracy")
                        else:
                            # Fallback to original if feature generation fails
                            X_tr_augmented = X_tr
                            y_tr_augmented = y_tr
                            print("[EVOLUTION] Warning: Could not regenerate features, using original training data")
                    else:
                        # Not enough data for window
                        X_tr_augmented = X_tr
                        y_tr_augmented = y_tr
                        print("[EVOLUTION] Warning: Insufficient window data, using original training data")
                else:
                    # If no actual price available, use original training data
                    X_tr_augmented = X_tr
                    y_tr_augmented = y_tr
                
                # retrain with the possibly new model configuration
                # retrain on historical training portion with new hyperparams and evaluate on validation
                sample_weight_post = None
                try:
                    if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                        half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                        if half_life and half_life > 0 and len(X_tr_augmented) > 0:
                            idx = np.arange(len(X_tr_augmented))
                            sample_weight_post = 0.5 ** ((len(X_tr_augmented) - 1 - idx) / float(half_life))
                except Exception:
                    sample_weight_post = None

                if sample_weight_post is not None:
                    self.model.fit(X_tr_augmented, y_tr_augmented, sample_weight=sample_weight_post)
                else:
                    self.model.fit(X_tr_augmented, y_tr_augmented)

                # evaluate again on validation
                try:
                    y_val_pred = self.model.predict(X_val)
                    r2_post = r2_score(y_val, y_val_pred) if len(y_val) >= 2 else float('nan')
                    mae_post = mean_absolute_error(y_val, y_val_pred)
                    mse_post = mean_squared_error(y_val, y_val_pred)
                    acc_post = self.accuracy_rounded_to_dimes(y_val, y_val_pred)
                except Exception:
                    # fallback to training metrics
                    y_tr_pred = self.model.predict(X_tr)
                    r2_post = r2_score(y_tr, y_tr_pred) if len(y_tr) >= 2 else float('nan')
                    mae_post = mean_absolute_error(y_tr, y_tr_pred)
                    mse_post = mean_squared_error(y_tr, y_tr_pred)
                    acc_post = self.accuracy_rounded_to_dimes(y_tr, y_tr_pred)
                
                # CRITICAL: Also evaluate accuracy on the AUGMENTED training set (includes forced-tier sample)
                try:
                    if 'X_tr_augmented' in locals() and 'y_tr_augmented' in locals() and len(X_tr_augmented) > 0:
                        y_tr_aug_pred = self.model.predict(X_tr_augmented)
                        acc_augmented = self.accuracy_rounded_to_dimes(y_tr_augmented, y_tr_aug_pred)
                        print(f"[POST-EVOLUTION] Child Training Accuracy (includes forced-tier, overshoot rule): {acc_augmented*100:.6f}%")
                except Exception as e:
                    pass
                
                try:
                    overs_val_post = int(np.sum(np.asarray(y_val_pred) > np.asarray(y_val)))
                    total_val_post = int(len(y_val)) if y_val is not None else 0
                    rate_val_post = (overs_val_post / total_val_post) if total_val_post > 0 else 0.0
                    print(f"[POST-EVOLUTION] Overshoots (pred > actual) on validation: {overs_val_post}/{total_val_post} ({rate_val_post*100:.2f}%)")
                except Exception:
                    pass
                try:
                    if 'y_tr_aug_pred' in locals() and 'y_tr_augmented' in locals() and len(y_tr_augmented) == len(y_tr_aug_pred):
                        overs_tr_aug = int(np.sum(np.asarray(y_tr_aug_pred) > np.asarray(y_tr_augmented)))
                        total_tr_aug = int(len(y_tr_augmented))
                        rate_tr_aug = (overs_tr_aug / total_tr_aug) if total_tr_aug > 0 else 0.0
                        print(f"[POST-EVOLUTION] Overshoots (pred > actual) on augmented training: {overs_tr_aug}/{total_tr_aug} ({rate_tr_aug*100:.2f}%)")
                except Exception:
                    pass
                print(f"[POST-EVOLUTION] Validation R²: {r2_post:.8f}, MAE: {mae_post:.8f}, MSE: {mse_post:.8f}, Accuracy ($0.10 rounded): {acc_post*100:.6f}%")
                print(f"[POST-EVOLUTION] Child Validation Accuracy ($0.10 rounded, overshoot rule): {acc_post*100:.6f}%")
                print(f"[POST-EVOLUTION] Rule: Overshoot penalty ENABLED (predictions > actual count as 0 accuracy)")
                # If we trained on an augmented sample (with actual), also show prediction on that exact trained sample
                if 'evolved_feat_for_eval' in locals() and evolved_feat_for_eval is not None:
                    try:
                        trained_pred = float(self.model.predict(evolved_feat_for_eval)[0])
                        # Round to nearest $0.10
                        trained_pred_rounded = round(trained_pred * 10.0) / 10.0
                        actual_rounded = round(sample_actual_price * 10.0) / 10.0
                        match = "✓ MATCH" if trained_pred_rounded == actual_rounded else "✗ MISMATCH"
                        print(f"[POST-EVOLUTION] trained-sample predicted: {trained_pred:.6f} (rounded: ${trained_pred_rounded:.2f}) | actual fed: {sample_actual_price:.6f} (rounded: ${actual_rounded:.2f}) | delta: {trained_pred - float(sample_actual_price):+.6f} | {match}")
                        # Per-sample accuracy under overshoot rule
                        sample_overshoot = trained_pred > float(sample_actual_price)
                        if sample_overshoot:
                            sample_acc = 0.0
                        else:
                            sample_acc = 1.0 if trained_pred_rounded == actual_rounded else 0.0
                        print(f"[POST-EVOLUTION] trained-sample accuracy (overshoot rule): {sample_acc*100:.6f}%")
                    except Exception:
                        pass
                # Print the post-evolution representative predicted price
                try:
                    post_pred_price = float(y_val_pred[-1]) if len(y_val_pred) > 0 else None
                    # Round to nearest $0.10
                    if post_pred_price is not None:
                        post_pred_price_rounded = round(post_pred_price * 10.0) / 10.0
                        print(f"[POST-EVOLUTION] sample predicted price: {post_pred_price:.6f} (rounded to $0.10: ${post_pred_price_rounded:.2f})")
                    else:
                        print(f"[POST-EVOLUTION] sample predicted price: None")
                except Exception:
                    post_pred_price = None

                # Update the last evolution record on disk with post-evolution metrics
                try:
                    hp_dir = f"hyperparams/{self.asset_symbol}"
                    hist_path = f"{hp_dir}/evolution_history.json"
                    if os.path.exists(hist_path):
                        with open(hist_path, 'r') as hf:
                            records = json.load(hf)
                        if len(records) > 0:
                            # Store both raw and rounded predicted price
                            post_pred_rounded = round(post_pred_price * 10.0) / 10.0 if post_pred_price is not None else None
                            records[-1]['post_evolution_metrics'] = {
                                'r2': float(r2_post),
                                'mae': float(mae_post),
                                'mse': float(mse_post),
                                'accuracy': float(acc_post),
                                'predicted_price': float(post_pred_price) if post_pred_price is not None else None,
                                'predicted_price_rounded_to_dimes': float(post_pred_rounded) if post_pred_rounded is not None else None,
                                'timestamp': pd.Timestamp.now().isoformat()
                            }
                            with open(hist_path, 'w') as hf:
                                json.dump(records, hf, indent=2)
                except Exception as e:
                    print(f"Could not update evolution history with post-evolution metrics: {e}")

                # After validating evolved hyperparams, retrain final model on full data and save
                final_sample_weight = None
                try:
                    if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                        half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                        if half_life and half_life > 0 and len(X) > 0:
                            idx = np.arange(len(X))
                            final_sample_weight = 0.5 ** ((len(X) - 1 - idx) / float(half_life))
                except Exception:
                    final_sample_weight = None

                if final_sample_weight is not None:
                    self.model.fit(X, y, sample_weight=final_sample_weight)
                else:
                    self.model.fit(X, y)
                self.save_model()
            else:
                # No evolution happened; train final model on full data and save
                final_sample_weight = None
                try:
                    if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                        half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                        if half_life and half_life > 0 and len(X) > 0:
                            idx = np.arange(len(X))
                            final_sample_weight = 0.5 ** ((len(X) - 1 - idx) / float(half_life))
                except Exception:
                    final_sample_weight = None

                if final_sample_weight is not None:
                    self.model.fit(X, y, sample_weight=final_sample_weight)
                else:
                    self.model.fit(X, y)
                self.save_model()

            if current_date:
                self.metrics_log.append({
                    'date': current_date,
                    'r2': r2,
                    'mae': mae,
                    'mse': mse,
                    'accuracy': acc,
                    'emc_energy': emc_total_energy,
                    'model_version': self.model_version
                })

    def forecast_next(self, df):
        result = self.phi_fold_features_with_tiers(df)
        if result is None:
            return None
        feat, _ = result
        
        # Check feature dimension compatibility
        try:
            expected_features = self.model.n_features_in_
            actual_features = feat.shape[1]
            
            if expected_features != actual_features:
                print(f"[WARNING] Feature mismatch: model expects {expected_features}, got {actual_features}")
                print("[WARNING] Model needs retraining with current feature set. Returning None.")
                print("[INFO] Please run training first or use '--rebuild-cache' to retrain the model.")
                return None
                
        except AttributeError:
            # Older sklearn versions might not have n_features_in_
            pass
        except Exception as e:
            print(f"[WARNING] Could not check feature compatibility: {e}")
        
        try:
            return float(self.model.predict(feat)[0])
        except NotFittedError:
            print("[WARNING] Model is not fitted yet. Please run training before forecasting.")
            return None

    def forecast_next_period(self, df, horizon_label="next_period"):
        """Generate genuine next-period prediction based on training interval.
        
        This predicts the actual next step based on the model's training interval,
        not derived from weekly predictions.
        """
        result = self.phi_fold_features_with_tiers(df)
        if result is None:
            return None
        feat, _ = result
        
        # Check feature dimension compatibility
        try:
            expected_features = self.model.n_features_in_
            actual_features = feat.shape[1]
            
            if expected_features != actual_features:
                print(f"[HORIZON] Feature mismatch: model expects {expected_features}, got {actual_features}")
                return None
                
        except AttributeError:
            pass
        except Exception as e:
            print(f"[HORIZON] Could not check feature compatibility: {e}")
        
        try:
            prediction = float(self.model.predict(feat)[0])
            current_close = float(df['Close'].iloc[-1])
            
            # Determine the actual training interval from data_interval
            interval_str = getattr(self, 'data_interval', 'unknown')
            
            print(f"[HORIZON] {horizon_label.upper()} prediction for {self.asset_symbol}: {prediction:.4f}")
            print(f"[HORIZON] Current close: {current_close:.4f}")
            print(f"[HORIZON] Change: {prediction - current_close:.4f} ({((prediction/current_close - 1)*100):+.2f}%)")
            
            # Save the genuine next-period prediction
            pred_dir = f"forecasts/{self.asset_symbol}"
            os.makedirs(pred_dir, exist_ok=True)
            
            with open(f"{pred_dir}/{horizon_label}_prediction.txt", 'w') as f:
                f.write(f"{horizon_label.upper()} prediction for {self.asset_symbol}: {prediction:.6f}\n")
                f.write(f"Training interval: {interval_str}\n")
                f.write(f"Current close: {current_close:.6f}\n")
                f.write(f"Predicted change: {prediction - current_close:.6f} ({((prediction/current_close - 1)*100):+.2f}%)\n")
                f.write(f"Generated on: {pd.Timestamp.now()}\n")
            
            # Also save to prediction_price directory
            pdir = f"prediction_price/{self.asset_symbol}"
            os.makedirs(pdir, exist_ok=True)
            with open(f"{pdir}/{horizon_label}_prediction.txt", 'w') as f2:
                f2.write(f"{prediction:.6f}\n")
            
            return prediction
            
        except NotFittedError:
            print(f"[HORIZON] Model is not fitted yet. Please train before forecasting {horizon_label}.")
            return None
        except Exception as e:
            print(f"[HORIZON] Error generating {horizon_label} forecast: {e}")
            return None

    def generate_multi_horizon_from_weekly(self, weekly_forecast, current_close):
        """Derive 1D,1H,30M,5M forecasts from a weekly forecast using multiplicative scaling.

        This uses the weekly forecast as a 7-day horizon and converts that growth factor
        into per-day/hour/minute multiplicative factors assuming trading hours (~6.5h/day).
        """
        try:
            if current_close <= 0:
                raise ValueError("Invalid current_close <= 0")
            weekly_factor = weekly_forecast / float(current_close)
            # prevent non-positive factor
            if weekly_factor <= 0:
                raise ValueError("Non-positive weekly factor")

            # convert to daily multiplicative factor
            daily_factor = weekly_factor ** (1.0 / 7.0)

            # trading day parameters
            trading_hours = 6.5
            minutes_per_day = 390.0

            pred_1W = weekly_forecast
            pred_1D = current_close * (daily_factor ** 1.0)
            pred_1H = current_close * (daily_factor ** (1.0 / trading_hours))
            pred_30M = current_close * (daily_factor ** (30.0 / minutes_per_day))
            pred_5M = current_close * (daily_factor ** (5.0 / minutes_per_day))

            return {
                '1W': float(pred_1W),
                '1D': float(pred_1D),
                '1H': float(pred_1H),
                '30M': float(pred_30M),
                '5M': float(pred_5M)
            }
        except Exception as e:
            print(f"[ERROR] generate_multi_horizon_from_weekly failed: {e}")
            return None

    # Backwards-compatible alias; main may call this name
    def generate_multi_horizon_forecasts(self, df, weekly_forecast):
        try:
            current_close = float(df['Close'].iloc[-1])
        except Exception:
            return None
        return self.generate_multi_horizon_from_weekly(weekly_forecast, current_close)

    def simulate_weekly_predictions(self, df):
        logs = []
        start_index = 15 if len(df) > 20 else 10

        for i in range(start_index, len(df) - 1):
            training_df = df.iloc[:i]
            if i < 9:
                continue
            latest_df = df.iloc[i-9:i+1]
            current_date = df.index[i + 1].strftime("%Y-%m-%d-%H:%M:%S")
            if len(training_df) > 10:
                self.update(training_df, current_date=current_date)
            pred = self.forecast_next(latest_df)
            if pred is None:
                continue
            actual = df['Close'].iloc[i + 1].item()
            error = pred - actual
            pct_error = (error / actual) * 100
            logs.append({
                'date': current_date,
                'predicted': pred,
                'actual': actual,
                'error': error,
                'pct_error': pct_error
            })

        df_logs = pd.DataFrame(logs)
        
        if len(logs) == 0:
            print(f"[WARNING] No predictions generated for weekly simulation - insufficient data")
            return df_logs
            
        forecast_path = f"forecasts/{self.asset_symbol}/weekly_forecast.csv"
        df_logs.to_csv(forecast_path, index=False)
        print(f"Saved forecast to {forecast_path}")

        # Also save actual vs predicted for plotting/inspection
        try:
            plot_df = df_logs[['date', 'predicted', 'actual']].copy()
            plot_df['date'] = pd.to_datetime(plot_df['date'])
            plot_df = plot_df.sort_values('date')
            actual_path = f"actual_price/{self.asset_symbol}/actual_vs_predicted.csv"
            plot_df.to_csv(actual_path, index=False)
            print(f"Saved actual vs predicted to {actual_path}")
            # create a quick plot for visual comparison
            try:
                plt.figure(figsize=(10, 5))
                plt.plot(plot_df['date'], plot_df['actual'], label='Actual', marker='o')
                plt.plot(plot_df['date'], plot_df['predicted'], label='Predicted', marker='x')
                plt.legend()
                plt.xlabel('Date')
                plt.ylabel('Price')
                plt.title(f'{self.asset_symbol} Actual vs Predicted')
                plt.grid(True)
                img_path = f"actual_price/{self.asset_symbol}/actual_vs_predicted.png"
                plt.tight_layout()
                plt.savefig(img_path)
                plt.close()
                print(f"Saved plot to {img_path}")
            except Exception as e:
                print(f"Could not generate plot: {e}")
        except Exception as e:
            print(f"Could not save actual vs predicted CSV: {e}")
        return df_logs

    # ---- Intraday helpers ----
    def fetch_intraday_data(self, interval='60m', period='60d'):
        """Fetch intraday data for the current symbol using yfinance.
        interval: '60m'|'30m'|'5m' etc. period must be compatible with interval.
        """
        try:
            print(f"Fetching intraday data for {self.asset_symbol}, interval={interval}, period={period}...")
            data = yf.download(self.asset_symbol, period=period, interval=interval)
            data = data.dropna()
            if data.empty:
                print("No intraday data returned.")
                return None
            return data
        except Exception as e:
            print(f"Error fetching intraday data: {e}")
            return None

    def train_intraday_model(self, intraday_df, interval_label, skip_train=False, tune=False):
        """Train a model on intraday_df (same features pipeline) and save the model under models/<symbol>/<interval_label>_model.pkl"""
        try:
            # Use same rolling-feature construction but shorter window (10 frames)
            X = []
            y = []
            for i in range(10, len(intraday_df) - 1):
                window = intraday_df.iloc[i-9:i+1]
                result = self.phi_fold_features_with_tiers(window)
                if result is not None:
                    feat, _ = result
                    X.append(feat.flatten())
                    y.append(intraday_df['Close'].iloc[i+1].item())

            if not X:
                print(f"Insufficient intraday samples for {interval_label}")
                return None

            X = np.array(X)
            y = np.array(y)
            # enforce rolling window cap for intraday models
            max_samples = getattr(self, 'max_train_samples', DEFAULT_MAX_TRAIN_SAMPLES)
            if len(X) > max_samples:
                X = X[-max_samples:]
                y = y[-max_samples:]

            model_dir = f"models/{self.asset_symbol}"
            os.makedirs(model_dir, exist_ok=True)
            model_path = f"{model_dir}/{interval_label}_intraday_model.pkl"
            meta_path = f"{model_dir}/{interval_label}_intraday_model_meta.json"

            # If skip_train and model exists, load it
            if skip_train and os.path.exists(model_path):
                try:
                    model = joblib.load(model_path)
                    print(f"Loaded existing intraday model for {interval_label} from {model_path}")
                    return model
                except Exception as e:
                    print(f"Could not load existing model, will retrain: {e}")

            # Simple tuning option
            model_hyper = {'n_estimators': 200, 'random_state': 42, 'n_jobs': -1}
            tuning_results = None
            if tune:
                print(f"Tuning intraday model for {interval_label}...")
                # small grid to limit runtime
                n_estimators_options = [50, 100, 200]
                max_depth_options = [5, 10, 20]
                min_samples_leaf_options = [1, 2]
                best_score = float('inf')
                best_params = None
                X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)
                for n in n_estimators_options:
                    for md in max_depth_options:
                        for msl in min_samples_leaf_options:
                            try:
                                candidate = RandomForestRegressor(n_estimators=n, max_depth=md, min_samples_leaf=msl, random_state=42, n_jobs=-1)
                                candidate.fit(X_train, y_train)
                                yv = candidate.predict(X_val)
                                mse = mean_squared_error(y_val, yv)
                                if mse < best_score:
                                    best_score = mse
                                    best_params = {'n_estimators': n, 'max_depth': md, 'min_samples_leaf': msl}
                            except Exception:
                                continue
                if best_params:
                    model_hyper.update(best_params)
                    tuning_results = {'best_mse': best_score, 'best_params': best_params}
                    print(f"Tuning complete for {interval_label}: {tuning_results}")

            # Train model with selected hyperparameters
            model = RandomForestRegressor(**model_hyper)
            # train intraday model with recency weighting if enabled
            sample_weight = None
            try:
                if getattr(self, 'use_recency_weighting', DEFAULT_USE_RECENCY_WEIGHTING):
                    half_life = getattr(self, 'recency_half_life', DEFAULT_RECENCY_HALF_LIFE)
                    if half_life and half_life > 0:
                        idx = np.arange(len(X))
                        decay = 0.5 ** ((len(X) - 1 - idx) / float(half_life))
                        sample_weight = decay
            except Exception:
                sample_weight = None

            if sample_weight is not None:
                model.fit(X, y, sample_weight=sample_weight)
            else:
                model.fit(X, y)
            model_dir = f"models/{self.asset_symbol}"
            os.makedirs(model_dir, exist_ok=True)
            model_path = f"{model_dir}/{interval_label}_intraday_model.pkl"
            joblib.dump(model, model_path)
            # Save metadata for this intraday model
            meta = {
                'interval': interval_label,
                'trained_on': pd.Timestamp.now().isoformat(),
                'n_samples': len(X),
                'hyperparameters': model_hyper,
                'model_path': model_path
            }
            if tuning_results:
                meta['tuned'] = True
                meta['tuning_results'] = tuning_results
            else:
                meta['tuned'] = False

            # Save per-interval hyperparams to hyperparams folder
            try:
                hp_dir = f"hyperparams/{self.asset_symbol}"
                os.makedirs(hp_dir, exist_ok=True)
                with open(f"{hp_dir}/intraday_{interval_label}_hyperparams.json", 'w') as hpf:
                    json.dump(model_hyper, hpf, indent=2)
            except Exception as e:
                print(f"Could not save intraday hyperparams: {e}")
            try:
                with open(meta_path, 'w') as mf:
                    json.dump(meta, mf, indent=2)
            except Exception as e:
                print(f"Could not save intraday metadata: {e}")

            print(f"Saved intraday model to {model_path} (meta: {meta_path})")
            return model
        except Exception as e:
            print(f"Error training intraday model: {e}")
            return None

    def forecast_intraday(self, intraday_df, model, horizon_label):
        """Make a single-step forecast using the provided intraday model and the latest window."""
        try:
            latest_window = intraday_df.iloc[-10:]
            feat_res = self.phi_fold_features_with_tiers(latest_window)
            if feat_res is None:
                print(f"Not enough data for intraday forecast {horizon_label}")
                return None
            feat, _ = feat_res
            try:
                pred = float(model.predict(feat)[0])
            except NotFittedError:
                print(f"[INTRADAY] Model for {horizon_label} is not fitted yet. Train before forecasting.")
                return None
            # Save
            pred_dir = f"forecasts/{self.asset_symbol}"
            os.makedirs(pred_dir, exist_ok=True)
            with open(f"{pred_dir}/{horizon_label}_intraday_prediction.txt", 'w') as f:
                f.write(f"{horizon_label} intraday forecast: {pred:.6f}\n")
                f.write(f"Generated on: {pd.Timestamp.now()}\n")
            pdir = f"prediction_price/{self.asset_symbol}"
            os.makedirs(pdir, exist_ok=True)
            with open(f"{pdir}/{horizon_label}_intraday_prediction.txt", 'w') as f2:
                f2.write(f"{pred:.6f}\n")
            print(f"Saved intraday prediction for {horizon_label}: {pred:.6f}")
            return pred
        except Exception as e:
            print(f"Error forecasting intraday: {e}")
            return None

    def load_and_forecast_intraday(self, interval_label, horizon_label):
        """Load existing intraday model and make forecast using current data."""
        try:
            model_dir = f"models/{self.asset_symbol}"
            model_path = f"{model_dir}/{interval_label}_intraday_model.pkl"
            
            if not os.path.exists(model_path):
                print(f"[ERROR] No existing intraday model found for {interval_label} at {model_path}")
                return None
                
            # Load the model
            model = joblib.load(model_path)
            print(f"[CACHE] Loaded existing intraday model for {interval_label}")
            
            # Get fresh intraday data for the forecast
            if interval_label == '60m':
                interval_config = INTRADAY_DEFAULTS['1H']
            elif interval_label == '30m':
                interval_config = INTRADAY_DEFAULTS['30M']
            elif interval_label == '5m':
                interval_config = INTRADAY_DEFAULTS['5M']
            else:
                print(f"[ERROR] Unknown interval label: {interval_label}")
                return None
                
            intraday_df = self.fetch_intraday_data(
                interval=interval_config['interval'], 
                period=interval_config['period']
            )
            
            if intraday_df is None or len(intraday_df) < 10:
                print(f"[ERROR] Insufficient intraday data for {interval_label}")
                return None
                
            # Make forecast using existing model
            forecast = self.forecast_intraday(intraday_df, model, horizon_label)
            return forecast
            
        except Exception as e:
            print(f"[ERROR] Failed to load and forecast intraday {interval_label}: {e}")
            return None

    def list_saved_models(self):
        """List saved models and associated metadata for this symbol."""
        model_dir = f"models/{self.asset_symbol}"
        if not os.path.exists(model_dir):
            print(f"No models directory for {self.asset_symbol}")
            return []
        entries = []
        for fname in os.listdir(model_dir):
            path = os.path.join(model_dir, fname)
            if fname.endswith('.pkl'):
                meta_file = None
                base = fname.rsplit('.', 1)[0]
                candidate_meta = os.path.join(model_dir, f"{base}_meta.json")
                candidate_meta2 = os.path.join(model_dir, f"{base}_intraday_model_meta.json")
                if os.path.exists(candidate_meta):
                    meta_file = candidate_meta
                elif os.path.exists(candidate_meta2):
                    meta_file = candidate_meta2
                meta = None
                if meta_file:
                    try:
                        with open(meta_file, 'r') as mf:
                            meta = json.load(mf)
                    except Exception:
                        meta = None
                entries.append({'model': fname, 'path': path, 'meta': meta})
        for e in entries:
            print('Model:', e['model'])
            if e['meta']:
                print('  meta:', json.dumps(e['meta'], indent=2))
            else:
                print('  (no metadata)')
        return entries

    def show_evolution_history(self, n=10):
        """Print the last n hyperparameter evolution records from disk."""
        try:
            hp_dir = f"hyperparams/{self.asset_symbol}"
            hist_path = f"{hp_dir}/evolution_history.json"
            if not os.path.exists(hist_path):
                print(f"No evolution history found at {hist_path}")
                return []
            with open(hist_path, 'r') as hf:
                records = json.load(hf)
            recent = records[-n:]
            for r in recent:
                print(json.dumps(r, indent=2))
            return recent
        except Exception as e:
            print(f"Error reading evolution history: {e}")
            return []

    def show_sample_stats(self):
        """Display statistics about accumulated training samples."""
        try:
            current_count = len(self.X_train) if hasattr(self, 'X_train') else 0
            max_samples = getattr(self, 'max_train_samples', DEFAULT_MAX_TRAIN_SAMPLES)
            
            print(f"\n[SAMPLE STATS] for {self.asset_symbol}")
            print(f"Current accumulated samples: {current_count}")
            print(f"Maximum sample limit: {max_samples}")
            print(f"Memory usage: {current_count / max_samples * 100:.1f}% of limit")
            
            # Check if persistent file exists
            samples_path = f"models/{self.asset_symbol}/accumulated_training_samples.npz"
            if os.path.exists(samples_path):
                try:
                    data = np.load(samples_path)
                    disk_count = len(data['X'])
                    print(f"Samples on disk: {disk_count}")
                    if disk_count != current_count:
                        print(f"⚠️  Memory/disk mismatch - run a training session to sync")
                except Exception:
                    print("⚠️  Could not read disk samples")
            else:
                print("No persistent samples file found")
                
            return current_count
        except Exception as e:
            print(f"Error showing sample stats: {e}")
            return 0

    def clean_training_samples(self):
        """Clean training samples to ensure shape consistency."""
        if not self.X_train:
            return
            
        expected_features = self.get_expected_feature_count()
        clean_X = []
        clean_y = []
        
        for i, (sample, target) in enumerate(zip(self.X_train, self.y_train)):
            if len(sample) == expected_features:
                clean_X.append(sample)
                clean_y.append(target)
            else:
                print(f"[CLEAN] Removing sample {i}: has {len(sample)} features, expected {expected_features}")
        
        removed_count = len(self.X_train) - len(clean_X)
        if removed_count > 0:
            print(f"[CLEAN] Removed {removed_count} incompatible samples")
            self.X_train = clean_X
            self.y_train = clean_y
            # Re-save the cleaned samples
            self.save_training_samples()

    def show_performance_config(self):
        """Display current computational performance configuration."""
        try:
            print(f"\n[PERFORMANCE CONFIG] for {self.asset_symbol}")
            print(f"Model Type: {'Ensemble' if getattr(self, 'use_ensemble', False) else 'Single RandomForest'}")
            print(f"Cross-Validation: {'Enabled' if getattr(self, 'use_cross_validation', False) else 'Disabled'}")
            print(f"Number of Estimators: {self.hyperparams.get('n_estimators', 'Unknown')}")
            print(f"Max Depth: {self.hyperparams.get('max_depth', 'Unknown')}")
            print(f"Max Features: {self.hyperparams.get('max_features', 'Unknown')}")
            print(f"Min Samples Leaf: {self.hyperparams.get('min_samples_leaf', 'Unknown')}")
            print(f"Max Training Samples: {getattr(self, 'max_train_samples', 'Unknown')}")
            print(f"CPU Cores Used: {self.hyperparams.get('n_jobs', 'Unknown')}")
            print(f"Bootstrap Sampling: {self.hyperparams.get('bootstrap', 'Unknown')}")
            
            # Estimate computational complexity
            n_est = self.hyperparams.get('n_estimators', 1000)
            max_depth = self.hyperparams.get('max_depth', 10)
            max_samples = getattr(self, 'max_train_samples', DEFAULT_MAX_TRAIN_SAMPLES)
            
            complexity_score = (n_est * max_depth * max_samples) / 1000000
            print(f"Estimated Complexity Score: {complexity_score:.2f} (higher = more computational load)")
            
            if complexity_score > 100:
                print("🔥 EXTREME computational load - will use maximum CPU power")
            elif complexity_score > 50:
                print("⚡ HIGH computational load - will stress test your CPU")  
            elif complexity_score > 10:
                print("💪 MODERATE computational load - good CPU utilization")
            else:
                print("🐌 LOW computational load - consider --high-performance mode")
                
        except Exception as e:
            print(f"Error showing performance config: {e}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Run Phi Forest forecasts')
    parser.add_argument('symbol', nargs='?', default=None, help='Asset symbol, e.g., SPY')
    parser.add_argument('--period', default=DEFAULT_DOWNLOAD_PERIOD, help='Download period for historical data (yfinance format)')
    parser.add_argument('--interval', default=DEFAULT_DOWNLOAD_INTERVAL, help='Historical data interval for weekly simulation')
    parser.add_argument('mode', nargs='?', default='run', choices=['run', 'intraday', 'intraday-cache', 'list_models', 'evolution', 'samples', 'clean-samples', 'perf-config', 'save-config', 'cache', 'clear-cache', 'clear-samples-cache', 'ORBITAL CANNON RESET', 'train-from-cache', 'predict-next', 'predict-hour', 'predict-day'], help='Mode: run (weekly), intraday, intraday-cache (train intraday from cache and forecast), list_models, evolution, samples (show sample stats), clean-samples (remove incompatible samples), perf-config (show performance configuration), save-config, cache, clear-cache (delete cached training datasets), clear-samples-cache (delete accumulated training samples), ORBITAL CANNON RESET (purge all caches), train-from-cache, predict-next (genuine next-period prediction), predict-hour (next hour), predict-day (next day)')
    parser.add_argument('--evo-n', type=int, default=10, help='Number of evolution records to show')
    parser.add_argument('--skip-train', action='store_true', help='Skip intraday training if models exist')
    parser.add_argument('--tune', action='store_true', help='Tune intraday models (slower)')
    parser.add_argument('--high-performance', action='store_true', help='Enable high-performance mode (uses maximum computational power)')
    parser.add_argument('--ensemble', action='store_true', help='Use ensemble of multiple models (more compute intensive)')
    parser.add_argument('--cross-validate', action='store_true', help='Enable cross-validation (significantly more compute)')
    parser.add_argument('--no-adaptive-scaling', action='store_true', help='Disable automatic hyperparameter scaling based on dataset size')
    parser.add_argument('--max-samples', type=int, default=None, help='Maximum number of training samples to keep (rolling window)')
    parser.add_argument('--max-fib-mult', type=float, default=None, help='Maximum Fibonacci tier multiplier (both +/-)')
    parser.add_argument('--max-fib-levels', type=int, default=None, help='Safety cap on total Fibonacci tiers used')
    parser.add_argument('--no-negative-fib', action='store_true', help='Disable negative Fibonacci tiers (levels above high)')
    parser.add_argument('--no-recency-weighting', action='store_true', help='Disable recency sample weighting')
    parser.add_argument('--recency-half-life', type=int, default=None, help='Half-life (in samples) for recency weighting (default 250)')
    parser.add_argument('--no-cache', action='store_true', help='Disable training data caching/load')
    parser.add_argument('--rebuild-cache', action='store_true', help='Force rebuilding of training data cache')
    args = parser.parse_args()

    # load config if present and no explicit symbol provided
    cfg = load_config() if args.symbol is None else None
    if args.symbol is None and cfg:
        symbol = cfg.get('symbol', DEFAULT_ASSET_SYMBOL)
        args.period = args.period or cfg.get('period', DEFAULT_DOWNLOAD_PERIOD)
        args.interval = args.interval or cfg.get('interval', DEFAULT_DOWNLOAD_INTERVAL)
    else:
        symbol = (args.symbol or DEFAULT_ASSET_SYMBOL).strip().upper()
    data = yf.download(symbol, period=args.period, interval=args.interval)

    if args.mode == 'save-config':
        save_config(symbol, args.period, args.interval)
        print('Configuration saved; exiting.')
        raise SystemExit(0)
    data = data.dropna()

    # Validate sufficient data
    if len(data) < 20:
        print(f"[ERROR] Insufficient data for {symbol}. Got {len(data)} rows, need at least 20 for meaningful analysis.")
        print(f"[SUGGESTION] Try using a longer period (e.g., --period 2y) or different interval.")
        raise SystemExit(1)

    engine = PhiForestEngineWithVolumeTiers(symbol)
    # attach runtime context for dataset caching
    engine.data_interval = args.interval
    engine.data_period = args.period
    engine.use_training_cache = not args.no_cache
    engine.rebuild_training_cache = bool(args.rebuild_cache)
    
    # High-performance mode configuration
    if args.high_performance:
        print("[HIGH-PERF] Enabling maximum computational power mode...")
        engine.max_train_samples = 100000  # Even more samples
        engine.hyperparams.update({
            'n_estimators': 50000,      # Maximum trees
            'max_depth': 50,           # Very deep trees
            'min_samples_leaf': 1,     # Maximum granularity
            'max_features': None,      # Consider all features
            'min_samples_split': 2,    # Most aggressive splitting
        })
        # Force model recreation with new hyperparams
        allowed = RandomForestRegressor().get_params().keys()
        model_kwargs = {k: v for k, v in engine.hyperparams.items() if k in allowed}
        engine.model = RandomForestRegressor(**model_kwargs)
        print(f"[HIGH-PERF] Updated model with hyperparameters: {model_kwargs}")
    
    # Ensemble mode configuration
    if args.ensemble:
        print("[ENSEMBLE] Enabling ensemble mode for maximum accuracy...")
        # This will be used later in training to create ensemble models
        engine.use_ensemble = True
    else:
        engine.use_ensemble = False
        
    # Cross-validation mode
    engine.use_cross_validation = args.cross_validate
    if args.cross_validate:
        print("[CV] Cross-validation enabled - training will take significantly longer")
    
    # Adaptive scaling mode
    engine.use_adaptive_scaling = not args.no_adaptive_scaling
    if args.no_adaptive_scaling:
        print("[ADAPTIVE] Automatic hyperparameter scaling disabled - using fixed hyperparameters")
    
    # set max samples if provided (override high-perf if explicitly set)
    if args.max_samples is not None:
        engine.max_train_samples = int(args.max_samples)
    elif not args.high_performance:
        engine.max_train_samples = DEFAULT_MAX_TRAIN_SAMPLES
        
    engine.use_recency_weighting = not args.no_recency_weighting
    engine.recency_half_life = int(args.recency_half_life) if args.recency_half_life is not None else DEFAULT_RECENCY_HALF_LIFE
    # Fibonacci tier controls from CLI
    if args.max_fib_mult is not None:
        engine.max_fib_multiplier = float(args.max_fib_mult)
    if args.max_fib_levels is not None:
        engine.max_fib_levels = int(args.max_fib_levels)
    engine.include_negative_fib_tiers = not args.no_negative_fib

    if args.mode == 'evolution':
        engine.show_evolution_history(n=args.evo_n)
        raise SystemExit(0)
    if args.mode == 'samples':
        engine.show_sample_stats()
        raise SystemExit(0)
    if args.mode == 'clean-samples':
        engine.clean_training_samples()
        raise SystemExit(0)
    if args.mode == 'perf-config':
        engine.show_performance_config()
        raise SystemExit(0)
    if args.mode == 'cache':
        engine.prepare_and_save_training_dataset(data)
        raise SystemExit(0)
    if args.mode == 'clear-cache':
        ok = engine.clear_training_cache()
        if not ok:
            print("[ERROR] Failed to clear cache")
            raise SystemExit(1)
        print('[CACHE] Cache cleared successfully')
        raise SystemExit(0)
    if args.mode == 'clear-samples-cache':
        ok = engine.clear_samples_cache()
        if not ok:
            print("[ERROR] Failed to clear samples cache")
            raise SystemExit(1)
        print('[PERSIST] Samples cache cleared successfully')
        raise SystemExit(0)
    if args.mode == 'ORBITAL CANNON RESET':
        ok = engine.orbital_cannon_reset()
        if not ok:
            print('[ERROR] ORBITAL CANNON RESET completed with warnings')
            raise SystemExit(1)
        raise SystemExit(0)
    if args.mode == 'train-from-cache':
        ok = engine.train_from_cached_dataset()
        if not ok:
            print("[ERROR] Failed to train from cached dataset")
            raise SystemExit(1)
        print("[CACHE-TRAIN] Model successfully trained from cache. Proceeding to generate forecast...")
        # Continue execution to generate forecast instead of exiting
    if args.mode in ['predict-next', 'predict-hour', 'predict-day']:
        # Generate genuine next-period predictions based on actual model training
        latest_window = data.iloc[-WINDOW_SIZE:]
        if args.mode == 'predict-next':
            prediction = engine.forecast_next_period(latest_window, "next_period")
        elif args.mode == 'predict-hour':
            prediction = engine.forecast_next_period(latest_window, "next_hour")
        elif args.mode == 'predict-day':
            prediction = engine.forecast_next_period(latest_window, "next_day")
        
        if prediction:
            print(f"\n[RESULT] {args.mode.upper()} prediction completed: {prediction:.4f}")
        else:
            print(f"[ERROR] Could not generate {args.mode} prediction")
        raise SystemExit(0)
    # Check model compatibility with current feature set before proceeding
    if hasattr(engine, 'model') and engine.model is not None and args.mode != 'train-from-cache':
        compatibility_ok = engine.check_feature_compatibility_and_retrain_if_needed(data)
        if not compatibility_ok and getattr(engine, '_needs_retraining', False):
            print("[MAIN] Model needs retraining due to feature mismatch. Training with current data...")
            # Do a minimal training run to fix the mismatch
            if len(data) >= WINDOW_SIZE + 2:
                engine.update(data.iloc[:len(data)//2])  # Train on first half
    
    # proactively save training dataset for reuse
    if engine.use_training_cache and args.mode != 'train-from-cache':
        try:
            engine.prepare_and_save_training_dataset(data)
        except Exception as e:
            print(f"[CACHE] Pre-save failed: {e}")
    
    # Only run simulation if not in train-from-cache mode (since we already have a trained model)
    if args.mode != 'train-from-cache':
        forecast_logs = engine.simulate_weekly_predictions(data)

    print("\n[FORECAST] Predicting next week...")
    latest_window = data.iloc[-10:]
    forecast = engine.forecast_next(latest_window)

    if forecast:
        print(f"[RESULT] Next week prediction for {symbol}: {forecast:.2f}")
        forecast_path = f"forecasts/{symbol}/final_next_week_prediction.txt"
        with open(forecast_path, "w") as f:
            f.write(f"Next week prediction for {symbol}: {forecast:.4f}\n")
            f.write(f"Generated on: {pd.Timestamp.now()}\n")
        print(f"[LOGGED] Final forecast written to {forecast_path}")

        # Generate multi-horizon forecasts based on weekly forecast
        try:
            multi = engine.generate_multi_horizon_forecasts(data, weekly_forecast=forecast)
            if multi:
                print(f"Multi-horizon forecasts generated: {multi}")
        except Exception as e:
            print(f"[WARNING] Multi-horizon generation failed: {e}")
    else:
        print("[WARNING] Could not generate forecast - insufficient training data or model issues.")
    # Optionally run intraday models if requested via args.mode
    if args.mode == 'list_models':
        engine.list_saved_models()
        raise SystemExit(0)
    if args.mode == 'intraday':
        skip_train_flag = args.skip_train
        tune_flag = args.tune
        try:
            # hourly
            intraday_60m = engine.fetch_intraday_data(interval=INTRADAY_DEFAULTS['1H']['interval'], period=INTRADAY_DEFAULTS['1H']['period'])
            if intraday_60m is not None and len(intraday_60m) > 20:
                model_60 = engine.train_intraday_model(intraday_60m, '60m', skip_train=skip_train_flag, tune=tune_flag)
                if model_60:
                    engine.forecast_intraday(intraday_60m, model_60, '1H')

            # 30m
            intraday_30m = engine.fetch_intraday_data(interval=INTRADAY_DEFAULTS['30M']['interval'], period=INTRADAY_DEFAULTS['30M']['period'])
            if intraday_30m is not None and len(intraday_30m) > 20:
                model_30 = engine.train_intraday_model(intraday_30m, '30m', skip_train=skip_train_flag, tune=tune_flag)
                if model_30:
                    engine.forecast_intraday(intraday_30m, model_30, '30M')

            # 5m
            intraday_5m = engine.fetch_intraday_data(interval=INTRADAY_DEFAULTS['5M']['interval'], period=INTRADAY_DEFAULTS['5M']['period'])
            if intraday_5m is not None and len(intraday_5m) > 20:
                model_5 = engine.train_intraday_model(intraday_5m, '5m', skip_train=skip_train_flag, tune=tune_flag)
                if model_5:
                    engine.forecast_intraday(intraday_5m, model_5, '5M')
        except Exception as e:
            print(f"[WARNING] Intraday processing failed: {e}")
    
    if args.mode == 'intraday-cache':
        print("[INTRADAY-CACHE] Loading existing intraday models and generating forecasts...")
        try:
            # Try to load and forecast with existing models
            forecasts = {}
            
            # 1H forecast
            forecast_1h = engine.load_and_forecast_intraday('60m', '1H')
            if forecast_1h:
                forecasts['1H'] = forecast_1h
                
            # 30M forecast  
            forecast_30m = engine.load_and_forecast_intraday('30m', '30M')
            if forecast_30m:
                forecasts['30M'] = forecast_30m
                
            # 5M forecast
            forecast_5m = engine.load_and_forecast_intraday('5m', '5M')
            if forecast_5m:
                forecasts['5M'] = forecast_5m
                
            if forecasts:
                print(f"[INTRADAY-CACHE] Generated forecasts: {forecasts}")
                
                # Save summary of all intraday forecasts
                summary_path = f"forecasts/{symbol}/intraday_summary.txt"
                with open(summary_path, 'w') as f:
                    f.write(f"Intraday Forecasts for {symbol}\n")
                    f.write(f"Generated on: {pd.Timestamp.now()}\n")
                    f.write("-" * 40 + "\n")
                    for horizon, price in forecasts.items():
                        f.write(f"{horizon}: {price:.6f}\n")
                        
                print(f"[INTRADAY-CACHE] Summary saved to {summary_path}")
            else:
                print("[INTRADAY-CACHE] No forecasts could be generated - check if intraday models exist")
                
        except Exception as e:
            print(f"[WARNING] Intraday cache processing failed: {e}")
    
    else:
        print("[WARNING] Forecast could not be generated due to insufficient data.")