Toggleable scheduler for financial oracle, and tweak batch

Author: theomgdev

examples/advanced/experiment_financial_oracle.py

@@ -170,9 +170,10 @@
 
 # --- Training ---
 EPOCHS         = 100
-BATCH_SIZE     = 32
-EVAL_BATCH     = 1024    # Larger batch for eval (no gradients)
+BATCH_SIZE     = 1024
+EVAL_BATCH     = 16384     # Larger batch for eval (no gradients)
 LR             = 1e-4
+USE_SCHEDULER  = True
 LR_MIN         = 1e-6    # Cosine annealing floor
 VAL_EVERY      = 3       # Validate every N epochs (validation is expensive)
 PATIENCE       = 20      # Early stopping patience (in validation checks)
@@ -185,7 +186,7 @@
 
 # Feature scaling multiplier applied to return channels before float32 training.
 # Higher values increase numerical visibility of tiny moves.
-RETURN_SCALE = 100.0
+RETURN_SCALE = 1.0
 # Base outlier threshold in unscaled return space.
 # Threshold in feature space is this base multiplied by RETURN_SCALE.
 MAX_ABS_NORMALIZED_BASE = 20.0
@@ -540,7 +541,6 @@ def build_model():
         weight_init='resonant',        # Edge of Chaos initialization
         activation='tanh',             # Bounded oscillations
         hebb_type='synapse',           # Per-synapse plasticity -> workbenches
-        dropout_rate=0.05,             # Light regularization
         gradient_checkpointing=False,  # VRAM plentiful; skip recompute for ~2x speed
         device=DEVICE,
     )
@@ -1068,10 +1068,12 @@ def main():
     trainer = OdyssNetTrainer(model, device=DEVICE, lr=LR)
     trainer.loss_fn = HeteroscedasticNLL()
 
-    # Cosine annealing: LR decays smoothly from LR to LR_MIN
-    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
-        trainer.optimizer, T_max=EPOCHS, eta_min=LR_MIN,
-    )
+    # Optional cosine annealing scheduler.
+    scheduler = None
+    if USE_SCHEDULER:
+        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
+            trainer.optimizer, T_max=EPOCHS, eta_min=LR_MIN,
+        )
 
     history = TrainingHistory()
     best_val_mse = float('inf')
@@ -1081,7 +1083,10 @@ def main():
 
     print(f"\n{'='*70}")
     print(f"  TRAINING")
-    print(f"  Epochs: {EPOCHS}  |  Batch: {BATCH_SIZE}  |  LR: {LR} -> {LR_MIN} (cosine)")
+    if USE_SCHEDULER:
+        print(f"  Epochs: {EPOCHS}  |  Batch: {BATCH_SIZE}  |  LR: {LR} -> {LR_MIN} (cosine)")
+    else:
+        print(f"  Epochs: {EPOCHS}  |  Batch: {BATCH_SIZE}  |  LR: {LR} (fixed)")
     print(f"  Loss: Heteroscedastic Gaussian NLL (Kendall & Gal, 2017)")
     print(f"  Confident signal threshold: P(up) > {CONF_THRESH:.2f} or < {1.0 - CONF_THRESH:.2f}")
     print(f"  Validation every {VAL_EVERY} epochs  |  Early stop patience: {PATIENCE}")
@@ -1092,8 +1097,11 @@ def main():
     for epoch in range(1, EPOCHS + 1):
         # Train
         avg_loss = run_epoch(trainer, train_x, train_y)
-        scheduler.step()
-        current_lr = scheduler.get_last_lr()[0]
+        if scheduler is not None:
+            scheduler.step()
+            current_lr = scheduler.get_last_lr()[0]
+        else:
+            current_lr = trainer.optimizer.param_groups[0]['lr']
 
         # Validate (every VAL_EVERY epochs + first + last)
         do_val = (epoch == 1 or epoch % VAL_EVERY == 0 or epoch == EPOCHS)