Add baseline scenario configs and pipeline/modeling updates

Author: LMBooth

analysis_pipeline/build_trial_table.py

@@ -375,13 +375,18 @@ def main() -> None:
     subject_dirs = sorted(path for path in bids_root.glob("sub-*") if path.is_dir())
     if not subject_dirs:
         raise FileNotFoundError(f"No subject directories found in {bids_root}")
+    print(
+        f"Stage 0 starting. task={task} subjects={len(subject_dirs)} "
+        f"trial_duration_s={args.trial_duration}"
+    )
 
     all_rows: list[dict[str, str]] = []
     subject_summaries: list[dict[str, Any]] = []
     all_anomalies: list[str] = []
 
-    for subject_dir in subject_dirs:
+    for subject_idx, subject_dir in enumerate(subject_dirs, start=1):
         subject = subject_dir.name
+        print(f"[Subject {subject_idx}/{len(subject_dirs)}] {subject}")
         events_path = subject_dir / "eeg" / f"{subject}_task-{task}_events.tsv"
         if not events_path.exists():
             all_anomalies.append(f"{subject}: Missing events file {events_path.name}.")

analysis_pipeline/config/pipeline.yaml

@@ -47,10 +47,13 @@ stage6:
     clip_lower_quantile: 0.01
     clip_upper_quantile: 0.99
     random_seed: 42
+    torch_device: "auto"
   class_scenarios:
     - name: "all_bins"
     - name: "omit_easiest"
       drop_labels: ["0.6-1.5"]
+    - name: "omit_hardest"
+      drop_labels: ["6.0-6.9"]
     - name: "three_level_merged"
       merge_map:
         "0.6-1.5": "low"
@@ -67,3 +70,5 @@ stage6_confusions:
   args:
     metric: "balanced_accuracy_mean"
     top_k_per_protocol: 1
+    include_all: true
+    out_png_dir: "analysis_pipeline/reports/confusion_pngs"

analysis_pipeline/config/pipeline_baseline_advanced_nn.yaml

@@ -0,0 +1,80 @@
+version: 1
+
+paths:
+  bids_root: "./data/bids_arithmetic"
+  python_executable: "python"
+
+reports:
+  run_manifest: "analysis_pipeline/reports/run_manifest_baseline_advanced_nn.json"
+
+stages:
+  stage0: false
+  stage1: false
+  stage2: false
+  stage3: false
+  stage4: false
+  stage5: true
+  stage6: true
+  stage6_confusions: true
+
+stage_args:
+  stage5:
+    include_tutorial: true
+    dropout_policy: "absolute"
+    dropout_threshold: 35.0
+    fused_out: "analysis_pipeline/features/features_fused_tutorial_baseline.tsv"
+    split_manifest_out: "analysis_pipeline/features/split_manifest_tutorial_baseline.json"
+    summary_json: "analysis_pipeline/reports/fusion_summary_tutorial_baseline.json"
+    unimodal_tag: "tutorial_baseline"
+
+stage6:
+  run_tag_prefix: "baseline_adv_nn"
+  results_json_template: "analysis_pipeline/reports/ml_results_{scenario}_baseline_advanced_nn.json"
+  summary_md_template: "analysis_pipeline/reports/ml_summary_{scenario}_baseline_advanced_nn.md"
+  base_args:
+    split_manifest: "analysis_pipeline/features/split_manifest_tutorial_baseline.json"
+    datasets: ["eeg", "ecg", "pupil", "fused"]
+    protocols: ["loso", "group_holdout", "within_participant"]
+    models: ["lstm1d", "gru1d", "cnn1d", "transformer", "bilstm1d", "bigru1d", "cnn1d_deep", "transformer_xl"]
+    feature_selectors: ["none"]
+    inner_folds: 2
+    max_param_combos: 2
+    max_outer_splits_per_protocol: 2
+    clip_lower_quantile: 0.01
+    clip_upper_quantile: 0.99
+    random_seed: 42
+    torch_device: "auto"
+    baseline_from_tutorial_label: "baseline"
+  class_scenarios:
+    - name: "baseline_all_bins"
+    - name: "baseline_omit_hardest"
+      drop_labels: ["6.0-6.9"]
+    - name: "baseline_low_high_omit_hardest"
+      drop_labels: ["6.0-6.9"]
+      merge_map:
+        "0.6-1.5": "low_1_2_3"
+        "1.5-2.4": "low_1_2_3"
+        "2.4-3.3": "low_1_2_3"
+        "3.3-4.2": "high_4_5_6"
+        "4.2-5.1": "high_4_5_6"
+        "5.1-6.0": "high_4_5_6"
+    - name: "baseline_grouped_4class_omit_hardest"
+      drop_labels: ["6.0-6.9"]
+      merge_map:
+        "0.6-1.5": "low_1_2"
+        "1.5-2.4": "low_1_2"
+        "2.4-3.3": "mid_3_4"
+        "3.3-4.2": "mid_3_4"
+        "4.2-5.1": "high_5_6"
+        "5.1-6.0": "high_5_6"
+    - name: "baseline_omit_easiest"
+      drop_labels: ["0.6-1.5"]
+
+stage6_confusions:
+  out_json_template: "analysis_pipeline/reports/confusion_highlights_{scenario}_baseline_advanced_nn.json"
+  out_md_template: "analysis_pipeline/reports/confusion_highlights_{scenario}_baseline_advanced_nn.md"
+  args:
+    metric: "balanced_accuracy_mean"
+    top_k_per_protocol: 1
+    include_all: true
+    out_png_dir: "analysis_pipeline/reports/confusion_pngs"

analysis_pipeline/config/pipeline_baseline_variants.yaml

@@ -0,0 +1,80 @@
+version: 1
+
+paths:
+  bids_root: "./data/bids_arithmetic"
+  python_executable: "python"
+
+reports:
+  run_manifest: "analysis_pipeline/reports/run_manifest_baseline_variants.json"
+
+stages:
+  stage0: false
+  stage1: false
+  stage2: false
+  stage3: false
+  stage4: false
+  stage5: true
+  stage6: true
+  stage6_confusions: true
+
+stage_args:
+  stage5:
+    include_tutorial: true
+    dropout_policy: "absolute"
+    dropout_threshold: 35.0
+    fused_out: "analysis_pipeline/features/features_fused_tutorial_baseline.tsv"
+    split_manifest_out: "analysis_pipeline/features/split_manifest_tutorial_baseline.json"
+    summary_json: "analysis_pipeline/reports/fusion_summary_tutorial_baseline.json"
+    unimodal_tag: "tutorial_baseline"
+
+stage6:
+  run_tag_prefix: "baseline_variant"
+  results_json_template: "analysis_pipeline/reports/ml_results_{scenario}_baseline.json"
+  summary_md_template: "analysis_pipeline/reports/ml_summary_{scenario}_baseline.md"
+  base_args:
+    split_manifest: "analysis_pipeline/features/split_manifest_tutorial_baseline.json"
+    datasets: ["eeg", "ecg", "pupil", "fused"]
+    protocols: ["loso", "group_holdout", "within_participant"]
+    models: ["logreg", "knn", "svm", "gaussian_nb", "decision_tree", "mlp", "rf"]
+    feature_selectors: ["none"]
+    inner_folds: 2
+    max_param_combos: 2
+    max_outer_splits_per_protocol: 2
+    clip_lower_quantile: 0.01
+    clip_upper_quantile: 0.99
+    random_seed: 42
+    torch_device: "auto"
+    baseline_from_tutorial_label: "baseline"
+  class_scenarios:
+    - name: "baseline_all_bins"
+    - name: "baseline_omit_hardest"
+      drop_labels: ["6.0-6.9"]
+    - name: "baseline_low_high_omit_hardest"
+      drop_labels: ["6.0-6.9"]
+      merge_map:
+        "0.6-1.5": "low_1_2_3"
+        "1.5-2.4": "low_1_2_3"
+        "2.4-3.3": "low_1_2_3"
+        "3.3-4.2": "high_4_5_6"
+        "4.2-5.1": "high_4_5_6"
+        "5.1-6.0": "high_4_5_6"
+    - name: "baseline_grouped_4class_omit_hardest"
+      drop_labels: ["6.0-6.9"]
+      merge_map:
+        "0.6-1.5": "low_1_2"
+        "1.5-2.4": "low_1_2"
+        "2.4-3.3": "mid_3_4"
+        "3.3-4.2": "mid_3_4"
+        "4.2-5.1": "high_5_6"
+        "5.1-6.0": "high_5_6"
+    - name: "baseline_omit_easiest"
+      drop_labels: ["0.6-1.5"]
+
+stage6_confusions:
+  out_json_template: "analysis_pipeline/reports/confusion_highlights_{scenario}_baseline.json"
+  out_md_template: "analysis_pipeline/reports/confusion_highlights_{scenario}_baseline.md"
+  args:
+    metric: "balanced_accuracy_mean"
+    top_k_per_protocol: 1
+    include_all: true
+    out_png_dir: "analysis_pipeline/reports/confusion_pngs"

analysis_pipeline/config/pipeline_class_variants.yaml

@@ -63,3 +63,5 @@ stage6_confusions:
   args:
     metric: "balanced_accuracy_mean"
     top_k_per_protocol: 1
+    include_all: true
+    out_png_dir: "analysis_pipeline/reports/confusion_pngs"

analysis_pipeline/config/pipeline_model_feature_sweep.yaml

@@ -41,3 +41,5 @@ stage6_confusions:
   args:
     metric: "balanced_accuracy_mean"
     top_k_per_protocol: 1
+    include_all: true
+    out_png_dir: "analysis_pipeline/reports/confusion_pngs"

analysis_pipeline/config/pipeline_with_deep_models.yaml

@@ -30,13 +30,14 @@ stage6:
   base_args:
     datasets: ["eeg", "ecg", "pupil", "fused"]
     protocols: ["loso", "group_holdout", "within_participant"]
-    models: ["logreg", "knn", "svm", "gaussian_nb", "decision_tree", "mlp", "rf", "lstm1d", "gru1d", "cnn1d", "transformer"]
+    models: ["lstm1d", "gru1d", "cnn1d", "transformer", "bilstm1d", "bigru1d", "cnn1d_deep", "transformer_xl"]
     inner_folds: 2
     max_param_combos: 2
     max_outer_splits_per_protocol: 2
     clip_lower_quantile: 0.01
     clip_upper_quantile: 0.99
     random_seed: 42
+    torch_device: "auto"
   class_scenarios:
     - name: "all_bins"
     - name: "omit_easiest"
@@ -57,3 +58,5 @@ stage6_confusions:
   args:
     metric: "balanced_accuracy_mean"
     top_k_per_protocol: 1
+    include_all: true
+    out_png_dir: "analysis_pipeline/reports/confusion_pngs"

analysis_pipeline/run_pipeline.py

@@ -7,6 +7,7 @@
 import shlex
 import subprocess
 import sys
+import time
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
@@ -414,6 +415,14 @@ def main() -> None:
     run_stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
     logs_dir = _reports_dir() / "run_logs" / run_stamp
     workdir = _analysis_root().parent
+    pipeline_start = time.time()
+    total_steps = len(steps)
+
+    print("Pipeline run starting.")
+    print(f"  Config: {config_path}")
+    print(f"  Working directory: {workdir}")
+    print(f"  Dry run: {bool(args.dry_run)}")
+    print(f"  Planned steps: {total_steps}")
 
     manifest: dict[str, Any] = {
         "pipeline_started_utc": _utc_now(),
@@ -433,12 +442,19 @@ def main() -> None:
     }
 
     try:
-        for step in steps:
-            print(f"[{step.stage}] {step.name}")
+        for step_index, step in enumerate(steps, start=1):
+            step_start = time.time()
+            print(f"[{step_index}/{total_steps}] [{step.stage}] {step.name}")
             print("  " + " ".join(shlex.quote(part) for part in step.command))
             step_result = _run_step(step=step, workdir=workdir, logs_dir=logs_dir, dry_run=args.dry_run)
             manifest["steps"].append(step_result)
+            elapsed_s = time.time() - step_start
+            print(f"  Status: {step_result['status']} (elapsed={elapsed_s:.1f}s)")
             if step_result["return_code"] != 0:
+                if step_result.get("stdout_log"):
+                    print(f"  stdout log: {step_result['stdout_log']}")
+                if step_result.get("stderr_log"):
+                    print(f"  stderr log: {step_result['stderr_log']}")
                 raise RuntimeError(f"Step failed: {step.name} (return_code={step_result['return_code']})")
         manifest["status"] = "dry_run" if args.dry_run else "success"
     except Exception as exc:  # noqa: BLE001
@@ -449,6 +465,8 @@ def main() -> None:
         manifest["pipeline_finished_utc"] = _utc_now()
         manifest_out.parent.mkdir(parents=True, exist_ok=True)
         manifest_out.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+        print(f"Pipeline status: {manifest['status']}")
+        print(f"Pipeline elapsed seconds: {time.time() - pipeline_start:.1f}")
         print(f"Run manifest: {manifest_out}")
 
 

analysis_pipeline/stage1_qc_summary.py

@@ -988,12 +988,17 @@ def main() -> None:
     subject_dirs = sorted(path for path in bids_root.glob("sub-*") if path.is_dir())
     if not subject_dirs:
         raise FileNotFoundError(f"No subject directories found in {bids_root}")
+    print(
+        f"Stage 1 starting. task={task} subjects={len(subject_dirs)} "
+        f"trial_rows={len(trial_rows)}"
+    )
 
     subject_rows: list[dict[str, Any]] = []
     all_anomalies: list[str] = []
-    for subject_dir in subject_dirs:
+    for subject_idx, subject_dir in enumerate(subject_dirs, start=1):
         paths = _resolve_subject_paths(subject_dir, task)
         subject = paths.subject
+        print(f"[Subject {subject_idx}/{len(subject_dirs)}] {subject}")
         analysis_included = (
             (participants.get(subject, {}).get("analysis_included") or "n/a").strip().lower()
         )

analysis_pipeline/stage2_preprocess.py

@@ -700,9 +700,13 @@ def main() -> None:
         subject_dirs = [path for path in subject_dirs if path.name in wanted]
     if not subject_dirs:
         raise FileNotFoundError("No matching subject directories found.")
+    print(
+        f"Stage 2 starting. task={task} subjects={len(subject_dirs)} "
+        f"overwrite={bool(args.overwrite)}"
+    )
 
     logs: list[dict[str, str]] = []
-    for subject_dir in subject_dirs:
+    for subject_idx, subject_dir in enumerate(subject_dirs, start=1):
         paths = _resolve_subject_paths(subject_dir, task)
         analysis_included = (
             (participants.get(paths.subject, {}).get("analysis_included") or "n/a")
@@ -714,7 +718,8 @@ def main() -> None:
         log = _process_subject(paths, out_root, analysis_included, args)
         logs.append(log)
         print(
-            f"{paths.subject}: EEG={log['eeg_status']} ECG={log['ecg_status']} "
+            f"[Subject {subject_idx}/{len(subject_dirs)}] {paths.subject}: "
+            f"EEG={log['eeg_status']} ECG={log['ecg_status']} "
             f"Pupil={log['pupil_status']}"
         )