Add LibriBrain100 — unified loader for LibriBrain + LibriBrain2

LibriBrain100 is the user-facing dataset for the full LibriBrain
release: a virtual union of pnpl/LibriBrain (sub-0 × Sherlock1..7) and
pnpl/LibriBrain2 (sub-0 × Sherlock8/Sherlock9/TIMIT/MOCHATIMIT/
Podcasts, plus sub-1..32 × Sherlock1 ses-11/ses-12). Manifest-driven,
keeps the existing task-based API, adds two selectors:

  - subjects=  "all"|"deep"|"new"|0|"sub-0"|[1,2,3]|range(1, 33)
  - corpus=    "all"|"sherlock"|"timit"|"mocha"|"podcasts"|[...]

Implementation notes:

  - manifest.py declares one RunRecord per (subject, session, task,
    run) tuple expected by the paper, tagging each with corpus, repo,
    and partition. Records the upload hasn't reached yet are tolerated
    at runtime — the loader skips missing H5s with a single grouped
    warning, so the API is usable while LibriBrain2 fills in.
  - selectors.py validates obvious dead-end combinations up front
    (e.g. subjects='new' has only Sherlock data and only val/test by
    design — the paper assigns no train partition to broad subjects).
  - base.py overrides _schedule_download to pick the per-record
    primary HF repo, so most fetches hit on the first try; the
    inherited HFDownloadMixin retry chain falls back to the other
    repo. Catches huggingface_hub.errors.{Entry,Repository}NotFound
    in addition to FileNotFoundError so missing-but-coming records
    don't crash through the download stack.
  - LibriBrain100 + LibriBrain100Speech/Phoneme/Word wrappers are
    exposed at pnpl.datasets; existing LibriBrain* classes
    are unchanged.

Tests:

  - 25 rudimentary tests in tests/test_libribrain100.py (selector
    normalization + manifest invariants + validation rules + the
    download-disabled error path).
  - A comprehensive on-VM suite at /workspace/libribrain100-tests/
    (60 tests, including live HF tree fingerprints, multi-repo
    download, full-DataLoader walks) — not in this repo.

Verified end-to-end on the validation VM: Sherlock1 ses-11 download +
sample materialization, MOCHATIMIT ses-1 download (different repo),
graceful skip when a corpus is not yet uploaded.

Author: gereonelvers

.gitignore

@@ -3,4 +3,8 @@
 *.egg-info/
 scripts/*
 .DS_Store
-.idea
+.idea
+.claude/
+
+# Dataset paper draft — local only.
+/libribrain100-paper/

pnpl/datasets/__init__.py

@@ -20,6 +20,12 @@
     "LibriBrainWord": ("pnpl.datasets.libribrain2025.compat", "LibriBrainWord"),
     "LibriBrainSentence": ("pnpl.datasets.libribrain2025.sentence_dataset", "LibriBrainSentence"),
 
+    # LibriBrain100 — union of pnpl/LibriBrain + pnpl/LibriBrain2
+    "LibriBrain100": ("pnpl.datasets.libribrain100.dataset", "LibriBrain100"),
+    "LibriBrain100Speech": ("pnpl.datasets.libribrain100.compat", "LibriBrain100Speech"),
+    "LibriBrain100Phoneme": ("pnpl.datasets.libribrain100.compat", "LibriBrain100Phoneme"),
+    "LibriBrain100Word": ("pnpl.datasets.libribrain100.compat", "LibriBrain100Word"),
+
     # MEG-MASC (Gwilliams et al., 2022) — auto-downloads from OSF
     "Gwilliams2022": ("pnpl.datasets.gwilliams2022.dataset", "Gwilliams2022"),
 

pnpl/datasets/libribrain100/__init__.py

@@ -0,0 +1,88 @@
+"""
+LibriBrain100 — unified loader for LibriBrain + LibriBrain2.
+
+LibriBrain100 is the user-facing PyTorch Dataset wrapping the full
+LibriBrain release: a virtual union of the original
+``pnpl/LibriBrain`` Hugging Face dataset and the
+``pnpl/LibriBrain2`` extension. The deep component (~80h, sub-0)
+covers the entire Sherlock Holmes canon plus TIMIT, MOCHA-TIMIT, and
+30 'The Moth' podcasts; the broad component (~44 min × 32 subjects)
+covers Sherlock book 1 chapters 11 and 12.
+
+Public API:
+
+    from pnpl.datasets import (
+        LibriBrain100,
+        LibriBrain100Speech,
+        LibriBrain100Phoneme,
+        LibriBrain100Word,
+    )
+"""
+
+from __future__ import annotations
+
+from .compat import LibriBrain100Phoneme, LibriBrain100Speech, LibriBrain100Word
+from .constants import (
+    CORPORA,
+    CORPUS_MOCHA,
+    CORPUS_PODCASTS,
+    CORPUS_SHERLOCK,
+    CORPUS_TIMIT,
+    DEEP_SUBJECT,
+    NEW_SUBJECTS,
+    PARTITION_TEST,
+    PARTITION_TRAIN,
+    PARTITION_VALIDATION,
+    PARTITIONS,
+    SUBJECTS,
+)
+from .dataset import LibriBrain100
+from .manifest import (
+    RUN_KEYS,
+    RUN_RECORDS,
+    RunRecord,
+    TEST_RUN_KEYS,
+    VALIDATION_RUN_KEYS,
+    get_record,
+    select_records,
+)
+from .selectors import (
+    normalize_corpus,
+    normalize_partition,
+    normalize_subjects,
+    validate_selector_combination,
+)
+
+__all__ = [
+    # Dataset classes
+    "LibriBrain100",
+    "LibriBrain100Speech",
+    "LibriBrain100Phoneme",
+    "LibriBrain100Word",
+    # Manifest
+    "RunRecord",
+    "RUN_RECORDS",
+    "RUN_KEYS",
+    "VALIDATION_RUN_KEYS",
+    "TEST_RUN_KEYS",
+    "get_record",
+    "select_records",
+    # Selectors / normalisation
+    "normalize_subjects",
+    "normalize_corpus",
+    "normalize_partition",
+    "validate_selector_combination",
+    # Constants
+    "DEEP_SUBJECT",
+    "SUBJECTS",
+    "NEW_SUBJECTS",
+    "CORPORA",
+    "CORPUS_SHERLOCK",
+    "CORPUS_TIMIT",
+    "CORPUS_MOCHA",
+    "CORPUS_PODCASTS",
+    "PARTITIONS",
+    "PARTITION_TRAIN",
+    "PARTITION_VALIDATION",
+    "PARTITION_TEST",
+]