Skip to content

Commit 6178b8e

Browse files
ocg-goodfireclaude
andcommitted
feat(2pool): wire torch profiler into the 2-pool run path
two_pool_run never passed a profiler to trainer.run (only the 3-pool did), so PD_TORCH_PROFILE_* was a no-op for 2-pool runs. Broaden _maybe_build_torch_profiler to accept either trainer (it only reads ctx.kind / ctx.role.rank) and pass it on both the fresh and resume run calls. Drop the now-unused ThreePoolRuntimeConfig import (TwoPoolRuntimeConfig subclasses PooledRuntimeConfig directly). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 29c169f commit 6178b8e

2 files changed

Lines changed: 30 additions & 15 deletions

File tree

param_decomp_lab/experiments/lm/three_pool_run.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
from param_decomp_lab.three_pool.config import PooledRuntimeConfig
9797
from param_decomp_lab.three_pool.consolidate import SNAPSHOT_SCRATCH_DIRNAME, ppgd_shard_dirname
9898
from param_decomp_lab.three_pool.pd_config import ThreePoolConstrainedPDConfig
99+
from param_decomp_lab.three_pool.two_pool_optimize import TwoPoolTrainer
99100

100101
# Cross-pool NCCL p2p deadlock guard. An asymmetric topology (fanout>1 on a cross-pool
101102
# edge) at long sequence lengths wedges when a rendezvous-blocked NCCL send/recv kernel
@@ -291,14 +292,17 @@ def _excepthook(exctype: type[BaseException], value: BaseException, tb: Any) ->
291292
sys.excepthook = _excepthook
292293

293294

294-
def _maybe_build_torch_profiler(trainer: ThreePoolTrainer) -> "torch.profiler.profile | None":
295+
def _maybe_build_torch_profiler(
296+
trainer: ThreePoolTrainer | TwoPoolTrainer,
297+
) -> "torch.profiler.profile | None":
295298
"""Opt-in `torch.profiler.profile` for the listed ranks (typically one per pool).
296299
297300
Env: `PD_TORCH_PROFILE_RANKS=0,96,100` + `PD_TORCH_PROFILE_OUT=/abs/dir`, plus
298301
schedule / instrumentation knobs (`PD_TORCH_PROFILE_SKIP_FIRST` default 20,
299302
`_ACTIVE` default 3, `_MEMORY` default on, `_STACK` / `_MODULES` / `_SHAPES` off).
300303
Returns the profile context (caller passes it to `trainer.run(profiler=...)`) or
301304
`None` if this rank isn't profiled. Verified at 104 ranks (gpt2-xl) 2026-05-28.
305+
Works for both the 3-pool and 2-pool trainers (only reads `ctx.kind` / `ctx.role.rank`).
302306
"""
303307
prof_ranks_env = os.environ.get("PD_TORCH_PROFILE_RANKS", "").strip()
304308
if not prof_ranks_env:

param_decomp_lab/experiments/lm/two_pool_run.py

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@
5353
)
5454
from param_decomp_lab.experiments.lm.three_pool_run import (
5555
THREE_POOL_SLURM_ENV,
56-
ThreePoolRuntimeConfig,
5756
_agree_on_run_id,
5857
_install_first_fail_marker,
58+
_maybe_build_torch_profiler,
5959
_maybe_enable_memory_profile,
6060
)
6161
from param_decomp_lab.experiments.utils import (
@@ -84,17 +84,15 @@
8484
)
8585
from param_decomp_lab.run_sink import ThreePoolSink
8686
from param_decomp_lab.seed import set_seed
87+
from param_decomp_lab.three_pool.config import PooledRuntimeConfig
8788
from param_decomp_lab.three_pool.consolidate import SNAPSHOT_SCRATCH_DIRNAME, ppgd_shard_dirname
8889
from param_decomp_lab.three_pool.pd_config import ThreePoolConstrainedPDConfig
8990
from param_decomp_lab.three_pool.two_pool_config import TwoPoolTopology
9091
from param_decomp_lab.three_pool.two_pool_optimize import TwoPoolTrainer
9192

9293

93-
class TwoPoolRuntimeConfig(ThreePoolRuntimeConfig):
94-
"""Core's substrate scalars + a 2-pool ``topology``. Subclasses
95-
``ThreePoolRuntimeConfig`` only to narrow the topology field's type."""
96-
97-
topology: TwoPoolTopology # pyright: ignore[reportIncompatibleVariableOverride]
94+
class TwoPoolRuntimeConfig(PooledRuntimeConfig):
95+
topology: TwoPoolTopology
9896

9997

10098
class TwoPoolLMExperimentConfig(BaseConfig):
@@ -221,12 +219,13 @@ def main(
221219
)
222220
return
223221

224-
if resume is not None:
225-
assert config_path is None, "pass either config_path or --resume, not both"
226-
_resume_main(Path(resume), group=group, tags=tags, run_id=run_id)
227-
else:
228-
assert config_path is not None, "must provide either config_path or --resume"
229-
_fresh_or_requeue_main(Path(config_path), group=group, tags=tags, run_id=run_id)
222+
match (resume, config_path):
223+
case (None, str(config_path)):
224+
_fresh_or_requeue_main(Path(config_path), group=group, tags=tags, run_id=run_id)
225+
case (str(resume), None):
226+
_resume_main(Path(resume), group=group, tags=tags, run_id=run_id)
227+
case _:
228+
raise ValueError("must provide either config_path or --resume")
230229

231230

232231
def _fresh_or_requeue_main(
@@ -318,7 +317,14 @@ def _fresh_main(
318317
runtime_config=cfg.runtime,
319318
two_pool_config=cfg.runtime.topology,
320319
)
321-
trainer.run(train_loader, sink, cfg.cadence, scratch_dir=scratch_dir, eval_loop=eval_loop)
320+
trainer.run(
321+
train_loader,
322+
sink,
323+
cfg.cadence,
324+
scratch_dir=scratch_dir,
325+
eval_loop=eval_loop,
326+
profiler=_maybe_build_torch_profiler(trainer),
327+
)
322328
finally:
323329
sink.finish()
324330

@@ -449,7 +455,12 @@ def _run_resume(
449455
ppgd_shard_dir=from_run / ppgd_shard_dirname(resolved_step),
450456
)
451457
trainer.run(
452-
train_loader, sink, effective_cfg.cadence, scratch_dir=scratch_dir, eval_loop=eval_loop
458+
train_loader,
459+
sink,
460+
effective_cfg.cadence,
461+
scratch_dir=scratch_dir,
462+
eval_loop=eval_loop,
463+
profiler=_maybe_build_torch_profiler(trainer),
453464
)
454465
finally:
455466
sink.finish()

0 commit comments

Comments
 (0)