|
53 | 53 | ) |
54 | 54 | from param_decomp_lab.experiments.lm.three_pool_run import ( |
55 | 55 | THREE_POOL_SLURM_ENV, |
56 | | - ThreePoolRuntimeConfig, |
57 | 56 | _agree_on_run_id, |
58 | 57 | _install_first_fail_marker, |
| 58 | + _maybe_build_torch_profiler, |
59 | 59 | _maybe_enable_memory_profile, |
60 | 60 | ) |
61 | 61 | from param_decomp_lab.experiments.utils import ( |
|
84 | 84 | ) |
85 | 85 | from param_decomp_lab.run_sink import ThreePoolSink |
86 | 86 | from param_decomp_lab.seed import set_seed |
| 87 | +from param_decomp_lab.three_pool.config import PooledRuntimeConfig |
87 | 88 | from param_decomp_lab.three_pool.consolidate import SNAPSHOT_SCRATCH_DIRNAME, ppgd_shard_dirname |
88 | 89 | from param_decomp_lab.three_pool.pd_config import ThreePoolConstrainedPDConfig |
89 | 90 | from param_decomp_lab.three_pool.two_pool_config import TwoPoolTopology |
90 | 91 | from param_decomp_lab.three_pool.two_pool_optimize import TwoPoolTrainer |
91 | 92 |
|
92 | 93 |
|
93 | | -class TwoPoolRuntimeConfig(ThreePoolRuntimeConfig): |
94 | | - """Core's substrate scalars + a 2-pool ``topology``. Subclasses |
95 | | - ``ThreePoolRuntimeConfig`` only to narrow the topology field's type.""" |
96 | | - |
97 | | - topology: TwoPoolTopology # pyright: ignore[reportIncompatibleVariableOverride] |
| 94 | +class TwoPoolRuntimeConfig(PooledRuntimeConfig): |
| 95 | + topology: TwoPoolTopology |
98 | 96 |
|
99 | 97 |
|
100 | 98 | class TwoPoolLMExperimentConfig(BaseConfig): |
@@ -221,12 +219,13 @@ def main( |
221 | 219 | ) |
222 | 220 | return |
223 | 221 |
|
224 | | - if resume is not None: |
225 | | - assert config_path is None, "pass either config_path or --resume, not both" |
226 | | - _resume_main(Path(resume), group=group, tags=tags, run_id=run_id) |
227 | | - else: |
228 | | - assert config_path is not None, "must provide either config_path or --resume" |
229 | | - _fresh_or_requeue_main(Path(config_path), group=group, tags=tags, run_id=run_id) |
| 222 | + match (resume, config_path): |
| 223 | + case (None, str(config_path)): |
| 224 | + _fresh_or_requeue_main(Path(config_path), group=group, tags=tags, run_id=run_id) |
| 225 | + case (str(resume), None): |
| 226 | + _resume_main(Path(resume), group=group, tags=tags, run_id=run_id) |
| 227 | + case _: |
| 228 | + raise ValueError("must provide either config_path or --resume") |
230 | 229 |
|
231 | 230 |
|
232 | 231 | def _fresh_or_requeue_main( |
@@ -318,7 +317,14 @@ def _fresh_main( |
318 | 317 | runtime_config=cfg.runtime, |
319 | 318 | two_pool_config=cfg.runtime.topology, |
320 | 319 | ) |
321 | | - trainer.run(train_loader, sink, cfg.cadence, scratch_dir=scratch_dir, eval_loop=eval_loop) |
| 320 | + trainer.run( |
| 321 | + train_loader, |
| 322 | + sink, |
| 323 | + cfg.cadence, |
| 324 | + scratch_dir=scratch_dir, |
| 325 | + eval_loop=eval_loop, |
| 326 | + profiler=_maybe_build_torch_profiler(trainer), |
| 327 | + ) |
322 | 328 | finally: |
323 | 329 | sink.finish() |
324 | 330 |
|
@@ -449,7 +455,12 @@ def _run_resume( |
449 | 455 | ppgd_shard_dir=from_run / ppgd_shard_dirname(resolved_step), |
450 | 456 | ) |
451 | 457 | trainer.run( |
452 | | - train_loader, sink, effective_cfg.cadence, scratch_dir=scratch_dir, eval_loop=eval_loop |
| 458 | + train_loader, |
| 459 | + sink, |
| 460 | + effective_cfg.cadence, |
| 461 | + scratch_dir=scratch_dir, |
| 462 | + eval_loop=eval_loop, |
| 463 | + profiler=_maybe_build_torch_profiler(trainer), |
453 | 464 | ) |
454 | 465 | finally: |
455 | 466 | sink.finish() |
|
0 commit comments