feat: always-on UUID log directories with symlink

Logs are now always written to output_dir/runs/<uuid>/ with a
output_dir/logs symlink pointing to the current run's directory.
This isolates logs per run while preserving backward compatibility
through the symlink — all existing code that reads
get_log_dir(output_dir) / trainer.log continues to work unchanged.

When resuming (ckpt.resume_step is set), the existing log directory
is reused so logs append to the same run. On clean_output_dir, the
entire output_dir is wiped (existing behavior), so a fresh UUID is
generated.

No config changes needed — the behavior is transparent. SLURM
templates also work as-is since /logs resolves through
the symlink created by the launcher before sbatch.

Author: samsja

src/prime_rl/entrypoints/inference.py

@@ -10,7 +10,7 @@
 from prime_rl.configs.inference import InferenceConfig
 from prime_rl.utils.config import cli, to_toml_dict
 from prime_rl.utils.logger import setup_logger
-from prime_rl.utils.pathing import format_log_message, get_config_dir, get_log_dir
+from prime_rl.utils.pathing import format_log_message, get_config_dir, get_log_dir, setup_log_dir
 from prime_rl.utils.process import DEFAULT_COMMON_ENV_VARS, DEFAULT_INFERENCE_ENV_VARS, set_proc_title
 
 INFERENCE_TOML = "inference.toml"
@@ -168,6 +168,8 @@ def inference_local(config: InferenceConfig):
 
 
 def inference(config: InferenceConfig):
+    config.output_dir.mkdir(parents=True, exist_ok=True)
+    setup_log_dir(config.output_dir)
     if config.slurm is not None:
         inference_slurm(config)
     else:

src/prime_rl/entrypoints/rl.py

@@ -24,6 +24,7 @@
     get_ckpt_dir,
     get_log_dir,
     resolve_latest_ckpt_step,
+    setup_log_dir,
     validate_output_dir,
 )
 from prime_rl.utils.process import (
@@ -535,6 +536,10 @@ def rl(config: RLConfig):
     if ckpt_output_dir is not None:
         ckpt_output_dir.mkdir(parents=True, exist_ok=True)
 
+    # Create a UUID-isolated log directory and symlink output_dir/logs to it.
+    # When resuming, reuse the existing run's log directory.
+    setup_log_dir(config.output_dir, resuming=resuming)
+
     # Clean stale rollouts and broadcasts. When resuming, anything past the resume
     # step is stale. When training from scratch, every existing step directory is
     # stale — without this, a fresh run in a dirty output_dir would pick up rollouts

src/prime_rl/entrypoints/sft.py

@@ -11,7 +11,13 @@
 from prime_rl.configs.sft import SFTConfig
 from prime_rl.utils.config import cli, to_toml_dict
 from prime_rl.utils.logger import setup_logger
-from prime_rl.utils.pathing import format_log_message, get_config_dir, get_log_dir, validate_output_dir
+from prime_rl.utils.pathing import (
+    format_log_message,
+    get_config_dir,
+    get_log_dir,
+    setup_log_dir,
+    validate_output_dir,
+)
 from prime_rl.utils.process import (
     DEFAULT_COMMON_ENV_VARS,
     DEFAULT_TRAINER_ENV_VARS,
@@ -122,7 +128,7 @@ def sft_local(config: SFTConfig):
         logger.success("Dry run complete. To start an SFT run locally, remove --dry-run from your command.")
         return
 
-    log_dir = config.output_dir / "logs"
+    log_dir = get_log_dir(config.output_dir)
     log_dir.mkdir(parents=True, exist_ok=True)
 
     from prime_rl.utils.utils import get_free_port
@@ -132,7 +138,7 @@ def sft_local(config: SFTConfig):
         "--role=trainer",
         f"--rdzv-endpoint=localhost:{get_free_port()}",
         f"--rdzv-id={uuid.uuid4().hex}",
-        f"--log-dir={config.output_dir / 'logs' / 'trainer' / 'torchrun'}",
+        f"--log-dir={log_dir / 'trainer' / 'torchrun'}",
         f"--local-ranks-filter={','.join(map(str, config.log.ranks_filter))}",
         "--redirect=3",
         "--tee=3",
@@ -211,6 +217,10 @@ def sft(config: SFTConfig):
     validate_output_dir(config.output_dir, resuming=resuming, clean=clean)
     config.output_dir.mkdir(parents=True, exist_ok=True)
 
+    # Create a UUID-isolated log directory and symlink output_dir/logs to it.
+    # When resuming, reuse the existing run's log directory.
+    setup_log_dir(config.output_dir, resuming=resuming)
+
     if not config.dry_run:
         from prime_rl.trainer.model import pre_download_model
 

src/prime_rl/utils/pathing.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 import shutil
 import time
 from pathlib import Path
@@ -7,9 +8,57 @@
 
 
 def get_log_dir(output_dir: Path) -> Path:
+    """Return the log directory for ``output_dir``.
+
+    ``logs`` is a symlink to ``runs/<uuid>/`` (see :func:`setup_log_dir`).
+    Falls back to the plain ``output_dir / "logs"`` directory when no
+    symlink exists yet (e.g. in unit tests that don't call ``setup_log_dir``).
+    """
     return output_dir / "logs"
 
 
+def setup_log_dir(output_dir: Path, *, resuming: bool = False) -> Path:
+    """Create a per-run UUID log directory and symlink ``logs`` to it.
+
+    ``output_dir/runs/<uuid>/`` holds the actual log files for this run.
+    ``output_dir/logs`` is a symlink pointing there, so existing code that
+    reads ``get_log_dir(output_dir) / "trainer.log"`` continues to work.
+
+    When *resuming* and the symlink already exists, the existing run
+    directory is reused so logs from the resumed run are appended to the
+    same directory.
+
+    Returns the resolved log directory (the symlink path, i.e. what
+    ``get_log_dir`` would return).
+    """
+    import uuid
+
+    logger = get_logger()
+    logs_link = output_dir / "logs"
+    runs_dir = output_dir / "runs"
+
+    # Reuse existing run directory when resuming
+    if resuming and logs_link.is_symlink():
+        target = logs_link.resolve()
+        logger.debug(f"Reusing existing log directory: {target}")
+        return logs_link
+
+    runs_dir.mkdir(parents=True, exist_ok=True)
+    run_uuid = uuid.uuid4().hex
+    run_dir = runs_dir / run_uuid
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    # Atomically update the symlink: create a temp link then rename
+    tmp_link = output_dir / f".logs.tmp.{run_uuid}"
+    if tmp_link.exists() or tmp_link.is_symlink():
+        tmp_link.unlink()
+    os.symlink(run_dir, tmp_link)
+    os.replace(tmp_link, logs_link)
+
+    logger.debug(f"Created log directory: {run_dir} (symlinked at {logs_link})")
+    return logs_link
+
+
 def format_log_message(
     log_dir: Path,
     trainer: bool = False,

tests/unit/utils/test_pathing.py

@@ -1,6 +1,6 @@
 import pytest
 
-from prime_rl.utils.pathing import validate_output_dir
+from prime_rl.utils.pathing import get_log_dir, setup_log_dir, validate_output_dir
 
 
 def test_nonexistent_dir_passes(tmp_path):
@@ -55,3 +55,94 @@ def test_clean_on_nonexistent_dir_is_noop(tmp_path):
     output_dir = tmp_path / "does_not_exist"
     validate_output_dir(output_dir, resuming=False, clean=True)
     assert not output_dir.exists()
+
+
+def test_setup_log_dir_creates_symlink(tmp_path):
+    """setup_log_dir creates runs/<uuid>/ and symlinks logs to it."""
+    output_dir = tmp_path / "outputs"
+    output_dir.mkdir()
+
+    log_dir = setup_log_dir(output_dir)
+
+    assert log_dir == output_dir / "logs"
+    assert log_dir.is_symlink()
+    target = log_dir.resolve()
+    assert target.parent == (output_dir / "runs").resolve()
+    assert target.is_dir()
+    # The symlink target should be a UUID directory under runs/
+    assert target.parent.name == "runs"
+    assert len(target.name) == 32  # uuid4().hex length
+
+
+def test_setup_log_dir_writable_through_symlink(tmp_path):
+    """Files written through the symlink land in the UUID directory."""
+    output_dir = tmp_path / "outputs"
+    output_dir.mkdir()
+
+    log_dir = setup_log_dir(output_dir)
+    (log_dir / "trainer.log").write_text("test")
+
+    target = log_dir.resolve()
+    assert (target / "trainer.log").read_text() == "test"
+    assert (log_dir / "trainer.log").read_text() == "test"
+
+
+def test_setup_log_dir_creates_fresh_uuid_each_call(tmp_path):
+    """Each call to setup_log_dir creates a new UUID run directory."""
+    output_dir = tmp_path / "outputs"
+    output_dir.mkdir()
+
+    log_dir_1 = setup_log_dir(output_dir)
+    target_1 = log_dir_1.resolve()
+
+    # Write a file to the first run
+    (log_dir_1 / "trainer.log").write_text("run1")
+
+    log_dir_2 = setup_log_dir(output_dir)
+    target_2 = log_dir_2.resolve()
+
+    # New UUID directory
+    assert target_1 != target_2
+    # Both directories exist
+    assert target_1.is_dir()
+    assert target_2.is_dir()
+    # Symlink now points to the second run
+    assert log_dir_2.resolve() == target_2
+    # First run's files are preserved
+    assert (target_1 / "trainer.log").read_text() == "run1"
+
+
+def test_setup_log_dir_resuming_reuses_existing(tmp_path):
+    """When resuming and symlink exists, reuse the existing run directory."""
+    output_dir = tmp_path / "outputs"
+    output_dir.mkdir()
+
+    log_dir_1 = setup_log_dir(output_dir)
+    target_1 = log_dir_1.resolve()
+    (log_dir_1 / "trainer.log").write_text("run1")
+
+    # Resume should reuse the same directory
+    log_dir_2 = setup_log_dir(output_dir, resuming=True)
+    assert log_dir_2.resolve() == target_1
+    assert (log_dir_2 / "trainer.log").read_text() == "run1"
+
+
+def test_setup_log_dir_resuming_without_symlink_creates_new(tmp_path):
+    """When resuming but no symlink exists, create a new run directory."""
+    output_dir = tmp_path / "outputs"
+    output_dir.mkdir()
+
+    log_dir = setup_log_dir(output_dir, resuming=True)
+    assert log_dir.is_symlink()
+    assert log_dir.resolve().is_dir()
+
+
+def test_get_log_dir_returns_symlink_path(tmp_path):
+    """get_log_dir returns output_dir/logs (the symlink path)."""
+    output_dir = tmp_path / "outputs"
+    output_dir.mkdir()
+
+    setup_log_dir(output_dir)
+    log_dir = get_log_dir(output_dir)
+    assert log_dir == output_dir / "logs"
+    assert log_dir.is_symlink()