feat: algorithm abstraction — named algorithm classes + inline frozen-model references (grpo, opd, sft_distill, self_distill, echo)

configs/ci/integration/reverse_text_rl_opd/start.toml

@@ -1,10 +1,10 @@
-# Smoke test for the RL-entrypoint OPD (on-policy distillation) training mode.
-# The student inference deployment is launched by the rl entrypoint on GPU 0;
-# the teacher inference server is started externally by the test fixture (see
+# Smoke test for the RL-entrypoint opd algorithm (on-policy distillation).
+# The policy inference deployment is launched by the rl entrypoint on GPU 0;
+# the frozen reference server is started externally by the test fixture (see
 # tests/integration/test_reverse_text_rl_opd.py) on the same GPU. Trainer
 # takes GPU 1. Training config mirrors `reverse_text/start.toml`.
 
-max_steps = 5
+max_steps = 15
 seq_len = 2048
 
 [model]
@@ -15,10 +15,16 @@ project = "reverse-text-ci"
 name = "ci-rl-opd"
 
 [orchestrator]
-training_mode = "opd"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo]
+type = "opd"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
 [orchestrator.renderer]
 name = "qwen3"
 
@@ -38,12 +44,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 

configs/ci/integration/reverse_text_rl_sft/start.toml

@@ -1,10 +1,10 @@
-# Smoke test for the RL-entrypoint SFT (on-policy hard distillation) training
-# mode. The student inference deployment is launched by the rl entrypoint on
-# GPU 0; the teacher inference server is started externally by the test
-# fixture (see tests/integration/test_reverse_text_rl_sft.py) on the same GPU.
-# Trainer takes GPU 1. Training config mirrors `reverse_text/start.toml`.
+# Smoke test for the RL-entrypoint sft algorithm (hard distillation).
+# The policy inference deployment is launched by the rl entrypoint on GPU 0;
+# the frozen sampling server is started externally by the test fixture (see
+# tests/integration/test_reverse_text_rl_sft.py) on the same GPU. Trainer
+# takes GPU 1. Training config mirrors `reverse_text/start.toml`.
 
-max_steps = 5
+max_steps = 15
 seq_len = 2048
 
 [model]
@@ -15,13 +15,25 @@ project = "reverse-text-ci"
 name = "ci-rl-sft"
 
 [orchestrator]
-training_mode = "sft"
 batch_size = 128
 group_size = 16
 
-# Teacher rolls out over plain chat-completions (no tokens); the renderer backfills them.
+[orchestrator.algo]
+type = "sft"
+
+# sft's teacher IS the model it samples from.
+[orchestrator.algo.sampling.source]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
+# The student's renderer tokenizes the teacher's messages into the student's
+# token space (backfill). Use `default` (wraps the student tokenizer's
+# apply_chat_template) so the CE target is faithful to the student's actual
+# chat template. The stock `qwen3` renderer reimplements the Qwen format and
+# injects an empty `<think></think>` block, which this model's custom template
+# does not emit — corrupting the distillation target.
 [orchestrator.renderer]
-name = "qwen3"
+name = "default"
 
 [orchestrator.train.sampling]
 max_completion_tokens = 128
@@ -39,12 +51,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 

configs/debug/algorithms/README.md

@@ -0,0 +1,62 @@
+# Algorithm — Debug Configs
+
+Minimal end-to-end configs for the algorithms against bundled verifiers envs, using `PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT` as the policy.
+
+| Config | Algorithm | Frozen model | Notes |
+|---|---|---|---|
+| `grpo.toml` | `grpo` | none | |
+| `max_rl.toml` | `max_rl` | none | GRPO with mean-normalized advantages (maximum-likelihood RL) |
+| `opd.toml` | `opd` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | |
+| `opd_lora.toml` | `opd` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | trains a LoRA adapter (rank 8) |
+| `sft_distill.toml` | `sft` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | |
+| `sft_distill_lora.toml` | `sft` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | trains a LoRA adapter (rank 8) |
+| `self_distill.toml` | `opsd` | none (self-distills against the live policy) | SDFT; demo from reverse-text's `answer` field |
+| `echo.toml` | `echo` | none | multi-turn `alphabet-sort`; CE on observation tokens |
+| `mixed_grpo_opd.toml` | `grpo` + `opd` (per env) | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | two envs, one run; heterogeneous batches (with/without `ref_logprobs`) |
+
+The policy inference server is auto-launched on GPU 0 at `http://localhost:8000/v1` with `gpu_memory_utilization=0.5`. The local frozen model (used by `opd*.toml`, `sft_distill.toml` / `sft_distill_lora.toml`, and `mixed_grpo_opd.toml`) is **not** auto-launched — start it manually on GPU 1.
+
+Frozen models are declared inline on the algorithm, named where the model is used — `[orchestrator.algo.teacher]` for `opd`, `[orchestrator.algo.sampling.source]` for `sft` — with `name` + `base_url`. `opsd` declares none (it self-distills against the live policy). prime-rl never hosts them; only the trainable policy's server is managed by the `rl` entrypoint.
+
+## Start the local frozen model
+
+Needed for `opd*.toml`, `sft_distill.toml` / `sft_distill_lora.toml`, and `mixed_grpo_opd.toml`:
+
+```bash
+CUDA_VISIBLE_DEVICES=1 uv run inference \
+  --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
+  --server.port 8001 \
+  --gpu-memory-utilization 0.5 \
+  --model.enforce-eager
+```
+
+## Run the debug configs
+
+Every config writes to the default `outputs/` directory, so running two back-to-back — or re-running one — fails with `FileExistsError`. Pass a distinct `--output-dir outputs/<algo>` per config (recommended for a sweep) or `--clean-output-dir` to wipe and restart.
+
+```bash
+# GRPO (no frozen model)
+uv run rl @ configs/debug/algorithms/grpo.toml --output-dir outputs/grpo
+
+# MaxRL (no frozen model)
+uv run rl @ configs/debug/algorithms/max_rl.toml --output-dir outputs/max_rl
+
+# OPD (needs the frozen model on port 8001)
+uv run rl @ configs/debug/algorithms/opd.toml --output-dir outputs/opd
+uv run rl @ configs/debug/algorithms/opd_lora.toml --output-dir outputs/opd_lora
+
+# SFT distillation (needs the frozen model on port 8001)
+uv run rl @ configs/debug/algorithms/sft_distill.toml --output-dir outputs/sft_distill
+uv run rl @ configs/debug/algorithms/sft_distill_lora.toml --output-dir outputs/sft_distill_lora
+
+# Self-distillation against the live policy (no frozen model)
+uv run rl @ configs/debug/algorithms/self_distill.toml --output-dir outputs/self_distill
+
+# ECHO (no frozen model; multi-turn env)
+uv run rl @ configs/debug/algorithms/echo.toml --output-dir outputs/echo
+
+# Mixed per-env algorithms: GRPO + OPD in one run (needs the frozen model on port 8001)
+uv run rl @ configs/debug/algorithms/mixed_grpo_opd.toml --output-dir outputs/mixed_grpo_opd
+```
+
+See [docs/algorithms.md](../../../docs/algorithms.md) for what each algorithm does and how to author your own.

configs/debug/algorithms/echo.toml

@@ -0,0 +1,50 @@
+# ECHO on the multi-turn alphabet-sort env (bundled with verifiers): GRPO on
+# action tokens + weighted CE on the env's observation tokens.
+#   uv run rl @ configs/debug/algorithms/echo.toml
+
+max_steps = 20
+seq_len = 4096
+
+[model]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
+
+[wandb]
+project = "algorithms-debug"
+name = "debug-echo"
+
+[orchestrator]
+batch_size = 32
+group_size = 4
+
+# alphabet-sort's feedback arrives as user messages, so train the user role
+# instead of echo's tool default.
+[orchestrator.algo]
+type = "echo"
+
+[orchestrator.algo.roles.user]
+alpha = 0.1
+
+[[orchestrator.train.env]]
+id = "alphabet-sort"
+args = { min_turns = 3, max_turns = 5, power_per_turn = false }
+
+[orchestrator.train.sampling]
+max_completion_tokens = 512
+
+# ECHO learns from observation tokens even when the GRPO advantage collapses
+# to zero — keep zero-advantage rollouts in the batch.
+[[orchestrator.post_batch_filters]]
+type = "zero_advantage"
+enforce = false
+
+# Qwen3 finetune with the standard PI template patch; always re-emits prior
+# <think> blocks, matched by the qwen3 renderer's preserve_all_thinking.
+[orchestrator.renderer]
+name = "qwen3"
+preserve_all_thinking = true
+
+[trainer.optim]
+lr = 1e-6
+
+[inference]
+gpu_memory_utilization = 0.5

configs/debug/training_modes/rl.toml → configs/debug/algorithms/grpo.toml

@@ -5,14 +5,16 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-rl"
 
 [orchestrator]
-training_mode = "rl"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo]
+type = "grpo"
+
 [orchestrator.renderer]
 name = "qwen3"
 

configs/debug/algorithms/max_rl.toml

@@ -0,0 +1,43 @@
+max_steps = 20
+seq_len = 2048
+
+[model]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
+
+[wandb]
+project = "algorithms-debug"
+name = "debug-max-rl"
+
+[orchestrator]
+batch_size = 128
+group_size = 16
+
+[orchestrator.algo]
+type = "max_rl"
+
+[orchestrator.renderer]
+name = "qwen3"
+
+[orchestrator.train.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.train.env]]
+id = "reverse-text"
+
+[orchestrator.eval]
+interval = 1
+num_examples = 128
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.eval.env]]
+id = "reverse-text"
+
+[trainer.optim]
+lr = 3e-6
+
+[ckpt]
+
+[inference]
+gpu_memory_utilization = 0.5

configs/debug/training_modes/opd.toml → configs/debug/algorithms/mixed_grpo_opd.toml

@@ -1,9 +1,12 @@
-# Start the teacher inference server first (on a separate GPU):
+# Mixed per-env algorithms in one run: a GRPO env and an OPD env, both on
+# reverse-text. Exercises heterogeneous train batches — OPD samples ship
+# ref_logprobs, GRPO samples don't, and both pack into the same micro batches.
+# Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
 #     --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
 #     --server.port 8001 --gpu-memory-utilization 0.5 --model.enforce-eager
 # Then:
-#   uv run rl @ configs/debug/training_modes/opd.toml
+#   uv run rl @ configs/debug/algorithms/mixed_grpo_opd.toml
 
 max_steps = 20
 seq_len = 2048
@@ -12,14 +15,16 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
-name = "debug-opd"
+project = "algorithms-debug"
+name = "debug-mixed-grpo-opd"
 
 [orchestrator]
-training_mode = "opd"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo]
+type = "grpo"
+
 [orchestrator.renderer]
 name = "qwen3"
 
@@ -28,9 +33,21 @@ max_completion_tokens = 128
 
 [[orchestrator.train.env]]
 id = "reverse-text"
+name = "reverse-text-grpo"
+
+[[orchestrator.train.env]]
+id = "reverse-text"
+name = "reverse-text-opd"
+
+[orchestrator.train.env.algo]
+type = "opd"
+
+[orchestrator.train.env.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
 
 [orchestrator.eval]
-interval = 1
+interval = 5
 num_examples = 128
 
 [orchestrator.eval.sampling]
@@ -39,16 +56,8 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 
-[ckpt]
-
 [inference]
 gpu_memory_utilization = 0.5