Fix some remaining references of 'policy' to 'generator'

Differential Revision: D89990890

Pull Request resolved: #692

Author: daniellepintz

apps/grpo/qwen3_1_7b.yaml

@@ -11,7 +11,7 @@ off_by_n: 1 # Off by one by default
 compile: true # Enable torch.compile for trainer/ref_model, and CUDA graphs for vLLM
 
 # Main loop configuration
-rollout_threads: 1   # Recommended to set equal to policy.num_replicas
+rollout_threads: 1   # Recommended to set equal to generator.num_replicas
 
 
 # Observability configuration

benchmarks/generator/throughput.py

@@ -105,10 +105,12 @@ async def run_throughput_benchmark(
         )
 
     print("Spawning Generator service...")
-    generator = await Generator.options(**cfg.services.policy).as_service(**cfg.policy)
+    generator = await Generator.options(**cfg.services.generator).as_service(
+        **cfg.generator
+    )
 
     print(f"Generating {num_requests} benchmark requests...")
-    model_name = cfg.policy.engine_args.get("model", "unknown")
+    model_name = cfg.generator.engine_args.get("model", "unknown")
     tokenizer = get_tokenizer(
         model_name,
         tokenizer_mode="auto",

docs/source/index.md

@@ -100,12 +100,12 @@ Before diving in, check out {doc}`getting_started` and ensure your system meets
 With TorchForge, your RL logic looks like pseudocode:
 
 ```python
-async def generate_episode(dataloader, policy, reward, replay_buffer):
+async def generate_episode(dataloader, generator, reward, replay_buffer):
     # Sample a prompt
     prompt, target = await dataloader.sample.route()
 
     # Generate response
-    response = await policy.generate.route(prompt)
+    response = await generator.generate.route(prompt)
 
     # Score the response
     reward_value = await reward.evaluate_response.route(

docs/source/tutorial_sources/zero-to-forge/1_RL_and_Forge_Fundamentals.md

@@ -37,7 +37,7 @@ graph TD
 ### RL Components Defined (TorchForge Names)
 
 1. **Dataset**: Provides questions/prompts (like "What is 2+2?")
-2. **Policy**: The AI being trained (generates answers like "The answer is 4")
+2. **Generator**: The policy being trained (generates answers like "The answer is 4")
 3. **Reward Model**: Evaluates answer quality (gives scores like 0.95)
 4. **Reference Model**: Original policy copy (prevents drift from baseline)
 5. **Replay Buffer**: Stores experiences (question + answer + score)
@@ -53,7 +53,7 @@ def conceptual_rl_step():
     question = dataset.sample()  # "What is 2+2?"
 
     # 2. Student generates answer
-    answer = policy.generate(question)  # "The answer is 4"
+    answer = generator.generate(question)  # "The answer is 4"
 
     # 3. Teacher grades it
     score = reward_model.evaluate(question, answer)  # 0.95
@@ -289,7 +289,7 @@ async def real_rl_training_step(services, step):
 
 ### Automatic Resource Management
 ```python
-responses = await policy.generate.route(prompt=question)
+responses = await generator.generate.route(prompt=question)
 answer = responses[0].text  # responses is list[Completion]
 ```
 
@@ -333,7 +333,7 @@ group_size = 1
             model=model,
         ),
         # Policy service with GPU
-        Policy.options(procs=1, with_gpus=True, num_replicas=1).as_service(
+        Generator.options(procs=1, with_gpus=True, num_replicas=1).as_service(
             engine_config={
                 "model": model,
                 "tensor_parallel_size": 1,
@@ -381,15 +381,15 @@ TorchForge has two types of distributed components:
 - **Actors**: Single instances that handle their own internal distribution (like TitanTrainer, ReplayBuffer)
 
 We cover this distinction in detail in Part 2, but for now this explains the scaling patterns:
-- Policy service: num_replicas=8 for high inference demand
+- Generator service: num_replicas=8 for high inference demand
 - RewardActor service: num_replicas=16 for parallel evaluation
 - TitanTrainer actor: Single instance with internal distributed training
 
 
 ### Fault Tolerance
 ```python
-# If a policy replica fails:
-responses = await policy.generate.route(prompt=question)
+# If a generator replica fails:
+responses = await generator.generate.route(prompt=question)
 answer = responses[0].text
 # -> TorchForge automatically routes to healthy replica
 # -> Failed replica respawns in background

tests/integration_tests/test_vllm_policy_correctness.py

@@ -6,7 +6,7 @@
 
 import pytest
 
-from forge.actors.generator import Generator as Policy
+from forge.actors.generator import Generator
 from vllm import SamplingParams
 from vllm.engine.arg_utils import AsyncEngineArgs
 from vllm.sampling_params import RequestOutputKind
@@ -30,15 +30,15 @@
 
 @pytest.mark.asyncio
 async def test_same_output():
-    """Compare outputs between vLLM and Policy service"""
+    """Compare outputs between vLLM and Generator service"""
     test_prompts = [
         "Hello, how are you?",
         "What is 2+2?",
         "Tell me a joke.",
         "Explain machine learning briefly.",
         "What color is the sky?",
     ]
-    policy = None
+    generator = None
     try:
         # Setup vLLM directly
         args = AsyncEngineArgs(
@@ -50,8 +50,8 @@ async def test_same_output():
         )
         vllm_model = AsyncLLM.from_engine_args(args)
 
-        # Setup Policy service
-        policy = await Policy.options(
+        # Setup Generator service
+        generator = await Generator.options(
             procs=1, num_replicas=1, with_gpus=True
         ).as_service(
             engine_args={
@@ -72,7 +72,7 @@ async def test_same_output():
 
         print("Models ready. Generating outputs...\n")
         vllm_outputs = []
-        policy_outputs = []
+        generator_outputs = []
         sampling_params = SamplingParams(
             max_tokens=MAX_TOKENS,
             temperature=TEMPERATURE,
@@ -89,19 +89,19 @@ async def test_same_output():
                 vllm_outputs.append(res.outputs[0].text)
 
             # Policy generation
-            policy_result = await policy.generate.route(prompt)
+            policy_result = await generator.generate.route(prompt)
             policy_text = policy_result[0].text
-            policy_outputs.append(policy_text)
+            generator_outputs.append(policy_text)
 
         # Final check
-        for vllm_output, policy_output in zip(vllm_outputs, policy_outputs):
+        for vllm_output, generator_output in zip(vllm_outputs, generator_outputs):
             assert vllm_output != ""
-            assert policy_output != ""
-            assert vllm_output == policy_output
+            assert generator_output != ""
+            assert vllm_output == generator_output
 
     finally:
-        if policy is not None:
-            await policy.shutdown()
+        if generator is not None:
+            await generator.shutdown()
 
 
 @pytest.mark.asyncio
@@ -126,7 +126,7 @@ async def test_cache_usage():
     via the AsyncLLM interface.
     - We do not test different different block sizes.
     """
-    policy = None
+    generator = None
     try:
         # Setup vLLM directly
         args = AsyncEngineArgs(
@@ -139,8 +139,8 @@ async def test_cache_usage():
         )
         vllm_model = AsyncLLM.from_engine_args(args)
 
-        # Setup Policy service
-        policy = await Policy.options(
+        # Setup Generator service
+        generator = await Generator.options(
             procs=1, num_replicas=1, with_gpus=True
         ).as_service(
             engine_args={
@@ -170,7 +170,7 @@ async def test_cache_usage():
             output_kind=RequestOutputKind.FINAL_ONLY,
         )
         vllm_outputs = []
-        policy_outputs = []
+        generator_outputs = []
 
         # Exactly 16 tokens to fill up 1 block
         first_prompt = (
@@ -182,9 +182,9 @@ async def test_cache_usage():
         ):
             vllm_outputs.append(res.outputs[0].text)
             assert res.num_cached_tokens == expected_cached_tokens
-        res = await policy.generate.route(first_prompt)
+        res = await generator.generate.route(first_prompt)
         assert res[0].metadata["num_cached_tokens"] == expected_cached_tokens
-        policy_outputs.append(res[0].text)
+        generator_outputs.append(res[0].text)
 
         # Another 16 tokens to now populate 2 blocks (+ reuse the first block)
         second_prompt = (
@@ -197,9 +197,9 @@ async def test_cache_usage():
         ):
             vllm_outputs.append(res.outputs[0].text)
             assert res.num_cached_tokens == expected_cached_tokens
-        res = await policy.generate.route(second_prompt)
+        res = await generator.generate.route(second_prompt)
         assert res[0].metadata["num_cached_tokens"] == expected_cached_tokens
-        policy_outputs.append(res[0].text)
+        generator_outputs.append(res[0].text)
 
         # The first same 32 tokens should now be populated in blocks
         third_prompt = second_prompt
@@ -209,13 +209,13 @@ async def test_cache_usage():
         ):
             vllm_outputs.append(res.outputs[0].text)
             assert res.num_cached_tokens == expected_cached_tokens
-        res = await policy.generate.route(third_prompt)
+        res = await generator.generate.route(third_prompt)
         assert res[0].metadata["num_cached_tokens"] == expected_cached_tokens
-        policy_outputs.append(res[0].text)
+        generator_outputs.append(res[0].text)
 
         # Now, let's clear the cache
         await vllm_model.reset_prefix_cache()
-        await policy._reset_prefix_cache.route()
+        await generator._reset_prefix_cache.route()
 
         # And try the third prompt again (should not use any cached tokens)
         expected_cached_tokens = 0
@@ -224,16 +224,16 @@ async def test_cache_usage():
         ):
             vllm_outputs.append(res.outputs[0].text)
             assert res.num_cached_tokens == expected_cached_tokens
-        res = await policy.generate.route(third_prompt)
+        res = await generator.generate.route(third_prompt)
         assert res[0].metadata["num_cached_tokens"] == expected_cached_tokens
-        policy_outputs.append(res[0].text)
+        generator_outputs.append(res[0].text)
 
         # Sanity check that outputs are still the same
-        for vllm_output, policy_output in zip(vllm_outputs, policy_outputs):
+        for vllm_output, generator_output in zip(vllm_outputs, generator_outputs):
             assert vllm_output != ""
-            assert policy_output != ""
-            assert vllm_output == policy_output
+            assert generator_output != ""
+            assert vllm_output == generator_output
 
     finally:
-        if policy is not None:
-            await policy.shutdown()
+        if generator is not None:
+            await generator.shutdown()

tests/sandbox/weight_sync/main.py

@@ -130,14 +130,14 @@ async def main(cfg: DictConfig):
     print("Initializing trainer and generator...")
     init_start = time.time()
 
-    trainer, policy = await asyncio.gather(
+    trainer, generator = await asyncio.gather(
         RLTrainer.options(**cfg.actors.trainer).as_actor(
             **cfg.trainer,
             loss=lambda *args, **kwargs: torch.tensor(
                 1.0, requires_grad=True, device="cuda"
             ),
         ),
-        Generator.options(**cfg.actors.policy).as_actor(**cfg.policy),
+        Generator.options(**cfg.actors.generator).as_actor(**cfg.generator),
     )
 
     init_time = time.time() - init_start
@@ -172,7 +172,7 @@ async def main(cfg: DictConfig):
     print("Updating generator weights from store...")
     update_start = time.time()
 
-    await policy.update_weights.call(version=1)
+    await generator.update_weights.call(version=1)
 
     update_time = time.time() - update_start
     print(f"Updated generator weights ({update_time:.2f}s)\n")