updated more instances of seeding pattern

Author: cemde

docs/guides/seeding.md

@@ -38,6 +38,39 @@ results = benchmark.run(tasks, agent_data=config)
 
 This creates a `DefaultSeedGenerator` internally and passes it to all setup methods.
 
+### Disabling Seeding
+
+When you don't pass a `seed` parameter (or pass `seed=None`), seeding is disabled:
+
+```python
+# No seed = seeding disabled
+benchmark = MyBenchmark()
+
+# Explicit None = also disabled
+benchmark = MyBenchmark(seed=None)
+```
+
+When seeding is disabled:
+
+1. A `DefaultSeedGenerator(global_seed=None)` is still created internally
+2. Setup methods still receive a `seed_generator` parameter (it's never `None`)
+3. `derive_seed()` returns `None` instead of an integer
+4. This `None` flows directly to model adapters (which accept `Optional[int]`)
+
+This design simplifies setup method implementations—you don't need `if seed_generator is not None:` checks:
+
+```python
+def setup_agents(self, agent_data, environment, task, user, seed_generator):
+    # Always works - seed_generator is never None
+    agent_gen = seed_generator.child("agents")
+
+    # Returns None if seeding disabled, int if enabled
+    agent_seed = agent_gen.derive_seed("orchestrator")
+
+    # Model adapters accept Optional[int], so None works fine
+    model = self.get_model_adapter(model_id, seed=agent_seed)
+```
+
 ### Using Seeds in Setup Methods
 
 All setup methods receive a `seed_generator` parameter. Use it to derive seeds for your components. When seeding is disabled (no `seed` passed to benchmark), `derive_seed()` returns `None`:

examples/five_a_day_benchmark/five_a_day_benchmark.ipynb

@@ -523,7 +523,7 @@
    "id": "70c66cd0",
    "metadata": {},
    "outputs": [],
-   "source": "class FiveADayBenchmark(Benchmark):\n    \"\"\"5-A-Day benchmark with multi-agent support.\"\"\"\n\n    def setup_environment(self, agent_data: Dict[str, Any], task: Task, seed_generator: Optional[SeedGenerator] = None) -> Environment:\n        \"\"\"Create environment from task data.\"\"\"\n        task_data = {\n            \"environment_data\": task.environment_data,\n            \"query\": task.query,\n            \"evaluation_data\": task.evaluation_data,\n            \"metadata\": task.metadata,\n        }\n\n        environment = FiveADayEnvironment(task_data)\n\n        # Register all tools for tracing\n        for tool_name, tool_adapter in environment.get_tools().items():\n            self.register(\"tools\", tool_name, tool_adapter)\n\n        return environment\n\n    def setup_agents(\n        self,\n        agent_data: Dict[str, Any],\n        environment: Environment,\n        task: Task,\n        user=None,\n        seed_generator: Optional[SeedGenerator] = None,\n    ) -> tuple[list[SmolAgentAdapter], Dict[str, SmolAgentAdapter]]:\n        \"\"\"Create multi-agent system with orchestrator and specialists.\n\n        If seed_generator is provided, seeds are derived for each agent\n        using the benchmark's seeding system with hierarchical paths.\n        \"\"\"\n        # Build seeds dict if seed_generator is available\n        # Use child(\"agents\") to create logical paths like \"agents/primary_agent\"\n        seeds = None\n        if seed_generator is not None:\n            agent_gen = seed_generator.child(\"agents\")\n            seeds = {}\n            for agent_spec in agent_data[\"agents\"]:\n                seeds[agent_spec[\"agent_id\"]] = agent_gen.derive_seed(agent_spec[\"agent_id\"])\n\n        agents_to_run, agents_to_monitor = build_agents(agent_data, environment, seeds)\n\n        # Create adapters for the primary agent(s) to run\n        adapters_to_run = [SmolAgentAdapter(agent, agent.name) for agent in agents_to_run]\n\n        # This ensures all agent traces are collected by the benchmark\n        all_agents = {agent.name: agent for agent in agents_to_run} | agents_to_monitor\n        adapters_to_monitor = {name: SmolAgentAdapter(agent, name) for name, agent in all_agents.items()}\n        return adapters_to_run, adapters_to_monitor\n\n    def setup_evaluators(self, environment, task, agents, user, seed_generator: Optional[SeedGenerator] = None) -> Sequence[Evaluator]:\n        \"\"\"Create evaluators based on task's evaluation criteria.\"\"\"\n        if not task.evaluation_data[\"evaluators\"]:\n            return []\n\n        evaluator_instances = []\n        for name in task.evaluation_data[\"evaluators\"]:\n            evaluator_class = getattr(evaluators, name)\n            evaluator_instances.append(evaluator_class(task, environment, user))\n\n        return evaluator_instances\n\n    def run_agents(self, agents: Sequence[AgentAdapter], task: Task, environment: Environment, query: str) -> Sequence[Any]:\n        \"\"\"Execute agents and return their final answers.\"\"\"\n        answers = [agent.run(query) for agent in agents]\n        return answers\n\n    def get_model_adapter(self, model_id: str, **kwargs) -> ModelAdapter:\n        \"\"\"Return a model adapter for benchmark components that need LLM access.\n\n        This benchmark doesn't use simulated tools, user simulators, or LLM judges,\n        so this method is not called during execution.\n        \"\"\"\n        raise NotImplementedError(\"This benchmark doesn't use model adapters for tools/users/evaluators.\")\n\n    def evaluate(\n        self,\n        evaluators: Sequence[Evaluator],\n        agents: Dict[str, AgentAdapter],\n        final_answer: Any,\n        traces: Dict[str, Any],\n    ) -> list[Dict[str, Any]]:\n        \"\"\"Evaluate agent performance.\"\"\"\n        results = []\n        for evaluator in evaluators:\n            filtered_traces = evaluator.filter_traces(traces)\n            results.append(evaluator(filtered_traces, final_answer))\n        return results"
+   "source": "class FiveADayBenchmark(Benchmark):\n    \"\"\"5-A-Day benchmark with multi-agent support.\"\"\"\n\n    def setup_environment(self, agent_data: Dict[str, Any], task: Task, seed_generator: SeedGenerator) -> Environment:\n        \"\"\"Create environment from task data.\"\"\"\n        task_data = {\n            \"environment_data\": task.environment_data,\n            \"query\": task.query,\n            \"evaluation_data\": task.evaluation_data,\n            \"metadata\": task.metadata,\n        }\n\n        environment = FiveADayEnvironment(task_data)\n\n        # Register all tools for tracing\n        for tool_name, tool_adapter in environment.get_tools().items():\n            self.register(\"tools\", tool_name, tool_adapter)\n\n        return environment\n\n    def setup_agents(\n        self,\n        agent_data: Dict[str, Any],\n        environment: Environment,\n        task: Task,\n        user,\n        seed_generator: SeedGenerator,\n    ) -> tuple[list[SmolAgentAdapter], Dict[str, SmolAgentAdapter]]:\n        \"\"\"Create multi-agent system with orchestrator and specialists.\n\n        Seeds are derived for each agent using the benchmark's seeding system\n        with hierarchical paths. derive_seed() returns None if seeding is disabled.\n        \"\"\"\n        # Build seeds dict using seed_generator\n        # Use child(\"agents\") to create logical paths like \"agents/primary_agent\"\n        agent_gen = seed_generator.child(\"agents\")\n        seeds = {}\n        for agent_spec in agent_data[\"agents\"]:\n            seeds[agent_spec[\"agent_id\"]] = agent_gen.derive_seed(agent_spec[\"agent_id\"])\n\n        agents_to_run, agents_to_monitor = build_agents(agent_data, environment, seeds)\n\n        # Create adapters for the primary agent(s) to run\n        adapters_to_run = [SmolAgentAdapter(agent, agent.name) for agent in agents_to_run]\n\n        # This ensures all agent traces are collected by the benchmark\n        all_agents = {agent.name: agent for agent in agents_to_run} | agents_to_monitor\n        adapters_to_monitor = {name: SmolAgentAdapter(agent, name) for name, agent in all_agents.items()}\n        return adapters_to_run, adapters_to_monitor\n\n    def setup_evaluators(self, environment, task, agents, user, seed_generator: SeedGenerator) -> Sequence[Evaluator]:\n        \"\"\"Create evaluators based on task's evaluation criteria.\"\"\"\n        if not task.evaluation_data[\"evaluators\"]:\n            return []\n\n        evaluator_instances = []\n        for name in task.evaluation_data[\"evaluators\"]:\n            evaluator_class = getattr(evaluators, name)\n            evaluator_instances.append(evaluator_class(task, environment, user))\n\n        return evaluator_instances\n\n    def run_agents(self, agents: Sequence[AgentAdapter], task: Task, environment: Environment, query: str) -> Sequence[Any]:\n        \"\"\"Execute agents and return their final answers.\"\"\"\n        answers = [agent.run(query) for agent in agents]\n        return answers\n\n    def get_model_adapter(self, model_id: str, **kwargs) -> ModelAdapter:\n        \"\"\"Return a model adapter for benchmark components that need LLM access.\n\n        This benchmark doesn't use simulated tools, user simulators, or LLM judges,\n        so this method is not called during execution.\n        \"\"\"\n        raise NotImplementedError(\"This benchmark doesn't use model adapters for tools/users/evaluators.\")\n\n    def evaluate(\n        self,\n        evaluators: Sequence[Evaluator],\n        agents: Dict[str, AgentAdapter],\n        final_answer: Any,\n        traces: Dict[str, Any],\n    ) -> list[Dict[str, Any]]:\n        \"\"\"Evaluate agent performance.\"\"\"\n        results = []\n        for evaluator in evaluators:\n            filtered_traces = evaluator.filter_traces(traces)\n            results.append(evaluator(filtered_traces, final_answer))\n        return results"
   },
   {
    "cell_type": "markdown",

examples/five_a_day_benchmark/five_a_day_benchmark.py

@@ -729,7 +729,7 @@ class FiveADayBenchmark(Benchmark):
     Supports single-agent and multi-agent (orchestrator+specialist) configurations.
     """
 
-    def setup_environment(self, agent_data: Dict[str, Any], task: Task, seed_generator: Optional[SeedGenerator] = None) -> Environment:
+    def setup_environment(self, agent_data: Dict[str, Any], task: Task, seed_generator: SeedGenerator) -> Environment:
         """Create environment from task data."""
         # Pass full task data to environment
         task_data = {
@@ -753,8 +753,8 @@ def setup_agents(
         agent_data: Dict[str, Any],
         environment: Environment,
         task: Task,
-        user=None,
-        seed_generator: Optional[SeedGenerator] = None,
+        user,
+        seed_generator: SeedGenerator,
     ) -> tuple[List[AgentAdapter], Dict[str, AgentAdapter]]:
         """Create framework-specific agent with tools from environment.
 
@@ -775,14 +775,13 @@ def setup_agents(
         primary_spec = next(a for a in agents_specs if a["agent_id"] == primary_agent_id)
         specialist_specs = [a for a in agents_specs if a["agent_id"] != primary_agent_id]
 
-        # Derive seeds for agents using seed_generator if available
+        # Derive seeds for agents using seed_generator
         # Use child("agents") to create logical paths like "agents/primary_agent"
-        seeds = None
-        if seed_generator is not None:
-            agent_gen = seed_generator.child("agents")
-            seeds = {primary_spec["agent_id"]: agent_gen.derive_seed(primary_spec["agent_id"])}
-            for spec in specialist_specs:
-                seeds[spec["agent_id"]] = agent_gen.derive_seed(spec["agent_id"])
+        # derive_seed() returns None if seeding is disabled
+        agent_gen = seed_generator.child("agents")
+        seeds = {primary_spec["agent_id"]: agent_gen.derive_seed(primary_spec["agent_id"])}
+        for spec in specialist_specs:
+            seeds[spec["agent_id"]] = agent_gen.derive_seed(spec["agent_id"])
 
         # Build agent using unified interface - now returns (primary_adapter, all_adapters_dict)
         builder = get_agent_builder(framework, agent_type)
@@ -791,7 +790,7 @@ def setup_agents(
         # Return primary adapter to run, and all adapters for trace registration
         return [primary_adapter], all_adapters_dict
 
-    def setup_evaluators(self, environment, task, agents, user, seed_generator: Optional[SeedGenerator] = None) -> Sequence[Evaluator]:
+    def setup_evaluators(self, environment, task, agents, user, seed_generator: SeedGenerator) -> Sequence[Evaluator]:
         """Create evaluators based on task's evaluation_data.evaluators list."""
         if not task.evaluation_data["evaluators"]:
             return []

examples/macs_benchmark/macs_benchmark.py

@@ -179,7 +179,7 @@ def setup_user(
         agent_data: Dict[str, Any],
         environment: Environment,
         task: Task,
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> SmolagentsMACSUser:
         """Create smolagents-compatible user simulator.
 
@@ -210,7 +210,7 @@ def setup_agents(
         environment: MACSEnvironment,  # type: ignore[override]
         task: Task,
         user: Optional[User],
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> Tuple[List[AgentAdapter], Dict[str, AgentAdapter]]:
         """Create smolagents multi-agent hierarchy.
 
@@ -435,7 +435,7 @@ def setup_user(
         agent_data: Dict[str, Any],
         environment: Environment,
         task: Task,
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> LangGraphMACSUser:
         """Create langgraph-compatible user simulator.
 
@@ -466,7 +466,7 @@ def setup_agents(
         environment: MACSEnvironment,  # type: ignore[override]
         task: Task,
         user: Optional[User],
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> Tuple[List[AgentAdapter], Dict[str, AgentAdapter]]:
         """Create langgraph multi-agent hierarchy.
 

examples/tau2_benchmark/tau2_benchmark.py

@@ -173,7 +173,7 @@ def setup_user(
         agent_data: Dict[str, Any],
         environment: Tau2Environment,
         task: Task,
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> DefaultTau2User:
         """Create user simulator with tool support for default agent."""
         user_data = task.user_data
@@ -213,7 +213,7 @@ def setup_agents(
         environment: Tau2Environment,
         task: Task,
         user: Optional[DefaultTau2User],
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ):
         """Create the default agent with user tool support."""
         agents_to_run, agents_dict = super().setup_agents(agent_data, environment, task, user, seed_generator)
@@ -248,7 +248,7 @@ def setup_user(
         agent_data: Dict[str, Any],
         environment: Tau2Environment,
         task: Task,
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> DefaultTau2User:
         """Create user simulator with tool support for default agent."""
         user_data = task.user_data
@@ -288,7 +288,7 @@ def setup_agents(
         environment: Tau2Environment,
         task: Task,
         user: Optional[DefaultTau2User],
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ):
         """Create the default agent with user tool support."""
         agents_to_run, agents_dict = super().setup_agents(agent_data, environment, task, user, seed_generator)
@@ -372,7 +372,7 @@ def setup_user(
         agent_data: Dict[str, Any],
         environment: Tau2Environment,
         task: Task,
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> SmolagentsTau2User:
         """Create smolagents-compatible user simulator."""
         user_data = task.user_data
@@ -411,7 +411,7 @@ def setup_agents(
         environment: Tau2Environment,
         task: Task,
         user: Optional[User],
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> Tuple[List[AgentAdapter], Dict[str, AgentAdapter]]:
         """Create smolagents customer service agent."""
         model_id = agent_data.get("model_id", "gemini-2.5-flash")
@@ -546,7 +546,7 @@ def setup_user(
         agent_data: Dict[str, Any],
         environment: Tau2Environment,
         task: Task,
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> LangGraphTau2User:
         """Create langgraph-compatible user simulator."""
         user_data = task.user_data
@@ -585,7 +585,7 @@ def setup_agents(
         environment: Tau2Environment,
         task: Task,
         user: Optional[User],
-        seed_generator: Optional[SeedGenerator] = None,
+        seed_generator: SeedGenerator,
     ) -> Tuple[List[AgentAdapter], Dict[str, AgentAdapter]]:
         """Create langgraph customer service agent."""
         model_id = agent_data.get("model_id", "gemini-2.5-flash")