Align SystemPrompt alias with config fields (#1501)

Author: willccbb

docs/reference.md

@@ -77,7 +77,7 @@ v1 task loader return types. Override `load_tasks(split=...)` on a
 ### SystemPrompt
 
 ```python
-SystemPrompt = str | Sequence[Message | JsonData]
+SystemPrompt = PromptInput | SystemPromptConfig | None
 SystemPromptStrategy = Literal["HT", "TH", "H_OR_T", "T_OR_H", "H", "T", "REJECT"]
 
 class SystemPromptConfig:
@@ -1030,7 +1030,7 @@ class EnvConfig(Config):
 
 class TasksetConfig(Config):
     taskset_id: str | None = None  # `id` shorthand accepted
-    system_prompt: PromptInput | SystemPromptConfig | None = None
+    system_prompt: SystemPrompt = None
     user: UserConfig | None = None
     bindings: BindingsConfig = BindingsConfig()
     objects: ObjectsConfig = ObjectsConfig()
@@ -1040,7 +1040,7 @@ class HarnessConfig(Config):
     harness_id: str | None = None  # `id` shorthand accepted
     program: ProgramConfig = ProgramConfig()
     model: ModelConfig = ModelConfig()
-    system_prompt: PromptInput | SystemPromptConfig | None = None
+    system_prompt: SystemPrompt = None
     system_prompt_strategy: SystemPromptStrategy = "HT"
     sandbox: SandboxConfig | None = None
     user: UserConfig | None = None

tests/test_v1_config_extension.py

@@ -2310,16 +2310,14 @@ class ChildTaskset(BaseTaskset):
 
 def test_taskset_class_loader_owns_split_loading() -> None:
     class LoaderTasksetConfig(TasksetConfig):
-        system_prompt: vf.SystemPrompt | None = "class prompt"
+        system_prompt: vf.SystemPrompt = "class prompt"
 
     class LoaderTaskset(Taskset[LoaderTasksetConfig]):
         def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
             answer = "class eval" if split == "eval" else "class tasks"
             return [{"prompt": [], "answer": answer}]
 
-        def load_system_prompt(
-            self, config: LoaderTasksetConfig
-        ) -> vf.SystemPrompt | None:
+        def load_system_prompt(self, config: LoaderTasksetConfig) -> vf.SystemPrompt:
             return config.system_prompt
 
     defaulted = LoaderTaskset(config=LoaderTasksetConfig())
@@ -2341,6 +2339,25 @@ def load_system_prompt(
     assert disabled_prompt.system_prompt == []
 
 
+def test_system_prompt_alias_accepts_config_data(tmp_path) -> None:
+    prompt_path = tmp_path / "system_prompt.txt"
+    prompt_path.write_text("alias path system prompt", encoding="utf-8")
+
+    class PromptTasksetConfig(TasksetConfig):
+        system_prompt: vf.SystemPrompt = None
+
+    config = PromptTasksetConfig.model_validate(
+        {"system_prompt": {"path": str(prompt_path)}}
+    )
+    assert isinstance(config.system_prompt, vf.SystemPromptConfig)
+
+    taskset = Taskset(config=config)
+
+    assert taskset.system_prompt == [
+        {"role": "system", "content": "alias path system prompt"}
+    ]
+
+
 def test_taskset_load_tasks_can_return_empty_dataset() -> None:
     class LocalTasksetConfig(TasksetConfig):
         enabled: bool = True

verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev16"
+__version__ = "0.1.15.dev17"
 
 import importlib
 import os

verifiers/v1/__init__.py

@@ -59,14 +59,13 @@
 )
 from .utils.endpoint_utils import Endpoint
 from .utils.binding_utils import BindingsConfig, ObjectsConfig
-from .utils.prompt_utils import SystemPromptConfig, SystemPromptStrategy
+from .utils.prompt_utils import SystemPrompt, SystemPromptConfig, SystemPromptStrategy
 from .types import (
     ConfigData,
     Handler,
     JsonData,
     Objects,
     PromptInput,
-    SystemPrompt,
     TaskSplit,
     Tasks,
 )

verifiers/v1/harness.py

@@ -72,8 +72,8 @@
     run_sandbox_python_program,
 )
 from .utils.prompt_utils import (
+    SystemPrompt,
     SystemPromptStrategy,
-    SystemPromptConfig,
     normalize_prompt,
     normalize_system_prompt,
     resolve_system_prompt,
@@ -88,7 +88,6 @@
     ConfigData,
     JsonData,
     Objects,
-    PromptInput,
 )
 
 if TYPE_CHECKING:
@@ -106,7 +105,7 @@ class HarnessConfig(LifecycleConfig):
     )
     program: ProgramConfig = ProgramConfig()
     model: ModelConfig = ModelConfig()
-    system_prompt: PromptInput | SystemPromptConfig | None = None
+    system_prompt: SystemPrompt = None
     system_prompt_strategy: SystemPromptStrategy = "HT"
     sandbox: SandboxConfig | None = None
     user: UserConfig | None = None
@@ -217,9 +216,7 @@ def __init__(
             self.endpoint = self.load_endpoint()
             self.program = self.compile_program(self.program_config)
 
-    def load_system_prompt(
-        self, config: ConfigT
-    ) -> PromptInput | SystemPromptConfig | None:
+    def load_system_prompt(self, config: ConfigT) -> SystemPrompt:
         return config.system_prompt
 
     def load_sandbox(self, config: SandboxConfig | None) -> SandboxConfig | None:

verifiers/v1/taskset.py

@@ -18,7 +18,7 @@
     BindingsConfig,
     ObjectsConfig,
 )
-from .utils.prompt_utils import SystemPromptConfig, normalize_system_prompt
+from .utils.prompt_utils import SystemPrompt, normalize_system_prompt
 from .utils.config_utils import (
     coerce_config,
     config_ref_context,
@@ -36,7 +36,6 @@
 from .types import (
     JsonData,
     Objects,
-    PromptInput,
     TaskSplit,
     Tasks,
 )
@@ -48,7 +47,7 @@ class TasksetConfig(LifecycleConfig):
         default=None,
         validation_alias=AliasChoices("taskset_id", "id"),
     )
-    system_prompt: PromptInput | SystemPromptConfig | None = None
+    system_prompt: SystemPrompt = None
     user: UserConfig | None = None
     bindings: BindingsConfig = BindingsConfig()
     objects: ObjectsConfig = ObjectsConfig()
@@ -152,7 +151,5 @@ def __iter__(self):
     def __len__(self) -> int:
         return len(self.get_dataset())
 
-    def load_system_prompt(
-        self, config: ConfigT
-    ) -> PromptInput | SystemPromptConfig | None:
+    def load_system_prompt(self, config: ConfigT) -> SystemPrompt:
         return config.system_prompt

verifiers/v1/types.py

@@ -41,7 +41,6 @@
 
 PromptMessage: TypeAlias = Message | JsonData
 PromptInput: TypeAlias = str | Sequence[PromptMessage]
-SystemPrompt: TypeAlias = PromptInput
 
 ModelClient: TypeAlias = Client | ClientConfig
 RuntimeObject: TypeAlias = object

verifiers/v1/utils/prompt_utils.py

@@ -1,15 +1,15 @@
 import importlib.util
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Literal, cast
+from typing import TYPE_CHECKING, Literal, TypeAlias, cast
 
 from pydantic import model_validator
 from typing_extensions import Self
 from verifiers.types import Messages, SystemMessage
 from verifiers.utils.message_utils import normalize_messages
 
 from ..config import Config
-from ..types import JsonData, PromptInput, SystemPrompt
+from ..types import JsonData, PromptInput
 from .config_utils import current_config_ref_module
 
 if TYPE_CHECKING:
@@ -64,13 +64,15 @@ class SystemPromptConfig(Config):
     messages: list[JsonData] = []
 
     @model_validator(mode="after")
-    def validate_one_source(self) -> Self:
-        sources = [
+    def validate_one_input(self) -> Self:
+        inputs = [
             self.path is not None,
             bool(self.messages),
         ]
-        if sum(sources) != 1:
-            raise ValueError("SystemPromptConfig requires exactly one source.")
+        if sum(inputs) != 1:
+            raise ValueError(
+                "SystemPromptConfig requires exactly one of path or messages."
+            )
         return self
 
     def load(self, field_name: str) -> PromptInput | None:
@@ -81,6 +83,9 @@ def load(self, field_name: str) -> PromptInput | None:
         return self.messages
 
 
+SystemPrompt: TypeAlias = PromptInput | SystemPromptConfig | None
+
+
 def normalize_prompt(
     value: PromptInput | None, field_name: str = "prompt"
 ) -> list[JsonData]:
@@ -95,7 +100,7 @@ def normalize_prompt(
 
 
 def normalize_system_prompt(
-    value: SystemPrompt | SystemPromptConfig | None,
+    value: SystemPrompt,
     field_name: str = "system_prompt",
 ) -> list[JsonData]:
     value = resolve_system_prompt_input(value, field_name=field_name)
@@ -111,7 +116,7 @@ def normalize_system_prompt(
 
 
 def resolve_system_prompt_input(
-    value: PromptInput | SystemPromptConfig | None,
+    value: SystemPrompt,
     *,
     field_name: str,
 ) -> PromptInput | None: