fix(v1): config errors name the flag the user typed, and predictable

mistakes get pointers instead of tracebacks

- Errors raised inside the [env] block re-surface with their full loc, so
  the CLI renders --env.solver.model, not a bogus root-level --solver with
  an absurd did-you-mean (prefix_validation_error, applied in
  resolve_env_field and narrow_plugin_field).
- Curated refusals for the predictable mistakes: conflicting positional
  taskset vs --env.taskset.id (first-narrowed/last-parsed nonsense before);
  a dangling harness id (was a raw TypeError); legacy --id combined with a
  v1 taskset (was silently inert); env-level `harness` (points at the seat);
  the retired `default` harness id (renamed to bash — stale namespace dir
  deleted); unknown taskset/env/harness ids print their one-line message
  instead of a 35-71-line traceback, and unknown harnesses list the
  built-ins.
- The usage gate lets --env.* argv through to the typed parse, so a typo'd
  --env.takset gets a real suggestion instead of the bare usage line.
- The role guard keys on the default instance like the role machinery does,
  so AgentConfig | None and Annotated forms can't shadow base fields or
  skip the default-instance requirement.
- dry_run is excluded from saved configs (eval, replay, gepa) — re-running
  `@ config.toml` no longer silently dry-runs; --server with an unset
  --rich now just runs non-rich instead of demanding --no-rich.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: hallerite

tests/v1/test_configs.py

@@ -39,6 +39,109 @@ def test_output_path_compounds_env_id():
     assert output_path(paired).parent.name.startswith("best-of-n+echo-v1--")
 
 
+def test_conflicting_cli_taskset_ids_are_refused():
+    """Two `--env.taskset.id` occurrences naming different ids (the positional
+    shorthand counts as the first) are refused up front — the narrowing pins the
+    first while the config merge takes the last, so letting them through yields a
+    baffling wrong-config-type error."""
+    from verifiers.v1.cli.resolve import narrow_config, with_positional_taskset
+
+    argv = with_positional_taskset(["echo-v1", "--env.taskset.id", "other-v1"])
+    with pytest.raises(SystemExit, match="'echo-v1' and 'other-v1'"):
+        narrow_config(EvalConfig, argv)
+    # The same id twice is not a conflict.
+    narrow_config(
+        EvalConfig, with_positional_taskset(["echo-v1", "--env.taskset.id", "echo-v1"])
+    )
+
+
+def test_dangling_harness_id_is_refused_with_the_builtins():
+    """A dangling `--env.agent.harness.id` (no value) parses as boolean True; the
+    narrowing must answer with the field path and the built-in harness ids, not a
+    raw TypeError from deep inside the loader."""
+    from pydantic import ValidationError
+
+    with pytest.raises(ValidationError, match=r"harness\.id needs an id.*bash"):
+        EvalConfig.model_validate(
+            {"env": {"taskset": {"id": "echo-v1"}, "agent": {"harness": {"id": True}}}}
+        )
+
+
+def test_nested_env_errors_keep_their_flag_path():
+    """Sub-models validate inside `mode=\"before\"` narrowing validators, which
+    would surface their error locs without the `env` (and `harness`) segments —
+    the CLI would render a flag the user never typed."""
+    from pydantic import ValidationError
+
+    with pytest.raises(ValidationError) as e:
+        EvalConfig.model_validate(
+            {
+                "env": {
+                    "taskset": {"id": "echo-v1"},
+                    "agent": {"harness": {"id": "bash", "runtime": {"type": "dockr"}}},
+                }
+            }
+        )
+    assert e.value.errors()[0]["loc"][:4] == ("env", "agent", "harness", "runtime")
+
+
+def test_role_guard_keys_on_the_default_instance():
+    """The definition-time role guard must use the machinery's membership test
+    (`isinstance(field.default, AgentConfig)`): `timeout: AgentConfig | None =
+    AgentConfig()` IS a role to `_declared_agent_configs`, so it must be refused
+    as shadowing; an AgentConfig annotation without a default instance is never a
+    role, so it must be refused as missing one — whatever the annotation form."""
+    import verifiers.v1 as vf
+
+    with pytest.raises(TypeError, match="shadow"):
+
+        class Shadowing(vf.EnvConfig):
+            timeout: vf.AgentConfig | None = vf.AgentConfig()
+
+    with pytest.raises(TypeError, match="default"):
+
+        class Uninstantiated(vf.EnvConfig):
+            solver: vf.AgentConfig | None = None
+
+
+def test_legacy_id_with_v1_taskset_is_refused():
+    """A legacy `--id` next to a v1 `env.taskset` used to be silently inert
+    (`is_legacy` is False, the v0 env never loads); the mix is refused with a
+    pointer at `--env.id`, the likely intent."""
+    from pydantic import ValidationError
+
+    with pytest.raises(ValidationError, match="--env.id"):
+        EvalConfig.model_validate(
+            {"id": "wordle", "env": {"taskset": {"id": "echo-v1"}}}
+        )
+
+
+def test_env_level_harness_key_points_at_the_seat():
+    """`--env.harness.*` (v0 muscle-memory, one level up from the seat) gets a
+    pointer home instead of a bare extra_forbidden."""
+    from pydantic import ValidationError
+
+    with pytest.raises(ValidationError, match=r"--env\.agent\.harness"):
+        EvalConfig.model_validate(
+            {"env": {"taskset": {"id": "echo-v1"}, "harness": {"id": "bash"}}}
+        )
+
+
+def test_server_flips_an_unset_rich_default():
+    """`--rich` defaults on but is in-process only: an unset `rich` defaults off
+    under `--server`; only an explicitly set `rich` is refused with it."""
+    from pydantic import ValidationError
+
+    served = EvalConfig.model_validate(
+        {"env": {"taskset": {"id": "echo-v1"}}, "server": True}
+    )
+    assert served.rich is False
+    with pytest.raises(ValidationError, match="--rich"):
+        EvalConfig.model_validate(
+            {"env": {"taskset": {"id": "echo-v1"}}, "server": True, "rich": True}
+        )
+
+
 def test_replay_lifts_the_saved_eval_taskset():
     """Replay layers the source run's saved config as its base. An eval run keeps
     its taskset on the [env] block — replay's root taskset must pick it up; an

verifiers/v1/cli/debug.py

@@ -18,6 +18,7 @@
 from verifiers.v1.cli.output import append_trace, save_config
 from verifiers.v1.cli.resolve import (
     extract_id,
+    plugin_errors,
     references_config_file,
     with_positional_taskset,
 )
@@ -309,14 +310,16 @@ def main(argv: list[str] | None = None) -> None:
     if not argv or any(arg in ("-h", "--help") for arg in argv):
         print(USAGE)
         sys.argv = [sys.argv[0], "--help"]
-        cli(_narrow(argv))
+        with plugin_errors():
+            cli(_narrow(argv))
         return
     if not extract_id(argv, "taskset") and not references_config_file(argv):
         raise SystemExit(USAGE)
 
-    config_type = _narrow(argv)
-    sys.argv = [sys.argv[0], *argv]
-    config = cli(config_type)
+    with plugin_errors():
+        config_type = _narrow(argv)
+        sys.argv = [sys.argv[0], *argv]
+        config = cli(config_type)
     setup_logging("DEBUG" if config.verbose else "INFO")
     # Graceful shutdown: first Ctrl-C/SIGTERM unwinds each task's teardown `finally`;
     # a second is swallowed so it can't orphan containers/sandboxes mid-cleanup.

verifiers/v1/cli/eval/main.py

@@ -13,6 +13,7 @@
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_config,
+    plugin_errors,
     references_config_file,
     with_positional_taskset,
 )
@@ -34,9 +35,10 @@ def main(argv: list[str] | None = None) -> None:
     if not argv or any(arg in ("-h", "--help") for arg in argv):
         print(USAGE)
         sys.argv = [sys.argv[0], "--help"]
-        cli(
-            narrow_config(EvalConfig, argv)
-        )  # full option help, narrowed to the given ids
+        with plugin_errors():
+            cli(
+                narrow_config(EvalConfig, argv)
+            )  # full option help, narrowed to the given ids
         return
     resume_dir, rest = split_resume(argv)
     # re-run a previous run's missing/errored rollouts, in place
@@ -48,22 +50,29 @@ def main(argv: list[str] | None = None) -> None:
         config = load_resume_config(resume_dir)
     else:
         legacy_id = any(a == "--id" or a.startswith("--id=") for a in argv)  # v0 env id
-        # A retired flat axis (--taskset.*/--harness.*) skips the usage gate so the
-        # parse renders its pointer to the new flags instead of a bare usage line.
-        retired_axis = any(a.startswith(("--taskset.", "--harness.")) for a in argv)
+        # An env-block flag (or a retired flat axis) skips the usage gate so the
+        # typed parse renders its did-you-mean / pointer to the new flags instead
+        # of a bare usage line.
+        typed_axis = any(
+            a.startswith(("--env.", "--taskset.", "--harness.")) for a in argv
+        )
         if (
             not extract_id(argv, "env.taskset")
             and not legacy_id
             and not references_config_file(argv)
-            and not retired_axis
+            and not typed_axis
         ):
             raise SystemExit(
                 USAGE
             )  # need a taskset (positional / --env.taskset.id), a legacy --id, or a @ file.toml
 
-        config_type = narrow_config(EvalConfig, argv)
-        sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
-        config = cli(config_type)
+        with plugin_errors():
+            config_type = narrow_config(EvalConfig, argv)
+            sys.argv = [
+                sys.argv[0],
+                *argv,
+            ]  # let prime-pydantic-config render help/errors
+            config = cli(config_type)
         if config.dry_run:  # resolved + validated; write it to the output dir and exit
             setup_logging("DEBUG" if config.verbose else "INFO")
             logger.info("wrote config to %s", write_config(config, output_path(config)))

verifiers/v1/cli/gepa.py

@@ -19,6 +19,7 @@
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_config,
+    plugin_errors,
     references_config_file,
     with_positional_taskset,
 )
@@ -37,28 +38,30 @@ def main(argv: list[str] | None = None) -> None:
     if not argv or any(arg in ("-h", "--help") for arg in argv):
         print(USAGE)
         sys.argv = [sys.argv[0], "--help"]
-        cli(
-            narrow_config(GEPAConfig, argv)
-        )  # full option help, narrowed to the given ids
+        with plugin_errors():
+            cli(
+                narrow_config(GEPAConfig, argv)
+            )  # full option help, narrowed to the given ids
         return
     if any(a == "--id" or a.startswith("--id=") for a in argv):  # v0 env id
         raise SystemExit(
             "gepa optimizes native v1 tasksets; run a legacy (v0) environment through "
             "`vf-gepa` instead of `gepa`."
         )
-    retired_axis = any(a.startswith(("--taskset.", "--harness.")) for a in argv)
+    typed_axis = any(a.startswith(("--env.", "--taskset.", "--harness.")) for a in argv)
     if (
         not extract_id(argv, "env.taskset")
         and not references_config_file(argv)
-        and not retired_axis
+        and not typed_axis
     ):
         raise SystemExit(
             USAGE
         )  # need a taskset (positional / --env.taskset.id) or a @ file.toml
 
-    config_type = narrow_config(GEPAConfig, argv)
-    sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
-    config = cli(config_type)
+    with plugin_errors():
+        config_type = narrow_config(GEPAConfig, argv)
+        sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
+        config = cli(config_type)
     setup_logging("DEBUG" if config.verbose else "INFO")
     if config.dry_run:  # resolved + validated; write it to the output dir and exit
         logger.info("wrote config to %s", write_config(config, output_path(config)))

verifiers/v1/cli/resolve.py

@@ -7,9 +7,23 @@
 states explicitly is pre-narrowed — never to a type a config file could contradict.
 """
 
+import contextlib
+
 import verifiers.v1 as vf
 
 
+@contextlib.contextmanager
+def plugin_errors():
+    """Surface a plugin-resolution failure (unknown taskset/env/harness id, a bad
+    `__all__` export) as the clean one-line exit the CLI's config errors get,
+    instead of a raw traceback. Wrap the id-driven narrowing and the typed parse;
+    a `SystemExit` from inside passes through untouched."""
+    try:
+        yield
+    except (ModuleNotFoundError, AttributeError, TypeError, ValueError) as e:
+        raise SystemExit(str(e)) from e
+
+
 def with_positional_taskset(
     argv: list[str], flag: str = "--env.taskset.id"
 ) -> list[str]:
@@ -29,16 +43,29 @@ def references_config_file(argv: list[str]) -> bool:
 
 def extract_id(argv: list[str], field: str, default: str = "") -> str:
     """The chosen `<field>.id` from `--<field>.id <x>` (or `=<x>`) on the CLI, before
-    the typed parse (the positional taskset shorthand is applied upstream). Absent
-    here, the id can still come from a `@ file.toml` — validation resolves it from
-    the parsed data."""
+    the typed parse (the positional taskset shorthand is applied upstream). Two
+    occurrences naming different ids are refused — narrowing would pin the first
+    while the config merge takes the last. Absent here, the id can still come from
+    a `@ file.toml` — validation resolves it from the parsed data."""
     flag = f"--{field}.id"
+    found: list[str] = []
     for i, arg in enumerate(argv):
         if arg == flag and i + 1 < len(argv):
-            return argv[i + 1]
-        if arg.startswith(flag + "="):
-            return arg.split("=", 1)[1]
-    return default
+            found.append(argv[i + 1])
+        elif arg.startswith(flag + "="):
+            found.append(arg.split("=", 1)[1])
+    distinct = list(dict.fromkeys(found))
+    if len(distinct) > 1:
+        hint = (
+            " — the positional <taskset-id> shorthand sets it too"
+            if field.endswith("taskset")
+            else ""
+        )
+        raise SystemExit(
+            f"{flag} is set twice, with different ids: {distinct[0]!r} and "
+            f"{distinct[1]!r}{hint}; drop one"
+        )
+    return found[0] if found else default
 
 
 def narrow_config(base: type, argv: list[str]) -> type:

verifiers/v1/cli/serve.py

@@ -9,6 +9,7 @@
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_config,
+    plugin_errors,
     references_config_file,
     with_positional_taskset,
 )
@@ -25,25 +26,28 @@ def main(argv: list[str] | None = None) -> None:
     if not argv or any(arg in ("-h", "--help") for arg in argv):
         print(USAGE)
         sys.argv = [sys.argv[0], "--help"]
-        cli(narrow_config(ServeConfig, argv))
+        with plugin_errors():
+            cli(narrow_config(ServeConfig, argv))
         return
     legacy_id = any(a == "--id" or a.startswith("--id=") for a in argv)  # v0 env id
-    # A retired flat axis (--taskset.*/--harness.*) skips the usage gate so the
-    # parse renders its pointer to the new flags instead of a bare usage line.
-    retired_axis = any(a.startswith(("--taskset.", "--harness.")) for a in argv)
+    # An env-block flag (or a retired flat axis) skips the usage gate so the typed
+    # parse renders its did-you-mean / pointer to the new flags instead of a bare
+    # usage line.
+    typed_axis = any(a.startswith(("--env.", "--taskset.", "--harness.")) for a in argv)
     if (
         not extract_id(argv, "env.taskset")
         and not legacy_id
         and not references_config_file(argv)
-        and not retired_axis
+        and not typed_axis
     ):
         raise SystemExit(
             USAGE
         )  # need a taskset (positional / --env.taskset.id), a legacy --id, or @ file.toml
 
-    config_type = narrow_config(ServeConfig, argv)
-    sys.argv = [sys.argv[0], *argv]
-    config = cli(config_type)
+    with plugin_errors():
+        config_type = narrow_config(ServeConfig, argv)
+        sys.argv = [sys.argv[0], *argv]
+        config = cli(config_type)
     if config.dry_run:
         print(config.model_dump_json(indent=2, exclude_none=True))
         return

verifiers/v1/cli/validate.py

@@ -14,6 +14,7 @@
 from verifiers.v1.cli.dashboard import TaskProgress, validate_dashboard
 from verifiers.v1.cli.resolve import (
     extract_id,
+    plugin_errors,
     references_config_file,
     with_positional_taskset,
 )
@@ -255,16 +256,21 @@ def main(argv: list[str] | None = None) -> None:
     if not argv or any(arg in ("-h", "--help") for arg in argv):
         print(USAGE)
         sys.argv = [sys.argv[0], "--help"]
-        cli(_narrow(argv))  # full option help, narrowed to the given taskset
+        with plugin_errors():
+            cli(_narrow(argv))  # full option help, narrowed to the given taskset
         return
     if not extract_id(argv, "taskset") and not references_config_file(argv):
         raise SystemExit(
             USAGE
         )  # need a taskset (positional / --taskset.id) or a @ file.toml
 
-    config_type = _narrow(argv)
-    sys.argv = [sys.argv[0], *argv]  # let prime-pydantic-config render help/errors
-    config = cli(config_type)
+    with plugin_errors():
+        config_type = _narrow(argv)
+        sys.argv = [
+            sys.argv[0],
+            *argv,
+        ]  # let prime-pydantic-config render help/errors
+        config = cli(config_type)
     # Nothing is persisted, so logs are the whole output. Under `--rich` the dashboard owns the
     # screen, so keep logs off the console (else stray records print over the UI).
     setup_logging("DEBUG" if config.verbose else "INFO", console=not config.rich)