fix(evaluator): address PR #438 review feedback

- api/schemas.py: require a non-empty `type` on the inline wire payload so
  malformed inline metrics are rejected at the API boundary instead of failing
  at execution (mirrors the runtime InlineMetricPayload validator). The metric
  body stays an open object; concrete shape is still validated on hydration.
- docs/evaluator/test_doc_examples.py: stub the executor with a sentinel in the
  built-in-submit test instead of `except Exception: pass`, so the test proves
  packaging resolved (no packager required) without swallowing unrelated errors.
- docs/evaluator/metrics/llm-as-a-judge.mdx: replace the `{{ end_of_text }}`
  Fern template token with a literal `<end_of_text>` per docs guidelines.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Sandy Chapman <schapman@nvidia.com>

Author: SandyChapman

docs/evaluator/metrics/llm-as-a-judge.mdx

@@ -645,7 +645,7 @@ prompt_template = {
     "temperature": 0.1,  # Lower for more consistent scoring
     "max_tokens": 1024,  # Increase if judge needs more space
     "timeout": 30,  # Request timeout in seconds
-    "stop": ["<{{ end_of_text }}>"],  # Stop sequences
+    "stop": ["<end_of_text>"],  # Stop sequences
 }
 ```
 

docs/evaluator/test_doc_examples.py

@@ -105,23 +105,23 @@ def test_packager_param_is_submit_only() -> None:
 def test_builtin_submit_does_not_require_a_packager() -> None:
     """Built-in metrics bundle inline, so docs omit the packager on ``submit()``.
 
-    Submitting reaches the executor (which then needs a live service); the point
-    is only that no packager-policy error is raised for a built-in metric.
+    Packager resolution happens before delegating to the executor, so we stub the
+    executor with a sentinel: reaching it (rather than raising a packager-policy
+    error) proves the built-in metric bundled inline with no packager required —
+    without depending on a live service or swallowing unrelated failures.
     """
+    from unittest.mock import patch
+
     from nemo_evaluator_sdk import ExactMatchMetric
 
     evaluator = _evaluator()
     metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
     dataset = [{"expected": "Paris", "output": "Paris"}]
+    sentinel = RuntimeError("reached executor.submit (packaging resolved without a packager)")
 
-    try:
-        evaluator.submit(metric=metric, dataset=dataset)
-    except MetricBundlePackagerPolicyError as error:  # pragma: no cover - regression guard
-        pytest.fail(f"built-in submit should not require a packager: {error}")
-    except Exception:
-        # Any other failure (e.g. connection refused with no live service) is fine;
-        # it means the built-in metric was bundled inline and reached execution.
-        pass
+    with patch.object(evaluator._executor, "submit", side_effect=sentinel):
+        with pytest.raises(RuntimeError, match="reached executor.submit"):
+            evaluator.submit(metric=metric, dataset=dataset)
 
 
 def test_custom_submit_requires_an_explicit_packager() -> None:

plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py

@@ -15,7 +15,7 @@
 )
 from nemo_evaluator_sdk.values.common import SecretRef
 from nemo_platform_plugin.schema import DatetimeFilter, Filter
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
 class CloudpickleMetricPayload(BaseModel):
@@ -60,6 +60,20 @@ class InlineMetricPayload(BaseModel):
         description="SHA-256 digest of the canonical metric JSON. Informational; recomputed server-side.",
     )
 
+    @field_validator("metric")
+    @classmethod
+    def _metric_must_declare_type(cls, value: dict[str, Any]) -> dict[str, Any]:
+        """Reject payloads without a metric ``type`` discriminator at the API boundary.
+
+        The metric body stays an open object (the concrete shape is validated when
+        the bundle is hydrated against the metric type union), but a non-empty
+        ``type`` is required so malformed payloads fail fast rather than at execution.
+        """
+        metric_type = value.get("type")
+        if not isinstance(metric_type, str) or not metric_type:
+            raise ValueError("inline metric payload must include a non-empty 'type'")
+        return value
+
 
 # Discriminated on ``kind`` so additional payload formats can join the union
 # without changing the field type.