Fix CI

Signed-off-by: Jash Gulabrai <jgulabrai@nvidia.com>

Author: JashG

.github/workflows/ci.yaml

@@ -1177,7 +1177,7 @@ jobs:
         # is defensive in case that ever stops being true (e.g. an artifact
         # ever leaks in from a previous workflow attempt).
         run: |
-          RUN_DIR=$(ls -td plugins/nemo-guardrails/benchmarks/artifacts/runs/*/ | head -1)
+          RUN_DIR=$(find plugins/nemo-guardrails/benchmarks/artifacts/runs -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
           echo "Analyzing run directory: $RUN_DIR"
           uv run --frozen --package nemo-guardrails-plugin --extra bench \
             python -m nemo_guardrails_plugin.benchmarks.analyze "$RUN_DIR"

plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/analyze.py

@@ -243,10 +243,7 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument(
         "run_dir",
         type=Path,
-        help=(
-            "Path to a run directory under "
-            "`plugins/nemo-guardrails/benchmarks/artifacts/runs/<timestamp>/`."
-        ),
+        help=("Path to a run directory under `plugins/nemo-guardrails/benchmarks/artifacts/runs/<timestamp>/`."),
     )
     parser.add_argument(
         "--log-level",

plugins/nemo-guardrails/tests/unit/benchmarks/test_seeding.py

@@ -14,6 +14,7 @@
     CS_MODEL_NAME,
     CS_PROVIDER,
     GUARDRAIL_CONFIG,
+    NO_GUARDRAILS_VM_NAME,
     VM_NAME,
     WORKSPACE,
 )
@@ -145,21 +146,33 @@ def test_calls_sdk_with_expected_payloads(self, fake_client: MagicMock, tmp_path
         cs_entity = seeded.cs_model_entity
         assert gc_call.kwargs["data"]["models"][0]["model"] == cs_entity
 
-        # VirtualModel uses the discovered app entity and points middleware at the
-        # guardrail config we just created.
-        vm_call = fake_client.inference.virtual_models.create.call_args
-        assert vm_call.kwargs["name"] == VM_NAME
-        assert vm_call.kwargs["default_model_entity"] == seeded.app_model_entity
-        assert vm_call.kwargs["models"] == [{"model": seeded.app_model_entity, "backend_format": "OPENAI_CHAT"}]
+        # Two VirtualModels are created: the guardrails VM (with middleware) and
+        # a control VM (no middleware) used by the without-guardrails benchmark
+        # variant.
+        vm_calls = fake_client.inference.virtual_models.create.call_args_list
+        assert len(vm_calls) == 2
+
+        guardrails_vm_call = vm_calls[0]
+        assert guardrails_vm_call.kwargs["name"] == VM_NAME
+        assert guardrails_vm_call.kwargs["default_model_entity"] == seeded.app_model_entity
+        assert guardrails_vm_call.kwargs["models"] == [
+            {"model": seeded.app_model_entity, "backend_format": "OPENAI_CHAT"}
+        ]
         expected_middleware = [
             {
                 "name": "nemo-guardrails",
                 "config_type": "guardrail_config",
                 "config_id": f"{WORKSPACE}/{GUARDRAIL_CONFIG}",
             }
         ]
-        assert vm_call.kwargs["request_middleware"] == expected_middleware
-        assert vm_call.kwargs["response_middleware"] == expected_middleware
+        assert guardrails_vm_call.kwargs["request_middleware"] == expected_middleware
+        assert guardrails_vm_call.kwargs["response_middleware"] == expected_middleware
+
+        control_vm_call = vm_calls[1]
+        assert control_vm_call.kwargs["name"] == NO_GUARDRAILS_VM_NAME
+        assert control_vm_call.kwargs["default_model_entity"] == seeded.app_model_entity
+        assert control_vm_call.kwargs["request_middleware"] == []
+        assert control_vm_call.kwargs["response_middleware"] == []
 
     def test_generated_dir_contains_artifacts(self, fake_client: MagicMock, tmp_path: Path) -> None:
         ng_root = tmp_path / "NeMo-Guardrails"
@@ -176,6 +189,7 @@ def test_generated_dir_contains_artifacts(self, fake_client: MagicMock, tmp_path
         assert (generated_dir / "app_provider.json").is_file()
         assert (generated_dir / "content_safety_provider.json").is_file()
         assert (generated_dir / "virtual_model.json").is_file()
+        assert (generated_dir / "virtual_model_no_guardrails.json").is_file()
 
         request_payload = json.loads(
             (generated_dir / "content_safety_local_nmp_request.json").read_text(encoding="utf-8")
@@ -197,6 +211,7 @@ def test_returns_seeded_resources(self, fake_client: MagicMock, tmp_path: Path)
 
         assert seeded.workspace == WORKSPACE
         assert seeded.vm_ref == f"{WORKSPACE}/{VM_NAME}"
+        assert seeded.no_guardrails_vm_name == NO_GUARDRAILS_VM_NAME
         assert seeded.guardrail_config_ref == f"{WORKSPACE}/{GUARDRAIL_CONFIG}"
 
     def test_raises_if_served_models_never_populated(self, tmp_path: Path) -> None: