Fix smoketests (#1161)

* remove manage_test_secrets

* remove unauthorized meta llama model

* Test private downloads

Author: bkorycki

plugins/validation_tests/test_object_creation.py

@@ -1,22 +1,19 @@
 import os
-import re
 
 import pytest
 from flaky import flaky  # type: ignore
+
 from modelgauge.base_test import PromptResponseTest
 from modelgauge.caching import SqlDictCache
 from modelgauge.config import load_secrets_from_config
 from modelgauge.dependency_helper import FromSourceDependencyHelper
-from modelgauge.external_data import WebData
 from modelgauge.load_plugins import load_plugins
 from modelgauge.locales import EN_US  # see "workaround" below
 from modelgauge.prompt import TextPrompt
-from modelgauge.prompt_sets import demo_prompt_set_url
 from modelgauge.record_init import InitializationRecord
 from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_registry import SUTS
-
 from modelgauge.suts.baseten_api import BasetenSUT
 from modelgauge.suts.huggingface_chat_completion import HuggingFaceChatCompletionDedicatedSUT
 from modelgauge.suts.together_client import TogetherDedicatedChatSUT
@@ -32,21 +29,6 @@
 TIMEOUT = 25 * 60  # 25 minutes. Baseten is the slowest to start up.
 
 
-def ensure_public_dependencies(dependencies):
-    """Some tests are defined with dependencies that require an auth token to download them.
-    In this test context, we substitute public files instead."""
-    for k, d in dependencies.items():
-        if isinstance(d, WebData):
-            new_dependency = WebData(
-                source_url=demo_prompt_set_url(d.source_url),
-                headers=None,
-                decompressor=d.decompressor,
-                unpacker=d.unpacker,
-            )
-            dependencies[k] = new_dependency
-    return dependencies
-
-
 @pytest.fixture(scope="session")
 def shared_run_dir(tmp_path_factory):
     # Create a single tmpdir and have all `make_test_items` share it.
@@ -64,7 +46,7 @@ def shared_run_dir(tmp_path_factory):
 @flaky
 @pytest.mark.parametrize("test_name", [key for key, _ in TESTS.items() if key not in TOO_SLOW])
 def test_all_tests_make_test_items(test_name, shared_run_dir):
-    test = TESTS.make_instance(test_name, secrets=_FAKE_SECRETS)
+    test = TESTS.make_instance(test_name, secrets=load_secrets_from_config())
 
     # TODO remove when localized files are handled better
     # workaround
@@ -73,10 +55,9 @@ def test_all_tests_make_test_items(test_name, shared_run_dir):
 
     if isinstance(test, PromptResponseTest):
         test_data_path = os.path.join(shared_run_dir, test.__class__.__name__)
-        dependencies = ensure_public_dependencies(test.get_dependencies())
         dependency_helper = FromSourceDependencyHelper(
             test_data_path,
-            dependencies,
+            test.get_dependencies(),
             required_versions={},
         )
         test_items = test.make_test_items(dependency_helper)

src/modelgauge/suts/meta_llama_client.py

@@ -115,7 +115,8 @@ def translate_response(self, request: MetaLlamaChatRequest, response: MetaLlamaM
         return SUTResponse(text=text)
 
 
-CHAT_MODELS = ["Llama-4-Scout-17B-16E-Instruct-FP8", "Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama-3.3-8B-Instruct"]
+# Unauthorized models: ["Llama-4-Scout-17B-16E-Instruct-FP8"]
+CHAT_MODELS = ["Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama-3.3-8B-Instruct"]
 
 for model_name in CHAT_MODELS:
     SUTS.register(MetaLlamaSUT, "meta-" + model_name.lower() + "-llama", model_name, InjectSecret(MetaLlamaApiKey))

tests/modelbench_tests/test_run.py

@@ -1,6 +1,4 @@
-import functools
 import math
-import os
 import pathlib
 from datetime import datetime
 from typing import List, Mapping, Sequence
@@ -23,7 +21,6 @@
 from modelbench.scoring import ValueEstimate
 from modelgauge.base_test import PromptResponseTest
 from modelgauge.preflight import make_sut
-from modelgauge.config import SECRETS_PATH
 from modelgauge.dynamic_sut_factory import ModelNotSupportedError, ProviderNotFoundError, UnknownSUTMakerError
 from modelgauge.locales import DEFAULT_LOCALE, EN_US, FR_FR, LOCALES
 from modelgauge.prompt_sets import PROMPT_SETS
@@ -32,8 +29,6 @@
 from modelgauge.sut import PromptResponseSUT
 from modelgauge_tests.fake_sut import FakeSUT
 
-TEST_SECRETS_PATH = os.path.join("tests", "config", "secrets.toml")
-
 
 class AHazard(HazardDefinition):
     def tests(self, secrets: RawSecrets) -> List[PromptResponseTest]:
@@ -111,33 +106,6 @@ def mock_score(
             datetime.now(),
         )
 
-    def manage_test_secrets(func):
-        """Decorator that manages test secrets during test execution.
-
-        1. If a secrets file exists, it's backed up
-        2. The test secrets file is copied to the expected location
-        3. After the test completes, the original state is restored
-        """
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            secrets_src = pathlib.Path(TEST_SECRETS_PATH)
-            secrets_dst = pathlib.Path(SECRETS_PATH)
-            backup_dst = secrets_dst.with_suffix(".bak")
-
-            if secrets_dst.exists():
-                secrets_dst.replace(backup_dst)
-            secrets_src.replace(secrets_dst)
-
-            try:
-                return func(*args, **kwargs)
-            finally:
-                secrets_dst.replace(secrets_src)
-                if backup_dst.exists():
-                    backup_dst.replace(secrets_dst)
-
-        return wrapper
-
     @pytest.fixture(autouse=False)
     def mock_run_benchmarks(self, sut, monkeypatch, tmp_path):
         mock = MagicMock(return_value=fake_benchmark_run(AHazard(), sut, tmp_path))
@@ -168,7 +136,6 @@ def runner(self):
         ],
         # TODO add more locales as we add support for them
     )
-    @manage_test_secrets
     def test_benchmark_basic_run_produces_json(
         self, runner, mock_run_benchmarks, mock_score_benchmarks, sut_uid, version, locale, prompt_set, tmp_path
     ):
@@ -232,7 +199,6 @@ def test_security_benchmark_basic_run_produces_json(
         ],
         # TODO add more locales as we add support for them
     )
-    @manage_test_secrets
     def test_benchmark_multiple_suts_produces_json(
         self, mock_run_benchmarks, runner, version, locale, prompt_set, sut_uid, tmp_path, monkeypatch
     ):
@@ -368,7 +334,6 @@ def test_calls_score_benchmark_with_correct_v1_locale(self, runner, mock_run_ben
     #     benchmark_arg = mock_score_benchmarks.call_args.args[0][0]
     #     assert isinstance(benchmark_arg, GeneralPurposeAiChatBenchmark)
 
-    @manage_test_secrets
     def test_v1_en_us_demo_is_default(self, runner, mock_run_benchmarks, sut_uid):
         result = runner.invoke(cli, ["benchmark", "--sut", sut_uid])
 
@@ -383,7 +348,6 @@ def test_nonexistent_benchmark_prompt_sets_can_not_be_called(self, runner, sut_u
         assert "Invalid value for '--prompt-set'" in result.output
 
     @pytest.mark.parametrize("prompt_set", PROMPT_SETS.keys())
-    @manage_test_secrets
     def test_calls_score_benchmark_with_correct_prompt_set(self, runner, mock_run_benchmarks, prompt_set, sut_uid):
         result = runner.invoke(cli, ["benchmark", "--prompt-set", prompt_set, "--sut", sut_uid])