From 7717df0410db0366a0b5b551ad2a21b1c22953d0 Mon Sep 17 00:00:00 2001 From: Barbara Korycki Date: Thu, 10 Apr 2025 11:18:13 -0700 Subject: [PATCH 1/2] 25 min timeout --- plugins/validation_tests/test_object_creation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/validation_tests/test_object_creation.py b/plugins/validation_tests/test_object_creation.py index acfed903c..0c0e2a494 100644 --- a/plugins/validation_tests/test_object_creation.py +++ b/plugins/validation_tests/test_object_creation.py @@ -17,7 +17,6 @@ from modelgauge.sut_capabilities import AcceptsTextPrompt from modelgauge.sut_registry import SUTS -from modelgauge.suts.huggingface_chat_completion import HUGGING_FACE_TIMEOUT from modelgauge.test_registry import TESTS from modelgauge.tests.safe_v1 import BaseSafeTestVersion1 # see "workaround" below from modelgauge_tests.fake_secrets import fake_all_secrets @@ -27,6 +26,7 @@ load_plugins() _FAKE_SECRETS = fake_all_secrets() +TIMEOUT = 25 * 60 # 25 minutes. Baseten is the slowest to startup. def ensure_public_dependencies(dependencies): @@ -111,7 +111,7 @@ def test_all_suts_construct_and_record_init(sut_name): # but still fails if the external service really is flaky or slow, so we can # get a sense of a real user's experience. @expensive_tests -@pytest.mark.timeout(HUGGING_FACE_TIMEOUT + 45) # Hugging Face spinup, plus some time for the test itself +@pytest.mark.timeout(TIMEOUT) @pytest.mark.parametrize("sut_name", set(SUTS.keys()) - SUTS_THAT_WE_DONT_CARE_ABOUT_FAILING) def test_all_suts_can_evaluate(sut_name): @@ -131,7 +131,7 @@ def test_all_suts_can_evaluate(sut_name): @expensive_tests -@pytest.mark.timeout(HUGGING_FACE_TIMEOUT + 45) # Hugging Face spinup, plus some time for the test itself +@pytest.mark.timeout(TIMEOUT) @pytest.mark.parametrize("sut_name", set(SUTS.keys()) - SUTS_THAT_WE_DONT_CARE_ABOUT_FAILING) def test_can_cache_all_sut_responses(sut_name, tmpdir): sut = SUTS.make_instance(sut_name, secrets=load_secrets_from_config()) From b25c2e42f97a17ac4de0c5368f36138516b971d2 Mon Sep 17 00:00:00 2001 From: Barbara Korycki Date: Thu, 10 Apr 2025 11:21:34 -0700 Subject: [PATCH 2/2] lint --- plugins/validation_tests/test_object_creation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/validation_tests/test_object_creation.py b/plugins/validation_tests/test_object_creation.py index 0c0e2a494..62e6d626a 100644 --- a/plugins/validation_tests/test_object_creation.py +++ b/plugins/validation_tests/test_object_creation.py @@ -26,7 +26,7 @@ load_plugins() _FAKE_SECRETS = fake_all_secrets() -TIMEOUT = 25 * 60 # 25 minutes. Baseten is the slowest to startup. +TIMEOUT = 25 * 60 # 25 minutes. Baseten is the slowest to startup. def ensure_public_dependencies(dependencies):