diff --git a/plugins/google/modelgauge/suts/google_genai.py b/plugins/google/modelgauge/suts/google_genai.py index 6bf1f3614..3b361e4ab 100644 --- a/plugins/google/modelgauge/suts/google_genai.py +++ b/plugins/google/modelgauge/suts/google_genai.py @@ -2,6 +2,10 @@ This file defines google SUTs that use Google's genai python SDK. """ +import logging + +logger = logging.getLogger(__name__) + from typing import Optional from google import genai @@ -17,7 +21,7 @@ from modelgauge.general import APIException from modelgauge.prompt import TextPrompt from modelgauge.retry_decorator import retry -from modelgauge.secret_values import InjectSecret +from modelgauge.secret_values import InjectSecret, loggable_secret from modelgauge.sut import REFUSAL_RESPONSE, PromptResponseSUT, SUTOptions, SUTResponse # usort: skip from modelgauge.sut_capabilities import AcceptsTextPrompt from modelgauge.sut_decorator import modelgauge_sut @@ -44,7 +48,11 @@ def __init__(self, uid: str, model_name: str, reasoning: bool, api_key: GoogleAi self.api_key = api_key.value def _load_client(self) -> genai.Client: - return genai.Client(api_key=self.api_key) + try: + return genai.Client(api_key=self.api_key) + except: + logger.exception(f"Failed to load genai.Client with api_key='{loggable_secret(self.api_key)}'") + raise def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> GenAiRequest: optional = {} diff --git a/src/modelgauge/secret_values.py b/src/modelgauge/secret_values.py index c21569de1..8db1bd815 100644 --- a/src/modelgauge/secret_values.py +++ b/src/modelgauge/secret_values.py @@ -7,6 +7,18 @@ from modelgauge.general import get_concrete_subclasses +def loggable_secret(secret: str) -> str: + if str is None or not isinstance(secret, str): + return secret + if len(secret) == 0: + val = "''" + elif len(secret) < 10: + val = "'…'" + else: + val = f"'{secret[0:2]}…{secret[-2:]}'" + return f"[REDACTED, len={len(secret)}, val={val}]" + + class SecretDescription(BaseModel): """How to look up a secret and how to get the value if you don't have it.""" diff --git a/tests/modelgauge_tests/test_secret_values.py b/tests/modelgauge_tests/test_secret_values.py index a872f33fc..d7da20a96 100644 --- a/tests/modelgauge_tests/test_secret_values.py +++ b/tests/modelgauge_tests/test_secret_values.py @@ -1,4 +1,5 @@ import pytest + from modelgauge.general import get_class from modelgauge.secret_values import ( InjectSecret, @@ -8,6 +9,7 @@ SecretDescription, SerializedSecret, get_all_secrets, + loggable_secret, ) @@ -109,3 +111,14 @@ def test_inject_required_missing(): injector = InjectSecret(SomeRequiredSecret) with pytest.raises(MissingSecretValues): injector.inject({"some-scope": {"different-key": "some-value"}}) + + +def test_loggable_secret_string(): + assert loggable_secret("abcdefghijklmnopqrstuvwxyz") == "[REDACTED, len=26, val='ab…yz']" + + assert loggable_secret(None) is None + assert loggable_secret(1234) is 1234 + assert loggable_secret("") == "[REDACTED, len=0, val='']" + assert loggable_secret("az") == "[REDACTED, len=2, val='…']" + assert loggable_secret("abcd1wxyz") == "[REDACTED, len=9, val='…']" + assert loggable_secret("abcdevwxyz") == "[REDACTED, len=10, val='ab…yz']"