Fix smoke test (#1057)

* see about fixing the smoke test.

* fiddling

* tidy up

* Redact secret

Author: wpietri

plugins/google/modelgauge/suts/google_genai.py

@@ -2,6 +2,10 @@
 This file defines google SUTs that use Google's genai python SDK.
 """
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 from typing import Optional
 
 from google import genai
@@ -17,7 +21,7 @@
 from modelgauge.general import APIException
 from modelgauge.prompt import TextPrompt
 from modelgauge.retry_decorator import retry
-from modelgauge.secret_values import InjectSecret
+from modelgauge.secret_values import InjectSecret, loggable_secret
 from modelgauge.sut import REFUSAL_RESPONSE, PromptResponseSUT, SUTOptions, SUTResponse  # usort: skip
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
@@ -44,7 +48,11 @@ def __init__(self, uid: str, model_name: str, reasoning: bool, api_key: GoogleAi
         self.api_key = api_key.value
 
     def _load_client(self) -> genai.Client:
-        return genai.Client(api_key=self.api_key)
+        try:
+            return genai.Client(api_key=self.api_key)
+        except:
+            logger.exception(f"Failed to load genai.Client with api_key='{loggable_secret(self.api_key)}'")
+            raise
 
     def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> GenAiRequest:
         optional = {}

src/modelgauge/secret_values.py

@@ -7,6 +7,18 @@
 from modelgauge.general import get_concrete_subclasses
 
 
+def loggable_secret(secret: str) -> str:
+    if str is None or not isinstance(secret, str):
+        return secret
+    if len(secret) == 0:
+        val = "''"
+    elif len(secret) < 10:
+        val = "'…'"
+    else:
+        val = f"'{secret[0:2]}…{secret[-2:]}'"
+    return f"[REDACTED, len={len(secret)}, val={val}]"
+
+
 class SecretDescription(BaseModel):
     """How to look up a secret and how to get the value if you don't have it."""
 

tests/modelgauge_tests/test_secret_values.py

@@ -1,4 +1,5 @@
 import pytest
+
 from modelgauge.general import get_class
 from modelgauge.secret_values import (
     InjectSecret,
@@ -8,6 +9,7 @@
     SecretDescription,
     SerializedSecret,
     get_all_secrets,
+    loggable_secret,
 )
 
 
@@ -109,3 +111,14 @@ def test_inject_required_missing():
     injector = InjectSecret(SomeRequiredSecret)
     with pytest.raises(MissingSecretValues):
         injector.inject({"some-scope": {"different-key": "some-value"}})
+
+
+def test_loggable_secret_string():
+    assert loggable_secret("abcdefghijklmnopqrstuvwxyz") == "[REDACTED, len=26, val='ab…yz']"
+
+    assert loggable_secret(None) is None
+    assert loggable_secret(1234) is 1234
+    assert loggable_secret("") == "[REDACTED, len=0, val='']"
+    assert loggable_secret("az") == "[REDACTED, len=2, val='…']"
+    assert loggable_secret("abcd1wxyz") == "[REDACTED, len=9, val='…']"
+    assert loggable_secret("abcdevwxyz") == "[REDACTED, len=10, val='ab…yz']"