test: pool Gemini API keys to spread eval load across quota (#84)

* test: pool Gemini API keys to spread eval load across quota

Collect every env var whose name starts with GEMINI_API_KEY into a pool
and pick one at random per eval request, spreading load across the
free-tier quota to avoid RESOURCE_EXHAUSTED. Build a fresh judge model
per test method so each scenario can draw a different key.

Co-Authored-By: Claude <svc-devxp-claude@slack-corp.com>

* Update env vars

* Improve token collection

---------

Co-authored-by: Claude <svc-devxp-claude@slack-corp.com>

Author: WilliamBergamin

.env.example

@@ -1,6 +1,9 @@
-# Google Gemini API key, required for the LLM-judged eval tests (make test-eval).
-# The eval suite fails when this is unset.
+# Google Gemini API key(s), required for the LLM-judged eval tests (make test-eval).
+# The eval suite fails when no key is set. To spread requests across the quota,
+# add more keys under any GEMINI_API_KEY* name.
 GEMINI_API_KEY=
+# GEMINI_API_KEY_BOB=
+# GEMINI_API_KEY_MIC=
 
 # Gemini model used as the DeepEval judge. Defaults to gemini-3.1-flash-lite if unset.
 GEMINI_MODEL_NAME=gemini-3.1-flash-lite

.github/workflows/ci-build.yml

@@ -91,8 +91,11 @@ jobs:
       - name: Run eval tests
         run: make test-eval
         env:
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           SLACK_MCP_TOKEN: ${{ secrets.SLACK_MCP_TOKEN }}
+          GEMINI_API_KEY_B: ${{ secrets.GEMINI_API_KEY_BOB }}
+          GEMINI_API_KEY_M: ${{ secrets.GEMINI_API_KEY_MWBROOKS }}
+          GEMINI_API_KEY_E: ${{ secrets.GEMINI_API_KEY_ELAINE }}
+          GEMINI_API_KEY_Z: ${{ secrets.GEMINI_API_KEY_ZIMEG }}
 
   notifications:
     name: Regression notifications

AGENTS.md

@@ -29,7 +29,7 @@ Requires Python 3.14+. Run `make install` before first use to set up the virtual
 | `make cursor-install` | Install this plugin into a local Cursor for development |
 | `make cursor-uninstall` | Uninstall this plugin from the local Cursor install |
 
-The LLM tests read `GEMINI_API_KEY` (required — the eval suite fails when it's unset) and `SLACK_MCP_TOKEN` (a Slack MCP bearer token; the MCP tool-selection test is skipped when it's unset). The DeepEval judge model defaults to `gemini-3.1-flash-lite`, overridable via `GEMINI_MODEL_NAME`. Copy `.env.example` to `.env` and fill in values — the test suite loads `.env` from the repo root via `python-dotenv` (`tests/config.py`), so values load the same however tests are launched. Real environment variables take precedence over `.env`, so you can also override inline, e.g. `GEMINI_MODEL_NAME=<model> make test-eval`.
+The LLM tests read at least one Gemini API key (required — the eval suite fails when none is set) and `SLACK_MCP_TOKEN` (a Slack MCP bearer token; the MCP tool-selection test is skipped when it's unset). To spread requests across the free-tier quota and avoid `RESOURCE_EXHAUSTED`, any env var whose name starts with `GEMINI_API_KEY` (e.g. `GEMINI_API_KEY`, `GEMINI_API_KEY_BOB`) contributes a key to a pool (blank values are skipped), and each eval request picks one at random. The DeepEval judge model defaults to `gemini-3.1-flash-lite`, overridable via `GEMINI_MODEL_NAME`. Copy `.env.example` to `.env` and fill in values — the test suite loads `.env` from the repo root via `python-dotenv` (`tests/config.py`), so values load the same however tests are launched. Real environment variables take precedence over `.env`, so you can also override inline, e.g. `GEMINI_MODEL_NAME=<model> make test-eval`.
 
 ## Cross-Skill References
 
@@ -60,7 +60,7 @@ GitHub Actions (`.github/workflows/ci-build.yml`) gates every PR with:
 - **Test** — `make test-unit` (pytest)
 - **Eval** — `make test-eval` (DeepEval + Gemini)
 
-The eval job reads the `GEMINI_API_KEY` and `SLACK_MCP_TOKEN` repository secrets; it skips on PRs from forks, which don't receive secrets. The workflow also runs nightly on a schedule, and a `notifications` job posts to Slack (via `SLACK_REGRESSION_FAILURES_WEBHOOK_URL`) when a job fails on `main`.
+The eval job reads the `GEMINI_API_KEY_*` (e.g. `GEMINI_API_KEY_BOB`, `GEMINI_API_KEY_MIC`) and `SLACK_MCP_TOKEN` repository secrets; it skips on PRs from forks, which don't receive secrets. The workflow also runs nightly on a schedule, and a `notifications` job posts to Slack (via `SLACK_REGRESSION_FAILURES_WEBHOOK_URL`) when a job fails on `main`.
 
 ## Releasing
 

tests/config.py

@@ -1,12 +1,18 @@
 import json
 import os
+from collections.abc import Mapping
 from pathlib import Path
 
 from dotenv import load_dotenv
 
 # Load .env from the repo root (if present) before reading the environment
 load_dotenv(Path(__file__).parent.parent / ".env")
 
+
+def get_gemini_api_key_pool(environ: Mapping[str, str]) -> list[str]:
+    return [environ[name].strip() for name in environ if name.startswith("GEMINI_API_KEY") and environ[name].strip()]
+
+
 # Filesystem
 SKILLS_ROOT = Path(__file__).parent.parent / "skills"
 
@@ -17,8 +23,9 @@
 # Skill inventory (single source of truth)
 EXPECTED_SKILLS = ("create-slack-app", "block-kit", "slack-api", "slack-cli")
 
-# Gemini judge model
-GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
+# Gemini judge model. Any env var whose name starts with GEMINI_API_KEY contributes a
+# key to the pool; blank values are skipped so an empty GEMINI_API_KEY= doesn't sneak in.
+GEMINI_API_KEY_POOL = get_gemini_api_key_pool(os.environ)
 GEMINI_MODEL = os.environ.get("GEMINI_MODEL_NAME", "gemini-3.1-flash-lite")
 
 # Slack MCP server

tests/eval/test_tool_selection.py

@@ -6,7 +6,7 @@
 from deepeval.test_case import ToolCall
 from pydantic import BaseModel
 
-from tests.config import GEMINI_API_KEY, SLACK_MCP_TOKEN
+from tests.config import GEMINI_API_KEY_POOL, SLACK_MCP_TOKEN
 from tests.support.judge import make_judge_model
 from tests.support.tools import get_all_skill_tools, get_slack_mcp_tools
 
@@ -189,15 +189,17 @@ class TestToolSelection:
 
     @classmethod
     def setup_class(cls) -> None:
-        if not GEMINI_API_KEY:
-            pytest.fail("GEMINI_API_KEY not set")
+        if not GEMINI_API_KEY_POOL:
+            pytest.fail("No Gemini API key set (set GEMINI_API_KEY or GEMINI_API_KEY_*)")
         if not SLACK_MCP_TOKEN:
             pytest.fail("SLACK_MCP_TOKEN not set")
         # Fetch tools once for the whole class: the MCP list is one network
         # round-trip, and skills are read from disk.
-        cls.model = make_judge_model()
         cls.available_tools = get_slack_mcp_tools() + get_all_skill_tools()
 
+    def setup_method(self) -> None:
+        self.model = make_judge_model()
+
     def teardown_method(self) -> None:
         # Gemini's free tier allows only 15 requests/minute. Each scenario makes one
         # model.generate() call, so sleep between scenarios to stay well under the

tests/support/judge.py

@@ -1,8 +1,10 @@
+import random
+
 from deepeval.models import GeminiModel
 
-from tests.config import GEMINI_API_KEY, GEMINI_MODEL
+from tests.config import GEMINI_API_KEY_POOL, GEMINI_MODEL
 
 
 def make_judge_model() -> GeminiModel:
     """Gemini model used as the DeepEval judge for LLM-graded eval tests."""
-    return GeminiModel(model=GEMINI_MODEL, api_key=GEMINI_API_KEY, temperature=0)
+    return GeminiModel(model=GEMINI_MODEL, api_key=random.choice(GEMINI_API_KEY_POOL), temperature=0)

tests/unit/test_config.py

@@ -0,0 +1,27 @@
+from tests.config import get_gemini_api_key_pool
+
+
+class TestGeminiApiKeyPool:
+    def test_collects_all_prefixed_vars(self) -> None:
+        environ = {"GEMINI_API_KEY": "primary", "GEMINI_API_KEY_BOB": "bob"}
+        assert set(get_gemini_api_key_pool(environ)) == {"primary", "bob"}
+
+    def test_bare_prefix_included(self) -> None:
+        assert get_gemini_api_key_pool({"GEMINI_API_KEY": "primary"}) == ["primary"]
+
+    def test_blank_and_whitespace_values_skipped(self) -> None:
+        environ = {"GEMINI_API_KEY": "", "GEMINI_API_KEY_X": "   ", "GEMINI_API_KEY_Y": "real"}
+        assert get_gemini_api_key_pool(environ) == ["real"]
+
+    def test_values_are_stripped(self) -> None:
+        assert get_gemini_api_key_pool({"GEMINI_API_KEY": "  abc  "}) == ["abc"]
+
+    def test_non_matching_vars_ignored(self) -> None:
+        environ = {"SLACK_MCP_TOKEN": "token", "PATH": "/usr/bin", "GEMINI_MODEL_NAME": "flash"}
+        assert get_gemini_api_key_pool(environ) == []
+
+    def test_prefix_must_be_at_start(self) -> None:
+        assert get_gemini_api_key_pool({"MY_GEMINI_API_KEY": "nope"}) == []
+
+    def test_empty_environment(self) -> None:
+        assert get_gemini_api_key_pool({}) == []