ci(eval): migrate gatekeeper eval jobs to Vertex AI and update model … (#450)

ci(eval): migrate gatekeeper eval jobs to Vertex AI and update model lineup

Add gatekeeper evaluation jobs with Vertex AI, using service account
credentials downloaded via GitLab secure files. Add a new setup job
(download-secure-files) to fetch credentials, and configure
GOOGLE_APPLICATION_CREDENTIALS in the eval base template.

Model changes:
  - Add Vertex AI jobs: gpt-oss-120b, gemini-3.1-pro-preview
  - Add Models.corp jobs: granite-4.0-h-small
  - Remove claude-4.6-opus (OpenRouter) job

Author: Jazzcort

.gitlab/ci/CREDENTIALS.md

@@ -0,0 +1,36 @@
+# GitLab CI Credentials
+
+This document describes the credentials used by the GitLab CI pipelines and how they were provisioned.
+
+## CI/CD Variables
+
+These are configured in **Settings > CI/CD > Variables** on the GitLab project.
+
+### `GITLAB_TOKEN`
+
+- **Used by:** `mirror.yml` (PR mirroring workflow)
+- **Purpose:** Pushes mirrored GitHub PR branches into the GitLab repo.
+- **Type:** Personal or Project Access Token with `write_repository` scope.
+- **How to create:** GitLab > Settings > Access Tokens > create a token with `write_repository`. Then add it as a CI/CD variable (Protected + Masked).
+
+### `MODELS_CORP_*_API_KEY`
+
+- **Used by:** `eval-gatekeeper.yml` (models.corp eval jobs)
+- **Purpose:** Authenticates against the internal Red Hat model platform (`models.corp.redhat.com`) to run gatekeeper evals. Each variable follows the pattern `MODELS_CORP_<MODEL_NAME>_API_KEY` (e.g. `MODELS_CORP_GPT_OSS_20B_API_KEY`, `MODELS_CORP_GRANITE_4_0_H_SMALL_API_KEY`).
+- **Type:** API key issued by models.corp.
+- **How to create:** Check [models.corp user documentation](https://gitlab.cee.redhat.com/models-corp/user-documentation) for requesting an API key. Add it as a CI/CD variable (Protected + Masked). When adding a new models.corp eval job, create a corresponding variable following the naming convention.
+
+## Secure Files
+
+These are uploaded via **Settings > CI/CD > Secure Files** and downloaded at runtime by the `download-secure-files` job using `glab securefile download`.
+
+### `gatekeeper-eval-service-account-credential.json`
+
+- **Used by:** `eval-gatekeeper.yml` (all Vertex AI eval jobs, via `GOOGLE_APPLICATION_CREDENTIALS`)
+- **Purpose:** Authenticates to the Google Cloud project `rhel-lightspeed-650189` to call Vertex AI model endpoints (e.g. `gpt-oss-120b-maas`, `gemini-3.1-pro-preview`).
+- **Type:** Google Cloud Service Account key (JSON).
+- **How it was created:**
+  1. In the GCP console for project `rhel-lightspeed-650189`, a service account was created with the Vertex AI User role (or equivalent).
+  2. A JSON key was exported for that service account.
+  3. The JSON key file was uploaded to GitLab at **Settings > CI/CD > Secure Files**.
+- **Rotation:** Generate a new JSON key in GCP, re-upload to Secure Files, and delete the old key.

.gitlab/ci/eval-gatekeeper.yml

@@ -24,6 +24,19 @@ setup-uv-and-mcp-apps:
       - mcp-app/dist/
     expire_in: 1 hour
 
+download-secure-files:
+  stage: setup
+  image: registry.gitlab.com/gitlab-org/cli:latest
+  script:
+    - mkdir -p securefiles
+    - glab auth login --job-token $CI_JOB_TOKEN --hostname $CI_SERVER_FQDN --api-protocol $CI_SERVER_PROTOCOL
+    - glab -R $CI_PROJECT_PATH securefile download --all --output-dir="./securefiles"
+    - cd securefiles && ls
+  artifacts:
+    paths:
+      - securefiles/
+
+
 # ==========================================
 # EVAL WORKFLOW - TEMPLATES
 # ==========================================
@@ -34,13 +47,26 @@ setup-uv-and-mcp-apps:
   needs:
     - job: setup-uv-and-mcp-apps
       artifacts: true
+    - job: download-secure-files
+      artifacts: true
+
   before_script:
+    # Add path of UV executable to PATH & Install dependencies
     - export PATH="$CI_PROJECT_DIR/.local/bin:$PATH"
     - uv sync --locked
+
+    # Create data/ to store the evaluation result
     - mkdir -p "$CI_PROJECT_DIR/data"
+
+    # Download Red Hat CA certifacate and combine it with root CA
     - export SSL_CERT_FILE="$CI_PROJECT_DIR/cert.pem"
     - cp /etc/ssl/certs/ca-certificates.crt $SSL_CERT_FILE
     - curl https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem >> $SSL_CERT_FILE
+
+    # Setup environment variables for Vertex AI
+    - export GOOGLE_APPLICATION_CREDENTIALS="$CI_PROJECT_DIR/securefiles/gatekeeper-eval-service-account-credential.json"
+    - export VERTEXAI_PROJECT="rhel-lightspeed-650189"
+    - export VERTEXAI_LOCATION="global"
   artifacts:
     paths: 
       - data/
@@ -49,26 +75,36 @@ setup-uv-and-mcp-apps:
 # ==========================================
 # EVAL WORKFLOW - JOBS
 # ==========================================
-gatekeeper-eval-claude-4.6-opus: 
+
+# Vertex AI
+gatekeeper-eval-gpt-oss-120b: 
   extends: .eval-base
   variables:
-    MODEL_NAME: "claude-4.6-opus"
+    MODEL_NAME: "gpt-oss-120b"
   script:
-    - export LINUX_MCP_GATEKEEPER_MODEL="openrouter/anthropic/$MODEL_NAME"
-    - uv run eval/gatekeeper/run-eval.py --all -f json --output-all -o "$CI_PROJECT_DIR/data/$MODEL_NAME.json"
-  rules:
-    - if: $OPENROUTER_API_KEY
+    - export LINUX_MCP_GATEKEEPER_MODEL="vertex_ai/openai/$MODEL_NAME-maas"
+    - uv run --extra gcp eval/gatekeeper/run-eval.py --all -f json --output-all -o "$CI_PROJECT_DIR/data/$MODEL_NAME.json"
 
-gatekeeper-eval-models-corp-gemini:
+gatekeeper-eval-gemini-3.1-pro-preview: 
   extends: .eval-base
   variables:
     MODEL_NAME: "gemini-3.1-pro-preview"
   script:
-    - export OPENAI_API_KEY="$MODELS_CORP_GEMINI_API_KEY"
-    - export MODEL="$MODEL_NAME"
+    - export LINUX_MCP_GATEKEEPER_MODEL="vertex_ai/$MODEL_NAME"
+    - uv run --extra gcp eval/gatekeeper/run-eval.py --all -f json --output-all -o "$CI_PROJECT_DIR/data/$MODEL_NAME.json"
+
+
+# Models.corp
+gatekeeper-eval-models-corp-granite-4.0-h-small:
+  extends: .eval-base
+  variables:
+    MODEL_NAME: "granite-4.0-h-small"
+  script:
+    - export OPENAI_API_KEY="$MODELS_CORP_GRANITE_4_0_H_SMALL_API_KEY"
+    - export MODEL="ibm-granite/$MODEL_NAME"
     - ./eval/gatekeeper/run-eval-models-corp.sh --all -f json --output-all -o "$CI_PROJECT_DIR/data/$MODEL_NAME.json"
   rules:
-    - if: $MODELS_CORP_GEMINI_API_KEY
+    - if: $MODELS_CORP_GRANITE_4_0_H_SMALL_API_KEY
 
 gatekeeper-eval-models-corp-gpt-oss-20b:
   extends: .eval-base