diff --git a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml index 6d64f49ed9b2e..300a9694694e7 100644 --- a/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml +++ b/ci-operator/config/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main.yaml @@ -38,6 +38,34 @@ tests: skip_if_only_changed: ^\.tekton/|\.md$|^(LICENSE|OWNERS)$ steps: workflow: openshift-ci-security +- as: mcpchecker-eval-google + optional: true + steps: + cluster_profile: openshift-org-aws + env: + ANTHROPIC_USE_VERTEX: "" + EVAL_CONFIG: evals/core-eval-testing/builtin-google/eval-core.yaml + EVAL_LABEL_SELECTOR: suite=core + GEMINI_USE_VERTEX: "1" + GOOGLE_CLOUD_LOCATION: us-east5 + GOOGLE_CLOUD_PROJECT: ocp-mcp-server-team + test: + - chain: openshift-mcp-server-mcpchecker-eval-vertex + workflow: ipi-aws +- as: mcpchecker-eval-anthropic + optional: true + steps: + cluster_profile: openshift-org-aws + env: + ANTHROPIC_USE_VERTEX: "1" + EVAL_CONFIG: evals/core-eval-testing/builtin-anthropic/eval-core.yaml + EVAL_LABEL_SELECTOR: suite=core + GEMINI_USE_VERTEX: "" + GOOGLE_CLOUD_LOCATION: us-east5 + GOOGLE_CLOUD_PROJECT: ocp-mcp-server-team + test: + - chain: openshift-mcp-server-mcpchecker-eval-vertex + workflow: ipi-aws zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml index e59d2f74652b6..593ca9d4386d5 100644 --- a/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/openshift-mcp-server/openshift-openshift-mcp-server-main-presubmits.yaml @@ -199,6 +199,162 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )lint,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/mcpchecker-eval-anthropic + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.ci + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-openshift-mcp-server-main-mcpchecker-eval-anthropic + optional: true + rerun_command: /test mcpchecker-eval-anthropic + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=mcpchecker-eval-anthropic + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )mcpchecker-eval-anthropic,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build13 + context: ci/prow/mcpchecker-eval-google + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile.ci + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-openshift-mcp-server-main-mcpchecker-eval-google + optional: true + rerun_command: /test mcpchecker-eval-google + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=mcpchecker-eval-google + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )mcpchecker-eval-google,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/step-registry/openshift/mcp-server/OWNERS b/ci-operator/step-registry/openshift/mcp-server/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md new file mode 100644 index 0000000000000..eb46862c82514 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/README.md @@ -0,0 +1,112 @@ +# openshift-mcp-server-mcpchecker-eval step registry + +Reusable CI steps for running [mcpchecker](https://github.com/mcpchecker/mcpchecker) +evaluations against the [openshift-mcp-server](https://github.com/openshift/openshift-mcp-server) +on a live OCP cluster. Designed to support both pre-merge (presubmit) and scheduled +(periodic) workflows via env-var and eval-target parameterization. + +## Structure + +``` +mcpchecker-eval/ +├── openshift-mcp-server-mcpchecker-eval # core eval step (shared) +├── vertex-setup/ # credential setup: Vertex AI +├── credentials-setup/ # credential setup: model API key +├── vertex/ # chain: vertex-setup → eval +└── model-api/ # chain: credentials-setup → eval +``` + +## Design intent + +All evaluation logic lives in the single **`openshift-mcp-server-mcpchecker-eval`** step, +parameterized entirely by environment variables. Credential sourcing is separated into +provider-specific **setup steps** that write exports to `${SHARED_DIR}/mcpchecker-creds.env`, +which the eval step sources at startup. This keeps the eval step provider-agnostic and +makes adding a new provider a matter of adding a new setup step and chain. + +The two chains are: + +| Chain | Credential source | Typical use | +|---|---|---| +| `openshift-mcp-server-mcpchecker-eval-vertex` | `ocp-mcp` secret (GCP service account) | Gemini or Anthropic via Vertex AI | +| `openshift-mcp-server-mcpchecker-eval-model-api` | `openai-token` secret | Any non-Vertex provider | + +Agent, model, and judge selection are specified entirely within the in-repo eval config +pointed to by `EVAL_CONFIG`. The credential setup steps exist only to surface authentication +material (service account files, API keys) that the eval config's agent definition requires; +they do not drive model routing themselves. + +Both chains run against a live OCP cluster provisioned by the `ipi-aws` workflow. +`kubectl` is unavailable in the CI image; the eval step symlinks `oc` (provided via +`cli: latest`) to satisfy eval setup scripts that call `kubectl`. + +## Eval step parameters + +All set via `steps.env` in the calling CI config: + +| Variable | Default | Purpose | +|---|---|---| +| `EVAL_CONFIG` | `evals/openai-agent/eval.yaml` | Repo-relative path to the eval YAML | +| `EVAL_LABEL_SELECTOR` | `suite=core` | Label filter passed to `mcpchecker check` | +| `TOOLSETS` | `core,config` | MCP server toolsets to start | +| `TASK_PASS_RATE` | `0.0` | Minimum fraction of tasks that must pass (0.0 = no gate) | +| `ASSERTION_PASS_RATE` | `0.0` | Minimum fraction of assertions that must pass (0.0 = no gate) | +| `GOOGLE_CLOUD_LOCATION` | `` | Vertex AI region (e.g. `us-east5`); Vertex chain only | +| `GOOGLE_CLOUD_PROJECT` | `` | GCP project ID; Vertex chain only | +| `GEMINI_USE_VERTEX` | `` | Set to `"1"` for Google Gemini via Vertex AI | +| `ANTHROPIC_USE_VERTEX` | `` | Set to `"1"` for Anthropic Claude via Vertex AI | + +Artifacts written to `${ARTIFACT_DIR}`: `mcpchecker-out.json` and `junit_mcpchecker.xml`. + +## Usage + +### Presubmit — Vertex AI provider (Gemini or Anthropic) + +```yaml +- as: mcpchecker-eval-google + optional: true + steps: + cluster_profile: openshift-org-aws + env: + EVAL_CONFIG: evals/core-eval-testing/builtin-google/eval-core.yaml + EVAL_LABEL_SELECTOR: suite=core + GEMINI_USE_VERTEX: "1" + GOOGLE_CLOUD_LOCATION: us-east5 + GOOGLE_CLOUD_PROJECT: ocp-mcp-server-team + test: + - chain: openshift-mcp-server-mcpchecker-eval-vertex + workflow: ipi-aws +``` + +Switch to Anthropic by setting `ANTHROPIC_USE_VERTEX: "1"` (and clearing `GEMINI_USE_VERTEX`) +and pointing `EVAL_CONFIG` at the Anthropic eval YAML. The chain is identical. + +### Periodic — non-Vertex provider + +```yaml +- as: periodic-mcpchecker-eval + cron: 0 9 * * 1 + steps: + cluster_profile: openshift-org-aws + env: + EVAL_CONFIG: evals/openai-agent/eval.yaml + EVAL_LABEL_SELECTOR: suite=core + TASK_PASS_RATE: "0.8" + ASSERTION_PASS_RATE: "0.8" + test: + - chain: openshift-mcp-server-mcpchecker-eval-model-api + workflow: ipi-aws +``` + +The agent, model, and judge are configured inside the eval YAML (`EVAL_CONFIG`). The +`openai-token` secret in the `test-credentials` namespace must contain whatever +authentication keys that eval config's agent definition requires. + +### Adding a new provider + +1. Create a new `-setup/` directory with a ref that mounts the relevant secret + and appends exports to `${SHARED_DIR}/mcpchecker-creds.env`. +2. Create a `/` directory with a chain that runs your setup step followed by + `openshift-mcp-server-mcpchecker-eval`. +3. Add an `OWNERS` file to both directories. +4. Run `make registry-metadata` to regenerate metadata. diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh new file mode 100644 index 0000000000000..2156a5ba3036a --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/credentials-setup/openshift-mcp-server-mcpchecker-eval-credentials-setup-commands.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +# Disable tracing while handling credential values +[[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false +set +x + +cat >> "${SHARED_DIR}/mcpchecker-creds.env" </dev/null 2>&1; then + mkdir -p /tmp/bin + ln -s "$(command -v oc)" /tmp/bin/kubectl + export PATH="/tmp/bin:${PATH}" +fi + +PATH="${PATH}:$(pwd)/_output/tools/bin" +export PATH +export MCP_CONFIG_DIR=dev/config/mcp-configs + +trap 'make stop-server || true' EXIT + +GOFLAGS='-mod=readonly' make build +GOFLAGS='' make mcpchecker + +make run-server TOOLSETS="${TOOLSETS}" +make run-evals EVAL_CONFIG="${EVAL_CONFIG}" EVAL_LABEL_SELECTOR="${EVAL_LABEL_SELECTOR}" + +RESULTS_FILE="$(find . -maxdepth 1 -name 'mcpchecker-*-out.json' | sort | tail -1)" +if [[ -z "${RESULTS_FILE}" ]]; then + echo "ERROR: no mcpchecker results file found" >&2 + exit 1 +fi + +MCPCHECKER="$(pwd)/_output/tools/bin/mcpchecker" +cp "${RESULTS_FILE}" "${ARTIFACT_DIR}/mcpchecker-out.json" + +"${MCPCHECKER}" result convert junit "${ARTIFACT_DIR}/mcpchecker-out.json" \ + --output-file "${ARTIFACT_DIR}/junit_mcpchecker.xml" + +if [[ "${TASK_PASS_RATE}" != "0.0" || "${ASSERTION_PASS_RATE}" != "0.0" ]]; then + "${MCPCHECKER}" result verify "${ARTIFACT_DIR}/mcpchecker-out.json" \ + --task "${TASK_PASS_RATE}" --assertion "${ASSERTION_PASS_RATE}" +fi diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json new file mode 100644 index 0000000000000..2e115e738c289 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.metadata.json @@ -0,0 +1,22 @@ +{ + "path": "openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml", + "owners": { + "approvers": [ + "Cali0707", + "manusa", + "matzew", + "Kaustubh-pande" + ], + "reviewers": [ + "2uasimojo", + "bentito", + "cajieh", + "Cali0707", + "dlom", + "grokspawn", + "manusa", + "matzew", + "Kaustubh-pande" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml new file mode 100644 index 0000000000000..3e9a4932eb99f --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/openshift-mcp-server-mcpchecker-eval-ref.yaml @@ -0,0 +1,44 @@ +ref: + as: openshift-mcp-server-mcpchecker-eval + from: src + cli: latest + commands: openshift-mcp-server-mcpchecker-eval-commands.sh + env: + - name: EVAL_CONFIG + default: evals/openai-agent/eval.yaml + documentation: Repo-relative path to the mcpchecker eval YAML + - name: EVAL_LABEL_SELECTOR + default: suite=core + documentation: Label selector for task filtering + - name: TOOLSETS + default: core,config + documentation: MCP server toolsets to enable + - name: TASK_PASS_RATE + default: "0.0" + documentation: Minimum task pass rate (0.0 disables the gate) + - name: ASSERTION_PASS_RATE + default: "0.0" + documentation: Minimum assertion pass rate (0.0 disables the gate) + - name: GOOGLE_CLOUD_LOCATION + default: "" + documentation: Vertex AI region (e.g. us-east5); leave empty for non-Vertex runs + - name: GOOGLE_CLOUD_PROJECT + default: "" + documentation: GCP project ID for Vertex AI; leave empty for non-Vertex runs + - name: GEMINI_USE_VERTEX + default: "" + documentation: Set to "1" to route Google Gemini calls through Vertex AI + - name: ANTHROPIC_USE_VERTEX + default: "" + documentation: Set to "1" to route Anthropic Claude calls through Vertex AI + resources: + requests: + cpu: "2" + memory: 4Gi + grace_period: 5m0s + timeout: 4h0m0s + documentation: |- + Runs mcpchecker evaluations against the openshift-mcp-server on a live OCP cluster. + Credentials must be written to ${SHARED_DIR}/mcpchecker-creds.env by a preceding + setup step. Produces JSON and JUnit artifacts under ${ARTIFACT_DIR}. + An optional pass-rate gate exits non-zero if results fall below the threshold. diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS new file mode 100644 index 0000000000000..3c93b553e96a1 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/OWNERS @@ -0,0 +1,15 @@ +approvers: +- Cali0707 +- manusa +- matzew +- Kaustubh-pande +reviewers: +- 2uasimojo +- bentito +- cajieh +- Cali0707 +- dlom +- grokspawn +- manusa +- matzew +- Kaustubh-pande diff --git a/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh new file mode 100644 index 0000000000000..ad4bc8406c5c4 --- /dev/null +++ b/ci-operator/step-registry/openshift/mcp-server/mcpchecker-eval/vertex-setup/openshift-mcp-server-mcpchecker-eval-vertex-setup-commands.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -euo pipefail + +# Copy the service account JSON into SHARED_DIR so it is accessible to the +# eval step, which runs in a separate container where /var/run/ocp-mcp is not +# mounted. GOOGLE_CLOUD_LOCATION, GOOGLE_CLOUD_PROJECT, and GEMINI_USE_VERTEX / +# ANTHROPIC_USE_VERTEX are passed via steps.env and flow automatically into all +# subsequent steps. +cp /var/run/ocp-mcp/service-account.json "${SHARED_DIR}/service-account.json" +chmod 0600 "${SHARED_DIR}/service-account.json" + +cat >> "${SHARED_DIR}/mcpchecker-creds.env" <