feat(bench): agent-effectiveness benchmark pilot, A0 vs A2 on S1/S3 with seeded ground truth and audit-derived metrics

.github/workflows/bench.yml

@@ -0,0 +1,49 @@
+# Benchmark harness module checks (issue #930, phase 1: #942).
+#
+# Build, vet, and unit-test the bench/ module — the deterministic regression
+# guard for the harness code itself (the pipeline integration test runs
+# against an in-process fake platform, no Docker or model API). The benchmark
+# SMOKE workflow (scripted run against a live compose stack, and eventually
+# model-backed runs with an API-key secret) is phase 4 (#945).
+#
+# Not part of `make verify`; runs when bench/ changes.
+name: Bench Harness
+
+on:
+  pull_request:
+    paths:
+      - "bench/**"
+      - ".github/workflows/bench.yml"
+  push:
+    branches: [main]
+    paths:
+      - "bench/**"
+      - ".github/workflows/bench.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  harness:
+    name: Harness module checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Set up Go
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: "1.26.5"
+          cache: true
+          cache-dependency-path: bench/go.sum
+
+      - name: Build, vet, and test the harness
+        run: make bench-test
+
+      - name: Lint the harness
+        uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0
+        with:
+          version: v2.11.4
+          working-directory: bench

Makefile

@@ -786,3 +786,120 @@ load-down:
 load-test:
 	@echo "Testing the load harness module..."
 	@cd test/load && $(GO) build ./... && $(GO) vet ./... && $(GO) test ./...
+
+# =============================================================================
+# Agent-Effectiveness Benchmark (issue #930, phase 1: #942)
+# =============================================================================
+#
+# The benchmark harness lives in bench/ (a separate Go module, kept out of the
+# root coverage/test/lint denominator, like test/load). It ablates the PLATFORM
+# (arms a0/a2 as config profiles) while holding the model, prompt scaffold,
+# seed data, and task set constant, and reads efficiency metrics back from the
+# platform's own audit API.
+#
+# Like `mutate` and `load-*`, benchmarking is DELIBERATELY NOT part of
+# `make verify`: it stands up Docker services, a real server binary, and (for
+# real runs) a model API. Do NOT add bench-* to the `verify` target.
+#
+# Arms: a0 (raw tools, no enrichment/search) and a2 (full knowledge platform;
+# requires a DataHub quickstart seeded via `make bench-seed-datahub`).
+
+# BENCH_ARM selects the platform config profile; BENCH_KEY is the admin API key.
+BENCH_ARM ?= a0
+BENCH_KEY ?= bench-admin-key
+BENCH_CONFIG ?= bench/config/platform.bench.$(BENCH_ARM).yaml
+BENCH_ADDR ?= :8098
+BENCH_METRICS_ADDR ?= :9092
+BENCH_URL ?= http://localhost:8098
+BENCH_PID := build/mcp-data-platform-bench.pid
+BENCH_LOG := build/mcp-data-platform-bench.log
+BENCH_COMPOSE := DOCKER_DEFAULT_PLATFORM= docker compose -f docker-compose.e2e.yml
+
+## bench-gen: Regenerate seed artifacts and the task set from the fixed seed
+bench-gen:
+	@cd bench && $(GO) run ./seedgen -seed-dir seed -tasks-dir tasks
+
+## bench-up: Start the compose stack, seed the bench warehouse, and run the platform (ARM=a0|a2 via BENCH_ARM)
+bench-up: e2e-up
+	@echo "Seeding bench warehouse in Trino..."
+	@$(BENCH_COMPOSE) cp bench/seed/trino/setup.sql trino:/tmp/bench-setup.sql
+	@$(BENCH_COMPOSE) exec -T trino trino --file /tmp/bench-setup.sql
+	@echo "Building release-style platform binary (no -race)..."
+	@mkdir -p $(BUILD_DIR)
+	$(GOBUILD) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-bench $(CMD_DIR)
+	@echo "Building benchrun..."
+	@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
+	@if [ -f $(BENCH_PID) ]; then \
+		echo "Stopping previous bench platform (pid $$(cat $(BENCH_PID)))..."; \
+		kill $$(cat $(BENCH_PID)) 2>/dev/null || true; \
+		while kill -0 $$(cat $(BENCH_PID)) 2>/dev/null; do sleep 1; done; \
+		rm -f $(BENCH_PID); \
+	fi
+	@if curl -fsS $(BENCH_URL)/readyz >/dev/null 2>&1; then \
+		echo "ERROR: something else is already serving $(BENCH_URL); run 'make bench-down' first"; exit 1; fi
+	@echo "Starting platform ($(BENCH_CONFIG)) on $(BENCH_ADDR)..."
+	@API_KEY_ADMIN=$(BENCH_KEY) LOG_LEVEL=info OTEL_METRICS_ADDR=$(BENCH_METRICS_ADDR) \
+		$(BUILD_DIR)/$(BINARY_NAME)-bench --config $(BENCH_CONFIG) --transport http --address $(BENCH_ADDR) \
+		> $(BENCH_LOG) 2>&1 & echo $$! > $(BENCH_PID)
+	@echo "Waiting for readiness on $(BENCH_URL)/readyz ..."
+	@for i in $$(seq 1 30); do \
+		if curl -fsS $(BENCH_URL)/readyz >/dev/null 2>&1; then break; fi; \
+		sleep 1; \
+	done; \
+	if ! curl -fsS $(BENCH_URL)/readyz >/dev/null 2>&1; then \
+		echo "ERROR: platform did not become ready; see $(BENCH_LOG)"; tail -20 $(BENCH_LOG); exit 1; fi; \
+	if ! kill -0 $$(cat $(BENCH_PID)) 2>/dev/null; then \
+		echo "ERROR: bench platform exited after start (another server answered readiness?); see $(BENCH_LOG)"; \
+		tail -20 $(BENCH_LOG); exit 1; fi
+	@echo "Seeding knowledge pages (requires platform migrations, just applied on boot)..."
+	@$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \
+		< bench/seed/postgres/knowledge_pages.sql
+	@echo "Platform ready (pid $$(cat $(BENCH_PID)), arm $(BENCH_ARM))."
+
+## bench-seed-datahub: Push bench metadata into a running DataHub quickstart (a2 arm)
+bench-seed-datahub:
+	@command -v datahub >/dev/null 2>&1 || { echo "ERROR: datahub CLI not found (pip install acryl-datahub)"; exit 1; }
+	datahub put --file bench/seed/datahub/bench_mces.json
+
+## bench-run: Run the benchmark (ARM must match bench-up; LLM=anthropic|scripted, SUITE=, K=, MODEL=)
+bench-run:
+	@mkdir -p build/bench-results
+	@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
+	@echo "Resetting search-first gate state (discovery scopes persist in Postgres across runs)..."
+	@$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \
+		-c "TRUNCATE search_gate_discovery"
+	$(BUILD_DIR)/benchrun \
+		-url $(BENCH_URL) \
+		-credential $(BENCH_KEY) \
+		-arm $(BENCH_ARM) \
+		-tasks bench/tasks \
+		-git-commit $$(git rev-parse HEAD) \
+		-out build/bench-results/results-$(BENCH_ARM).json \
+		$(if $(LLM),-llm $(LLM),) \
+		$(if $(SCRIPT),-script $(SCRIPT),) \
+		$(if $(SUITE),-suite $(SUITE),) \
+		$(if $(K),-k $(K),) \
+		$(if $(MODEL),-model $(MODEL),)
+
+## bench-smoke: Run the scripted (no-API-key) smoke against the running platform
+bench-smoke:
+	@$(MAKE) bench-run LLM=scripted SCRIPT=bench/tasks/scripted-smoke.json K=1
+
+## bench-report: Print the human summary of the last run for BENCH_ARM
+bench-report:
+	@cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun
+	$(BUILD_DIR)/benchrun -summarize build/bench-results/results-$(BENCH_ARM).json
+
+## bench-down: Stop the bench platform and the compose stack
+bench-down:
+	@if [ -f $(BENCH_PID) ]; then \
+		echo "Stopping platform (pid $$(cat $(BENCH_PID)))..."; \
+		kill $$(cat $(BENCH_PID)) 2>/dev/null || true; \
+		rm -f $(BENCH_PID); \
+	fi
+	@$(MAKE) e2e-down
+
+## bench-test: Build, vet, and unit-test the benchmark module itself
+bench-test:
+	@echo "Testing the benchmark module..."
+	@cd bench && $(GO) build ./... && $(GO) vet ./... && $(GO) test ./...

README.md

@@ -233,6 +233,12 @@ Go gates (`complexity <= 10` ≈ `gocyclo <= 10`, `cognitive-complexity <= 15` 
 `gocognit <= 15`) plus an import-cycle rule. See [`ui/README.md`](ui/README.md)
 for the thresholds and the ratchet baseline.
 
+Two measurement harnesses live outside `make verify` (each is its own Go
+module): [`test/load`](test/load/README.md) measures throughput and resource
+limits ("how much"), and [`bench/`](bench/README.md) measures agent
+effectiveness — arm-ablated accuracy and efficiency with audit-derived metrics
+("how well"). Run them via `make load-*` and `make bench-*` targets.
+
 Contributions for bug fixes, tests, and documentation are welcome. Please run `make verify` (formatting, race-detected tests, coverage, linting, security scanning) before opening a pull request.
 
 ## License

bench/.golangci.yml

@@ -0,0 +1,97 @@
+# golangci-lint configuration for the benchmark harness module (bench).
+#
+# This is a SEPARATE module from the repository root and gets its own profile.
+# It keeps the root's correctness, security, and complexity gates (the ones that
+# catch real bugs) but drops two families that are specific to the main module
+# rather than a self-contained CLI test tool:
+#
+#   - depguard / wrapcheck: the root's import-boundary and error-wrapping gates
+#     are derived from the platform's layered package graph; they do not apply to
+#     a standalone command that imports only its own internal packages.
+#   - revive `exported` / `add-constant`: library-surface documentation pedantry
+#     (doc comments on every interface one-liner, named constants for file
+#     permissions) that adds noise without value in a test harness.
+#
+# The complexity ceilings match the root (gocyclo 10, gocognit 15).
+version: "2"
+
+run:
+  timeout: 5m
+  modules-download-mode: readonly
+
+linters:
+  default: none
+  enable:
+    - errcheck
+    - govet
+    - staticcheck
+    - ineffassign
+    - unused
+    - bodyclose
+    - noctx
+    - nilerr
+    - nilnil
+    - durationcheck
+    - makezero
+    - wastedassign
+    - misspell
+    - unconvert
+    - whitespace
+    - perfsprint
+    - gocritic
+    - revive
+    - gocyclo
+    - gocognit
+    - nestif
+    - prealloc
+    - predeclared
+    - unparam
+    - errorlint
+    - gosec
+    - copyloopvar
+    - modernize
+
+  settings:
+    govet:
+      disable:
+        - fieldalignment
+        - shadow
+    misspell:
+      locale: US
+    gocyclo:
+      min-complexity: 10
+    gocognit:
+      min-complexity: 15
+    nestif:
+      min-complexity: 5
+    revive:
+      rules:
+        - name: blank-imports
+        - name: context-as-argument
+        - name: context-keys-type
+        - name: error-return
+        - name: error-strings
+        - name: error-naming
+        - name: if-return
+        - name: increment-decrement
+        - name: var-declaration
+        - name: package-comments
+        - name: range
+        - name: receiver-naming
+        - name: indent-error-flow
+        - name: superfluous-else
+        - name: unreachable-code
+        - name: redefines-builtin-id
+
+  exclusions:
+    generated: lax
+    rules:
+      # Test files legitimately use loopback HTTP and unchecked helpers, and
+      # table-driven tests with many assertions exceed the complexity ceilings
+      # that matter for production code.
+      - path: _test\.go
+        linters:
+          - gosec
+          - noctx
+          - gocyclo
+          - gocognit