diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000..474dcf5b --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,49 @@ +# Benchmark harness module checks (issue #930, phase 1: #942). +# +# Build, vet, and unit-test the bench/ module — the deterministic regression +# guard for the harness code itself (the pipeline integration test runs +# against an in-process fake platform, no Docker or model API). The benchmark +# SMOKE workflow (scripted run against a live compose stack, and eventually +# model-backed runs with an API-key secret) is phase 4 (#945). +# +# Not part of `make verify`; runs when bench/ changes. +name: Bench Harness + +on: + pull_request: + paths: + - "bench/**" + - ".github/workflows/bench.yml" + push: + branches: [main] + paths: + - "bench/**" + - ".github/workflows/bench.yml" + +permissions: + contents: read + +jobs: + harness: + name: Harness module checks + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: "1.26.5" + cache: true + cache-dependency-path: bench/go.sum + + - name: Build, vet, and test the harness + run: make bench-test + + - name: Lint the harness + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + version: v2.11.4 + working-directory: bench diff --git a/Makefile b/Makefile index 93c3c863..d2a64bcd 100644 --- a/Makefile +++ b/Makefile @@ -786,3 +786,120 @@ load-down: load-test: @echo "Testing the load harness module..." @cd test/load && $(GO) build ./... && $(GO) vet ./... && $(GO) test ./... + +# ============================================================================= +# Agent-Effectiveness Benchmark (issue #930, phase 1: #942) +# ============================================================================= +# +# The benchmark harness lives in bench/ (a separate Go module, kept out of the +# root coverage/test/lint denominator, like test/load). It ablates the PLATFORM +# (arms a0/a2 as config profiles) while holding the model, prompt scaffold, +# seed data, and task set constant, and reads efficiency metrics back from the +# platform's own audit API. +# +# Like `mutate` and `load-*`, benchmarking is DELIBERATELY NOT part of +# `make verify`: it stands up Docker services, a real server binary, and (for +# real runs) a model API. Do NOT add bench-* to the `verify` target. +# +# Arms: a0 (raw tools, no enrichment/search) and a2 (full knowledge platform; +# requires a DataHub quickstart seeded via `make bench-seed-datahub`). + +# BENCH_ARM selects the platform config profile; BENCH_KEY is the admin API key. +BENCH_ARM ?= a0 +BENCH_KEY ?= bench-admin-key +BENCH_CONFIG ?= bench/config/platform.bench.$(BENCH_ARM).yaml +BENCH_ADDR ?= :8098 +BENCH_METRICS_ADDR ?= :9092 +BENCH_URL ?= http://localhost:8098 +BENCH_PID := build/mcp-data-platform-bench.pid +BENCH_LOG := build/mcp-data-platform-bench.log +BENCH_COMPOSE := DOCKER_DEFAULT_PLATFORM= docker compose -f docker-compose.e2e.yml + +## bench-gen: Regenerate seed artifacts and the task set from the fixed seed +bench-gen: + @cd bench && $(GO) run ./seedgen -seed-dir seed -tasks-dir tasks + +## bench-up: Start the compose stack, seed the bench warehouse, and run the platform (ARM=a0|a2 via BENCH_ARM) +bench-up: e2e-up + @echo "Seeding bench warehouse in Trino..." + @$(BENCH_COMPOSE) cp bench/seed/trino/setup.sql trino:/tmp/bench-setup.sql + @$(BENCH_COMPOSE) exec -T trino trino --file /tmp/bench-setup.sql + @echo "Building release-style platform binary (no -race)..." + @mkdir -p $(BUILD_DIR) + $(GOBUILD) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME)-bench $(CMD_DIR) + @echo "Building benchrun..." + @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun + @if [ -f $(BENCH_PID) ]; then \ + echo "Stopping previous bench platform (pid $$(cat $(BENCH_PID)))..."; \ + kill $$(cat $(BENCH_PID)) 2>/dev/null || true; \ + while kill -0 $$(cat $(BENCH_PID)) 2>/dev/null; do sleep 1; done; \ + rm -f $(BENCH_PID); \ + fi + @if curl -fsS $(BENCH_URL)/readyz >/dev/null 2>&1; then \ + echo "ERROR: something else is already serving $(BENCH_URL); run 'make bench-down' first"; exit 1; fi + @echo "Starting platform ($(BENCH_CONFIG)) on $(BENCH_ADDR)..." + @API_KEY_ADMIN=$(BENCH_KEY) LOG_LEVEL=info OTEL_METRICS_ADDR=$(BENCH_METRICS_ADDR) \ + $(BUILD_DIR)/$(BINARY_NAME)-bench --config $(BENCH_CONFIG) --transport http --address $(BENCH_ADDR) \ + > $(BENCH_LOG) 2>&1 & echo $$! > $(BENCH_PID) + @echo "Waiting for readiness on $(BENCH_URL)/readyz ..." + @for i in $$(seq 1 30); do \ + if curl -fsS $(BENCH_URL)/readyz >/dev/null 2>&1; then break; fi; \ + sleep 1; \ + done; \ + if ! curl -fsS $(BENCH_URL)/readyz >/dev/null 2>&1; then \ + echo "ERROR: platform did not become ready; see $(BENCH_LOG)"; tail -20 $(BENCH_LOG); exit 1; fi; \ + if ! kill -0 $$(cat $(BENCH_PID)) 2>/dev/null; then \ + echo "ERROR: bench platform exited after start (another server answered readiness?); see $(BENCH_LOG)"; \ + tail -20 $(BENCH_LOG); exit 1; fi + @echo "Seeding knowledge pages (requires platform migrations, just applied on boot)..." + @$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \ + < bench/seed/postgres/knowledge_pages.sql + @echo "Platform ready (pid $$(cat $(BENCH_PID)), arm $(BENCH_ARM))." + +## bench-seed-datahub: Push bench metadata into a running DataHub quickstart (a2 arm) +bench-seed-datahub: + @command -v datahub >/dev/null 2>&1 || { echo "ERROR: datahub CLI not found (pip install acryl-datahub)"; exit 1; } + datahub put --file bench/seed/datahub/bench_mces.json + +## bench-run: Run the benchmark (ARM must match bench-up; LLM=anthropic|scripted, SUITE=, K=, MODEL=) +bench-run: + @mkdir -p build/bench-results + @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun + @echo "Resetting search-first gate state (discovery scopes persist in Postgres across runs)..." + @$(BENCH_COMPOSE) exec -T postgres psql -q -U platform -d mcp_platform -v ON_ERROR_STOP=1 \ + -c "TRUNCATE search_gate_discovery" + $(BUILD_DIR)/benchrun \ + -url $(BENCH_URL) \ + -credential $(BENCH_KEY) \ + -arm $(BENCH_ARM) \ + -tasks bench/tasks \ + -git-commit $$(git rev-parse HEAD) \ + -out build/bench-results/results-$(BENCH_ARM).json \ + $(if $(LLM),-llm $(LLM),) \ + $(if $(SCRIPT),-script $(SCRIPT),) \ + $(if $(SUITE),-suite $(SUITE),) \ + $(if $(K),-k $(K),) \ + $(if $(MODEL),-model $(MODEL),) + +## bench-smoke: Run the scripted (no-API-key) smoke against the running platform +bench-smoke: + @$(MAKE) bench-run LLM=scripted SCRIPT=bench/tasks/scripted-smoke.json K=1 + +## bench-report: Print the human summary of the last run for BENCH_ARM +bench-report: + @cd bench && $(GO) build -o ../$(BUILD_DIR)/benchrun ./benchrun + $(BUILD_DIR)/benchrun -summarize build/bench-results/results-$(BENCH_ARM).json + +## bench-down: Stop the bench platform and the compose stack +bench-down: + @if [ -f $(BENCH_PID) ]; then \ + echo "Stopping platform (pid $$(cat $(BENCH_PID)))..."; \ + kill $$(cat $(BENCH_PID)) 2>/dev/null || true; \ + rm -f $(BENCH_PID); \ + fi + @$(MAKE) e2e-down + +## bench-test: Build, vet, and unit-test the benchmark module itself +bench-test: + @echo "Testing the benchmark module..." + @cd bench && $(GO) build ./... && $(GO) vet ./... && $(GO) test ./... diff --git a/README.md b/README.md index 429ad5a2..4b1d984f 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,12 @@ Go gates (`complexity <= 10` ≈ `gocyclo <= 10`, `cognitive-complexity <= 15` `gocognit <= 15`) plus an import-cycle rule. See [`ui/README.md`](ui/README.md) for the thresholds and the ratchet baseline. +Two measurement harnesses live outside `make verify` (each is its own Go +module): [`test/load`](test/load/README.md) measures throughput and resource +limits ("how much"), and [`bench/`](bench/README.md) measures agent +effectiveness — arm-ablated accuracy and efficiency with audit-derived metrics +("how well"). Run them via `make load-*` and `make bench-*` targets. + Contributions for bug fixes, tests, and documentation are welcome. Please run `make verify` (formatting, race-detected tests, coverage, linting, security scanning) before opening a pull request. ## License diff --git a/bench/.golangci.yml b/bench/.golangci.yml new file mode 100644 index 00000000..207a026b --- /dev/null +++ b/bench/.golangci.yml @@ -0,0 +1,97 @@ +# golangci-lint configuration for the benchmark harness module (bench). +# +# This is a SEPARATE module from the repository root and gets its own profile. +# It keeps the root's correctness, security, and complexity gates (the ones that +# catch real bugs) but drops two families that are specific to the main module +# rather than a self-contained CLI test tool: +# +# - depguard / wrapcheck: the root's import-boundary and error-wrapping gates +# are derived from the platform's layered package graph; they do not apply to +# a standalone command that imports only its own internal packages. +# - revive `exported` / `add-constant`: library-surface documentation pedantry +# (doc comments on every interface one-liner, named constants for file +# permissions) that adds noise without value in a test harness. +# +# The complexity ceilings match the root (gocyclo 10, gocognit 15). +version: "2" + +run: + timeout: 5m + modules-download-mode: readonly + +linters: + default: none + enable: + - errcheck + - govet + - staticcheck + - ineffassign + - unused + - bodyclose + - noctx + - nilerr + - nilnil + - durationcheck + - makezero + - wastedassign + - misspell + - unconvert + - whitespace + - perfsprint + - gocritic + - revive + - gocyclo + - gocognit + - nestif + - prealloc + - predeclared + - unparam + - errorlint + - gosec + - copyloopvar + - modernize + + settings: + govet: + disable: + - fieldalignment + - shadow + misspell: + locale: US + gocyclo: + min-complexity: 10 + gocognit: + min-complexity: 15 + nestif: + min-complexity: 5 + revive: + rules: + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: error-return + - name: error-strings + - name: error-naming + - name: if-return + - name: increment-decrement + - name: var-declaration + - name: package-comments + - name: range + - name: receiver-naming + - name: indent-error-flow + - name: superfluous-else + - name: unreachable-code + - name: redefines-builtin-id + + exclusions: + generated: lax + rules: + # Test files legitimately use loopback HTTP and unchecked helpers, and + # table-driven tests with many assertions exceed the complexity ceilings + # that matter for production code. + - path: _test\.go + linters: + - gosec + - noctx + - gocyclo + - gocognit diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..a66c8c43 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,155 @@ +# Agent-effectiveness benchmark harness + +The benchmark for issue #930 (phase 1 pilot: #942). It measures whether an +agent connected to this platform answers real data questions more correctly +and more efficiently than the same agent connected to bare data tools — by +ablating the PLATFORM, not the model: every run holds the model, prompt +scaffold, seed data, and task set constant and varies only the platform +configuration (arms). + +This is distinct from the load harness (`test/load`): that suite answers "how +much" (throughput, latency, memory); this one answers "how well" (accuracy, +tool-call efficiency, knowledge-trap resistance). + +## Why a separate module + +`bench/` is its own Go module (same rationale as `test/load`): the repository +root runs coverage, test, and lint gates over `./...`, and a nested module is +never matched by the root's `./...`. Run its checks from this directory: + +```bash +cd bench +go build ./... && go vet ./... && go test ./... +golangci-lint run ./... +``` + +Or from the repo root: `make bench-test`. + +Like mutation and load testing, benchmarking is **deliberately not part of +`make verify`** — it stands up Docker services, a real server binary, and (for +real runs) a model API. Do not add `bench-*` to the `verify` target. + +## Arms + +Arms are platform config profiles (`config/`), not code forks — the config +surface is the ablation mechanism: + +| Arm | Profile | What the agent gets | +| --- | --- | --- | +| `a0` | `platform.bench.a0.yaml` | Raw toolkit tools only (`trino_*`, `s3_*`); no semantic provider, no search, all cross-enrichment off, search-first gate off. Equivalent to wiring the standalone toolkit libraries. | +| `a2` | `platform.bench.a2.yaml` | The shipped semantic-first platform: DataHub enrichment, the `search` tool, the search-first gate, seeded metadata and knowledge pages. | + +Arms `a1` (enrichment-only) and `a3` (memory lifecycle) join in phase 2/3 +(#943, #944). The search-first gate is not persona-aware, so `a0` must disable +`workflow.require_search`; the profile documents this. + +## Seeded ground truth + +`seedgen` generates everything from one fixed-seed dataset model +(`internal/gen`, seed 930): Trino DDL/DML (memory catalog), DataHub metadata +proposals (descriptions, column units, deprecation), knowledge-page SQL, and +the task YAML whose ground truths are computed from the generated rows — +derived, never hand-typed. `TestCommittedArtifactsMatch` fails if the +committed artifacts drift from regeneration (`make bench-gen` refreshes them). + +The pilot task set (10 tasks): + +- **S1 discovery** (5): "which dataset answers X", graded by entity alias + match. +- **S3 knowledge traps** (5): answerable plausibly-but-wrongly without the + knowledge layer. Two seeded trap classes: `units_cents` (monetary columns + are integer cents; the fact lives only in metadata) and `net_revenue` + (policy revenue = amount - discount over completed orders only; the fact + lives in a knowledge page and dataset description; the gross leader and the + net leader differ by construction). Graded numerically or by entity; each + task carries rubric notes recorded into transcripts for manual review + (LLM-judged in phase 2). + +## Measurement + +The platform's audit log is the measurement instrument. The harness calls +`platform_info` to mint the `dps_` session handle, strips the injected +`session_id` property from every tool schema shown to the model, and threads +the handle itself — measurement plumbing invisible to the agent, uniform +across arms. Efficiency metrics are read back from +`GET /api/v1/admin/audit/events?session_id=...`; arm profiles run audit with +`delivery: sync`, and a run **fails loudly** when a session's audit rows fall +outside the bounds of the harness's client-side accounting: confirmed calls +must all have rows, platform refusals (authz, the gates, the per-user +limiter) short-circuit outer to the audit middleware and have none, and +transport-level failures are counted as indeterminate (the platform may have +audited before the error surfaced). + +Two independence guarantees keep attempts comparable: + +- **Identity pool.** The search-first gate keys discovery on the + authenticated USER, not the MCP session, so every attempt authenticates as + its own pool identity (`-01`..`-32`, defined in the arm configs; + `-identity-keys` must match, and a run refuses to start when tasks x k + exceeds the pool). `make bench-run` additionally resets the persisted + discovery state (`search_gate_discovery`) so repeated runs start clean. +- **Harness failures never grade.** Attempts that fail at the harness level + (connect, adapter, audit read-back) are excluded from accuracy and reported + separately; pass^k requires all k attempts graded and correct. + +Grading is deterministic and scores only the first line after the mandated +"FINAL ANSWER:" marker: numeric answers prefer the first decimal-bearing +number (a restated year is a bare integer and is skipped when a decimal +candidate exists); entity answers must name a correct alias and must not name +any of the task's known trap answers (`wrong_aliases`, generated with the +task). Judgment-call scoring is deliberately deferred to the phase-2 judge. + +## Running + +From the repository root: + +```bash +make bench-up BENCH_ARM=a0 # compose stack + seeded warehouse + platform +make bench-smoke # scripted no-API-key end-to-end validation +make bench-run BENCH_ARM=a0 K=3 # real run (needs ANTHROPIC_API_KEY) +make bench-report BENCH_ARM=a0 +make bench-down +``` + +For `a2`, start a DataHub quickstart first (same external convention as e2e +and load), then `make bench-seed-datahub` and `make bench-up BENCH_ARM=a2`. + +The **scripted adapter** (`-llm scripted`) plays a deterministic script +(`tasks/scripted-smoke.json`, generated) instead of a model: SQL-backed tasks +run their reference SQL through the live `trino_query` and answer with the +live result. One smoke run therefore validates the seed data, the stored +ground truths, handle threading, audit read-back, and both graders against the +real platform — with no API key and no model variance. Real runs use the +Anthropic adapter (`-llm anthropic`, model pinned in the manifest, no sampling +parameters; run-to-run variance is handled by k-repeats and pass^k). + +## Output + +`benchrun` writes a results JSON (manifest: git commit, platform version, +model, dataset seed, task-set hash, arm, k; per-attempt records; per-task and +per-suite aggregates) plus per-attempt transcripts, and prints a human +summary. Pilot-phase reporting is manual from these files; the report +generator, CIs, and the published `docs/reference/benchmarks.md` page land in +phases 2 and 4 (#943, #945). + +## Layout + +``` +bench/ +├── benchrun/ CLI entry point +├── seedgen/ deterministic artifact/task generator CLI +├── config/ arm profiles (platform configs) +├── seed/ generated seed artifacts (committed; bench-gen) +├── tasks/ generated task YAML + smoke script (committed) +└── internal/ + ├── gen/ dataset model, emitters, ground-truth computation + ├── task/ task schema, loader, task-set hash + ├── llm/ adapter interface + anthropic + scripted + ├── agent/ model-driven tool loop with budget + ├── mcpc/ MCP session, handle mint, session_id threading + ├── auditapi/ admin audit API read-back + metrics + ├── grade/ deterministic graders (numeric, entity) + ├── pipeline/ task x k orchestration + ├── report/ results model, aggregates, human summary + └── target/ endpoint + Bearer auth +``` diff --git a/bench/benchrun/main.go b/bench/benchrun/main.go new file mode 100644 index 00000000..1e05862b --- /dev/null +++ b/bench/benchrun/main.go @@ -0,0 +1,157 @@ +// Command benchrun executes the agent-effectiveness benchmark against a +// running mcp-data-platform deployment (issue #930 phase 1, #942): one arm, +// the pilot task set, k repeats, results JSON plus a human summary. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/pipeline" + "github.com/txn2/mcp-data-platform/bench/internal/report" + "github.com/txn2/mcp-data-platform/bench/internal/target" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// config carries the parsed flags. +type config struct { + url string + credential string + arm string + suite string + k int + llmProvider string + model string + maxTokens int64 + script string + tasksDir string + out string + gitCommit string + httpTimeout time.Duration + llmTimeout time.Duration + auditTimeout time.Duration + identityKeys int + summarize string +} + +func main() { + cfg := parseFlags() + if err := run(cfg); err != nil { + fmt.Fprintln(os.Stderr, "benchrun:", err) + os.Exit(1) + } +} + +// parseFlags reads the CLI configuration. +func parseFlags() config { + var cfg config + flag.StringVar(&cfg.url, "url", "http://localhost:8098", "platform base URL (MCP + REST)") + flag.StringVar(&cfg.credential, "credential", "", "admin API key (Bearer)") + flag.StringVar(&cfg.arm, "arm", "", "benchmark arm (a0|a2), required") + flag.StringVar(&cfg.suite, "suite", "", "suite filter (s1|s3), empty = all") + flag.IntVar(&cfg.k, "k", 3, "repeats per task (pass^k)") + flag.StringVar(&cfg.llmProvider, "llm", "anthropic", "model adapter: anthropic|scripted") + flag.StringVar(&cfg.model, "model", "claude-opus-4-8", "model id for -llm anthropic") + flag.Int64Var(&cfg.maxTokens, "max-tokens", 8192, "max tokens per completion") + flag.StringVar(&cfg.script, "script", "", "playback script for -llm scripted") + flag.StringVar(&cfg.tasksDir, "tasks", "tasks", "task YAML directory") + flag.StringVar(&cfg.out, "out", "results.json", "results JSON output path") + flag.StringVar(&cfg.gitCommit, "git-commit", "", "repository commit for the manifest") + flag.DurationVar(&cfg.httpTimeout, "http-timeout", 120*time.Second, "platform HTTP timeout") + flag.DurationVar(&cfg.llmTimeout, "llm-timeout", 5*time.Minute, "model API request timeout") + flag.DurationVar(&cfg.auditTimeout, "audit-timeout", 15*time.Second, "audit read-back timeout per session") + flag.IntVar(&cfg.identityKeys, "identity-keys", 32, "per-attempt identity pool size matching the arm config (0 = single identity)") + flag.StringVar(&cfg.summarize, "summarize", "", "print the human summary of an existing results JSON and exit") + flag.Parse() + return cfg +} + +// run dispatches summarize-only mode or a full benchmark run. +func run(cfg config) error { + if cfg.summarize != "" { + res, err := report.LoadJSON(cfg.summarize) + if err != nil { + return err + } + fmt.Print(res.HumanSummary()) + return nil + } + return runBenchmark(cfg) +} + +// runBenchmark executes the pipeline and writes outputs. The results JSON is +// written even when the run fails so partial evidence is never discarded. +func runBenchmark(cfg config) error { + if cfg.arm == "" { + return errors.New("-arm is required") + } + factory, err := buildFactory(cfg) + if err != nil { + return err + } + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + res, runErr := pipeline.Run(context.Background(), pipeline.Options{ + Target: target.Target{BaseURL: cfg.url, Credential: cfg.credential}, + HTTPTimeout: cfg.httpTimeout, + Arm: cfg.arm, + Suite: cfg.suite, + K: cfg.k, + TasksDir: cfg.tasksDir, + TranscriptDir: transcriptDir(cfg.out), + Factory: factory, + LLMProvider: cfg.llmProvider, + GitCommit: cfg.gitCommit, + AuditTimeout: cfg.auditTimeout, + IdentityKeys: cfg.identityKeys, + Log: log, + }) + if res != nil { + if err := res.WriteJSON(cfg.out); err != nil { + return err + } + fmt.Print(res.HumanSummary()) + fmt.Println("results:", cfg.out) + } + return runErr +} + +// transcriptDir derives the transcript directory from the results path. +func transcriptDir(out string) string { + return filepath.Join(filepath.Dir(out), "transcripts") +} + +// buildFactory constructs the adapter factory for the selected provider. +func buildFactory(cfg config) (pipeline.AdapterFactory, error) { + switch cfg.llmProvider { + case "anthropic": + adapter, err := llm.NewAnthropic(cfg.model, cfg.maxTokens, cfg.llmTimeout) + if err != nil { + return nil, err + } + return func(task.Task) (llm.Adapter, error) { return adapter, nil }, nil + case "scripted": + if cfg.script == "" { + return nil, errors.New("-script is required for -llm scripted") + } + script, err := llm.LoadScript(cfg.script) + if err != nil { + return nil, err + } + return func(t task.Task) (llm.Adapter, error) { + steps, ok := script[t.ID] + if !ok { + return nil, fmt.Errorf("script has no steps for task %s", t.ID) + } + return llm.NewScripted(steps), nil + }, nil + default: + return nil, fmt.Errorf("unknown -llm provider %q", cfg.llmProvider) + } +} diff --git a/bench/config/platform.bench.a0.yaml b/bench/config/platform.bench.a0.yaml new file mode 100644 index 00000000..99c79eea --- /dev/null +++ b/bench/config/platform.bench.a0.yaml @@ -0,0 +1,197 @@ +# Benchmark arm A0 — baseline (issue #930 / #942). +# +# Raw toolkit tools only: the persona exposes trino_* and s3_* (plus +# platform_info, which the HARNESS calls to mint the session handle; the agent +# never sees it — the harness strips it from the tool list). No semantic +# provider, no DataHub toolkit, all cross-enrichment off, search-first gate +# off (the gate is not persona-aware, so a persona without a discovery tool +# would deadlock on SEARCH_REQUIRED). Equivalent to wiring the standalone +# toolkit libraries. +# +# Audit runs with delivery: sync so the harness's audit read-back is +# deterministic: when a tool call returns, its audit row is committed. +# +# Env: +# API_KEY_ADMIN admin API key the harness authenticates with (Bearer) +apiVersion: v1 + +server: + name: mcp-data-platform-bench-a0 + transport: http + address: ":8098" + +auth: + api_keys: + enabled: true + keys: + # Base key: the harness's audit read-back (admin REST) credential. + - key: "${API_KEY_ADMIN}" + name: "bench-admin" + roles: ["admin"] + # Identity pool: one key per task attempt. The search-first gate keys + # discovery on the authenticated USER identity (apikey:), so + # attempts must not share a credential or the first search would open + # the gate for every later attempt. The harness rotates through these + # (-identity-keys must match the pool size). + - key: "${API_KEY_ADMIN}-01" + name: "bench-agent-01" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-02" + name: "bench-agent-02" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-03" + name: "bench-agent-03" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-04" + name: "bench-agent-04" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-05" + name: "bench-agent-05" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-06" + name: "bench-agent-06" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-07" + name: "bench-agent-07" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-08" + name: "bench-agent-08" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-09" + name: "bench-agent-09" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-10" + name: "bench-agent-10" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-11" + name: "bench-agent-11" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-12" + name: "bench-agent-12" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-13" + name: "bench-agent-13" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-14" + name: "bench-agent-14" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-15" + name: "bench-agent-15" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-16" + name: "bench-agent-16" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-17" + name: "bench-agent-17" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-18" + name: "bench-agent-18" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-19" + name: "bench-agent-19" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-20" + name: "bench-agent-20" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-21" + name: "bench-agent-21" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-22" + name: "bench-agent-22" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-23" + name: "bench-agent-23" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-24" + name: "bench-agent-24" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-25" + name: "bench-agent-25" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-26" + name: "bench-agent-26" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-27" + name: "bench-agent-27" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-28" + name: "bench-agent-28" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-29" + name: "bench-agent-29" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-30" + name: "bench-agent-30" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-31" + name: "bench-agent-31" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-32" + name: "bench-agent-32" + roles: ["admin"] + +database: + dsn: "postgres://platform:platform_secret@localhost:5432/mcp_platform?sslmode=disable" + max_open_conns: 10 + +audit: + enabled: true + log_tool_calls: true + delivery: sync + +# The per-user tool-call limiter would throttle the single benchmark identity; +# the benchmark measures agent effectiveness, not the limiter. +rate_limit: + enabled: false + +workflow: + require_search: false + +enrichment: + trino_semantic_enrichment: false + datahub_query_enrichment: false + s3_semantic_enrichment: false + datahub_storage_enrichment: false + +# The persona is NAMED admin because the admin REST API (the harness's audit +# read-back path) requires the caller's resolved persona to be the deployment's +# admin persona; the name grants nothing by itself — MCP tool access is still +# the allowlist below, which is the arm's ablation surface. +personas: + admin: + display_name: "Bench Baseline (A0)" + roles: ["admin"] + tools: + allow: ["platform_info", "trino_*", "s3_*"] + connections: + allow: ["*"] + default_persona: admin + +toolkits: + trino: + enabled: true + default: e2e + instances: + e2e: + host: "localhost" + port: 8090 + user: "bench" + catalog: "memory" + schema: "bench" + connection_name: "e2e-trino" + config: + default_limit: 1000 + max_limit: 10000 + read_only: true + + s3: + enabled: true + default: e2e + instances: + e2e: + endpoint: "http://localhost:9000" + access_key_id: "admin" + secret_access_key: "admin_secret" + region: "us-east-1" + use_path_style: true + connection_name: "e2e-s3" diff --git a/bench/config/platform.bench.a2.yaml b/bench/config/platform.bench.a2.yaml new file mode 100644 index 00000000..e8d207d8 --- /dev/null +++ b/bench/config/platform.bench.a2.yaml @@ -0,0 +1,211 @@ +# Benchmark arm A2 — knowledge (issue #930 / #942). +# +# The full discovery-and-knowledge platform: semantic provider (DataHub), +# cross-enrichment on (defaults), the search tool, the search-first gate on +# (default), seeded DataHub metadata and knowledge pages live. Requires a +# DataHub quickstart (same external convention as the e2e and load stacks) +# seeded with bench/seed/datahub/bench_mces.json. +# +# Audit runs with delivery: sync so the harness's audit read-back is +# deterministic: when a tool call returns, its audit row is committed. +# +# Env: +# API_KEY_ADMIN admin API key the harness authenticates with (Bearer) +apiVersion: v1 + +server: + name: mcp-data-platform-bench-a2 + transport: http + address: ":8098" + +auth: + api_keys: + enabled: true + keys: + # Base key: the harness's audit read-back (admin REST) credential. + - key: "${API_KEY_ADMIN}" + name: "bench-admin" + roles: ["admin"] + # Identity pool: one key per task attempt. The search-first gate keys + # discovery on the authenticated USER identity (apikey:), so + # attempts must not share a credential or the first search would open + # the gate for every later attempt. The harness rotates through these + # (-identity-keys must match the pool size). + - key: "${API_KEY_ADMIN}-01" + name: "bench-agent-01" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-02" + name: "bench-agent-02" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-03" + name: "bench-agent-03" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-04" + name: "bench-agent-04" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-05" + name: "bench-agent-05" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-06" + name: "bench-agent-06" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-07" + name: "bench-agent-07" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-08" + name: "bench-agent-08" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-09" + name: "bench-agent-09" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-10" + name: "bench-agent-10" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-11" + name: "bench-agent-11" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-12" + name: "bench-agent-12" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-13" + name: "bench-agent-13" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-14" + name: "bench-agent-14" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-15" + name: "bench-agent-15" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-16" + name: "bench-agent-16" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-17" + name: "bench-agent-17" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-18" + name: "bench-agent-18" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-19" + name: "bench-agent-19" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-20" + name: "bench-agent-20" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-21" + name: "bench-agent-21" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-22" + name: "bench-agent-22" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-23" + name: "bench-agent-23" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-24" + name: "bench-agent-24" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-25" + name: "bench-agent-25" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-26" + name: "bench-agent-26" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-27" + name: "bench-agent-27" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-28" + name: "bench-agent-28" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-29" + name: "bench-agent-29" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-30" + name: "bench-agent-30" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-31" + name: "bench-agent-31" + roles: ["admin"] + - key: "${API_KEY_ADMIN}-32" + name: "bench-agent-32" + roles: ["admin"] + +database: + dsn: "postgres://platform:platform_secret@localhost:5432/mcp_platform?sslmode=disable" + max_open_conns: 10 + +audit: + enabled: true + log_tool_calls: true + delivery: sync + +# The per-user tool-call limiter would throttle the single benchmark identity; +# the benchmark measures agent effectiveness, not the limiter. +rate_limit: + enabled: false + +# workflow.require_search and enrichment.* stay at their defaults (on): the A2 +# arm IS the shipped semantic-first configuration. + +# The persona is NAMED admin because the admin REST API (the harness's audit +# read-back path) requires the caller's resolved persona to be the deployment's +# admin persona; the name grants nothing by itself — MCP tool access is still +# the allowlist below, which is the arm's ablation surface. +personas: + admin: + display_name: "Bench Knowledge (A2)" + roles: ["admin"] + tools: + allow: ["platform_info", "search", "trino_*", "datahub_*", "s3_*"] + connections: + allow: ["*"] + default_persona: admin + +semantic: + provider: datahub + instance: primary + cache: + enabled: true + ttl: 5m + +query: + provider: trino + instance: e2e + +toolkits: + trino: + enabled: true + default: e2e + instances: + e2e: + host: "localhost" + port: 8090 + user: "bench" + catalog: "memory" + schema: "bench" + connection_name: "e2e-trino" + config: + default_limit: 1000 + max_limit: 10000 + read_only: true + + datahub: + enabled: true + default: primary + instances: + primary: + url: "http://localhost:8080/api/graphql" + # DataHub quickstart ships with metadata-service auth disabled; the + # provider requires a non-empty token, so a placeholder is supplied. + token: "bench-placeholder-token" + read_only: true + + s3: + enabled: true + default: e2e + instances: + e2e: + endpoint: "http://localhost:9000" + access_key_id: "admin" + secret_access_key: "admin_secret" + region: "us-east-1" + use_path_style: true + connection_name: "e2e-s3" diff --git a/bench/go.mod b/bench/go.mod new file mode 100644 index 00000000..b6892e70 --- /dev/null +++ b/bench/go.mod @@ -0,0 +1,30 @@ +// Module github.com/txn2/mcp-data-platform/bench is the agent-effectiveness +// benchmark harness (issue #930, phase 1: #942). +// +// It is a SEPARATE Go module on purpose, mirroring test/load (issue #921): the +// root module's coverage, test, and lint gates run over `./...`, and a nested +// module is never matched by the root's `./...`, so the harness stays out of +// the root coverage denominator. Run its own checks from this directory: +// `go test ./...`, `go vet ./...`, `golangci-lint run ./...`. +module github.com/txn2/mcp-data-platform/bench + +go 1.26.2 + +require ( + github.com/anthropics/anthropic-sdk-go v1.19.0 + github.com/google/jsonschema-go v0.4.3 + github.com/modelcontextprotocol/go-sdk v1.6.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/segmentio/asm v1.1.3 // indirect + github.com/segmentio/encoding v0.5.4 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sys v0.41.0 // indirect +) diff --git a/bench/go.sum b/bench/go.sum new file mode 100644 index 00000000..16a14914 --- /dev/null +++ b/bench/go.sum @@ -0,0 +1,42 @@ +github.com/anthropics/anthropic-sdk-go v1.19.0 h1:mO6E+ffSzLRvR/YUH9KJC0uGw0uV8GjISIuzem//3KE= +github.com/anthropics/anthropic-sdk-go v1.19.0/go.mod h1:WTz31rIUHUHqai2UslPpw5CwXrQP3geYBioRV4WOLvE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/modelcontextprotocol/go-sdk v1.6.1 h1:0zOSupjKUxPKSocPT1Wtago+mUHU2/uZ4xSOY0FGReU= +github.com/modelcontextprotocol/go-sdk v1.6.1/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= +github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/bench/internal/agent/agent.go b/bench/internal/agent/agent.go new file mode 100644 index 00000000..932d34ba --- /dev/null +++ b/bench/internal/agent/agent.go @@ -0,0 +1,111 @@ +// Package agent runs the benchmark's model-driven tool loop: present the +// arm's tools, execute the model's tool calls against the live MCP session, +// and stop at a final text answer or the tool-call budget. +package agent + +import ( + "context" + "fmt" + + "github.com/txn2/mcp-data-platform/bench/internal/llm" +) + +// extraIterations bounds the wind-down after budget exhaustion: the model gets +// this many further completions to produce a final answer before the loop +// stops with whatever text it has. +const extraIterations = 2 + +// ToolExecutor executes one tool call against the live session and returns its +// result. Transport-level failures are reported as error results so the model +// can adapt, mirroring how a real client surfaces them. +type ToolExecutor func(ctx context.Context, name string, args map[string]any) llm.ToolResult + +// Config is one task attempt's loop parameters. +type Config struct { + // System is the fixed prompt scaffold (identical across arms). + System string + // Prompt is the task prompt. + Prompt string + // Tools is the arm's tool surface. + Tools []llm.ToolDef + // Budget is the maximum number of tool calls executed. + Budget int +} + +// Result is the outcome of one attempt's loop. +type Result struct { + // FinalAnswer is the model's last assistant text. + FinalAnswer string + // Transcript is the full provider-agnostic conversation, for the + // per-attempt transcript file (manual rubric review reads these). + Transcript []llm.Message + // ToolCalls counts tool calls executed (budget-refused calls excluded). + ToolCalls int + // ToolErrors counts executed calls whose result was an error. + ToolErrors int + // BudgetExhausted is true when the loop hit the tool-call budget. + BudgetExhausted bool + // Usage is the summed token usage across completions. + Usage llm.Usage +} + +// Run drives the loop to completion. It returns an error only for adapter or +// protocol failures; a wrong answer is a graded outcome, not an error. +func Run(ctx context.Context, a llm.Adapter, cfg Config, exec ToolExecutor) (Result, error) { + res := Result{Transcript: []llm.Message{{Role: "user", Text: cfg.Prompt}}} + maxIterations := cfg.Budget + extraIterations + 1 + windDown := 0 + for iter := range maxIterations { + // The wind-down bound is a separate counter, not derived from the + // iteration cap: a model that burns the whole budget with multi-call + // turns would otherwise keep receiving completions (each answered + // only with budget-refusal results) until the iteration cap. + if res.BudgetExhausted { + windDown++ + if windDown > extraIterations { + return res, fmt.Errorf("no final answer within %d completions after budget exhaustion", extraIterations) + } + } + msg, usage, err := a.Complete(ctx, cfg.System, res.Transcript, cfg.Tools) + if err != nil { + return res, fmt.Errorf("completion %d: %w", iter+1, err) + } + res.Usage.Add(usage) + res.Transcript = append(res.Transcript, msg) + if len(msg.ToolCalls) == 0 { + res.FinalAnswer = msg.Text + return res, nil + } + res.Transcript = append(res.Transcript, executeCalls(ctx, msg.ToolCalls, cfg.Budget, exec, &res)) + } + return res, fmt.Errorf("loop exceeded %d iterations without a final answer", maxIterations) +} + +// executeCalls runs the requested calls up to the budget and builds the user +// turn answering them. Calls past the budget receive error results, and the +// turn carries an instruction to answer with what was gathered. +func executeCalls(ctx context.Context, calls []llm.ToolCall, budget int, exec ToolExecutor, res *Result) llm.Message { + reply := llm.Message{Role: "user"} + for _, call := range calls { + if res.ToolCalls >= budget { + res.BudgetExhausted = true + reply.ToolResults = append(reply.ToolResults, llm.ToolResult{ + CallID: call.ID, + Text: "tool-call budget exhausted; no further tool calls are executed", + IsError: true, + }) + continue + } + res.ToolCalls++ + result := exec(ctx, call.Name, call.Args) + result.CallID = call.ID + if result.IsError { + res.ToolErrors++ + } + reply.ToolResults = append(reply.ToolResults, result) + } + if res.BudgetExhausted { + reply.Text = "The tool-call budget is exhausted. Give your FINAL ANSWER now using the information gathered so far." + } + return reply +} diff --git a/bench/internal/agent/agent_test.go b/bench/internal/agent/agent_test.go new file mode 100644 index 00000000..1cbbc430 --- /dev/null +++ b/bench/internal/agent/agent_test.go @@ -0,0 +1,149 @@ +package agent + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/txn2/mcp-data-platform/bench/internal/llm" +) + +// fakeAdapter plays canned assistant turns. +type fakeAdapter struct { + turns []llm.Message + pos int + seen [][]llm.Message // transcript snapshot per Complete call +} + +func (f *fakeAdapter) Model() string { return "fake" } + +func (f *fakeAdapter) Complete(_ context.Context, _ string, msgs []llm.Message, _ []llm.ToolDef) (llm.Message, llm.Usage, error) { + snapshot := make([]llm.Message, len(msgs)) + copy(snapshot, msgs) + f.seen = append(f.seen, snapshot) + if f.pos >= len(f.turns) { + return llm.Message{}, llm.Usage{}, errors.New("no more turns") + } + m := f.turns[f.pos] + f.pos++ + return m, llm.Usage{InputTokens: 10, OutputTokens: 5}, nil +} + +// okExec answers every call successfully. +func okExec(_ context.Context, name string, _ map[string]any) llm.ToolResult { + return llm.ToolResult{Text: "result of " + name} +} + +func TestRunToolLoopThenFinal(t *testing.T) { + a := &fakeAdapter{turns: []llm.Message{ + {Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "1", Name: "search", Args: map[string]any{"intent": "x"}}}}, + {Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "2", Name: "trino_query", Args: map[string]any{"sql": "SELECT 1"}}}}, + {Role: "assistant", Text: "FINAL ANSWER: 42"}, + }} + res, err := Run(context.Background(), a, Config{Prompt: "q", Budget: 5}, okExec) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.FinalAnswer != "FINAL ANSWER: 42" || res.ToolCalls != 2 || res.ToolErrors != 0 || res.BudgetExhausted { + t.Errorf("unexpected result: %+v", res) + } + if res.Usage.InputTokens != 30 || res.Usage.OutputTokens != 15 { + t.Errorf("usage not accumulated: %+v", res.Usage) + } + // The tool result must be visible to the next completion. + last := a.seen[len(a.seen)-1] + if got := last[len(last)-1].ToolResults[0].Text; got != "result of trino_query" { + t.Errorf("tool result not threaded: %q", got) + } +} + +func TestRunBudgetExhaustion(t *testing.T) { + twoCalls := []llm.ToolCall{ + {ID: "a", Name: "t1", Args: map[string]any{}}, + {ID: "b", Name: "t2", Args: map[string]any{}}, + } + a := &fakeAdapter{turns: []llm.Message{ + {Role: "assistant", ToolCalls: twoCalls}, + {Role: "assistant", ToolCalls: twoCalls}, // second call of this turn exceeds budget 3 + {Role: "assistant", Text: "FINAL ANSWER: partial"}, + }} + res, err := Run(context.Background(), a, Config{Prompt: "q", Budget: 3}, okExec) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !res.BudgetExhausted || res.ToolCalls != 3 { + t.Errorf("budget accounting wrong: %+v", res) + } + // The over-budget call must be answered with an error result plus the + // wind-down instruction. + turn := a.seen[2][len(a.seen[2])-1] + if len(turn.ToolResults) != 2 || !turn.ToolResults[1].IsError || !strings.Contains(turn.Text, "FINAL ANSWER") { + t.Errorf("budget turn malformed: %+v", turn) + } + if res.FinalAnswer != "FINAL ANSWER: partial" { + t.Errorf("final answer: %q", res.FinalAnswer) + } +} + +func TestRunToolErrorCounted(t *testing.T) { + a := &fakeAdapter{turns: []llm.Message{ + {Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "1", Name: "bad", Args: map[string]any{}}}}, + {Role: "assistant", Text: "FINAL ANSWER: none"}, + }} + exec := func(context.Context, string, map[string]any) llm.ToolResult { + return llm.ToolResult{Text: "boom", IsError: true} + } + res, err := Run(context.Background(), a, Config{Prompt: "q", Budget: 5}, exec) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.ToolErrors != 1 { + t.Errorf("tool errors = %d, want 1", res.ToolErrors) + } +} + +func TestRunAdapterErrorPropagates(t *testing.T) { + a := &fakeAdapter{} // immediately exhausted + if _, err := Run(context.Background(), a, Config{Prompt: "q", Budget: 1}, okExec); err == nil { + t.Fatal("expected adapter error") + } +} + +func TestRunWindDownBounded(t *testing.T) { + // A model that burns the whole budget with one multi-call turn and keeps + // requesting tools gets at most extraIterations completions after + // exhaustion, not the whole iteration cap. + burst := make([]llm.ToolCall, 6) + for i := range burst { + burst[i] = llm.ToolCall{ID: "b", Name: "t", Args: map[string]any{}} + } + keepCalling := llm.Message{Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "x", Name: "t", Args: map[string]any{}}}} + a := &fakeAdapter{turns: []llm.Message{ + {Role: "assistant", ToolCalls: burst}, // consumes budget 6 in one turn + keepCalling, // first refused call flags exhaustion + keepCalling, keepCalling, keepCalling, // would run to the iteration cap without the wind-down bound + }} + _, err := Run(context.Background(), a, Config{Prompt: "q", Budget: 6}, okExec) + if err == nil || !strings.Contains(err.Error(), "budget exhaustion") { + t.Fatalf("want wind-down bound error, got %v", err) + } + // Burst turn + the turn that trips exhaustion + extraIterations wind-down + // completions; the iteration cap alone would have allowed 9. + if got := len(a.seen); got != 4 { + t.Errorf("completions = %d, want 4 (burst + tripping turn + 2 wind-down)", got) + } +} + +func TestRunNeverEndingLoopBounded(t *testing.T) { + call := llm.Message{Role: "assistant", ToolCalls: []llm.ToolCall{{ID: "x", Name: "t", Args: map[string]any{}}}} + turns := make([]llm.Message, 50) + for i := range turns { + turns[i] = call + } + a := &fakeAdapter{turns: turns} + _, err := Run(context.Background(), a, Config{Prompt: "q", Budget: 2}, okExec) + if err == nil || !strings.Contains(err.Error(), "iterations") { + t.Fatalf("expected iteration bound error, got %v", err) + } +} diff --git a/bench/internal/auditapi/auditapi.go b/bench/internal/auditapi/auditapi.go new file mode 100644 index 00000000..f4d87f76 --- /dev/null +++ b/bench/internal/auditapi/auditapi.go @@ -0,0 +1,196 @@ +// Package auditapi reads efficiency metrics back from the platform's admin +// audit API. The benchmark does not instrument its own timing: the audit log +// is the measurement instrument, and a run fails loudly when audit rows are +// missing for one of its sessions (that missing data is itself a platform +// defect the benchmark must surface, not paper over). +package auditapi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// pageSize is the per_page used when fetching events. +const pageSize = 200 + +// pollInterval is the delay between polls while waiting for rows to land. +// Arm profiles run audit with delivery: sync, so convergence is immediate in +// practice; the poll covers request scheduling slack only. +const pollInterval = 250 * time.Millisecond + +// Event is the subset of the platform audit event the benchmark scores. +// Field names mirror pkg/audit.Event's JSON encoding. +type Event struct { + Timestamp time.Time `json:"timestamp"` + DurationMS int64 `json:"duration_ms"` + SessionID string `json:"session_id"` + ToolName string `json:"tool_name"` + Success bool `json:"success"` + ErrorMessage string `json:"error_message,omitempty"` + EnrichmentApplied bool `json:"enrichment_applied"` + EnrichmentTokensFull int `json:"enrichment_tokens_full"` + EnrichmentTokensDedup int `json:"enrichment_tokens_dedup"` + EnrichmentMode string `json:"enrichment_mode,omitempty"` + EventKind string `json:"event_kind,omitempty"` +} + +// envelope is the admin list response. +type envelope struct { + Data []Event `json:"data"` + Total int `json:"total"` + Page int `json:"page"` + PerPage int `json:"per_page"` +} + +// Client queries the admin audit API with the harness's admin credential. +type Client struct { + base string + http *http.Client +} + +// New returns a Client for the platform base URL using the supplied +// authenticated HTTP client. +func New(baseURL string, httpClient *http.Client) *Client { + return &Client{base: strings.TrimRight(baseURL, "/"), http: httpClient} +} + +// EventsForSession fetches every audit event recorded for a session handle. +func (c *Client) EventsForSession(ctx context.Context, sessionID string) ([]Event, error) { + var all []Event + for page := 1; ; page++ { + env, err := c.fetchPage(ctx, sessionID, page) + if err != nil { + return nil, err + } + all = append(all, env.Data...) + if len(all) >= env.Total || len(env.Data) == 0 { + return all, nil + } + } +} + +// WaitForSession polls until the session's audit rows land within +// [minCount, maxCount] and returns them. The bounds come from the harness's +// client-side accounting: minCount is the calls it CONFIRMED reached the +// handler chain (each must have a row — fewer means audit lost data, which +// fails the run loudly), while maxCount adds the indeterminate calls +// (transport-level errors such as a protocol error or client timeout, where +// the platform may or may not have audited the call before the failure +// surfaced). More rows than maxCount means the accounting itself is wrong and +// the result is not publishable. +func (c *Client) WaitForSession(ctx context.Context, sessionID string, minCount, maxCount int, timeout time.Duration) ([]Event, error) { + deadline := time.Now().Add(timeout) + got := -1 + for { + events, err := c.EventsForSession(ctx, sessionID) + if err != nil { + return nil, err + } + done, err := settled(sessionID, len(events), got, minCount, maxCount) + if done || err != nil { + return events, err + } + got = len(events) + if time.Now().After(deadline) { + if got >= minCount { + return events, nil + } + return nil, fmt.Errorf("audit rows for session %s: got %d of at least %d within %s (missing rows fail the run)", + sessionID, got, minCount, timeout) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(pollInterval): + } + } +} + +// settled decides whether a poll's row count terminates the wait: an +// overcount is a hard error, hitting maxCount is complete, and an in-bounds +// count stable across one poll interval means the indeterminate calls that +// could still add rows did not. +func settled(sessionID string, count, previous, minCount, maxCount int) (bool, error) { + switch { + case count > maxCount: + return false, fmt.Errorf("audit rows for session %s: got %d, expected at most %d (overcount)", + sessionID, count, maxCount) + case count == maxCount: + return true, nil + case count >= minCount && count == previous: + return true, nil + } + return false, nil +} + +// fetchPage fetches one page of events for a session. +func (c *Client) fetchPage(ctx context.Context, sessionID string, page int) (*envelope, error) { + q := url.Values{} + q.Set("session_id", sessionID) + q.Set("page", strconv.Itoa(page)) + q.Set("per_page", strconv.Itoa(pageSize)) + q.Set("sort_by", "timestamp") + q.Set("sort_order", "asc") + endpoint := c.base + "/api/v1/admin/audit/events?" + q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("build audit request: %w", err) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("audit request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, fmt.Errorf("read audit response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("audit API status %d: %.300s", resp.StatusCode, string(body)) + } + var env envelope + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("parse audit response: %w", err) + } + return &env, nil +} + +// Metrics summarizes one session's audit trail for the report. +type Metrics struct { + // AuditedCalls is the number of audited tool calls under the session + // handle. platform_info is not among them: its own audit row carries the + // transport session id because the handle is minted inside its handler. + AuditedCalls int `json:"audited_calls"` + // Errors counts audited calls with success=false. + Errors int `json:"errors"` + // TotalDurationMS sums server-side handler time across the session. + TotalDurationMS int64 `json:"total_duration_ms"` + // EnrichedCalls counts calls where cross-enrichment was applied. + EnrichedCalls int `json:"enriched_calls"` + // EnrichmentTokensDedup sums the deduplicated enrichment token volume. + EnrichmentTokensDedup int `json:"enrichment_tokens_dedup"` +} + +// Summarize folds a session's events into Metrics. +func Summarize(events []Event) Metrics { + var m Metrics + for _, e := range events { + m.AuditedCalls++ + if !e.Success { + m.Errors++ + } + m.TotalDurationMS += e.DurationMS + if e.EnrichmentApplied { + m.EnrichedCalls++ + m.EnrichmentTokensDedup += e.EnrichmentTokensDedup + } + } + return m +} diff --git a/bench/internal/auditapi/auditapi_test.go b/bench/internal/auditapi/auditapi_test.go new file mode 100644 index 00000000..085d5afe --- /dev/null +++ b/bench/internal/auditapi/auditapi_test.go @@ -0,0 +1,175 @@ +package auditapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// fakeAudit serves paginated events per session with optional delayed +// visibility (simulating rows landing over time). +type fakeAudit struct { + mu sync.Mutex + events []Event +} + +func (f *fakeAudit) add(e Event) { + f.mu.Lock() + defer f.mu.Unlock() + f.events = append(f.events, e) +} + +func (f *fakeAudit) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer key" { + t.Errorf("missing bearer auth: %q", r.Header.Get("Authorization")) + } + session := r.URL.Query().Get("session_id") + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + perPage, _ := strconv.Atoi(r.URL.Query().Get("per_page")) + f.mu.Lock() + var matched []Event + for _, e := range f.events { + if e.SessionID == session { + matched = append(matched, e) + } + } + f.mu.Unlock() + start := (page - 1) * perPage + end := min(start+perPage, len(matched)) + var pageData []Event + if start < len(matched) { + pageData = matched[start:end] + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": pageData, "total": len(matched), "page": page, "per_page": perPage, + }) + } +} + +// authedClient wraps the test server with the Bearer credential. +type authedTransport struct{ base http.RoundTripper } + +func (a authedTransport) RoundTrip(r *http.Request) (*http.Response, error) { + r.Header.Set("Authorization", "Bearer key") + return a.base.RoundTrip(r) +} + +func newTestClient(t *testing.T, f *fakeAudit) *Client { + t.Helper() + srv := httptest.NewServer(f.handler(t)) + t.Cleanup(srv.Close) + return New(srv.URL, &http.Client{Transport: authedTransport{base: http.DefaultTransport}}) +} + +func TestEventsForSessionPaginates(t *testing.T) { + f := &fakeAudit{} + for range 450 { // > 2 pages at pageSize 200 + f.add(Event{SessionID: "dps_x", ToolName: "t", Success: true, DurationMS: 1}) + } + f.add(Event{SessionID: "dps_other", ToolName: "t"}) + c := newTestClient(t, f) + events, err := c.EventsForSession(context.Background(), "dps_x") + if err != nil { + t.Fatal(err) + } + if len(events) != 450 { + t.Errorf("got %d events, want 450", len(events)) + } +} + +func TestWaitForSessionConverges(t *testing.T) { + f := &fakeAudit{} + f.add(Event{SessionID: "dps_x", Success: true}) + c := newTestClient(t, f) + go func() { + time.Sleep(300 * time.Millisecond) + f.add(Event{SessionID: "dps_x", Success: true}) + }() + events, err := c.WaitForSession(context.Background(), "dps_x", 2, 2, 5*time.Second) + if err != nil { + t.Fatal(err) + } + if len(events) != 2 { + t.Errorf("got %d, want 2", len(events)) + } +} + +func TestWaitForSessionTimeoutIsLoud(t *testing.T) { + f := &fakeAudit{} + f.add(Event{SessionID: "dps_x", Success: true}) + c := newTestClient(t, f) + _, err := c.WaitForSession(context.Background(), "dps_x", 3, 3, 600*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "got 1 of at least 3") { + t.Fatalf("want loud missing-rows error, got %v", err) + } +} + +func TestWaitForSessionOvercountFails(t *testing.T) { + f := &fakeAudit{} + f.add(Event{SessionID: "dps_x"}) + f.add(Event{SessionID: "dps_x"}) + c := newTestClient(t, f) + if _, err := c.WaitForSession(context.Background(), "dps_x", 1, 1, time.Second); err == nil || + !strings.Contains(err.Error(), "overcount") { + t.Fatalf("want overcount error, got %v", err) + } +} + +func TestWaitForSessionZeroExpected(t *testing.T) { + c := newTestClient(t, &fakeAudit{}) + events, err := c.WaitForSession(context.Background(), "dps_x", 0, 0, time.Second) + if err != nil || len(events) != 0 { + t.Fatalf("zero-expected should return immediately: %v %v", events, err) + } +} + +func TestWaitForSessionIndeterminateBand(t *testing.T) { + // min=1 confirmed, max=2 (one indeterminate call): with exactly one row + // present and stable, the wait accepts without demanding the second. + f := &fakeAudit{} + f.add(Event{SessionID: "dps_x", Success: true}) + c := newTestClient(t, f) + events, err := c.WaitForSession(context.Background(), "dps_x", 1, 2, 5*time.Second) + if err != nil || len(events) != 1 { + t.Fatalf("stable in-band count rejected: %v %v", events, err) + } + // A third row exceeds max: the accounting is wrong and must fail. + f.add(Event{SessionID: "dps_x"}) + f.add(Event{SessionID: "dps_x"}) + if _, err := c.WaitForSession(context.Background(), "dps_x", 1, 2, time.Second); err == nil || + !strings.Contains(err.Error(), "overcount") { + t.Fatalf("want overcount error, got %v", err) + } +} + +func TestClientErrorStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + c := New(srv.URL, srv.Client()) + if _, err := c.EventsForSession(context.Background(), "dps_x"); err == nil || + !strings.Contains(err.Error(), "status 500") { + t.Fatalf("want status error, got %v", err) + } +} + +func TestSummarize(t *testing.T) { + events := []Event{ + {Success: true, DurationMS: 10, EnrichmentApplied: true, EnrichmentTokensDedup: 100}, + {Success: false, DurationMS: 5}, + {Success: true, DurationMS: 20, EnrichmentApplied: true, EnrichmentTokensDedup: 50}, + } + m := Summarize(events) + want := Metrics{AuditedCalls: 3, Errors: 1, TotalDurationMS: 35, EnrichedCalls: 2, EnrichmentTokensDedup: 150} + if m != want { + t.Errorf("Summarize = %+v, want %+v", m, want) + } +} diff --git a/bench/internal/gen/emit_datahub.go b/bench/internal/gen/emit_datahub.go new file mode 100644 index 00000000..45a22037 --- /dev/null +++ b/bench/internal/gen/emit_datahub.go @@ -0,0 +1,116 @@ +package gen + +import ( + "encoding/json" + "fmt" +) + +// DataHub URN construction for the bench warehouse. +func benchURN(table string) string { + return fmt.Sprintf("urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.%s,PROD)", table) +} + +// mcp is one metadata change proposal in the `datahub put --file` format the +// e2e testdata uses (test/e2e/testdata/datahub/datasets.json). +type mcp struct { + EntityURN string `json:"entityUrn"` + EntityType string `json:"entityType"` + AspectName string `json:"aspectName"` + ChangeType string `json:"changeType"` + Aspect any `json:"aspect"` +} + +// Dataset descriptions are the A2 knowledge channel: they carry the facts the +// schema alone does not (units in cents, the net-revenue policy, deprecation, +// the gross-only nature of the pre-aggregated index). +const ( + ordersDescription = "Current order transactions, one row per order. Monetary columns (amount, discount) " + + "are stored as INTEGERS IN US CENTS; divide by 100 for USD. Company revenue reporting policy: " + + "revenue = amount - discount, COMPLETED orders only (refunded and pending orders are excluded). " + + "See the 'Revenue Reporting Policy' knowledge page for the authoritative definition." + customersDescription = "Customer directory: one row per customer with profile attributes " + + "(name, region, tier, account created_at). Join key: customer_id." + legacyDescription = "DEPRECATED order extract from the retired ingestion pipeline. Partial coverage, " + + "totals in dollars. Use memory.bench.orders for all order analysis." + dailyDescription = "Pre-aggregated daily revenue by region, derived from completed orders. Values are " + + "GROSS of discounts (USD), so this index must not be used for policy net-revenue figures; " + + "use memory.bench.orders per the Revenue Reporting Policy." +) + +// DataHubMCEs emits the metadata proposals for `datahub put --file`. +func (d *Dataset) DataHubMCEs() ([]byte, error) { + proposals := make([]mcp, 0, 10) + proposals = append(proposals, + datasetProps("orders", ordersDescription, map[string]string{"team": "bench", "unit": "cents"}), + editableSchema("orders", map[string]string{ + "amount": "Order amount in US cents (integer). Divide by 100 for USD.", + "discount": "Discount in US cents (integer), non-zero on completed orders only.", + }), + datasetProps("customers", customersDescription, map[string]string{"team": "bench"}), + datasetProps("legacy_orders", legacyDescription, map[string]string{"team": "bench"}), + deprecation("legacy_orders", "Deprecated. Use memory.bench.orders instead."), + datasetProps("daily_region_revenue", dailyDescription, map[string]string{"team": "bench", "grain": "day,region"}), + ) + for _, table := range []string{"orders", "customers", "legacy_orders", "daily_region_revenue"} { + proposals = append(proposals, tag(table)) + } + return json.MarshalIndent(proposals, "", " ") +} + +// datasetProps builds a datasetProperties proposal. +func datasetProps(table, description string, custom map[string]string) mcp { + return mcp{ + EntityURN: benchURN(table), + EntityType: "dataset", + AspectName: "datasetProperties", + ChangeType: "UPSERT", + Aspect: map[string]any{ + "name": table, + "description": description, + "customProperties": custom, + }, + } +} + +// editableSchema builds column descriptions. +func editableSchema(table string, fields map[string]string) mcp { + infos := make([]map[string]any, 0, len(fields)) + for _, field := range []string{"amount", "discount"} { + if desc, ok := fields[field]; ok { + infos = append(infos, map[string]any{"fieldPath": field, "description": desc}) + } + } + return mcp{ + EntityURN: benchURN(table), + EntityType: "dataset", + AspectName: "editableSchemaMetadata", + ChangeType: "UPSERT", + Aspect: map[string]any{"editableSchemaFieldInfo": infos}, + } +} + +// deprecation builds the deprecation aspect (the S1 deprecated-table trap). +func deprecation(table, note string) mcp { + return mcp{ + EntityURN: benchURN(table), + EntityType: "dataset", + AspectName: "deprecation", + ChangeType: "UPSERT", + Aspect: map[string]any{ + "deprecated": true, + "note": note, + "actor": "urn:li:corpuser:bench-seed", + }, + } +} + +// tag applies the bench tag so seeded entities are identifiable. +func tag(table string) mcp { + return mcp{ + EntityURN: benchURN(table), + EntityType: "dataset", + AspectName: "globalTags", + ChangeType: "UPSERT", + Aspect: map[string]any{"tags": []map[string]string{{"tag": "urn:li:tag:bench"}}}, + } +} diff --git a/bench/internal/gen/emit_kp.go b/bench/internal/gen/emit_kp.go new file mode 100644 index 00000000..df783484 --- /dev/null +++ b/bench/internal/gen/emit_kp.go @@ -0,0 +1,94 @@ +package gen + +import ( + "fmt" + "strings" +) + +// Knowledge pages are the second A2 knowledge channel: portal-stored markdown +// the search tool surfaces. The insert mirrors dev/seed.sql's direct-SQL +// pattern (embeddings NULL, search falls back to lexical until the indexer +// fills them) and is idempotent via ON CONFLICT. + +const revenuePolicyBody = `# Revenue Reporting Policy + +This is the authoritative definition of "revenue" for all bench warehouse reporting. + +**Revenue = amount - discount, over COMPLETED orders only.** Refunded and pending +orders are excluded entirely. Any figure that includes refunded orders or ignores +discounts is gross volume, not revenue, and must not be reported as revenue. + +Two mechanical rules when computing revenue from [orders](urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)): + +1. The amount and discount columns are stored as integers in **US cents**. + Divide by 100 for USD. +2. Filter on status = 'completed' before summing. + +The pre-aggregated daily_region_revenue index is **gross of discounts** and must +not be used for policy revenue figures.` + +const warehouseGuideBody = `# Bench Warehouse Guide + +The bench schema (memory.bench) holds four tables: + +- **orders** — current order transactions, one row per order. The single source + of truth for order analysis. Monetary columns are integers in US cents. +- **customers** — customer profiles: name, region, tier, account created_at. +- **legacy_orders** — DEPRECATED extract from the retired pipeline; partial + coverage, dollar totals. Do not use; query orders instead. +- **daily_region_revenue** — pre-aggregated daily gross revenue (USD) by region, + completed orders only, gross of discounts. Convenient for trend charts; not + valid for policy revenue (see the Revenue Reporting Policy page).` + +// kpRow is one portal_knowledge_pages row. +type kpRow struct { + id string + slug string + title string + summary string + body string + tags string +} + +// KnowledgePagesSQL emits idempotent inserts for the benchmark's knowledge +// pages, applied via psql after platform migrations have run. +func (d *Dataset) KnowledgePagesSQL() string { + rows := []kpRow{ + { + id: "kp-bench-1", + slug: "revenue-reporting-policy", + title: "Revenue Reporting Policy", + summary: "Revenue = amount - discount over completed orders only; amounts are stored in US cents. " + + "The authoritative definition for all bench warehouse reporting.", + body: revenuePolicyBody, + tags: `["finance","policy","bench"]`, + }, + { + id: "kp-bench-2", + slug: "bench-warehouse-guide", + title: "Bench Warehouse Guide", + summary: "Which bench table to use: orders is current, legacy_orders is deprecated, " + + "daily_region_revenue is gross-only pre-aggregation.", + body: warehouseGuideBody, + tags: `["warehouse","bench"]`, + }, + } + var b strings.Builder + fmt.Fprintf(&b, "-- Generated by bench/seedgen (seed %d). Do not edit; regenerate with `make bench-gen`.\n", Seed) + b.WriteString("-- Requires platform migrations (table portal_knowledge_pages); apply after the platform has booted.\n") + for _, r := range rows { + writeKPInsert(&b, r) + } + return b.String() +} + +// writeKPInsert emits one upsert using dollar-quoted bodies. +func writeKPInsert(b *strings.Builder, r kpRow) { + b.WriteString("\nINSERT INTO portal_knowledge_pages\n") + b.WriteString(" (id, slug, title, summary, body, tags, created_by, created_email, updated_by, current_version, created_at, updated_at)\nVALUES\n") + fmt.Fprintf(b, " ('%s', '%s', '%s',\n '%s',\n $benchkp$%s$benchkp$,\n '%s'::jsonb, 'bench-seed@example.com', 'bench-seed@example.com', 'bench-seed@example.com', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')\n", + r.id, r.slug, sqlEscape(r.title), sqlEscape(r.summary), r.body, r.tags) + b.WriteString("ON CONFLICT (id) DO UPDATE SET\n" + + " slug = EXCLUDED.slug, title = EXCLUDED.title, summary = EXCLUDED.summary,\n" + + " body = EXCLUDED.body, tags = EXCLUDED.tags, updated_at = EXCLUDED.updated_at;\n") +} diff --git a/bench/internal/gen/emit_tasks.go b/bench/internal/gen/emit_tasks.go new file mode 100644 index 00000000..334e78bb --- /dev/null +++ b/bench/internal/gen/emit_tasks.go @@ -0,0 +1,142 @@ +package gen + +import ( + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// pilotBudget is the per-task tool-call budget (MCP-Atlas's budget approach, +// starting value from the #930 design). +const pilotBudget = 30 + +// numericTolerance is the absolute grading tolerance for numeric tasks: +// sums are exact to the cent, so a correct answer is within a cent. +const numericTolerance = 0.01 + +// pilotArms is the phase-1 arm pair. +var pilotArms = []string{"a0", "a2"} + +// Tasks derives the pilot task set (S1 discovery + S3 knowledge traps) with +// ground truths computed from the generated rows. +func (d *Dataset) Tasks() []task.Task { + tasks := d.s1Tasks() + return append(tasks, d.s3Tasks()...) +} + +// s1Tasks are discovery tasks graded by entity match. +func (d *Dataset) s1Tasks() []task.Task { + entity := func(id, prompt string, aliases, wrongAliases []string) task.Task { + return task.Task{ + ID: id, Suite: "s1", Prompt: prompt, Arms: pilotArms, BudgetToolCalls: pilotBudget, + Grading: task.Grading{Kind: task.GradeEntity, Aliases: aliases, WrongAliases: wrongAliases}, + } + } + return []task.Task{ + entity("s1-order-transactions", + "Which table in the bench warehouse would you query for individual customer order transactions (one row per order)?", + []string{"memory.bench.orders", "bench.orders"}, []string{"legacy_orders"}), + entity("s1-daily-aggregate", + "Which table in the bench warehouse provides pre-aggregated daily revenue by region, so you do not have to aggregate raw orders yourself?", + []string{"daily_region_revenue"}, nil), + entity("s1-customer-profile", + "Which table in the bench warehouse holds customer profile attributes such as tier and region?", + []string{"memory.bench.customers", "bench.customers"}, nil), + entity("s1-current-orders", + "The bench warehouse contains more than one table of order data. Which one is the current, supported table for order analysis?", + []string{"memory.bench.orders", "bench.orders"}, []string{"legacy_orders"}), + entity("s1-account-created", + "Which table in the bench warehouse would you use to look up when a customer's account was created?", + []string{"memory.bench.customers", "bench.customers"}, nil), + } +} + +// s3Tasks are the knowledge-trap tasks: each is answerable +// plausibly-but-wrongly without the knowledge layer (dollars-vs-cents, gross +// vs policy net revenue) and correctly with it. +func (d *Dataset) s3Tasks() []task.Task { + return []task.Task{ + d.numericTask("s3-units-q1-total", + "Using the bench warehouse, what was the total order amount in USD across ALL orders (any status) placed in Q1 2025 (2025-01-01 through 2025-03-31)? Round to the nearest cent.", + []string{"units_cents"}, + d.TotalAmountQ1USD(), + "SELECT ROUND(SUM(amount) / 100e0, 2) AS total_usd FROM memory.bench.orders WHERE order_ts >= TIMESTAMP '2025-01-01 00:00:00' AND order_ts < TIMESTAMP '2025-04-01 00:00:00'", + task.RubricItem{ID: "caveat-units", Note: "Answer should note that amounts are stored in cents and were converted to USD."}), + d.numericTask("s3-units-avg-enterprise", + "What is the average order amount in USD across all orders placed by enterprise-tier customers in the bench warehouse? Round to the nearest cent.", + []string{"units_cents"}, + d.AvgAmountEnterpriseUSD(), + "SELECT ROUND(AVG(o.amount) / 100e0, 2) AS avg_usd FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE c.tier = 'enterprise'", + task.RubricItem{ID: "caveat-units", Note: "Answer should note that amounts are stored in cents and were converted to USD."}), + d.numericTask("s3-net-east-march", + "Per the company revenue reporting policy, what was the revenue in USD for the East region in March 2025? Round to the nearest cent.", + []string{"net_revenue", "units_cents"}, + d.NetEastMarchUSD(), + "SELECT ROUND(SUM(o.amount - o.discount) / 100e0, 2) AS revenue_usd FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE o.status = 'completed' AND c.region = 'East' AND o.order_ts >= TIMESTAMP '2025-03-01 00:00:00' AND o.order_ts < TIMESTAMP '2025-04-01 00:00:00'", + task.RubricItem{ID: "caveat-policy", Note: "Answer should state that refunded/pending orders were excluded and discounts subtracted per policy."}), + d.topRegionTask(), + d.numericTask("s3-net-total-2025", + "Per the company revenue reporting policy, what was the company's total revenue in USD for calendar year 2025? Round to the nearest cent.", + []string{"net_revenue", "units_cents"}, + d.NetTotal2025USD(), + "SELECT ROUND(SUM(amount - discount) / 100e0, 2) AS revenue_usd FROM memory.bench.orders WHERE status = 'completed' AND order_ts >= TIMESTAMP '2025-01-01 00:00:00' AND order_ts < TIMESTAMP '2026-01-01 00:00:00'", + task.RubricItem{ID: "caveat-policy", Note: "Answer should state that refunded/pending orders were excluded and discounts subtracted per policy."}), + } +} + +// numericTask builds one numeric S3 task. +func (d *Dataset) numericTask(id, prompt string, traps []string, value float64, sql string, rubric task.RubricItem) task.Task { + return task.Task{ + ID: id, Suite: "s3", Prompt: prompt, Arms: pilotArms, TrapClasses: traps, + BudgetToolCalls: pilotBudget, ExpectedSQL: sql, + Grading: task.Grading{Kind: task.GradeNumeric, Value: new(value), AbsTolerance: numericTolerance}, + Rubric: []task.RubricItem{rubric}, + } +} + +// topRegionTask is the entity-graded trap: the gross leader and the policy +// net-revenue leader differ by construction (the generator asserts it). +func (d *Dataset) topRegionTask() task.Task { + return task.Task{ + ID: "s3-net-top-region", + Suite: "s3", + Prompt: "Per the company revenue reporting policy, which region had the highest revenue in calendar year 2025? " + + "Answer with the region name.", + Arms: pilotArms, TrapClasses: []string{"net_revenue"}, + BudgetToolCalls: pilotBudget, + ExpectedSQL: "SELECT c.region FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id " + + "WHERE o.status = 'completed' AND o.order_ts >= TIMESTAMP '2025-01-01 00:00:00' AND o.order_ts < TIMESTAMP '2026-01-01 00:00:00' " + + "GROUP BY c.region ORDER BY SUM(o.amount - o.discount) DESC LIMIT 1", + Grading: task.Grading{Kind: task.GradeEntity, Aliases: []string{d.TopRegionNet2025()}, WrongAliases: d.losingRegions()}, + Rubric: []task.RubricItem{{ + ID: "caveat-policy", + Note: "Answer should state the ranking uses policy net revenue (completed orders, discounts subtracted).", + }}, + } +} + +// ScriptedSmoke derives the deterministic playback script: tasks with a +// reference SQL run it through trino_query and answer with the live result +// (validating seed data, ground truth, and grading against the running +// platform in one pass); pure discovery tasks answer directly (validating the +// entity grading path). Every scripted path opens with a search call so the +// a2 arm's search-first gate is satisfied; under a0 (no search tool) that +// call fails harmlessly and the script proceeds. +func ScriptedSmoke(tasks []task.Task) llm.Script { + script := llm.Script{} + for _, t := range tasks { + search := llm.Step{ToolCalls: []llm.ToolCall{{Name: "search", Args: map[string]any{"intent": t.Prompt}}}} + if t.ExpectedSQL != "" { + script[t.ID] = []llm.Step{ + search, + {ToolCalls: []llm.ToolCall{{Name: "trino_query", Args: map[string]any{"sql": t.ExpectedSQL}}}}, + {FinalText: "FINAL ANSWER: {{last_result}}"}, + } + continue + } + script[t.ID] = []llm.Step{ + search, + {FinalText: "FINAL ANSWER: " + t.Grading.Aliases[0]}, + } + } + return script +} diff --git a/bench/internal/gen/emit_trino.go b/bench/internal/gen/emit_trino.go new file mode 100644 index 00000000..9be49d94 --- /dev/null +++ b/bench/internal/gen/emit_trino.go @@ -0,0 +1,117 @@ +package gen + +import ( + "fmt" + "sort" + "strings" +) + +// insertChunk bounds rows per INSERT statement to keep statements parseable. +const insertChunk = 250 + +// TrinoSQL emits the warehouse DDL/DML for the memory catalog. Statements are +// idempotent (DROP + CREATE) so re-seeding a running stack is safe. +func (d *Dataset) TrinoSQL() string { + var b strings.Builder + b.WriteString("-- Generated by bench/seedgen (seed ") + fmt.Fprintf(&b, "%d", Seed) + b.WriteString("). Do not edit; regenerate with `make bench-gen`.\n") + b.WriteString("CREATE SCHEMA IF NOT EXISTS memory.bench;\n\n") + d.emitCustomersSQL(&b) + d.emitOrdersSQL(&b) + d.emitLegacyOrdersSQL(&b) + d.emitDailyRevenueSQL(&b) + return b.String() +} + +// emitCustomersSQL writes the customer dimension. +func (d *Dataset) emitCustomersSQL(b *strings.Builder) { + b.WriteString("DROP TABLE IF EXISTS memory.bench.customers;\n") + b.WriteString("CREATE TABLE memory.bench.customers (\n" + + " customer_id INTEGER,\n name VARCHAR,\n region VARCHAR,\n tier VARCHAR,\n created_at TIMESTAMP\n);\n") + rows := make([]string, len(d.Customers)) + for i, c := range d.Customers { + rows[i] = fmt.Sprintf("(%d, '%s', '%s', '%s', TIMESTAMP '%s')", + c.ID, sqlEscape(c.Name), c.Region, c.Tier, c.CreatedAt.Format("2006-01-02 15:04:05")) + } + writeInserts(b, "memory.bench.customers", rows) +} + +// emitOrdersSQL writes the order fact. The amount and discount columns carry +// the units trap: plain INTEGER columns named without a unit suffix. +func (d *Dataset) emitOrdersSQL(b *strings.Builder) { + b.WriteString("\nDROP TABLE IF EXISTS memory.bench.orders;\n") + b.WriteString("CREATE TABLE memory.bench.orders (\n" + + " order_id INTEGER,\n customer_id INTEGER,\n order_ts TIMESTAMP,\n status VARCHAR,\n amount INTEGER,\n discount INTEGER\n);\n") + rows := make([]string, len(d.Orders)) + for i, o := range d.Orders { + rows[i] = fmt.Sprintf("(%d, %d, TIMESTAMP '%s', '%s', %d, %d)", + o.ID, o.CustomerID, o.TS.Format("2006-01-02 15:04:05"), o.Status, o.Amount, o.Discount) + } + writeInserts(b, "memory.bench.orders", rows) +} + +// emitLegacyOrdersSQL writes the deprecated order table (the S1 deprecation +// distractor): an older projection of the first orders, totals in dollars. +func (d *Dataset) emitLegacyOrdersSQL(b *strings.Builder) { + b.WriteString("\nDROP TABLE IF EXISTS memory.bench.legacy_orders;\n") + b.WriteString("CREATE TABLE memory.bench.legacy_orders (\n" + + " order_id INTEGER,\n customer_id INTEGER,\n order_date DATE,\n total DOUBLE\n);\n") + rows := make([]string, 0, legacyCount) + for _, o := range d.Orders[:legacyCount] { + rows = append(rows, fmt.Sprintf("(%d, %d, DATE '%s', %.2f)", + o.ID, o.CustomerID, o.TS.Format("2006-01-02"), float64(o.Amount)/100.0)) + } + writeInserts(b, "memory.bench.legacy_orders", rows) +} + +// emitDailyRevenueSQL writes the pre-aggregated index (an S1 discovery target +// and an S3 distractor: gross of discounts, so it cannot answer policy +// revenue questions). +func (d *Dataset) emitDailyRevenueSQL(b *strings.Builder) { + b.WriteString("\nDROP TABLE IF EXISTS memory.bench.daily_region_revenue;\n") + b.WriteString("CREATE TABLE memory.bench.daily_region_revenue (\n" + + " day DATE,\n region VARCHAR,\n gross_revenue_usd DOUBLE\n);\n") + byID := d.customerByID() + type key struct { + day string + region string + } + sums := map[key]int64{} + for _, o := range d.Orders { + if o.Status != "completed" { + continue + } + sums[key{o.TS.Format("2006-01-02"), byID[o.CustomerID].Region}] += o.Amount + } + keys := make([]key, 0, len(sums)) + for k := range sums { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].day != keys[j].day { + return keys[i].day < keys[j].day + } + return keys[i].region < keys[j].region + }) + rows := make([]string, len(keys)) + for i, k := range keys { + rows[i] = fmt.Sprintf("(DATE '%s', '%s', %.2f)", k.day, k.region, float64(sums[k])/100.0) + } + writeInserts(b, "memory.bench.daily_region_revenue", rows) +} + +// writeInserts emits chunked INSERT statements. +func writeInserts(b *strings.Builder, table string, rows []string) { + for start := 0; start < len(rows); start += insertChunk { + end := min(start+insertChunk, len(rows)) + b.WriteString("INSERT INTO " + table + " VALUES\n") + b.WriteString(strings.Join(rows[start:end], ",\n")) + b.WriteString(";\n") + } +} + +// sqlEscape doubles single quotes for SQL string literals. +func sqlEscape(s string) string { + return strings.ReplaceAll(s, "'", "''") +} diff --git a/bench/internal/gen/gen.go b/bench/internal/gen/gen.go new file mode 100644 index 00000000..9fc09cc4 --- /dev/null +++ b/bench/internal/gen/gen.go @@ -0,0 +1,177 @@ +// Package gen is the benchmark's deterministic dataset generator (issue #930 +// principle 3: seeded ground truth). One fixed-seed dataset model emits every +// seed artifact — Trino DDL/DML, DataHub metadata proposals, knowledge-page +// SQL — plus the task set whose ground truths are computed directly from the +// generated rows, so truth is derived, never hand-typed. Regeneration is +// byte-identical; a test diffs the committed artifacts against a fresh run. +package gen + +import ( + "fmt" + // nosemgrep: go.lang.security.audit.crypto.math_random.math-random-used -- deterministic fixture generation from a fixed seed is the point; crypto/rand would break reproducibility + "math/rand" + "time" +) + +// Seed is the fixed RNG seed. Changing it changes every artifact and ground +// truth together; the committed task-set hash pins the current value. +const Seed = 930 + +// Dataset dimensions. Small enough to seed in seconds, large enough that no +// answer is guessable without querying. +const ( + customerCount = 80 + orderCount = 1200 + legacyCount = 60 +) + +// regions and tiers are the categorical dimensions. +var ( + regions = []string{"North", "South", "East", "West"} + tiers = []string{"basic", "plus", "enterprise"} + + firstNames = []string{"Avery", "Blake", "Casey", "Devon", "Emery", "Finley", "Gray", "Harper", "Indigo", "Jules", + "Kai", "Lane", "Morgan", "Noel", "Oakley", "Parker", "Quinn", "Reese", "Sage", "Tatum"} + lastNames = []string{"Alvarez", "Brooks", "Chen", "Dawson", "Ellis", "Foster", "Garcia", "Hayes", "Iwata", "Jensen", + "Kim", "Lopez", "Mercer", "Nguyen", "Okafor", "Price", "Ramos", "Silva", "Turner", "Vargas"} +) + +// Customer is one row of memory.bench.customers. +type Customer struct { + ID int + Name string + Region string + Tier string + CreatedAt time.Time +} + +// Order is one row of memory.bench.orders. Amount and Discount are integer US +// cents — the units trap: only the metadata layer records the unit. +type Order struct { + ID int + CustomerID int + TS time.Time + Status string // completed | refunded | pending + Amount int64 // cents + Discount int64 // cents +} + +// Dataset is the generated world every artifact derives from. +type Dataset struct { + Customers []Customer + Orders []Order +} + +// Generate builds the dataset from the fixed seed and asserts the trap +// invariants hold. It panics on invariant violation: the seed is a constant, +// so a violation is a code bug caught by the generator's own tests, never a +// runtime roll of the dice. +func Generate() *Dataset { + rng := rand.New(rand.NewSource(Seed)) // #nosec G404 -- deterministic fixture generation, not crypto + ds := &Dataset{ + Customers: genCustomers(rng), + } + ds.Orders = genOrders(rng, ds.Customers) + ds.assertTrapInvariants() + return ds +} + +// genCustomers builds the customer dimension. +func genCustomers(rng *rand.Rand) []Customer { + base := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + customers := make([]Customer, customerCount) + for i := range customers { + customers[i] = Customer{ + ID: i + 1, + Name: fmt.Sprintf("%s %s", firstNames[rng.Intn(len(firstNames))], lastNames[rng.Intn(len(lastNames))]), + Region: regions[rng.Intn(len(regions))], + Tier: pickTier(rng), + CreatedAt: base.Add(time.Duration(rng.Intn(700*24)) * time.Hour), + } + } + return customers +} + +// pickTier weights tiers toward basic. +func pickTier(rng *rand.Rand) string { + switch r := rng.Float64(); { + case r < 0.5: + return tiers[0] + case r < 0.8: + return tiers[1] + default: + return tiers[2] + } +} + +// genOrders builds the order fact. Regional biases engineer the net-revenue +// trap: East runs high gross with heavy refunds and discounts, West runs +// moderate gross but clean, so the gross-revenue leader and the policy +// net-revenue leader differ. assertTrapInvariants verifies the result. +func genOrders(rng *rand.Rand, customers []Customer) []Order { + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + orders := make([]Order, orderCount) + for i := range orders { + c := customers[rng.Intn(len(customers))] + amount := int64(500 + rng.Intn(249500)) // 5.00 .. 2500.00 USD in cents + bias := regionBias(c.Region) + amount = int64(float64(amount) * bias.amountFactor) + status := pickStatus(rng, bias.refundP) + var discount int64 + if status == "completed" { + discount = int64(float64(amount) * bias.discountFrac * rng.Float64()) + } + orders[i] = Order{ + ID: 1000 + i, + CustomerID: c.ID, + TS: start.Add(time.Duration(rng.Intn(365*24*60)) * time.Minute), + Status: status, + Amount: amount, + Discount: discount, + } + } + return orders +} + +// regionTrapBias shapes one region's order economics. +type regionTrapBias struct { + amountFactor float64 + refundP float64 + discountFrac float64 +} + +// regionBias returns the trap-engineering bias for a region. +func regionBias(region string) regionTrapBias { + switch region { + case "East": + return regionTrapBias{amountFactor: 1.6, refundP: 0.35, discountFrac: 0.5} + case "West": + return regionTrapBias{amountFactor: 1.25, refundP: 0.05, discountFrac: 0.08} + default: + return regionTrapBias{amountFactor: 1.0, refundP: 0.10, discountFrac: 0.15} + } +} + +// pickStatus draws an order status with the region's refund probability. +func pickStatus(rng *rand.Rand, refundP float64) string { + switch r := rng.Float64(); { + case r < refundP: + return "refunded" + case r < refundP+0.08: + return "pending" + default: + return "completed" + } +} + +// assertTrapInvariants verifies the properties the S3 tasks depend on. +func (d *Dataset) assertTrapInvariants() { + grossLeader := d.topRegionGrossAll() + netLeader := d.TopRegionNet2025() + if grossLeader == netLeader { + panic(fmt.Sprintf("trap invariant violated: gross leader %s equals net leader %s", grossLeader, netLeader)) + } + if d.enterpriseOrderCount() == 0 { + panic("trap invariant violated: no enterprise orders") + } +} diff --git a/bench/internal/gen/gen_test.go b/bench/internal/gen/gen_test.go new file mode 100644 index 00000000..7d963b17 --- /dev/null +++ b/bench/internal/gen/gen_test.go @@ -0,0 +1,149 @@ +package gen + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +func TestGenerateDeterministic(t *testing.T) { + a, b := Generate(), Generate() + if !reflect.DeepEqual(a, b) { + t.Fatal("Generate is not deterministic") + } + if a.TrinoSQL() != b.TrinoSQL() || a.KnowledgePagesSQL() != b.KnowledgePagesSQL() { + t.Fatal("emitters are not deterministic") + } + am, _ := a.DataHubMCEs() + bm, _ := b.DataHubMCEs() + if string(am) != string(bm) { + t.Fatal("mce emitter is not deterministic") + } +} + +func TestTrapInvariants(t *testing.T) { + ds := Generate() + gross, net := ds.topRegionGrossAll(), ds.TopRegionNet2025() + if gross == net { + t.Fatalf("gross leader %q must differ from net leader %q", gross, net) + } + if ds.enterpriseOrderCount() == 0 { + t.Fatal("no enterprise orders") + } + // Ground truths must be non-trivial (nonzero and distinct) so a task is + // never accidentally answerable by zero or by another task's value. + values := []float64{ds.TotalAmountQ1USD(), ds.AvgAmountEnterpriseUSD(), ds.NetEastMarchUSD(), ds.NetTotal2025USD()} + seen := map[float64]bool{} + for _, v := range values { + if v <= 0 { + t.Errorf("ground truth %v not positive", v) + } + if seen[v] { + t.Errorf("duplicate ground truth %v", v) + } + seen[v] = true + } +} + +func TestTasksAreValid(t *testing.T) { + ds := Generate() + tasks := ds.Tasks() + if len(tasks) != 10 { + t.Fatalf("pilot task set = %d tasks, want 10", len(tasks)) + } + suites := map[string]int{} + for _, tk := range tasks { + if err := tk.Validate(); err != nil { + t.Errorf("task %s invalid: %v", tk.ID, err) + } + suites[tk.Suite]++ + } + if suites["s1"] != 5 || suites["s3"] != 5 { + t.Errorf("suite split %v, want 5 s1 / 5 s3", suites) + } +} + +func TestScriptedSmokeCoversAllTasks(t *testing.T) { + ds := Generate() + tasks := ds.Tasks() + script := ScriptedSmoke(tasks) + for _, tk := range tasks { + steps, ok := script[tk.ID] + if !ok || len(steps) == 0 { + t.Errorf("no smoke steps for %s", tk.ID) + continue + } + last := steps[len(steps)-1] + if last.FinalText == "" { + t.Errorf("%s: smoke script does not end in a final answer", tk.ID) + } + if tk.ExpectedSQL != "" && !strings.Contains(last.FinalText, "{{last_result}}") { + t.Errorf("%s: sql-backed task must answer from the live result", tk.ID) + } + } +} + +// TestCommittedArtifactsMatch is the reproducibility gate: the committed seed +// artifacts and task set must regenerate byte-identically from the fixed seed. +func TestCommittedArtifactsMatch(t *testing.T) { + ds := Generate() + root := "../.." + mces, err := ds.DataHubMCEs() + if err != nil { + t.Fatal(err) + } + compareFile(t, filepath.Join(root, "seed/trino/setup.sql"), []byte(ds.TrinoSQL())) + compareFile(t, filepath.Join(root, "seed/datahub/bench_mces.json"), mces) + compareFile(t, filepath.Join(root, "seed/postgres/knowledge_pages.sql"), []byte(ds.KnowledgePagesSQL())) + + committed, err := task.Load(filepath.Join(root, "tasks")) + if err != nil { + t.Fatalf("load committed tasks: %v", err) + } + if got, want := task.Hash(committed), task.Hash(ds.Tasks()); got != want { + t.Errorf("committed task set hash %s != regenerated %s; run `make bench-gen`", got, want) + } + smoke, err := json.MarshalIndent(ScriptedSmoke(ds.Tasks()), "", " ") + if err != nil { + t.Fatal(err) + } + compareFile(t, filepath.Join(root, "tasks/scripted-smoke.json"), append(smoke, '\n')) +} + +func compareFile(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) // #nosec G304 -- repo-relative test fixture + if err != nil { + t.Fatalf("read %s: %v (run `make bench-gen`)", path, err) + } + if string(got) != string(want) { + t.Errorf("%s differs from regeneration; run `make bench-gen`", path) + } +} + +func TestEmittedContentCarriesTraps(t *testing.T) { + ds := Generate() + sql := ds.TrinoSQL() + for _, table := range []string{"customers", "orders", "legacy_orders", "daily_region_revenue"} { + if !strings.Contains(sql, "CREATE TABLE memory.bench."+table) { + t.Errorf("trino sql missing table %s", table) + } + } + mces, _ := ds.DataHubMCEs() + for _, needle := range []string{"US CENTS", "deprecation", "GROSS of discounts"} { + if !strings.Contains(string(mces), needle) { + t.Errorf("mces missing %q", needle) + } + } + kp := ds.KnowledgePagesSQL() + for _, needle := range []string{"revenue-reporting-policy", "bench-warehouse-guide", "ON CONFLICT"} { + if !strings.Contains(kp, needle) { + t.Errorf("knowledge pages missing %q", needle) + } + } +} diff --git a/bench/internal/gen/truth.go b/bench/internal/gen/truth.go new file mode 100644 index 00000000..ad887ae8 --- /dev/null +++ b/bench/internal/gen/truth.go @@ -0,0 +1,158 @@ +package gen + +import ( + "math" + "time" +) + +// Ground-truth computations. Each mirrors the reference SQL recorded on its +// task (expected_sql) exactly: integer-cent sums divided by 100 at the end, so +// sums are exact to the cent and only averages round. + +// centsToUSD converts an exact cent sum to dollars. +func centsToUSD(cents int64) float64 { + return float64(cents) / 100.0 +} + +// round2 rounds to the nearest cent, matching ROUND(x, 2). +func round2(v float64) float64 { + return math.Round(v*100) / 100 +} + +// customerByID indexes the customer dimension. +func (d *Dataset) customerByID() map[int]Customer { + m := make(map[int]Customer, len(d.Customers)) + for _, c := range d.Customers { + m[c.ID] = c + } + return m +} + +// inRange reports ts in [from, to). +func inRange(ts, from, to time.Time) bool { + return !ts.Before(from) && ts.Before(to) +} + +// TotalAmountQ1USD is the units-trap truth: total order amount (any status) +// for Q1 2025, in USD. +func (d *Dataset) TotalAmountQ1USD() float64 { + from := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + to := time.Date(2025, 4, 1, 0, 0, 0, 0, time.UTC) + var cents int64 + for _, o := range d.Orders { + if inRange(o.TS, from, to) { + cents += o.Amount + } + } + return centsToUSD(cents) +} + +// AvgAmountEnterpriseUSD is the units-trap truth with a join: average order +// amount across enterprise-tier customers' orders, in USD, rounded to cents. +func (d *Dataset) AvgAmountEnterpriseUSD() float64 { + byID := d.customerByID() + var cents, n int64 + for _, o := range d.Orders { + if byID[o.CustomerID].Tier == "enterprise" { + cents += o.Amount + n++ + } + } + if n == 0 { + return 0 + } + return round2(float64(cents) / float64(n) / 100.0) +} + +// enterpriseOrderCount supports the generator's invariant check. +func (d *Dataset) enterpriseOrderCount() int { + byID := d.customerByID() + n := 0 + for _, o := range d.Orders { + if byID[o.CustomerID].Tier == "enterprise" { + n++ + } + } + return n +} + +// netCentsByRegion sums policy net revenue (amount - discount, completed only) +// per region over [from, to). +func (d *Dataset) netCentsByRegion(from, to time.Time) map[string]int64 { + byID := d.customerByID() + sums := map[string]int64{} + for _, o := range d.Orders { + if o.Status == "completed" && inRange(o.TS, from, to) { + sums[byID[o.CustomerID].Region] += o.Amount - o.Discount + } + } + return sums +} + +// year2025 returns the calendar-2025 range. +func year2025() (time.Time, time.Time) { + return time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) +} + +// NetEastMarchUSD is the policy-trap truth: net revenue for East, March 2025. +func (d *Dataset) NetEastMarchUSD() float64 { + from := time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC) + to := time.Date(2025, 4, 1, 0, 0, 0, 0, time.UTC) + return centsToUSD(d.netCentsByRegion(from, to)["East"]) +} + +// NetTotal2025USD is the policy-trap truth: total net revenue for 2025. +func (d *Dataset) NetTotal2025USD() float64 { + from, to := year2025() + var total int64 + for _, cents := range d.netCentsByRegion(from, to) { + total += cents + } + return centsToUSD(total) +} + +// TopRegionNet2025 is the region with the highest policy net revenue in 2025. +func (d *Dataset) TopRegionNet2025() string { + from, to := year2025() + return argmax(d.netCentsByRegion(from, to)) +} + +// topRegionGrossAll is the plausible-but-wrong reading the trap is built +// against: highest gross amount, all statuses, 2025. +func (d *Dataset) topRegionGrossAll() string { + byID := d.customerByID() + from, to := year2025() + sums := map[string]int64{} + for _, o := range d.Orders { + if inRange(o.TS, from, to) { + sums[byID[o.CustomerID].Region] += o.Amount + } + } + return argmax(sums) +} + +// losingRegions returns every region except the policy net-revenue leader — +// the top-region task's wrong-alias set (any answer naming one is incorrect). +func (d *Dataset) losingRegions() []string { + winner := d.TopRegionNet2025() + losers := make([]string, 0, len(regions)-1) + for _, r := range regions { + if r != winner { + losers = append(losers, r) + } + } + return losers +} + +// argmax returns the key with the highest value, ties broken by name order so +// the result is deterministic (the generator asserts a unique leader anyway). +func argmax(sums map[string]int64) string { + var best string + var bestV int64 + for _, r := range regions { + if v := sums[r]; best == "" || v > bestV { + best, bestV = r, v + } + } + return best +} diff --git a/bench/internal/grade/grade.go b/bench/internal/grade/grade.go new file mode 100644 index 00000000..1987aa69 --- /dev/null +++ b/bench/internal/grade/grade.go @@ -0,0 +1,88 @@ +// Package grade implements the deterministic graders: numeric tolerance match +// and entity alias match, plus the FINAL ANSWER extraction convention shared +// with the system-prompt scaffold. +// +// Both graders score only the first line after the FINAL ANSWER marker (the +// convention the system prompt mandates), so trailing commentary cannot flip a +// grade. They are deliberately simple and documented rather than clever; +// judgment-call scoring is the phase-2 LLM judge's job, and every attempt's +// transcript is persisted for manual audit. +package grade + +import ( + "math" + "regexp" + "strconv" + "strings" +) + +// finalMarker matches the answer convention the system prompt mandates. The +// last occurrence wins so a model that restates the marker while reasoning is +// graded on its actual final line. +var finalMarker = regexp.MustCompile(`(?i)FINAL ANSWER:\s*`) + +// numberPattern matches a decimal number, tolerating $ prefixes and thousands +// separators in the surrounding text. +var numberPattern = regexp.MustCompile(`-?\d[\d,]*(?:\.\d+)?`) + +// ExtractFinal returns the text after the last "FINAL ANSWER:" marker, or the +// whole trimmed text when the marker is absent. +func ExtractFinal(text string) string { + locs := finalMarker.FindAllStringIndex(text, -1) + if len(locs) == 0 { + return strings.TrimSpace(text) + } + return strings.TrimSpace(text[locs[len(locs)-1][1]:]) +} + +// firstLine isolates the answer line the graders score. +func firstLine(final string) string { + line, _, _ := strings.Cut(final, "\n") + return strings.TrimSpace(line) +} + +// Numeric grades a final answer against an expected value. Candidates come +// from the first line only; among them, numbers carrying a decimal point or +// thousands separator are preferred over bare integers (the tasks demand +// cent-rounded USD, while bare integers in a verbose answer are usually +// restated years or counts). The first such candidate is graded; with no +// decimal-bearing candidate, the first number is. ok is false when the line +// carries no number at all. +func Numeric(final string, expected, absTolerance float64) (got float64, ok, correct bool) { + matches := numberPattern.FindAllString(firstLine(final), -1) + if len(matches) == 0 { + return 0, false, false + } + candidate := matches[0] + for _, m := range matches { + if strings.ContainsAny(m, ".,") { + candidate = m + break + } + } + got, err := strconv.ParseFloat(strings.ReplaceAll(candidate, ",", ""), 64) + if err != nil { + return 0, false, false + } + return got, true, math.Abs(got-expected) <= absTolerance +} + +// Entity grades a final answer's first line against accepted aliases, +// case-insensitively. An answer is correct only when a correct alias appears +// AND no wrong alias does: the wrong-alias list enumerates the task's known +// trap answers (the deprecated table, the gross-revenue region), so a verbose +// answer that names the trap while mentioning the truth is not credited. +func Entity(final string, aliases, wrongAliases []string) (matched string, correct bool) { + line := strings.ToLower(firstLine(final)) + for _, w := range wrongAliases { + if w != "" && strings.Contains(line, strings.ToLower(w)) { + return "", false + } + } + for _, a := range aliases { + if a != "" && strings.Contains(line, strings.ToLower(a)) { + return a, true + } + } + return "", false +} diff --git a/bench/internal/grade/grade_test.go b/bench/internal/grade/grade_test.go new file mode 100644 index 00000000..607a8fc0 --- /dev/null +++ b/bench/internal/grade/grade_test.go @@ -0,0 +1,94 @@ +package grade + +import "testing" + +func TestExtractFinal(t *testing.T) { + cases := []struct { + name, in, want string + }{ + {"marker", "reasoning...\nFINAL ANSWER: 42.50", "42.50"}, + {"last marker wins", "FINAL ANSWER: draft\nmore\nFINAL ANSWER: real", "real"}, + {"case insensitive", "final answer: memory.bench.orders", "memory.bench.orders"}, + {"no marker", " just text ", "just text"}, + {"multiline tail", "FINAL ANSWER: 12.30\nbecause of X", "12.30\nbecause of X"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := ExtractFinal(c.in); got != c.want { + t.Errorf("ExtractFinal(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +func TestNumeric(t *testing.T) { + cases := []struct { + name string + final string + expected float64 + tol float64 + wantGot float64 + wantOK bool + wantHit bool + }{ + {"exact", "42.50", 42.5, 0.01, 42.5, true, true}, + {"dollar and commas", "$1,234,567.89", 1234567.89, 0.01, 1234567.89, true, true}, + {"within tolerance", "100.004", 100.0, 0.01, 100.004, true, true}, + {"outside tolerance", "100.02", 100.0, 0.01, 100.02, true, false}, + {"hundredfold cents miss", "4250", 42.5, 0.01, 4250, true, false}, + {"negative", "-12.5", -12.5, 0.01, -12.5, true, true}, + {"first decimal wins", "12345.67 (from 890 orders)", 12345.67, 0.01, 12345.67, true, true}, + {"restated year skipped", "The Q1 2025 total is 1,077,853.21 USD", 1077853.21, 0.01, 1077853.21, true, true}, + {"bare integer answer", "4250 USD", 4250, 0.01, 4250, true, true}, + {"second line ignored", "42.50\nbut 99.99 was gross", 42.5, 0.01, 42.5, true, true}, + {"no number", "unknown", 1, 0.01, 0, false, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ok, hit := Numeric(c.final, c.expected, c.tol) + if ok != c.wantOK || hit != c.wantHit || (ok && got != c.wantGot) { + t.Errorf("Numeric(%q, %v, %v) = (%v, %v, %v), want (%v, %v, %v)", + c.final, c.expected, c.tol, got, ok, hit, c.wantGot, c.wantOK, c.wantHit) + } + }) + } +} + +func TestEntity(t *testing.T) { + aliases := []string{"memory.bench.orders", "bench.orders"} + wrong := []string{"legacy_orders"} + cases := []struct { + name string + final string + want string + correct bool + }{ + {"qualified", "memory.bench.orders", "memory.bench.orders", true}, + {"case", "MEMORY.BENCH.ORDERS", "memory.bench.orders", true}, + {"embedded", "the table memory.bench.orders is best", "memory.bench.orders", true}, + {"legacy does not match", "memory.bench.legacy_orders", "", false}, + {"bare name does not match", "orders", "", false}, + {"wrong alias vetoes", "memory.bench.legacy_orders (not memory.bench.orders)", "", false}, + {"second line ignored", "memory.bench.orders\nlegacy_orders is deprecated", "memory.bench.orders", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, correct := Entity(c.final, aliases, wrong) + if got != c.want || correct != c.correct { + t.Errorf("Entity(%q) = (%q, %v), want (%q, %v)", c.final, got, correct, c.want, c.correct) + } + }) + } +} + +func TestEntityRegionTrap(t *testing.T) { + // The top-region trap: naming a losing region anywhere on the answer line + // is incorrect even when the winner is also mentioned. + got, correct := Entity("East - though West leads after discounts", []string{"West"}, []string{"North", "South", "East"}) + if correct || got != "" { + t.Errorf("trap answer graded correct (matched %q)", got) + } + if _, ok := Entity("West", []string{"West"}, []string{"North", "South", "East"}); !ok { + t.Error("clean winner answer graded incorrect") + } +} diff --git a/bench/internal/llm/anthropic.go b/bench/internal/llm/anthropic.go new file mode 100644 index 00000000..66526d8f --- /dev/null +++ b/bench/internal/llm/anthropic.go @@ -0,0 +1,177 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/anthropics/anthropic-sdk-go/option" +) + +// Anthropic is an Adapter over the Anthropic Messages API via the official Go +// SDK. It sends no sampling parameters (removed on current models); run-to-run +// variance is handled by the harness's k-repeats and pass^k reporting, per the +// benchmark design. Thinking configuration is omitted so the request shape is +// valid across the full current model range. +type Anthropic struct { + client anthropic.Client + model string + maxTokens int64 +} + +// NewAnthropic builds an adapter for the given model. The API key is read from +// ANTHROPIC_API_KEY (SDK default); a missing key is an immediate error so a +// misconfigured run fails before any session is minted. +func NewAnthropic(model string, maxTokens int64, timeout time.Duration) (*Anthropic, error) { + if os.Getenv("ANTHROPIC_API_KEY") == "" { + return nil, errors.New("ANTHROPIC_API_KEY is not set; required for -llm anthropic") + } + if model == "" { + return nil, errors.New("model must not be empty") + } + return &Anthropic{ + client: anthropic.NewClient(option.WithRequestTimeout(timeout)), + model: model, + maxTokens: maxTokens, + }, nil +} + +// Model implements Adapter. +func (a *Anthropic) Model() string { return a.model } + +// Complete implements Adapter with one Messages API call. The SDK retries +// 429/5xx with backoff (default max_retries). +func (a *Anthropic) Complete(ctx context.Context, system string, msgs []Message, tools []ToolDef) (Message, Usage, error) { + params := anthropic.MessageNewParams{ + Model: anthropic.Model(a.model), + MaxTokens: a.maxTokens, + } + if system != "" { + params.System = []anthropic.TextBlockParam{{Text: system}} + } + apiTools, err := toAPITools(tools) + if err != nil { + return Message{}, Usage{}, err + } + params.Tools = apiTools + apiMsgs, err := toAPIMessages(msgs) + if err != nil { + return Message{}, Usage{}, err + } + params.Messages = apiMsgs + + resp, err := a.client.Messages.New(ctx, params) + if err != nil { + return Message{}, Usage{}, fmt.Errorf("anthropic messages: %w", err) + } + usage := Usage{InputTokens: resp.Usage.InputTokens, OutputTokens: resp.Usage.OutputTokens} + if resp.StopReason == anthropic.StopReasonRefusal { + return Message{}, usage, errors.New("anthropic: request refused (stop_reason refusal)") + } + return fromAPIContent(resp), usage, nil +} + +// toAPITools converts ToolDefs, passing the MCP-provided JSON Schema through +// verbatim so the model sees exactly what the platform advertises. +func toAPITools(tools []ToolDef) ([]anthropic.ToolUnionParam, error) { + out := make([]anthropic.ToolUnionParam, 0, len(tools)) + for _, t := range tools { + var schema struct { + Properties map[string]any `json:"properties"` + Required []string `json:"required"` + } + if len(t.InputSchema) > 0 { + if err := json.Unmarshal(t.InputSchema, &schema); err != nil { + return nil, fmt.Errorf("tool %s: parse input schema: %w", t.Name, err) + } + } + tp := anthropic.ToolParam{ + Name: t.Name, + Description: anthropic.String(t.Description), + InputSchema: anthropic.ToolInputSchemaParam{ + Properties: schema.Properties, + Required: schema.Required, + }, + } + out = append(out, anthropic.ToolUnionParam{OfTool: &tp}) + } + return out, nil +} + +// toAPIMessages converts the provider-agnostic transcript to API params. +func toAPIMessages(msgs []Message) ([]anthropic.MessageParam, error) { + out := make([]anthropic.MessageParam, 0, len(msgs)) + for _, m := range msgs { + switch m.Role { + case "user": + out = append(out, anthropic.NewUserMessage(userBlocks(m)...)) + case "assistant": + blocks, err := assistantBlocks(m) + if err != nil { + return nil, err + } + out = append(out, anthropic.MessageParam{ + Role: anthropic.MessageParamRoleAssistant, + Content: blocks, + }) + default: + return nil, fmt.Errorf("unsupported transcript role %q", m.Role) + } + } + return out, nil +} + +// userBlocks renders a user turn: tool results first (answering the preceding +// assistant tool calls), then any text. +func userBlocks(m Message) []anthropic.ContentBlockParamUnion { + blocks := make([]anthropic.ContentBlockParamUnion, 0, len(m.ToolResults)+1) + for _, r := range m.ToolResults { + blocks = append(blocks, anthropic.NewToolResultBlock(r.CallID, r.Text, r.IsError)) + } + if m.Text != "" { + blocks = append(blocks, anthropic.NewTextBlock(m.Text)) + } + return blocks +} + +// assistantBlocks renders an assistant turn with text and tool_use blocks. +func assistantBlocks(m Message) ([]anthropic.ContentBlockParamUnion, error) { + blocks := make([]anthropic.ContentBlockParamUnion, 0, len(m.ToolCalls)+1) + if m.Text != "" { + blocks = append(blocks, anthropic.NewTextBlock(m.Text)) + } + for _, c := range m.ToolCalls { + raw, err := json.Marshal(c.Args) + if err != nil { + return nil, fmt.Errorf("marshal tool call args for %s: %w", c.Name, err) + } + tu := anthropic.ToolUseBlockParam{ID: c.ID, Name: c.Name, Input: json.RawMessage(raw)} + blocks = append(blocks, anthropic.ContentBlockParamUnion{OfToolUse: &tu}) + } + return blocks, nil +} + +// fromAPIContent converts a response into the transcript model. +func fromAPIContent(resp *anthropic.Message) Message { + out := Message{Role: "assistant"} + for _, block := range resp.Content { + switch v := block.AsAny().(type) { + case anthropic.TextBlock: + if out.Text != "" { + out.Text += "\n" + } + out.Text += v.Text + case anthropic.ToolUseBlock: + var args map[string]any + if err := json.Unmarshal([]byte(v.JSON.Input.Raw()), &args); err != nil { + args = map[string]any{} + } + out.ToolCalls = append(out.ToolCalls, ToolCall{ID: v.ID, Name: v.Name, Args: args}) + } + } + return out +} diff --git a/bench/internal/llm/llm.go b/bench/internal/llm/llm.go new file mode 100644 index 00000000..0e472fb8 --- /dev/null +++ b/bench/internal/llm/llm.go @@ -0,0 +1,65 @@ +// Package llm defines the provider-pluggable model adapter the benchmark agent +// loop drives. The transcript model is provider-agnostic so the same loop runs +// against a real model (anthropic.go) and a deterministic playback script +// (scripted.go); the adapter owns the translation to its provider's wire shape. +package llm + +import ( + "context" + "encoding/json" +) + +// ToolDef is one tool as presented to the model: the name, description, and +// full JSON Schema for its input, taken from the live tools/list of the +// benchmark session (so each arm's persona shapes what the model sees). +type ToolDef struct { + Name string + Description string + InputSchema json.RawMessage +} + +// ToolCall is one tool invocation requested by the model. +type ToolCall struct { + ID string `json:"id"` + Name string `json:"name"` + Args map[string]any `json:"args"` +} + +// ToolResult is the outcome of executing one ToolCall, keyed by its ID. +type ToolResult struct { + CallID string `json:"call_id"` + Text string `json:"text"` + IsError bool `json:"is_error"` +} + +// Message is one turn of the provider-agnostic transcript. An assistant turn +// carries Text and/or ToolCalls; a user turn carries Text and/or ToolResults. +type Message struct { + Role string `json:"role"` // "user" or "assistant" + Text string `json:"text,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolResults []ToolResult `json:"tool_results,omitempty"` +} + +// Usage counts tokens for one completion, for the run manifest and cost audit. +type Usage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` +} + +// Add accumulates another completion's usage. +func (u *Usage) Add(o Usage) { + u.InputTokens += o.InputTokens + u.OutputTokens += o.OutputTokens +} + +// Adapter produces one assistant turn given the system prompt, transcript, and +// available tools. Implementations must be safe to use for sequential calls +// within one task attempt; they need not be safe for concurrent use. +type Adapter interface { + // Model identifies the model for the run manifest (e.g. "claude-opus-4-8" + // or "scripted"). + Model() string + // Complete returns the next assistant message. + Complete(ctx context.Context, system string, msgs []Message, tools []ToolDef) (Message, Usage, error) +} diff --git a/bench/internal/llm/scripted.go b/bench/internal/llm/scripted.go new file mode 100644 index 00000000..1415784c --- /dev/null +++ b/bench/internal/llm/scripted.go @@ -0,0 +1,86 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" +) + +// Step is one scripted assistant turn: either a set of tool calls or a final +// text answer. FinalText may contain the placeholder "{{last_result}}", which +// is replaced with the text of the most recent tool result in the transcript — +// this lets the smoke script answer with whatever the seeded platform actually +// returned (validating seed data, ground truth, and grading in one pass). The +// substituted result is flattened onto one line because the graders score only +// the FINAL ANSWER line, exactly as a compliant model answers. +type Step struct { + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + FinalText string `json:"final_text,omitempty"` +} + +// Script maps a task ID to its ordered playback steps. +type Script map[string][]Step + +// LoadScript reads a Script from a JSON file. +func LoadScript(path string) (Script, error) { + raw, err := os.ReadFile(path) // #nosec G304 -- operator-supplied script path + if err != nil { + return nil, fmt.Errorf("read script: %w", err) + } + var s Script + if err := json.Unmarshal(raw, &s); err != nil { + return nil, fmt.Errorf("parse script %s: %w", path, err) + } + return s, nil +} + +// Scripted is a deterministic Adapter that plays back a fixed step sequence. +// It exists so the harness pipeline (session mint, handle threading, tool +// execution, audit read-back, grading, reporting) is provable end-to-end with +// no LLM API key and no model variance. +type Scripted struct { + steps []Step + pos int +} + +// NewScripted returns an adapter that plays the given steps in order. +func NewScripted(steps []Step) *Scripted { + return &Scripted{steps: steps} +} + +// Model implements Adapter. +func (s *Scripted) Model() string { return "scripted" } + +// Complete implements Adapter by returning the next scripted step. +func (s *Scripted) Complete(_ context.Context, _ string, msgs []Message, _ []ToolDef) (Message, Usage, error) { + if s.pos >= len(s.steps) { + return Message{}, Usage{}, fmt.Errorf("scripted adapter exhausted after %d steps", len(s.steps)) + } + step := s.steps[s.pos] + s.pos++ + if len(step.ToolCalls) > 0 { + calls := make([]ToolCall, len(step.ToolCalls)) + copy(calls, step.ToolCalls) + for i := range calls { + if calls[i].ID == "" { + calls[i].ID = fmt.Sprintf("scripted_call_%d_%d", s.pos, i) + } + } + return Message{Role: "assistant", ToolCalls: calls}, Usage{}, nil + } + text := strings.ReplaceAll(step.FinalText, "{{last_result}}", lastToolResult(msgs)) + return Message{Role: "assistant", Text: text}, Usage{}, nil +} + +// lastToolResult returns the text of the most recent tool result in the +// transcript flattened onto one line, or "" when none exists. +func lastToolResult(msgs []Message) string { + for i := len(msgs) - 1; i >= 0; i-- { + if n := len(msgs[i].ToolResults); n > 0 { + return strings.Join(strings.Fields(msgs[i].ToolResults[n-1].Text), " ") + } + } + return "" +} diff --git a/bench/internal/llm/scripted_test.go b/bench/internal/llm/scripted_test.go new file mode 100644 index 00000000..93ca1000 --- /dev/null +++ b/bench/internal/llm/scripted_test.go @@ -0,0 +1,74 @@ +package llm + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestScriptedPlayback(t *testing.T) { + s := NewScripted([]Step{ + {ToolCalls: []ToolCall{{Name: "trino_query", Args: map[string]any{"sql": "SELECT 1"}}}}, + {FinalText: "FINAL ANSWER: {{last_result}}"}, + }) + msg, _, err := s.Complete(context.Background(), "", nil, nil) + if err != nil { + t.Fatal(err) + } + if len(msg.ToolCalls) != 1 || msg.ToolCalls[0].ID == "" { + t.Fatalf("first step should be a tool call with a synthesized id: %+v", msg) + } + transcript := []Message{ + {Role: "user", Text: "q"}, + msg, + {Role: "user", ToolResults: []ToolResult{{CallID: msg.ToolCalls[0].ID, Text: "42.5"}}}, + } + final, _, err := s.Complete(context.Background(), "", transcript, nil) + if err != nil { + t.Fatal(err) + } + if final.Text != "FINAL ANSWER: 42.5" { + t.Errorf("placeholder not replaced: %q", final.Text) + } + if _, _, err := s.Complete(context.Background(), "", nil, nil); err == nil { + t.Error("expected exhaustion error") + } +} + +func TestScriptedPlaceholderWithoutResult(t *testing.T) { + s := NewScripted([]Step{{FinalText: "FINAL ANSWER: {{last_result}}"}}) + msg, _, err := s.Complete(context.Background(), "", []Message{{Role: "user", Text: "q"}}, nil) + if err != nil { + t.Fatal(err) + } + if msg.Text != "FINAL ANSWER: " { + t.Errorf("empty placeholder handling: %q", msg.Text) + } +} + +func TestLoadScript(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "script.json") + content := `{"t1": [{"final_text": "FINAL ANSWER: x"}]}` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + script, err := LoadScript(path) + if err != nil { + t.Fatal(err) + } + if len(script["t1"]) != 1 || script["t1"][0].FinalText != "FINAL ANSWER: x" { + t.Errorf("script parsed wrong: %+v", script) + } + if _, err := LoadScript(filepath.Join(dir, "missing.json")); err == nil { + t.Error("expected error for missing file") + } + bad := filepath.Join(dir, "bad.json") + if err := os.WriteFile(bad, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadScript(bad); err == nil { + t.Error("expected error for malformed json") + } +} diff --git a/bench/internal/mcpc/mcpc.go b/bench/internal/mcpc/mcpc.go new file mode 100644 index 00000000..282bd350 --- /dev/null +++ b/bench/internal/mcpc/mcpc.go @@ -0,0 +1,240 @@ +// Package mcpc wraps the official MCP SDK client for the benchmark harness: it +// connects an authenticated session over streamable HTTP, mints the platform's +// dps_ session handle via platform_info, lists the tools the arm's persona +// exposes (with measurement plumbing stripped), and threads the handle into +// every tool call so audit rows correlate to the run. Adapted from +// test/load/internal/mcpc with the session-handle layer added. +package mcpc + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/txn2/mcp-data-platform/bench/internal/llm" +) + +// sessionHandleArg is the tool-call argument the platform's session-handle +// middleware injects into every tool schema and expects threaded back. The +// harness owns this plumbing so it is invisible to the model and uniform +// across arms. +const sessionHandleArg = "session_id" + +// infoToolName mints the handle and reports the platform version. It is +// excluded from the tool set shown to the model: it is platform +// infrastructure the harness calls, not a data tool under test. +const infoToolName = "platform_info" + +// Client connects MCP sessions to a target endpoint using a shared, +// authenticated HTTP client. +type Client struct { + endpoint string + http *http.Client + impl *mcp.Implementation +} + +// New returns a Client that dials endpoint (the platform's MCP base URL) with +// the supplied authenticated HTTP client. +func New(endpoint string, httpClient *http.Client) *Client { + return &Client{ + endpoint: endpoint, + http: httpClient, + impl: &mcp.Implementation{Name: "mcp-data-platform-bench", Version: "1.0.0"}, + } +} + +// Connect establishes one MCP session (performing the initialize handshake). +// The caller must Close the returned session. +func (c *Client) Connect(ctx context.Context) (*mcp.ClientSession, error) { + client := mcp.NewClient(c.impl, nil) + session, err := client.Connect(ctx, &mcp.StreamableClientTransport{ + Endpoint: c.endpoint, + HTTPClient: c.http, + DisableStandaloneSSE: true, + }, nil) + if err != nil { + return nil, fmt.Errorf("mcp connect: %w", err) + } + return session, nil +} + +// SessionInfo is what the harness needs from platform_info. +type SessionInfo struct { + // Handle is the dps_ session handle used to correlate audit rows. + Handle string + // PlatformVersion is the deployment's version string, for the manifest. + PlatformVersion string +} + +// infoPayload is the subset of the platform_info result the harness reads. +type infoPayload struct { + SessionID string `json:"session_id"` + Version string `json:"version"` +} + +// Mint calls platform_info and extracts the dps_ session handle and platform +// version. The handle is a tool-result value by design (the MCP spec removed +// the session header), read from structured content with a text-JSON fallback. +func Mint(ctx context.Context, s *mcp.ClientSession) (SessionInfo, error) { + res, err := s.CallTool(ctx, &mcp.CallToolParams{Name: infoToolName, Arguments: map[string]any{}}) + if err != nil { + return SessionInfo{}, fmt.Errorf("platform_info: %w", err) + } + if res.IsError { + return SessionInfo{}, fmt.Errorf("platform_info returned error: %s", firstText(res)) + } + var payload infoPayload + if res.StructuredContent != nil { + raw, err := json.Marshal(res.StructuredContent) + if err == nil { + _ = json.Unmarshal(raw, &payload) + } + } + if payload.SessionID == "" { + _ = json.Unmarshal([]byte(firstText(res)), &payload) + } + if payload.SessionID == "" { + return SessionInfo{}, fmt.Errorf("platform_info result carries no session_id (text: %.200s)", firstText(res)) + } + return SessionInfo{Handle: payload.SessionID, PlatformVersion: payload.Version}, nil +} + +// ListTools returns the session's tools as adapter ToolDefs, dropping +// platform_info and stripping the injected session_id property from each input +// schema so the model never sees the measurement plumbing. +func ListTools(ctx context.Context, s *mcp.ClientSession) ([]llm.ToolDef, error) { + res, err := s.ListTools(ctx, &mcp.ListToolsParams{}) + if err != nil { + return nil, fmt.Errorf("tools/list: %w", err) + } + out := make([]llm.ToolDef, 0, len(res.Tools)) + for _, t := range res.Tools { + if t.Name == infoToolName { + continue + } + schema, err := stripSessionArg(t.InputSchema) + if err != nil { + return nil, fmt.Errorf("tool %s: %w", t.Name, err) + } + out = append(out, llm.ToolDef{Name: t.Name, Description: t.Description, InputSchema: schema}) + } + return out, nil +} + +// stripSessionArg removes the session_id property (and its required-list +// entry) from a tool input schema. +func stripSessionArg(schema any) (json.RawMessage, error) { + raw, err := json.Marshal(schema) + if err != nil { + return nil, fmt.Errorf("marshal input schema: %w", err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("parse input schema: %w", err) + } + if props, ok := m["properties"].(map[string]any); ok { + delete(props, sessionHandleArg) + } + if req, ok := m["required"].([]any); ok { + kept := make([]any, 0, len(req)) + for _, r := range req { + if s, ok := r.(string); ok && s == sessionHandleArg { + continue + } + kept = append(kept, r) + } + m["required"] = kept + } + out, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("re-marshal input schema: %w", err) + } + return out, nil +} + +// CallResult reports the outcome of one tool call. +type CallResult struct { + // TransportErr is a protocol/transport failure; the call did not complete + // and produces no audit row. + TransportErr error + // ToolErr is true when the call completed but the tool returned isError. + ToolErr bool + // ErrorCode is the platform error contract's stable code from the result's + // structured error envelope ("unauthorized", "search_required", ...), or + // "" when the result carries none. The pipeline uses it to tell platform + // refusals (short-circuited outer to the audit middleware, so no audit + // row) from handler-level tool errors (audited). + ErrorCode string + // Text is the concatenation of all text content blocks. + Text string +} + +// errorEnvelope mirrors pkg/middleware's structured error payload. +type errorEnvelope struct { + Error struct { + Code string `json:"code"` + } `json:"error"` +} + +// Call invokes a tool, injecting the session handle. It never returns a Go +// error directly; inspect the CallResult. +func Call(ctx context.Context, s *mcp.ClientSession, name string, args map[string]any, handle string) CallResult { + sent := make(map[string]any, len(args)+1) + maps.Copy(sent, args) + if handle != "" { + sent[sessionHandleArg] = handle + } + res, err := s.CallTool(ctx, &mcp.CallToolParams{Name: name, Arguments: sent}) + if err != nil { + return CallResult{TransportErr: err} + } + return CallResult{ToolErr: res.IsError, ErrorCode: errorCode(res), Text: allText(res)} +} + +// errorCode extracts the structured error code from a tool result, if any. +func errorCode(res *mcp.CallToolResult) string { + if res == nil || !res.IsError || res.StructuredContent == nil { + return "" + } + raw, err := json.Marshal(res.StructuredContent) + if err != nil { + return "" + } + var env errorEnvelope + if err := json.Unmarshal(raw, &env); err != nil { + return "" + } + return env.Error.Code +} + +// firstText returns the first text content block of a tool result, or "". +func firstText(res *mcp.CallToolResult) string { + if res == nil { + return "" + } + for _, c := range res.Content { + if tc, ok := c.(*mcp.TextContent); ok { + return tc.Text + } + } + return "" +} + +// allText joins every text content block, preserving order. +func allText(res *mcp.CallToolResult) string { + if res == nil { + return "" + } + var parts []string + for _, c := range res.Content { + if tc, ok := c.(*mcp.TextContent); ok { + parts = append(parts, tc.Text) + } + } + return strings.Join(parts, "\n") +} diff --git a/bench/internal/mcpc/mcpc_test.go b/bench/internal/mcpc/mcpc_test.go new file mode 100644 index 00000000..e1fcb04f --- /dev/null +++ b/bench/internal/mcpc/mcpc_test.go @@ -0,0 +1,94 @@ +package mcpc + +// Connect, Mint, ListTools, and Call are exercised end-to-end against a real +// streamable-HTTP MCP server by internal/pipeline's integration test; these +// unit tests cover the pure transformations. + +import ( + "encoding/json" + "testing" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func TestStripSessionArg(t *testing.T) { + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "sql": map[string]any{"type": "string"}, + "session_id": map[string]any{"type": "string"}, + }, + "required": []any{"sql", "session_id"}, + } + out, err := stripSessionArg(schema) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil { + t.Fatal(err) + } + props := m["properties"].(map[string]any) + if _, ok := props["session_id"]; ok { + t.Error("session_id property not stripped") + } + if _, ok := props["sql"]; !ok { + t.Error("sql property lost") + } + req := m["required"].([]any) + if len(req) != 1 || req[0] != "sql" { + t.Errorf("required = %v, want [sql]", req) + } +} + +func TestStripSessionArgNoProperties(t *testing.T) { + out, err := stripSessionArg(map[string]any{"type": "object"}) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil { + t.Fatal(err) + } + if m["type"] != "object" { + t.Errorf("schema mangled: %v", m) + } +} + +func TestErrorCode(t *testing.T) { + cases := []struct { + name string + res *mcp.CallToolResult + want string + }{ + {"nil result", nil, ""}, + {"not an error", &mcp.CallToolResult{}, ""}, + {"no structured content", &mcp.CallToolResult{IsError: true}, ""}, + {"envelope", &mcp.CallToolResult{IsError: true, StructuredContent: map[string]any{ + "error": map[string]any{"code": "unauthorized"}, + }}, "unauthorized"}, + {"foreign structured content", &mcp.CallToolResult{IsError: true, StructuredContent: map[string]any{ + "rows": 3, + }}, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := errorCode(c.res); got != c.want { + t.Errorf("errorCode = %q, want %q", got, c.want) + } + }) + } +} + +func TestAllTextJoins(t *testing.T) { + res := &mcp.CallToolResult{Content: []mcp.Content{ + &mcp.TextContent{Text: "a"}, + &mcp.TextContent{Text: "b"}, + }} + if got := allText(res); got != "a\nb" { + t.Errorf("allText = %q", got) + } + if got := allText(nil); got != "" { + t.Errorf("allText(nil) = %q", got) + } +} diff --git a/bench/internal/pipeline/pipeline.go b/bench/internal/pipeline/pipeline.go new file mode 100644 index 00000000..c13846ed --- /dev/null +++ b/bench/internal/pipeline/pipeline.go @@ -0,0 +1,332 @@ +// Package pipeline orchestrates a benchmark run: for each applicable task and +// repeat, a fresh MCP session is minted, the agent loop runs against the arm's +// live tool surface, efficiency metrics are read back from the platform audit +// API, and the attempt is deterministically graded. +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/txn2/mcp-data-platform/bench/internal/agent" + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" + "github.com/txn2/mcp-data-platform/bench/internal/gen" + "github.com/txn2/mcp-data-platform/bench/internal/grade" + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/mcpc" + "github.com/txn2/mcp-data-platform/bench/internal/report" + "github.com/txn2/mcp-data-platform/bench/internal/target" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// systemScaffold is the fixed prompt held constant across arms (#930 +// principle 1: the platform is the ablation variable, never the prompt). +const systemScaffold = `You are a data analyst agent connected to a data platform over MCP. Answer the user's question using the available tools. + +Rules: +- Ground every answer in tool results from this session; do not answer from prior knowledge about any dataset. +- Verify which tables and columns exist before querying them. +- When you have the answer, end your reply with a single line: "FINAL ANSWER: ".` + +// Format instructions appended per grading kind so deterministic graders have +// a stable convention to parse. +const ( + numericFormat = `- For this numeric question the FINAL ANSWER line must contain exactly one number (USD unless the question states otherwise), for example "FINAL ANSWER: 12345.67".` + entityFormat = `- For this question the FINAL ANSWER line must name the single best answer: a fully qualified table name (catalog.schema.table) for dataset questions, or the exact name requested.` +) + +// AdapterFactory builds the model adapter for one task attempt. +type AdapterFactory func(t task.Task) (llm.Adapter, error) + +// Options configures a run. +type Options struct { + Target target.Target + HTTPTimeout time.Duration + Arm string + Suite string // "" = all suites + K int + TasksDir string + TranscriptDir string + Factory AdapterFactory + LLMProvider string + GitCommit string + AuditTimeout time.Duration + // IdentityKeys is the size of the per-attempt identity pool. The + // search-first gate keys discovery on the authenticated USER (not the + // MCP session or dps_ handle), so attempts sharing one credential would + // contaminate each other: the first attempt's search opens the gate for + // every later attempt. Each attempt therefore authenticates as + // "-NN" (NN = 01..IdentityKeys), matching the identity-pool + // keys the arm configs define; the run refuses to start when the pool is + // smaller than tasks x k. Zero disables rotation (single identity) for + // targets without the pool. + IdentityKeys int + Log *slog.Logger +} + +// Run executes the benchmark and returns aggregated results. Attempts that +// fail at the harness level (adapter error, missing audit rows) are recorded +// with their error and reported in the returned error so the process exits +// nonzero: a run with unaccounted attempts is not publishable. +func Run(ctx context.Context, opts Options) (*report.Results, error) { + tasks, err := loadApplicable(opts) + if err != nil { + return nil, err + } + res := &report.Results{Manifest: report.Manifest{ + StartedAt: time.Now().UTC(), + GitCommit: opts.GitCommit, + Target: opts.Target.BaseURL, + Arm: opts.Arm, + LLMProvider: opts.LLMProvider, + Seed: gen.Seed, + TaskSetHash: task.Hash(tasks), + K: opts.K, + Suite: opts.Suite, + }} + total := len(tasks) * opts.K + if opts.IdentityKeys > 0 && total > opts.IdentityKeys { + return nil, fmt.Errorf("%d attempts exceed the identity pool of %d keys; attempts would share a discovery scope (raise -identity-keys and the config pool)", total, opts.IdentityKeys) + } + env := &runEnv{ + opts: opts, + // Audit read-back always authenticates as the base (admin) key. + audit: auditapi.New(opts.Target.BaseURL, opts.Target.HTTPClient(opts.HTTPTimeout)), + } + failures := 0 + seq := 0 + for _, t := range tasks { + for attempt := 1; attempt <= opts.K; attempt++ { + seq++ + a := env.runAttempt(ctx, t, attempt, seq, res) + if a.Error != "" { + failures++ + } + res.Attempts = append(res.Attempts, a) + } + } + res.Manifest.FinishedAt = time.Now().UTC() + res.Aggregate() + if failures > 0 { + return res, fmt.Errorf("%d attempt(s) failed at the harness level; see results attempts[].error", failures) + } + return res, nil +} + +// loadApplicable loads the task set filtered to the run's arm and suite. +func loadApplicable(opts Options) ([]task.Task, error) { + all, err := task.Load(opts.TasksDir) + if err != nil { + return nil, err + } + var tasks []task.Task + for _, t := range all { + if t.AppliesTo(opts.Arm) && (opts.Suite == "" || t.Suite == opts.Suite) { + tasks = append(tasks, t) + } + } + if len(tasks) == 0 { + return nil, fmt.Errorf("no tasks apply to arm %q suite %q", opts.Arm, opts.Suite) + } + return tasks, nil +} + +// runEnv holds per-run clients. +type runEnv struct { + opts Options + audit *auditapi.Client +} + +// attemptClient builds the MCP client for one attempt, authenticating as the +// attempt's pool identity (or the base credential when rotation is off). +func (e *runEnv) attemptClient(seq int) *mcpc.Client { + t := e.opts.Target + if e.opts.IdentityKeys > 0 { + t.Credential = fmt.Sprintf("%s-%02d", t.Credential, seq) + } + return mcpc.New(t.BaseURL, t.HTTPClient(e.opts.HTTPTimeout)) +} + +// runAttempt executes one task attempt end to end. Harness failures land in +// Attempt.Error; graded outcomes (right or wrong) do not. +func (e *runEnv) runAttempt(ctx context.Context, t task.Task, attempt, seq int, res *report.Results) report.Attempt { + a := report.Attempt{TaskID: t.ID, Suite: t.Suite, Attempt: attempt} + log := e.opts.Log.With("task", t.ID, "attempt", attempt, "arm", e.opts.Arm) + + session, err := e.attemptClient(seq).Connect(ctx) + if err != nil { + a.Error = fmt.Sprintf("connect: %v", err) + return a + } + defer func() { _ = session.Close() }() + + info, err := mcpc.Mint(ctx, session) + if err != nil { + a.Error = fmt.Sprintf("mint session handle: %v", err) + return a + } + a.SessionID = info.Handle + if res.Manifest.PlatformVersion == "" { + res.Manifest.PlatformVersion = info.PlatformVersion + } + + tools, err := mcpc.ListTools(ctx, session) + if err != nil { + a.Error = fmt.Sprintf("list tools: %v", err) + return a + } + + adapter, err := e.opts.Factory(t) + if err != nil { + a.Error = fmt.Sprintf("build adapter: %v", err) + return a + } + if res.Manifest.Model == "" { + res.Manifest.Model = adapter.Model() + } + + // audited counts calls CONFIRMED to produce an audit row under this + // session's handle: completed calls minus platform refusals (authz, the + // gates, the per-user limiter), which short-circuit in middleware OUTER + // to the audit middleware and leave no row. indeterminate counts + // transport-level failures where the platform MAY have audited the call + // before the error surfaced client-side (a protocol error for an unknown + // tool name is logged by the audit middleware; a client timeout races a + // server that finishes and audits). platform_info's own row carries the + // transport session id (the handle is minted inside its handler), so it + // is in neither count. + audited, indeterminate := 0, 0 + exec := func(ctx context.Context, name string, args map[string]any) llm.ToolResult { + r := mcpc.Call(ctx, session, name, args, info.Handle) + if r.TransportErr != nil { + indeterminate++ + return llm.ToolResult{Text: "transport error: " + r.TransportErr.Error(), IsError: true} + } + if !preAuditRefusal(r.ErrorCode) { + audited++ + } + return llm.ToolResult{Text: r.Text, IsError: r.ToolErr} + } + + start := time.Now() + result, err := agent.Run(ctx, adapter, agent.Config{ + System: systemScaffold + "\n" + formatInstruction(t), + Prompt: t.Prompt, + Tools: tools, + Budget: t.BudgetToolCalls, + }, exec) + a.WallMS = time.Since(start).Milliseconds() + fillAgentResult(&a, result) + e.writeTranscript(&a, t, result, log) + if err != nil { + a.Error = fmt.Sprintf("agent loop: %v", err) + return a + } + e.settleAndGrade(ctx, &a, t, info.Handle, audited, audited+indeterminate, log) + return a +} + +// settleAndGrade reads the session's audit trail back (failing the attempt +// loudly when rows are missing) and applies the deterministic grader. +func (e *runEnv) settleAndGrade(ctx context.Context, a *report.Attempt, t task.Task, handle string, minAudited, maxAudited int, log *slog.Logger) { + events, err := e.audit.WaitForSession(ctx, handle, minAudited, maxAudited, e.opts.AuditTimeout) + if err != nil { + a.Error = fmt.Sprintf("audit read-back: %v", err) + return + } + a.Audit = auditapi.Summarize(events) + gradeAttempt(a, t) + log.Info("attempt graded", "correct", a.Correct, "tool_calls", a.ToolCalls, "wall_ms", a.WallMS) +} + +// fillAgentResult copies loop outcomes onto the attempt. +func fillAgentResult(a *report.Attempt, r agent.Result) { + a.FinalAnswer = grade.ExtractFinal(r.FinalAnswer) + a.ToolCalls = r.ToolCalls + a.ToolErrors = r.ToolErrors + a.BudgetExhausted = r.BudgetExhausted + a.InputTokens = r.Usage.InputTokens + a.OutputTokens = r.Usage.OutputTokens +} + +// gradeAttempt applies the task's deterministic grader. +func gradeAttempt(a *report.Attempt, t task.Task) { + switch t.Grading.Kind { + case task.GradeNumeric: + got, ok, correct := grade.Numeric(a.FinalAnswer, *t.Grading.Value, t.Grading.AbsTolerance) + if ok { + a.GotValue = &got + } + a.Correct = correct + case task.GradeEntity: + a.MatchedAlias, a.Correct = grade.Entity(a.FinalAnswer, t.Grading.Aliases, t.Grading.WrongAliases) + } +} + +// preAuditRefusal reports whether a structured error code marks a platform +// refusal issued outer to the audit middleware (pkg/middleware error contract: +// auth, authz, session gate, search-first gate). Such calls complete from the +// client's view but leave no audit row. +func preAuditRefusal(code string) bool { + switch code { + case "unauthenticated", "unauthorized", "session_required", "session_expired", + "search_required", "setup_required", "rate_limited": + return true + } + return false +} + +// formatInstruction returns the per-grading-kind answer format rule. +func formatInstruction(t task.Task) string { + if t.Grading.Kind == task.GradeNumeric { + return numericFormat + } + return entityFormat +} + +// transcript is the per-attempt file layout (manual rubric review reads these). +type transcript struct { + TaskID string `json:"task_id"` + Arm string `json:"arm"` + Attempt int `json:"attempt"` + SessionID string `json:"session_id"` + Rubric []string `json:"rubric,omitempty"` + Transcript []llm.Message `json:"transcript"` +} + +// writeTranscript persists the attempt transcript with the task's rubric notes +// attached (the pilot's rubric items are reviewed manually from these files); +// failure to write is logged, not fatal (the graded outcome stands). +func (e *runEnv) writeTranscript(a *report.Attempt, t task.Task, r agent.Result, log *slog.Logger) { + if e.opts.TranscriptDir == "" { + return + } + if err := os.MkdirAll(e.opts.TranscriptDir, 0o750); err != nil { + log.Warn("transcript dir", "error", err) + return + } + rubric := make([]string, 0, len(t.Rubric)) + for _, item := range t.Rubric { + rubric = append(rubric, item.ID+": "+item.Note) + } + path := filepath.Join(e.opts.TranscriptDir, fmt.Sprintf("%s-%s-k%d.json", a.TaskID, e.opts.Arm, a.Attempt)) + raw, err := json.MarshalIndent(transcript{ + TaskID: a.TaskID, Arm: e.opts.Arm, Attempt: a.Attempt, SessionID: a.SessionID, + Rubric: rubric, + Transcript: r.Transcript, + }, "", " ") + if err != nil { + log.Warn("marshal transcript", "error", err) + return + } + if err := os.WriteFile(path, raw, 0o600); err != nil { + log.Warn("write transcript", "error", err) + return + } + a.TranscriptPath = path +} diff --git a/bench/internal/pipeline/pipeline_test.go b/bench/internal/pipeline/pipeline_test.go new file mode 100644 index 00000000..9b0abb08 --- /dev/null +++ b/bench/internal/pipeline/pipeline_test.go @@ -0,0 +1,316 @@ +package pipeline + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/jsonschema-go/jsonschema" + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" + "github.com/txn2/mcp-data-platform/bench/internal/llm" + "github.com/txn2/mcp-data-platform/bench/internal/report" + "github.com/txn2/mcp-data-platform/bench/internal/target" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +// fakePlatform assembles a real MCP server over streamable HTTP plus the admin +// audit API surface, so the pipeline is exercised through genuine protocol +// wiring: initialize handshake, platform_info handle mint, session_id +// threading (the audit rows are keyed by the session_id argument the tools +// receive — if threading breaks, audit read-back times out and the test +// fails), refusal classification, and audit-derived metrics. +type fakePlatform struct { + mu sync.Mutex + events []auditapi.Event + identities map[string]bool // Authorization headers seen on MCP traffic + minted atomic.Int64 + httpSrv *httptest.Server +} + +func newFakePlatform(t *testing.T) *fakePlatform { + t.Helper() + fp := &fakePlatform{} + server := mcp.NewServer(&mcp.Implementation{Name: "fake-platform", Version: "1.0.0"}, nil) + fp.addPlatformInfo(server) + fp.addTrinoQuery(server) + fp.addDeniedTool(server) + + fp.identities = map[string]bool{} + mcpHandler := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return server }, nil) + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/admin/audit/events", fp.serveAudit) + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fp.mu.Lock() + fp.identities[r.Header.Get("Authorization")] = true + fp.mu.Unlock() + mcpHandler.ServeHTTP(w, r) + })) + fp.httpSrv = httptest.NewServer(mux) + t.Cleanup(fp.httpSrv.Close) + return fp +} + +// addPlatformInfo mints dps_ handles like the real init tool. +func (fp *fakePlatform) addPlatformInfo(server *mcp.Server) { + mcp.AddTool(server, &mcp.Tool{Name: "platform_info", Description: "platform orientation"}, + func(context.Context, *mcp.CallToolRequest, map[string]any) (*mcp.CallToolResult, any, error) { + handle := fmt.Sprintf("dps_fake_%d", fp.minted.Add(1)) + payload := map[string]any{"session_id": handle, "version": "fake-1.0.0"} + raw, _ := json.Marshal(payload) + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: string(raw)}}, + StructuredContent: payload, + }, nil, nil + }) +} + +// addTrinoQuery records an audit event under the threaded session_id and +// returns a single-value result. +func (fp *fakePlatform) addTrinoQuery(server *mcp.Server) { + schema := &jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "sql": {Type: "string"}, + "session_id": {Type: "string", Description: "platform-injected session handle"}, + }, + Required: []string{"sql", "session_id"}, + } + mcp.AddTool(server, &mcp.Tool{Name: "trino_query", Description: "run sql", InputSchema: schema}, + func(_ context.Context, _ *mcp.CallToolRequest, args map[string]any) (*mcp.CallToolResult, any, error) { + sessionID, _ := args["session_id"].(string) + fp.record(auditapi.Event{ + Timestamp: time.Now().UTC(), DurationMS: 7, SessionID: sessionID, + ToolName: "trino_query", Success: true, EventKind: "mcp_tool_call", + EnrichmentApplied: true, EnrichmentTokensDedup: 42, + }) + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: `[{"total_usd": 42.5}]`}}, + }, nil, nil + }) +} + +// addDeniedTool mimics a persona-refused tool: an error result carrying the +// platform error contract's structured envelope, with NO audit row (refusals +// short-circuit outer to the audit middleware on the real platform). +func (fp *fakePlatform) addDeniedTool(server *mcp.Server) { + mcp.AddTool(server, &mcp.Tool{Name: "denied_tool", Description: "always refused"}, + func(context.Context, *mcp.CallToolRequest, map[string]any) (*mcp.CallToolResult, any, error) { + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: "Access denied"}}, + StructuredContent: map[string]any{"error": map[string]any{ + "code": "unauthorized", "category": "authorization_denied", "message": "Access denied", + }}, + }, nil, nil + }) +} + +func (fp *fakePlatform) record(e auditapi.Event) { + fp.mu.Lock() + defer fp.mu.Unlock() + fp.events = append(fp.events, e) +} + +// serveAudit implements the admin audit list endpoint (session_id filter). +func (fp *fakePlatform) serveAudit(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") { + http.Error(w, "unauthenticated", http.StatusUnauthorized) + return + } + sessionID := r.URL.Query().Get("session_id") + fp.mu.Lock() + var matched []auditapi.Event + for _, e := range fp.events { + if e.SessionID == sessionID { + matched = append(matched, e) + } + } + fp.mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": matched, "total": len(matched), "page": 1, "per_page": 200, + }) +} + +// writeTaskFiles writes a two-task set: a numeric trap answered via the fake +// trino_query result, and an entity discovery task answered directly. +func writeTaskFiles(t *testing.T, dir string) { + t.Helper() + numeric := task.Task{ + ID: "t-numeric", Suite: "s3", Prompt: "total?", Arms: []string{"a0"}, BudgetToolCalls: 5, + ExpectedSQL: "SELECT 42.5", + Grading: task.Grading{Kind: task.GradeNumeric, Value: new(42.5), AbsTolerance: 0.01}, + } + entity := task.Task{ + ID: "t-entity", Suite: "s1", Prompt: "which table?", Arms: []string{"a0"}, BudgetToolCalls: 5, + Grading: task.Grading{Kind: task.GradeEntity, Aliases: []string{"memory.bench.orders"}}, + } + for _, tk := range []task.Task{numeric, entity} { + raw, err := json.Marshal(tk) // Task yaml tags match json; YAML is a JSON superset + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, tk.ID+".yaml"), raw, 0o600); err != nil { + t.Fatal(err) + } + } +} + +// testScript plays: refused call, then the real query, then answer from the +// live result (numeric); direct answer (entity). +func testScript() llm.Script { + return llm.Script{ + "t-numeric": { + {ToolCalls: []llm.ToolCall{{Name: "denied_tool", Args: map[string]any{}}}}, + {ToolCalls: []llm.ToolCall{{Name: "trino_query", Args: map[string]any{"sql": "SELECT 42.5"}}}}, + {FinalText: "FINAL ANSWER: {{last_result}}"}, + }, + "t-entity": { + {FinalText: "FINAL ANSWER: memory.bench.orders"}, + }, + } +} + +// TestRunRefusesUndersizedIdentityPool verifies the run fails before spending +// any session when attempts would share a discovery scope. +func TestRunRefusesUndersizedIdentityPool(t *testing.T) { + fp := newFakePlatform(t) + tasksDir := t.TempDir() + writeTaskFiles(t, tasksDir) + _, err := Run(context.Background(), Options{ + Target: target.Target{BaseURL: fp.httpSrv.URL, Credential: "test-key"}, + HTTPTimeout: time.Second, + Arm: "a0", + K: 2, + TasksDir: tasksDir, + Factory: func(task.Task) (llm.Adapter, error) { return llm.NewScripted(nil), nil }, + LLMProvider: "scripted", + AuditTimeout: time.Second, + IdentityKeys: 3, // 2 tasks x k=2 = 4 attempts > 3 keys + Log: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + if err == nil || !strings.Contains(err.Error(), "identity pool") { + t.Fatalf("want identity-pool error, got %v", err) + } + if fp.minted.Load() != 0 { + t.Errorf("sessions were minted despite the pool refusal") + } +} + +// TestRunEndToEnd wires the real pipeline against the fake platform. +func TestRunEndToEnd(t *testing.T) { + fp := newFakePlatform(t) + tasksDir := t.TempDir() + writeTaskFiles(t, tasksDir) + outDir := t.TempDir() + script := testScript() + + res, err := Run(context.Background(), Options{ + Target: target.Target{BaseURL: fp.httpSrv.URL, Credential: "test-key"}, + HTTPTimeout: 10 * time.Second, + Arm: "a0", + K: 2, + TasksDir: tasksDir, + TranscriptDir: filepath.Join(outDir, "transcripts"), + Factory: func(tk task.Task) (llm.Adapter, error) { + return llm.NewScripted(script[tk.ID]), nil + }, + LLMProvider: "scripted", + GitCommit: "deadbeef", + AuditTimeout: 5 * time.Second, + IdentityKeys: 32, + Log: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + + // Every attempt must authenticate as its own pool identity (the + // search-first gate keys discovery on the user, so shared credentials + // would contaminate attempts). + fp.mu.Lock() + identities := len(fp.identities) + wantRotated := map[bool]bool{} + for id := range fp.identities { + wantRotated[strings.HasPrefix(id, "Bearer test-key-")] = true + } + fp.mu.Unlock() + if identities != 4 { // 2 tasks x k=2 + t.Errorf("distinct MCP identities = %d, want 4", identities) + } + if !wantRotated[true] || wantRotated[false] { + t.Errorf("identities not drawn from the pool: %v", fp.identities) + } + + if got := len(res.Attempts); got != 4 { // 2 tasks x k=2 + t.Fatalf("attempts = %d, want 4", got) + } + for _, a := range res.Attempts { + if a.Error != "" { + t.Errorf("%s attempt %d: harness error %q", a.TaskID, a.Attempt, a.Error) + } + if !a.Correct { + t.Errorf("%s attempt %d: graded incorrect (final %q)", a.TaskID, a.Attempt, a.FinalAnswer) + } + if !strings.HasPrefix(a.SessionID, "dps_fake_") { + t.Errorf("%s attempt %d: session id %q not minted", a.TaskID, a.Attempt, a.SessionID) + } + if a.TranscriptPath == "" { + t.Errorf("%s attempt %d: no transcript written", a.TaskID, a.Attempt) + } + } + verifyNumericAttempt(t, res) + verifyManifestAndAggregates(t, res) +} + +// verifyNumericAttempt checks the refusal-vs-audit accounting: the denied call +// executed (2 tool calls, 1 error) but only trino_query is audited, carrying +// the enrichment volume. +func verifyNumericAttempt(t *testing.T, res *report.Results) { + t.Helper() + for _, a := range res.Attempts { + if a.TaskID != "t-numeric" { + continue + } + if a.ToolCalls != 2 || a.ToolErrors != 1 { + t.Errorf("t-numeric: tool_calls=%d errors=%d, want 2/1", a.ToolCalls, a.ToolErrors) + } + if a.Audit.AuditedCalls != 1 || a.Audit.EnrichmentTokensDedup != 42 { + t.Errorf("t-numeric: audit=%+v, want 1 audited call with 42 dedup tokens", a.Audit) + } + if a.GotValue == nil || *a.GotValue != 42.5 { + t.Errorf("t-numeric: got value %v, want 42.5", a.GotValue) + } + } +} + +// verifyManifestAndAggregates checks manifest capture and roll-ups. +func verifyManifestAndAggregates(t *testing.T, res *report.Results) { + t.Helper() + m := res.Manifest + if m.PlatformVersion != "fake-1.0.0" || m.Model != "scripted" || m.GitCommit != "deadbeef" || m.TaskSetHash == "" { + t.Errorf("manifest incomplete: %+v", m) + } + if len(res.Tasks) != 2 || len(res.Suites) != 2 { + t.Fatalf("aggregates: %d tasks, %d suites, want 2/2", len(res.Tasks), len(res.Suites)) + } + for _, s := range res.Suites { + if s.Accuracy != 1.0 || s.PassKRate != 1.0 { + t.Errorf("suite %s: accuracy=%.2f passk=%.2f, want 1.0/1.0", s.Suite, s.Accuracy, s.PassKRate) + } + } + if sum := res.HumanSummary(); !strings.Contains(sum, "arm=a0") || !strings.Contains(sum, "t-numeric") { + t.Errorf("human summary missing expected content:\n%s", sum) + } +} diff --git a/bench/internal/report/report.go b/bench/internal/report/report.go new file mode 100644 index 00000000..682b92ee --- /dev/null +++ b/bench/internal/report/report.go @@ -0,0 +1,275 @@ +// Package report defines the benchmark results model: a manifest pinning +// everything that shaped the run, per-attempt records, and per-task/per-suite +// aggregates (accuracy, pass^k, tool-call and wall-clock distributions). +package report + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "strings" + "time" + + "github.com/txn2/mcp-data-platform/bench/internal/auditapi" +) + +// Manifest pins the run so results are attributable and reproducible. +type Manifest struct { + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + GitCommit string `json:"git_commit"` + PlatformVersion string `json:"platform_version"` + Target string `json:"target"` + Arm string `json:"arm"` + LLMProvider string `json:"llm_provider"` + Model string `json:"model"` + Seed int64 `json:"seed"` + TaskSetHash string `json:"task_set_hash"` + K int `json:"k"` + Suite string `json:"suite,omitempty"` // filter, "" = all +} + +// Attempt is one task execution. +type Attempt struct { + TaskID string `json:"task_id"` + Suite string `json:"suite"` + Attempt int `json:"attempt"` // 1..k + SessionID string `json:"session_id"` + Correct bool `json:"correct"` + FinalAnswer string `json:"final_answer"` + GotValue *float64 `json:"got_value,omitempty"` + MatchedAlias string `json:"matched_alias,omitempty"` + ToolCalls int `json:"tool_calls"` + ToolErrors int `json:"tool_errors"` + BudgetExhausted bool `json:"budget_exhausted"` + WallMS int64 `json:"wall_ms"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + Audit auditapi.Metrics `json:"audit"` + TranscriptPath string `json:"transcript_path,omitempty"` + Error string `json:"error,omitempty"` // harness/adapter failure, not a wrong answer +} + +// TaskSummary aggregates one task's k attempts. Harness-failed attempts +// (Attempt.Error set) are excluded from grading counts and reported in +// HarnessFailures: an infrastructure failure is not evidence about the +// platform's quality. PassK requires all of the manifest's k attempts graded +// AND correct, so a task with harness failures can never claim pass^k. +type TaskSummary struct { + TaskID string `json:"task_id"` + Suite string `json:"suite"` + Graded int `json:"graded"` + Correct int `json:"correct"` + HarnessFailures int `json:"harness_failures,omitempty"` + PassRate float64 `json:"pass_rate"` // correct / graded (0 when nothing graded) + PassK bool `json:"pass_k"` // all k attempts graded and correct (tau-bench pass^k) +} + +// SuiteSummary aggregates one suite under the run's arm over GRADED attempts; +// harness failures are counted separately and never fold into accuracy. +type SuiteSummary struct { + Suite string `json:"suite"` + Tasks int `json:"tasks"` + Graded int `json:"graded"` + HarnessFailures int `json:"harness_failures,omitempty"` + Accuracy float64 `json:"accuracy"` // correct graded attempts / graded attempts + PassKRate float64 `json:"pass_k_rate"` // tasks with all k graded and correct / tasks + MedianToolCalls float64 `json:"median_tool_calls"` + P90ToolCalls float64 `json:"p90_tool_calls"` + MedianWallMS float64 `json:"median_wall_ms"` + ToolErrors int `json:"tool_errors"` + EnrichmentTokensDedup int `json:"enrichment_tokens_dedup"` +} + +// Results is the full run output. +type Results struct { + Manifest Manifest `json:"manifest"` + Attempts []Attempt `json:"attempts"` + Tasks []TaskSummary `json:"tasks"` + Suites []SuiteSummary `json:"suites"` +} + +// Aggregate builds the task and suite summaries from the attempts. +func (r *Results) Aggregate() { + r.Tasks = taskSummaries(r.Attempts, r.Manifest.K) + r.Suites = suiteSummaries(r.Attempts, r.Tasks) +} + +// taskSummaries folds attempts per task; k is the manifest's repeat count. +func taskSummaries(attempts []Attempt, k int) []TaskSummary { + byTask := map[string]*TaskSummary{} + var order []string + for _, a := range attempts { + s, ok := byTask[a.TaskID] + if !ok { + s = &TaskSummary{TaskID: a.TaskID, Suite: a.Suite} + byTask[a.TaskID] = s + order = append(order, a.TaskID) + } + if a.Error != "" { + s.HarnessFailures++ + continue + } + s.Graded++ + if a.Correct { + s.Correct++ + } + } + out := make([]TaskSummary, 0, len(order)) + for _, id := range order { + s := byTask[id] + if s.Graded > 0 { + s.PassRate = float64(s.Correct) / float64(s.Graded) + } + s.PassK = s.Graded == k && s.Correct == k + out = append(out, *s) + } + return out +} + +// suiteSummaries folds attempts per suite. +func suiteSummaries(attempts []Attempt, tasks []TaskSummary) []SuiteSummary { + bySuite := map[string][]Attempt{} + var order []string + for _, a := range attempts { + if _, ok := bySuite[a.Suite]; !ok { + order = append(order, a.Suite) + } + bySuite[a.Suite] = append(bySuite[a.Suite], a) + } + sort.Strings(order) + out := make([]SuiteSummary, 0, len(order)) + for _, suite := range order { + out = append(out, summarizeSuite(suite, bySuite[suite], tasks)) + } + return out +} + +// summarizeSuite computes one suite's aggregate row over graded attempts. +func summarizeSuite(suite string, attempts []Attempt, tasks []TaskSummary) SuiteSummary { + s := SuiteSummary{Suite: suite} + toolCalls := make([]float64, 0, len(attempts)) + wall := make([]float64, 0, len(attempts)) + correct := 0 + for _, a := range attempts { + if a.Error != "" { + s.HarnessFailures++ + continue + } + s.Graded++ + if a.Correct { + correct++ + } + toolCalls = append(toolCalls, float64(a.ToolCalls)) + wall = append(wall, float64(a.WallMS)) + s.ToolErrors += a.ToolErrors + s.EnrichmentTokensDedup += a.Audit.EnrichmentTokensDedup + } + if s.Graded > 0 { + s.Accuracy = float64(correct) / float64(s.Graded) + } + s.MedianToolCalls = percentile(toolCalls, 0.5) + s.P90ToolCalls = percentile(toolCalls, 0.9) + s.MedianWallMS = percentile(wall, 0.5) + passK := 0 + for _, t := range tasks { + if t.Suite != suite { + continue + } + s.Tasks++ + if t.PassK { + passK++ + } + } + if s.Tasks > 0 { + s.PassKRate = float64(passK) / float64(s.Tasks) + } + return s +} + +// percentile computes the nearest-rank percentile of values. +func percentile(values []float64, p float64) float64 { + if len(values) == 0 { + return 0 + } + sorted := slices.Clone(values) + sort.Float64s(sorted) + idx := int(p * float64(len(sorted)-1)) + return sorted[idx] +} + +// WriteJSON persists the results. +func (r *Results) WriteJSON(path string) error { + raw, err := json.MarshalIndent(r, "", " ") + if err != nil { + return fmt.Errorf("marshal results: %w", err) + } + if err := os.WriteFile(path, raw, 0o600); err != nil { + return fmt.Errorf("write results: %w", err) + } + return nil +} + +// LoadJSON reads results written by WriteJSON (for -summarize). +func LoadJSON(path string) (*Results, error) { + raw, err := os.ReadFile(path) // #nosec G304 -- operator-supplied results path + if err != nil { + return nil, fmt.Errorf("read results: %w", err) + } + var r Results + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("parse results: %w", err) + } + return &r, nil +} + +// HumanSummary renders the run for a terminal. +func (r *Results) HumanSummary() string { + var b strings.Builder + m := r.Manifest + fmt.Fprintf(&b, "bench run: arm=%s model=%s (%s) k=%d\n", m.Arm, m.Model, m.LLMProvider, m.K) + fmt.Fprintf(&b, " platform %s @ %s | commit %s | seed %d | tasks %s\n", + m.PlatformVersion, m.Target, short(m.GitCommit), m.Seed, short(m.TaskSetHash)) + fmt.Fprintf(&b, " %s .. %s\n\n", m.StartedAt.Format(time.RFC3339), m.FinishedAt.Format(time.RFC3339)) + b.WriteString("suite tasks graded accuracy pass^k med calls p90 calls med wall ms tool errs enrich tok harness fails\n") + for _, s := range r.Suites { + fmt.Fprintf(&b, "%-7s %5d %6d %7.1f%% %5.1f%% %9.1f %9.1f %11.0f %9d %10d %13d\n", + s.Suite, s.Tasks, s.Graded, s.Accuracy*100, s.PassKRate*100, + s.MedianToolCalls, s.P90ToolCalls, s.MedianWallMS, s.ToolErrors, s.EnrichmentTokensDedup, s.HarnessFailures) + } + b.WriteString("\ntask graded correct pass^k\n") + for _, t := range r.Tasks { + fmt.Fprintf(&b, "%-30s %6d %7d %v\n", t.TaskID, t.Graded, t.Correct, t.PassK) + } + if failures := r.harnessFailures(); len(failures) > 0 { + b.WriteString("\nharness failures (excluded from grading):\n") + for _, f := range failures { + fmt.Fprintf(&b, " %s attempt %d: %s\n", f.TaskID, f.Attempt, f.Error) + } + } + return b.String() +} + +// harnessFailures lists attempts that errored rather than being graded. +func (r *Results) harnessFailures() []Attempt { + var out []Attempt + for _, a := range r.Attempts { + if a.Error != "" { + out = append(out, a) + } + } + return out +} + +// short truncates a hash for display. +func short(s string) string { + if len(s) > 12 { + return s[:12] + } + if s == "" { + return "unknown" + } + return s +} diff --git a/bench/internal/report/report_test.go b/bench/internal/report/report_test.go new file mode 100644 index 00000000..b0bc5eac --- /dev/null +++ b/bench/internal/report/report_test.go @@ -0,0 +1,117 @@ +package report + +import ( + "path/filepath" + "strings" + "testing" + "time" +) + +func sampleResults() *Results { + r := &Results{Manifest: Manifest{ + StartedAt: time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC), + Arm: "a2", Model: "scripted", LLMProvider: "scripted", K: 2, + GitCommit: "abcdef1234567890", TaskSetHash: "hash1234567890", PlatformVersion: "1.0.0", + }} + r.Attempts = []Attempt{ + {TaskID: "t1", Suite: "s1", Attempt: 1, Correct: true, ToolCalls: 2, WallMS: 100}, + {TaskID: "t1", Suite: "s1", Attempt: 2, Correct: true, ToolCalls: 4, WallMS: 200}, + {TaskID: "t2", Suite: "s3", Attempt: 1, Correct: true, ToolCalls: 3, WallMS: 300}, + {TaskID: "t2", Suite: "s3", Attempt: 2, Correct: false, ToolCalls: 9, WallMS: 400, ToolErrors: 2}, + } + r.Aggregate() + return r +} + +func TestAggregate(t *testing.T) { + r := sampleResults() + if len(r.Tasks) != 2 || len(r.Suites) != 2 { + t.Fatalf("aggregates: %d tasks %d suites", len(r.Tasks), len(r.Suites)) + } + t1 := r.Tasks[0] + if t1.TaskID != "t1" || !t1.PassK || t1.PassRate != 1.0 || t1.Graded != 2 { + t.Errorf("t1 summary wrong: %+v", t1) + } + t2 := r.Tasks[1] + if t2.PassK || t2.Correct != 1 { + t.Errorf("t2 summary wrong: %+v", t2) + } + for _, s := range r.Suites { + switch s.Suite { + case "s1": + if s.Accuracy != 1.0 || s.PassKRate != 1.0 || s.MedianToolCalls != 2 { + t.Errorf("s1 summary wrong: %+v", s) + } + case "s3": + if s.Accuracy != 0.5 || s.PassKRate != 0.0 || s.ToolErrors != 2 { + t.Errorf("s3 summary wrong: %+v", s) + } + } + } +} + +func TestWriteAndLoadJSON(t *testing.T) { + r := sampleResults() + path := filepath.Join(t.TempDir(), "results.json") + if err := r.WriteJSON(path); err != nil { + t.Fatal(err) + } + loaded, err := LoadJSON(path) + if err != nil { + t.Fatal(err) + } + if loaded.Manifest.Arm != "a2" || len(loaded.Attempts) != 4 { + t.Errorf("round trip lost data: %+v", loaded.Manifest) + } +} + +func TestHarnessFailuresExcludedFromGrading(t *testing.T) { + r := sampleResults() + // A harness failure on an otherwise-perfect task must not dent accuracy, + // but must also deny pass^k (the task was not graded k times). + r.Attempts = []Attempt{ + r.Attempts[0], // t1 attempt 1, correct + {TaskID: "t1", Suite: "s1", Attempt: 2, Error: "audit read-back: missing"}, + } + r.Aggregate() + t1 := r.Tasks[0] + if t1.Graded != 1 || t1.Correct != 1 || t1.PassRate != 1.0 { + t.Errorf("failure folded into grading: %+v", t1) + } + if t1.PassK { + t.Errorf("pass^k claimed with a harness failure: %+v", t1) + } + if t1.HarnessFailures != 1 { + t.Errorf("harness failure not counted: %+v", t1) + } + for _, s := range r.Suites { + if s.Suite == "s1" && (s.Accuracy != 1.0 || s.Graded != 1 || s.HarnessFailures != 1) { + t.Errorf("s1 suite mixed failure into accuracy: %+v", s) + } + } +} + +func TestHumanSummary(t *testing.T) { + r := sampleResults() + r.Attempts = append(r.Attempts, Attempt{TaskID: "t3", Suite: "s3", Attempt: 1, Error: "audit read-back: missing"}) + r.Aggregate() + sum := r.HumanSummary() + for _, needle := range []string{"arm=a2", "s1", "s3", "t1", "harness failures (excluded from grading)", "audit read-back"} { + if !strings.Contains(sum, needle) { + t.Errorf("summary missing %q:\n%s", needle, sum) + } + } +} + +func TestPercentile(t *testing.T) { + if got := percentile(nil, 0.5); got != 0 { + t.Errorf("empty percentile = %v", got) + } + vals := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + if got := percentile(vals, 0.5); got != 5 { + t.Errorf("median = %v, want 5", got) + } + if got := percentile(vals, 0.9); got != 9 { + t.Errorf("p90 = %v, want 9", got) + } +} diff --git a/bench/internal/target/target.go b/bench/internal/target/target.go new file mode 100644 index 00000000..397f5094 --- /dev/null +++ b/bench/internal/target/target.go @@ -0,0 +1,45 @@ +// Package target describes the platform under test — its endpoint and auth — +// and builds the authenticated HTTP client the harness reuses for MCP and +// admin REST traffic. Adapted from test/load/internal/target. +package target + +import ( + "net/http" + "time" +) + +// Target is the platform under test. +type Target struct { + // BaseURL is the MCP + REST base, e.g. http://localhost:8098. The MCP + // streamable endpoint is BaseURL itself (the SDK client posts there). + BaseURL string + // Credential is the admin API key, sent as a Bearer token. + Credential string +} + +// authRoundTripper injects the Authorization header on every request. +type authRoundTripper struct { + header string + base http.RoundTripper +} + +func (a authRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + if a.header != "" { + r.Header.Set("Authorization", a.header) + } + return a.base.RoundTrip(r) +} + +// HTTPClient builds an authenticated *http.Client. Benchmark sessions run one +// at a time, so the transport is a plain default rather than the load +// harness's high-concurrency pool. +func (t Target) HTTPClient(timeout time.Duration) *http.Client { + var header string + if t.Credential != "" { + header = "Bearer " + t.Credential + } + return &http.Client{ + Timeout: timeout, + Transport: authRoundTripper{header: header, base: http.DefaultTransport}, + } +} diff --git a/bench/internal/task/task.go b/bench/internal/task/task.go new file mode 100644 index 00000000..ac538fbe --- /dev/null +++ b/bench/internal/task/task.go @@ -0,0 +1,180 @@ +// Package task defines the benchmark task schema, loader, and the canonical +// task-set hash recorded in every run manifest. +package task + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "gopkg.in/yaml.v3" +) + +// Grading kinds. +const ( + GradeNumeric = "numeric" + GradeEntity = "entity" +) + +// Task is one benchmark task. Ground truth is generated by the dataset +// generator (bench/seedgen), never hand-typed. +type Task struct { + // ID is unique across the task set, e.g. "s3-units-q1-total". + ID string `yaml:"id" json:"id"` + // Suite is the task suite, e.g. "s1" (discovery) or "s3" (knowledge traps). + Suite string `yaml:"suite" json:"suite"` + // Prompt is what the agent is asked. + Prompt string `yaml:"prompt" json:"prompt"` + // Arms lists the arm names this task applies to. + Arms []string `yaml:"arms" json:"arms"` + // TrapClasses tags S3 tasks with the seeded trap classes they exercise. + TrapClasses []string `yaml:"trap_classes,omitempty" json:"trap_classes,omitempty"` + // BudgetToolCalls caps tool calls per attempt. + BudgetToolCalls int `yaml:"budget_tool_calls" json:"budget_tool_calls"` + // ExpectedSQL records the reference query the generator used to compute + // ground truth (audit trail; also drives the scripted smoke). + ExpectedSQL string `yaml:"expected_sql,omitempty" json:"expected_sql,omitempty"` + // Grading is the deterministic grading spec. + Grading Grading `yaml:"grading" json:"grading"` + // Rubric lists judgment-call items recorded for manual review in the + // pilot (LLM-judged in phase 2). + Rubric []RubricItem `yaml:"rubric,omitempty" json:"rubric,omitempty"` +} + +// Grading is a deterministic grading spec. +type Grading struct { + // Kind is GradeNumeric or GradeEntity. + Kind string `yaml:"kind" json:"kind"` + // Value is the expected number (numeric only). + Value *float64 `yaml:"value,omitempty" json:"value,omitempty"` + // AbsTolerance is the absolute tolerance for numeric match. + AbsTolerance float64 `yaml:"abs_tolerance,omitempty" json:"abs_tolerance,omitempty"` + // Aliases are accepted entity identifiers (entity only), matched + // case-insensitively as substrings of the final answer's first line. + Aliases []string `yaml:"aliases,omitempty" json:"aliases,omitempty"` + // WrongAliases enumerate the task's known trap answers (entity only): a + // final answer mentioning any of them is graded incorrect even when a + // correct alias also appears. + WrongAliases []string `yaml:"wrong_aliases,omitempty" json:"wrong_aliases,omitempty"` +} + +// RubricItem is one judged criterion, recorded but not auto-scored in phase 1. +type RubricItem struct { + ID string `yaml:"id" json:"id"` + Note string `yaml:"note" json:"note"` +} + +// Validate rejects malformed tasks at load time so a broken task set fails +// before any session is spent on it. +func (t Task) Validate() error { + switch { + case t.ID == "": + return errors.New("task with empty id") + case t.Suite == "": + return fmt.Errorf("task %s: empty suite", t.ID) + case t.Prompt == "": + return fmt.Errorf("task %s: empty prompt", t.ID) + case len(t.Arms) == 0: + return fmt.Errorf("task %s: no arms", t.ID) + case t.BudgetToolCalls <= 0: + return fmt.Errorf("task %s: budget_tool_calls must be positive", t.ID) + } + return t.validateGrading() +} + +// validateGrading checks the grading spec matches its kind. +func (t Task) validateGrading() error { + switch t.Grading.Kind { + case GradeNumeric: + if t.Grading.Value == nil { + return fmt.Errorf("task %s: numeric grading requires value", t.ID) + } + if t.Grading.AbsTolerance < 0 { + return fmt.Errorf("task %s: negative tolerance", t.ID) + } + return nil + case GradeEntity: + return t.validateEntityGrading() + default: + return fmt.Errorf("task %s: unknown grading kind %q", t.ID, t.Grading.Kind) + } +} + +// validateEntityGrading checks alias sets. A correct alias containing a wrong +// alias would make a correct answer ungradeable (naming the truth always +// names the trap), so that combination is rejected. +func (t Task) validateEntityGrading() error { + if len(t.Grading.Aliases) == 0 { + return fmt.Errorf("task %s: entity grading requires aliases", t.ID) + } + for _, a := range t.Grading.Aliases { + for _, w := range t.Grading.WrongAliases { + if w != "" && strings.Contains(strings.ToLower(a), strings.ToLower(w)) { + return fmt.Errorf("task %s: alias %q contains wrong alias %q", t.ID, a, w) + } + } + } + return nil +} + +// AppliesTo reports whether the task runs under the given arm. +func (t Task) AppliesTo(arm string) bool { + return slices.Contains(t.Arms, arm) +} + +// Load reads every *.yaml task in dir (sorted by filename), validating each +// and rejecting duplicate IDs. +func Load(dir string) ([]Task, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read task dir: %w", err) + } + var tasks []Task + seen := map[string]bool{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yaml") { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) // #nosec G304 -- operator-supplied task dir + if err != nil { + return nil, fmt.Errorf("read %s: %w", e.Name(), err) + } + var t Task + if err := yaml.Unmarshal(raw, &t); err != nil { + return nil, fmt.Errorf("parse %s: %w", e.Name(), err) + } + if err := t.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", e.Name(), err) + } + if seen[t.ID] { + return nil, fmt.Errorf("duplicate task id %s", t.ID) + } + seen[t.ID] = true + tasks = append(tasks, t) + } + if len(tasks) == 0 { + return nil, fmt.Errorf("no tasks found in %s", dir) + } + return tasks, nil +} + +// Hash returns the canonical SHA-256 of the task set (sorted by ID, JSON +// encoded) for the run manifest. +func Hash(tasks []Task) string { + sorted := make([]Task, len(tasks)) + copy(sorted, tasks) + slices.SortFunc(sorted, func(a, b Task) int { return strings.Compare(a.ID, b.ID) }) + raw, err := json.Marshal(sorted) + if err != nil { + // Task is a plain data struct; marshal cannot fail on validated input. + panic(fmt.Sprintf("marshal task set: %v", err)) + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} diff --git a/bench/internal/task/task_test.go b/bench/internal/task/task_test.go new file mode 100644 index 00000000..56db4de5 --- /dev/null +++ b/bench/internal/task/task_test.go @@ -0,0 +1,97 @@ +package task + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func validYAML() string { + return ` +id: s3-test +suite: s3 +prompt: what is x? +arms: [a0, a2] +budget_tool_calls: 10 +grading: + kind: numeric + value: 42.5 + abs_tolerance: 0.01 +` +} + +func writeTask(t *testing.T, dir, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestLoadValid(t *testing.T) { + dir := t.TempDir() + writeTask(t, dir, "a.yaml", validYAML()) + tasks, err := Load(dir) + if err != nil { + t.Fatal(err) + } + if len(tasks) != 1 || tasks[0].ID != "s3-test" || *tasks[0].Grading.Value != 42.5 { + t.Errorf("loaded wrong: %+v", tasks) + } + if !tasks[0].AppliesTo("a0") || tasks[0].AppliesTo("a1") { + t.Error("arm applicability wrong") + } +} + +func TestLoadRejectsInvalid(t *testing.T) { + cases := []struct { + name, yaml, wantErr string + }{ + {"missing id", "suite: s1\nprompt: p\narms: [a0]\nbudget_tool_calls: 1\ngrading: {kind: entity, aliases: [x]}", "empty id"}, + {"no budget", "id: t\nsuite: s1\nprompt: p\narms: [a0]\ngrading: {kind: entity, aliases: [x]}", "budget"}, + {"numeric no value", "id: t\nsuite: s3\nprompt: p\narms: [a0]\nbudget_tool_calls: 1\ngrading: {kind: numeric}", "requires value"}, + {"entity no aliases", "id: t\nsuite: s1\nprompt: p\narms: [a0]\nbudget_tool_calls: 1\ngrading: {kind: entity}", "requires aliases"}, + {"unknown kind", "id: t\nsuite: s1\nprompt: p\narms: [a0]\nbudget_tool_calls: 1\ngrading: {kind: vibes}", "unknown grading kind"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + dir := t.TempDir() + writeTask(t, dir, "t.yaml", c.yaml) + _, err := Load(dir) + if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Errorf("want error containing %q, got %v", c.wantErr, err) + } + }) + } +} + +func TestLoadRejectsDuplicates(t *testing.T) { + dir := t.TempDir() + writeTask(t, dir, "a.yaml", validYAML()) + writeTask(t, dir, "b.yaml", validYAML()) + if _, err := Load(dir); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Errorf("want duplicate error, got %v", err) + } +} + +func TestLoadEmptyDir(t *testing.T) { + if _, err := Load(t.TempDir()); err == nil { + t.Error("want error for empty dir") + } +} + +func TestHashStableAndOrderIndependent(t *testing.T) { + a := Task{ID: "a", Suite: "s1", Prompt: "p", Arms: []string{"a0"}, BudgetToolCalls: 1, + Grading: Grading{Kind: GradeEntity, Aliases: []string{"x"}}} + b := Task{ID: "b", Suite: "s1", Prompt: "q", Arms: []string{"a0"}, BudgetToolCalls: 1, + Grading: Grading{Kind: GradeEntity, Aliases: []string{"y"}}} + h1 := Hash([]Task{a, b}) + h2 := Hash([]Task{b, a}) + if h1 != h2 { + t.Error("hash must be order independent") + } + b.Prompt = "changed" + if Hash([]Task{a, b}) == h1 { + t.Error("hash must change when a task changes") + } +} diff --git a/bench/seed/datahub/bench_mces.json b/bench/seed/datahub/bench_mces.json new file mode 100644 index 00000000..c38da5e2 --- /dev/null +++ b/bench/seed/datahub/bench_mces.json @@ -0,0 +1,137 @@ +[ + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "customProperties": { + "team": "bench", + "unit": "cents" + }, + "description": "Current order transactions, one row per order. Monetary columns (amount, discount) are stored as INTEGERS IN US CENTS; divide by 100 for USD. Company revenue reporting policy: revenue = amount - discount, COMPLETED orders only (refunded and pending orders are excluded). See the 'Revenue Reporting Policy' knowledge page for the authoritative definition.", + "name": "orders" + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)", + "entityType": "dataset", + "aspectName": "editableSchemaMetadata", + "changeType": "UPSERT", + "aspect": { + "editableSchemaFieldInfo": [ + { + "description": "Order amount in US cents (integer). Divide by 100 for USD.", + "fieldPath": "amount" + }, + { + "description": "Discount in US cents (integer), non-zero on completed orders only.", + "fieldPath": "discount" + } + ] + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.customers,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "customProperties": { + "team": "bench" + }, + "description": "Customer directory: one row per customer with profile attributes (name, region, tier, account created_at). Join key: customer_id.", + "name": "customers" + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.legacy_orders,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "customProperties": { + "team": "bench" + }, + "description": "DEPRECATED order extract from the retired ingestion pipeline. Partial coverage, totals in dollars. Use memory.bench.orders for all order analysis.", + "name": "legacy_orders" + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.legacy_orders,PROD)", + "entityType": "dataset", + "aspectName": "deprecation", + "changeType": "UPSERT", + "aspect": { + "actor": "urn:li:corpuser:bench-seed", + "deprecated": true, + "note": "Deprecated. Use memory.bench.orders instead." + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.daily_region_revenue,PROD)", + "entityType": "dataset", + "aspectName": "datasetProperties", + "changeType": "UPSERT", + "aspect": { + "customProperties": { + "grain": "day,region", + "team": "bench" + }, + "description": "Pre-aggregated daily revenue by region, derived from completed orders. Values are GROSS of discounts (USD), so this index must not be used for policy net-revenue figures; use memory.bench.orders per the Revenue Reporting Policy.", + "name": "daily_region_revenue" + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)", + "entityType": "dataset", + "aspectName": "globalTags", + "changeType": "UPSERT", + "aspect": { + "tags": [ + { + "tag": "urn:li:tag:bench" + } + ] + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.customers,PROD)", + "entityType": "dataset", + "aspectName": "globalTags", + "changeType": "UPSERT", + "aspect": { + "tags": [ + { + "tag": "urn:li:tag:bench" + } + ] + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.legacy_orders,PROD)", + "entityType": "dataset", + "aspectName": "globalTags", + "changeType": "UPSERT", + "aspect": { + "tags": [ + { + "tag": "urn:li:tag:bench" + } + ] + } + }, + { + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.daily_region_revenue,PROD)", + "entityType": "dataset", + "aspectName": "globalTags", + "changeType": "UPSERT", + "aspect": { + "tags": [ + { + "tag": "urn:li:tag:bench" + } + ] + } + } +] \ No newline at end of file diff --git a/bench/seed/postgres/knowledge_pages.sql b/bench/seed/postgres/knowledge_pages.sql new file mode 100644 index 00000000..1d29132a --- /dev/null +++ b/bench/seed/postgres/knowledge_pages.sql @@ -0,0 +1,50 @@ +-- Generated by bench/seedgen (seed 930). Do not edit; regenerate with `make bench-gen`. +-- Requires platform migrations (table portal_knowledge_pages); apply after the platform has booted. + +INSERT INTO portal_knowledge_pages + (id, slug, title, summary, body, tags, created_by, created_email, updated_by, current_version, created_at, updated_at) +VALUES + ('kp-bench-1', 'revenue-reporting-policy', 'Revenue Reporting Policy', + 'Revenue = amount - discount over completed orders only; amounts are stored in US cents. The authoritative definition for all bench warehouse reporting.', + $benchkp$# Revenue Reporting Policy + +This is the authoritative definition of "revenue" for all bench warehouse reporting. + +**Revenue = amount - discount, over COMPLETED orders only.** Refunded and pending +orders are excluded entirely. Any figure that includes refunded orders or ignores +discounts is gross volume, not revenue, and must not be reported as revenue. + +Two mechanical rules when computing revenue from [orders](urn:li:dataset:(urn:li:dataPlatform:trino,memory.bench.orders,PROD)): + +1. The amount and discount columns are stored as integers in **US cents**. + Divide by 100 for USD. +2. Filter on status = 'completed' before summing. + +The pre-aggregated daily_region_revenue index is **gross of discounts** and must +not be used for policy revenue figures.$benchkp$, + '["finance","policy","bench"]'::jsonb, 'bench-seed@example.com', 'bench-seed@example.com', 'bench-seed@example.com', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z') +ON CONFLICT (id) DO UPDATE SET + slug = EXCLUDED.slug, title = EXCLUDED.title, summary = EXCLUDED.summary, + body = EXCLUDED.body, tags = EXCLUDED.tags, updated_at = EXCLUDED.updated_at; + +INSERT INTO portal_knowledge_pages + (id, slug, title, summary, body, tags, created_by, created_email, updated_by, current_version, created_at, updated_at) +VALUES + ('kp-bench-2', 'bench-warehouse-guide', 'Bench Warehouse Guide', + 'Which bench table to use: orders is current, legacy_orders is deprecated, daily_region_revenue is gross-only pre-aggregation.', + $benchkp$# Bench Warehouse Guide + +The bench schema (memory.bench) holds four tables: + +- **orders** — current order transactions, one row per order. The single source + of truth for order analysis. Monetary columns are integers in US cents. +- **customers** — customer profiles: name, region, tier, account created_at. +- **legacy_orders** — DEPRECATED extract from the retired pipeline; partial + coverage, dollar totals. Do not use; query orders instead. +- **daily_region_revenue** — pre-aggregated daily gross revenue (USD) by region, + completed orders only, gross of discounts. Convenient for trend charts; not + valid for policy revenue (see the Revenue Reporting Policy page).$benchkp$, + '["warehouse","bench"]'::jsonb, 'bench-seed@example.com', 'bench-seed@example.com', 'bench-seed@example.com', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z') +ON CONFLICT (id) DO UPDATE SET + slug = EXCLUDED.slug, title = EXCLUDED.title, summary = EXCLUDED.summary, + body = EXCLUDED.body, tags = EXCLUDED.tags, updated_at = EXCLUDED.updated_at; diff --git a/bench/seed/trino/setup.sql b/bench/seed/trino/setup.sql new file mode 100644 index 00000000..7aacdf7f --- /dev/null +++ b/bench/seed/trino/setup.sql @@ -0,0 +1,2070 @@ +-- Generated by bench/seedgen (seed 930). Do not edit; regenerate with `make bench-gen`. +CREATE SCHEMA IF NOT EXISTS memory.bench; + +DROP TABLE IF EXISTS memory.bench.customers; +CREATE TABLE memory.bench.customers ( + customer_id INTEGER, + name VARCHAR, + region VARCHAR, + tier VARCHAR, + created_at TIMESTAMP +); +INSERT INTO memory.bench.customers VALUES +(1, 'Harper Silva', 'East', 'basic', TIMESTAMP '2023-02-02 15:00:00'), +(2, 'Quinn Ramos', 'East', 'basic', TIMESTAMP '2024-03-15 19:00:00'), +(3, 'Tatum Silva', 'South', 'plus', TIMESTAMP '2024-08-19 16:00:00'), +(4, 'Harper Dawson', 'North', 'plus', TIMESTAMP '2023-05-15 00:00:00'), +(5, 'Morgan Mercer', 'South', 'plus', TIMESTAMP '2023-07-27 23:00:00'), +(6, 'Reese Okafor', 'West', 'enterprise', TIMESTAMP '2023-04-08 22:00:00'), +(7, 'Blake Nguyen', 'South', 'enterprise', TIMESTAMP '2024-06-11 12:00:00'), +(8, 'Lane Lopez', 'North', 'basic', TIMESTAMP '2024-01-11 17:00:00'), +(9, 'Reese Iwata', 'South', 'basic', TIMESTAMP '2024-01-14 03:00:00'), +(10, 'Casey Kim', 'West', 'basic', TIMESTAMP '2024-02-17 23:00:00'), +(11, 'Noel Mercer', 'West', 'enterprise', TIMESTAMP '2023-05-19 00:00:00'), +(12, 'Morgan Iwata', 'West', 'basic', TIMESTAMP '2023-08-21 18:00:00'), +(13, 'Morgan Vargas', 'South', 'plus', TIMESTAMP '2024-01-31 17:00:00'), +(14, 'Jules Nguyen', 'West', 'basic', TIMESTAMP '2024-09-22 03:00:00'), +(15, 'Harper Nguyen', 'North', 'enterprise', TIMESTAMP '2024-05-06 11:00:00'), +(16, 'Oakley Foster', 'North', 'basic', TIMESTAMP '2024-04-26 11:00:00'), +(17, 'Oakley Vargas', 'South', 'basic', TIMESTAMP '2023-08-10 01:00:00'), +(18, 'Sage Price', 'East', 'basic', TIMESTAMP '2024-01-13 00:00:00'), +(19, 'Noel Ellis', 'West', 'basic', TIMESTAMP '2023-02-13 22:00:00'), +(20, 'Gray Dawson', 'West', 'basic', TIMESTAMP '2024-10-12 20:00:00'), +(21, 'Lane Jensen', 'South', 'plus', TIMESTAMP '2024-09-22 00:00:00'), +(22, 'Oakley Price', 'South', 'plus', TIMESTAMP '2024-10-03 15:00:00'), +(23, 'Lane Price', 'West', 'enterprise', TIMESTAMP '2023-04-04 20:00:00'), +(24, 'Sage Nguyen', 'North', 'plus', TIMESTAMP '2024-04-14 16:00:00'), +(25, 'Blake Lopez', 'West', 'basic', TIMESTAMP '2024-03-18 09:00:00'), +(26, 'Jules Brooks', 'West', 'basic', TIMESTAMP '2023-08-30 21:00:00'), +(27, 'Noel Iwata', 'North', 'plus', TIMESTAMP '2024-11-18 12:00:00'), +(28, 'Oakley Ramos', 'North', 'enterprise', TIMESTAMP '2024-06-30 23:00:00'), +(29, 'Lane Mercer', 'East', 'basic', TIMESTAMP '2024-05-30 18:00:00'), +(30, 'Oakley Okafor', 'South', 'enterprise', TIMESTAMP '2024-02-10 06:00:00'), +(31, 'Oakley Vargas', 'West', 'basic', TIMESTAMP '2024-07-05 22:00:00'), +(32, 'Blake Garcia', 'East', 'enterprise', TIMESTAMP '2023-01-14 19:00:00'), +(33, 'Kai Ellis', 'South', 'plus', TIMESTAMP '2023-05-17 11:00:00'), +(34, 'Parker Hayes', 'West', 'basic', TIMESTAMP '2023-08-25 21:00:00'), +(35, 'Sage Turner', 'East', 'plus', TIMESTAMP '2023-08-20 19:00:00'), +(36, 'Noel Price', 'South', 'plus', TIMESTAMP '2024-11-07 10:00:00'), +(37, 'Emery Vargas', 'East', 'basic', TIMESTAMP '2023-07-07 11:00:00'), +(38, 'Tatum Hayes', 'North', 'basic', TIMESTAMP '2023-02-16 22:00:00'), +(39, 'Oakley Nguyen', 'East', 'basic', TIMESTAMP '2024-05-17 19:00:00'), +(40, 'Noel Kim', 'South', 'plus', TIMESTAMP '2024-07-04 12:00:00'), +(41, 'Blake Dawson', 'West', 'enterprise', TIMESTAMP '2024-03-03 18:00:00'), +(42, 'Casey Garcia', 'North', 'plus', TIMESTAMP '2024-08-08 03:00:00'), +(43, 'Morgan Kim', 'East', 'basic', TIMESTAMP '2023-11-12 10:00:00'), +(44, 'Sage Ramos', 'West', 'basic', TIMESTAMP '2023-08-01 21:00:00'), +(45, 'Devon Price', 'East', 'plus', TIMESTAMP '2024-10-14 08:00:00'), +(46, 'Kai Vargas', 'North', 'basic', TIMESTAMP '2024-07-06 23:00:00'), +(47, 'Noel Garcia', 'East', 'basic', TIMESTAMP '2023-01-14 15:00:00'), +(48, 'Jules Vargas', 'North', 'plus', TIMESTAMP '2024-03-30 07:00:00'), +(49, 'Devon Dawson', 'North', 'enterprise', TIMESTAMP '2024-08-25 19:00:00'), +(50, 'Emery Jensen', 'South', 'basic', TIMESTAMP '2024-09-30 15:00:00'), +(51, 'Jules Turner', 'North', 'enterprise', TIMESTAMP '2024-11-28 15:00:00'), +(52, 'Blake Mercer', 'West', 'plus', TIMESTAMP '2024-04-20 08:00:00'), +(53, 'Lane Kim', 'South', 'basic', TIMESTAMP '2023-01-14 18:00:00'), +(54, 'Emery Alvarez', 'South', 'basic', TIMESTAMP '2023-03-10 23:00:00'), +(55, 'Jules Silva', 'West', 'basic', TIMESTAMP '2023-03-22 14:00:00'), +(56, 'Emery Kim', 'West', 'enterprise', TIMESTAMP '2024-09-27 01:00:00'), +(57, 'Blake Vargas', 'East', 'plus', TIMESTAMP '2023-10-10 02:00:00'), +(58, 'Finley Garcia', 'West', 'enterprise', TIMESTAMP '2024-11-27 09:00:00'), +(59, 'Blake Dawson', 'East', 'plus', TIMESTAMP '2023-04-17 13:00:00'), +(60, 'Noel Mercer', 'South', 'plus', TIMESTAMP '2023-08-12 09:00:00'), +(61, 'Avery Jensen', 'West', 'plus', TIMESTAMP '2024-06-04 12:00:00'), +(62, 'Lane Price', 'East', 'basic', TIMESTAMP '2023-03-26 13:00:00'), +(63, 'Tatum Lopez', 'North', 'basic', TIMESTAMP '2023-08-21 21:00:00'), +(64, 'Quinn Chen', 'South', 'plus', TIMESTAMP '2023-12-11 21:00:00'), +(65, 'Sage Silva', 'East', 'enterprise', TIMESTAMP '2024-09-14 00:00:00'), +(66, 'Devon Chen', 'South', 'plus', TIMESTAMP '2023-01-19 07:00:00'), +(67, 'Lane Mercer', 'South', 'plus', TIMESTAMP '2023-03-21 20:00:00'), +(68, 'Lane Nguyen', 'East', 'enterprise', TIMESTAMP '2024-02-07 11:00:00'), +(69, 'Finley Nguyen', 'East', 'enterprise', TIMESTAMP '2023-05-03 02:00:00'), +(70, 'Parker Alvarez', 'West', 'basic', TIMESTAMP '2023-01-07 13:00:00'), +(71, 'Indigo Alvarez', 'East', 'enterprise', TIMESTAMP '2024-01-26 17:00:00'), +(72, 'Oakley Okafor', 'North', 'basic', TIMESTAMP '2023-05-06 04:00:00'), +(73, 'Devon Hayes', 'West', 'basic', TIMESTAMP '2024-10-22 00:00:00'), +(74, 'Gray Foster', 'East', 'basic', TIMESTAMP '2023-03-29 14:00:00'), +(75, 'Avery Hayes', 'East', 'basic', TIMESTAMP '2024-10-13 21:00:00'), +(76, 'Oakley Hayes', 'North', 'basic', TIMESTAMP '2023-09-25 05:00:00'), +(77, 'Reese Lopez', 'West', 'enterprise', TIMESTAMP '2024-02-15 15:00:00'), +(78, 'Casey Nguyen', 'West', 'plus', TIMESTAMP '2024-10-21 02:00:00'), +(79, 'Oakley Nguyen', 'North', 'basic', TIMESTAMP '2024-09-16 14:00:00'), +(80, 'Tatum Jensen', 'West', 'enterprise', TIMESTAMP '2023-12-20 15:00:00'); + +DROP TABLE IF EXISTS memory.bench.orders; +CREATE TABLE memory.bench.orders ( + order_id INTEGER, + customer_id INTEGER, + order_ts TIMESTAMP, + status VARCHAR, + amount INTEGER, + discount INTEGER +); +INSERT INTO memory.bench.orders VALUES +(1000, 71, TIMESTAMP '2025-11-11 09:05:00', 'completed', 58692, 13269), +(1001, 55, TIMESTAMP '2025-11-07 12:28:00', 'completed', 252586, 6080), +(1002, 41, TIMESTAMP '2025-05-01 15:36:00', 'completed', 221661, 6124), +(1003, 41, TIMESTAMP '2025-07-15 17:18:00', 'completed', 176712, 4162), +(1004, 3, TIMESTAMP '2025-11-14 19:26:00', 'completed', 12979, 1570), +(1005, 54, TIMESTAMP '2025-11-04 22:49:00', 'completed', 117901, 4195), +(1006, 3, TIMESTAMP '2025-09-25 15:21:00', 'completed', 95307, 1223), +(1007, 23, TIMESTAMP '2025-03-21 09:32:00', 'completed', 300395, 15806), +(1008, 50, TIMESTAMP '2025-05-06 17:48:00', 'completed', 222446, 30510), +(1009, 7, TIMESTAMP '2025-12-28 07:58:00', 'completed', 83348, 2832), +(1010, 33, TIMESTAMP '2025-01-01 20:27:00', 'completed', 50441, 1712), +(1011, 45, TIMESTAMP '2025-03-16 18:13:00', 'completed', 398481, 76836), +(1012, 7, TIMESTAMP '2025-12-22 19:41:00', 'completed', 39674, 1188), +(1013, 78, TIMESTAMP '2025-10-11 00:49:00', 'completed', 232240, 2569), +(1014, 79, TIMESTAMP '2025-06-11 21:09:00', 'completed', 191114, 18228), +(1015, 36, TIMESTAMP '2025-09-08 02:48:00', 'completed', 115702, 1271), +(1016, 20, TIMESTAMP '2025-01-29 06:23:00', 'completed', 240760, 8183), +(1017, 5, TIMESTAMP '2025-05-24 11:42:00', 'refunded', 203835, 0), +(1018, 4, TIMESTAMP '2025-06-15 17:00:00', 'completed', 246672, 19066), +(1019, 16, TIMESTAMP '2025-09-16 17:30:00', 'completed', 127303, 13268), +(1020, 61, TIMESTAMP '2025-02-18 20:42:00', 'completed', 200288, 4611), +(1021, 11, TIMESTAMP '2025-03-22 14:15:00', 'completed', 90546, 6443), +(1022, 71, TIMESTAMP '2025-12-15 09:53:00', 'completed', 135595, 31947), +(1023, 67, TIMESTAMP '2025-03-22 15:28:00', 'refunded', 101742, 0), +(1024, 39, TIMESTAMP '2025-11-01 16:30:00', 'completed', 385681, 49917), +(1025, 66, TIMESTAMP '2025-02-05 12:09:00', 'completed', 44530, 4393), +(1026, 1, TIMESTAMP '2025-08-26 00:34:00', 'refunded', 160395, 0), +(1027, 31, TIMESTAMP '2025-08-30 18:33:00', 'completed', 287243, 21022), +(1028, 31, TIMESTAMP '2025-06-10 04:53:00', 'completed', 49832, 2189), +(1029, 28, TIMESTAMP '2025-04-12 15:27:00', 'completed', 135111, 18291), +(1030, 61, TIMESTAMP '2025-03-04 15:35:00', 'completed', 137016, 1849), +(1031, 40, TIMESTAMP '2025-10-23 21:07:00', 'completed', 200098, 28252), +(1032, 10, TIMESTAMP '2025-09-04 16:40:00', 'completed', 100276, 3745), +(1033, 10, TIMESTAMP '2025-04-12 22:24:00', 'completed', 287951, 14860), +(1034, 7, TIMESTAMP '2025-04-18 07:29:00', 'completed', 203953, 4802), +(1035, 13, TIMESTAMP '2025-08-25 10:55:00', 'refunded', 220198, 0), +(1036, 22, TIMESTAMP '2025-10-17 09:39:00', 'completed', 217243, 10342), +(1037, 54, TIMESTAMP '2025-09-07 23:03:00', 'completed', 73105, 6817), +(1038, 47, TIMESTAMP '2025-03-24 13:19:00', 'refunded', 62254, 0), +(1039, 73, TIMESTAMP '2025-01-12 22:03:00', 'completed', 246221, 150), +(1040, 76, TIMESTAMP '2025-07-09 17:30:00', 'completed', 210900, 12386), +(1041, 64, TIMESTAMP '2025-03-19 20:27:00', 'refunded', 119308, 0), +(1042, 35, TIMESTAMP '2025-11-21 17:44:00', 'refunded', 292145, 0), +(1043, 1, TIMESTAMP '2025-07-15 23:28:00', 'refunded', 112126, 0), +(1044, 77, TIMESTAMP '2025-02-14 01:17:00', 'completed', 234027, 590), +(1045, 51, TIMESTAMP '2025-07-01 21:41:00', 'completed', 155033, 9768), +(1046, 45, TIMESTAMP '2025-08-17 06:21:00', 'completed', 391089, 166658), +(1047, 71, TIMESTAMP '2025-02-05 12:25:00', 'refunded', 275484, 0), +(1048, 10, TIMESTAMP '2025-09-25 13:55:00', 'completed', 271276, 17332), +(1049, 35, TIMESTAMP '2025-09-15 20:29:00', 'completed', 160180, 59175), +(1050, 27, TIMESTAMP '2025-05-22 15:32:00', 'completed', 82214, 8271), +(1051, 43, TIMESTAMP '2025-05-11 08:50:00', 'completed', 230819, 27958), +(1052, 41, TIMESTAMP '2025-01-23 12:23:00', 'completed', 190470, 718), +(1053, 56, TIMESTAMP '2025-08-22 23:58:00', 'completed', 182250, 3910), +(1054, 2, TIMESTAMP '2025-11-01 06:26:00', 'completed', 356579, 93431), +(1055, 24, TIMESTAMP '2025-10-02 06:33:00', 'completed', 158562, 14558), +(1056, 48, TIMESTAMP '2025-07-22 12:54:00', 'completed', 120937, 2163), +(1057, 70, TIMESTAMP '2025-08-23 23:18:00', 'pending', 176440, 0), +(1058, 24, TIMESTAMP '2025-05-20 17:37:00', 'refunded', 125914, 0), +(1059, 22, TIMESTAMP '2025-05-02 08:11:00', 'completed', 121129, 10485), +(1060, 5, TIMESTAMP '2025-12-29 07:30:00', 'completed', 63626, 1175), +(1061, 55, TIMESTAMP '2025-05-09 17:38:00', 'completed', 274067, 20399), +(1062, 6, TIMESTAMP '2025-10-20 01:05:00', 'completed', 27742, 231), +(1063, 64, TIMESTAMP '2025-09-11 07:03:00', 'completed', 73109, 10110), +(1064, 46, TIMESTAMP '2025-05-14 04:24:00', 'completed', 8707, 803), +(1065, 28, TIMESTAMP '2025-11-17 07:42:00', 'completed', 41595, 5398), +(1066, 32, TIMESTAMP '2025-02-17 00:45:00', 'completed', 376126, 89192), +(1067, 71, TIMESTAMP '2025-03-11 06:41:00', 'pending', 312780, 0), +(1068, 48, TIMESTAMP '2025-11-14 11:02:00', 'completed', 8719, 635), +(1069, 41, TIMESTAMP '2025-01-07 05:47:00', 'completed', 266151, 6728), +(1070, 6, TIMESTAMP '2025-04-25 12:40:00', 'pending', 59311, 0), +(1071, 25, TIMESTAMP '2025-02-25 14:15:00', 'completed', 278958, 10981), +(1072, 33, TIMESTAMP '2025-06-28 01:39:00', 'refunded', 66708, 0), +(1073, 6, TIMESTAMP '2025-04-17 08:21:00', 'completed', 260827, 8755), +(1074, 44, TIMESTAMP '2025-06-06 08:48:00', 'completed', 89697, 5320), +(1075, 61, TIMESTAMP '2025-11-10 03:07:00', 'completed', 180095, 10368), +(1076, 25, TIMESTAMP '2025-09-22 15:58:00', 'completed', 222482, 3249), +(1077, 6, TIMESTAMP '2025-11-08 18:23:00', 'completed', 195085, 5442), +(1078, 80, TIMESTAMP '2025-07-13 01:55:00', 'completed', 203835, 8889), +(1079, 19, TIMESTAMP '2025-03-21 20:39:00', 'completed', 41853, 1749), +(1080, 61, TIMESTAMP '2025-08-17 17:32:00', 'completed', 134628, 7832), +(1081, 10, TIMESTAMP '2025-06-02 08:27:00', 'completed', 147503, 4216), +(1082, 73, TIMESTAMP '2025-05-31 15:03:00', 'completed', 42830, 2929), +(1083, 20, TIMESTAMP '2025-11-02 10:02:00', 'completed', 60340, 1293), +(1084, 80, TIMESTAMP '2025-01-27 12:54:00', 'completed', 264101, 5812), +(1085, 66, TIMESTAMP '2025-10-02 12:46:00', 'refunded', 208195, 0), +(1086, 26, TIMESTAMP '2025-08-06 11:50:00', 'completed', 123257, 7331), +(1087, 31, TIMESTAMP '2025-04-01 15:27:00', 'completed', 192043, 4192), +(1088, 76, TIMESTAMP '2025-10-04 08:45:00', 'completed', 151307, 11232), +(1089, 10, TIMESTAMP '2025-01-01 17:32:00', 'completed', 152471, 5384), +(1090, 33, TIMESTAMP '2025-12-26 08:02:00', 'completed', 199585, 27565), +(1091, 36, TIMESTAMP '2025-10-31 09:40:00', 'pending', 14133, 0), +(1092, 54, TIMESTAMP '2025-03-13 13:38:00', 'completed', 28735, 3620), +(1093, 55, TIMESTAMP '2025-09-08 04:20:00', 'completed', 236295, 16228), +(1094, 63, TIMESTAMP '2025-12-09 06:50:00', 'pending', 141541, 0), +(1095, 35, TIMESTAMP '2025-02-07 05:21:00', 'completed', 283640, 49272), +(1096, 46, TIMESTAMP '2025-12-24 20:49:00', 'refunded', 33969, 0), +(1097, 35, TIMESTAMP '2025-04-10 11:54:00', 'completed', 248265, 6090), +(1098, 36, TIMESTAMP '2025-06-24 00:03:00', 'completed', 108723, 1231), +(1099, 37, TIMESTAMP '2025-06-14 09:14:00', 'pending', 106144, 0), +(1100, 16, TIMESTAMP '2025-07-14 08:24:00', 'completed', 133056, 9325), +(1101, 36, TIMESTAMP '2025-05-12 22:48:00', 'completed', 156262, 5209), +(1102, 16, TIMESTAMP '2025-08-29 03:07:00', 'completed', 30691, 3644), +(1103, 57, TIMESTAMP '2025-03-07 11:08:00', 'refunded', 221961, 0), +(1104, 46, TIMESTAMP '2025-12-02 19:19:00', 'completed', 135088, 13725), +(1105, 14, TIMESTAMP '2025-09-03 05:14:00', 'completed', 41813, 1553), +(1106, 8, TIMESTAMP '2025-05-22 17:53:00', 'completed', 118724, 14999), +(1107, 1, TIMESTAMP '2025-02-14 15:15:00', 'pending', 30497, 0), +(1108, 3, TIMESTAMP '2025-03-27 21:33:00', 'completed', 170791, 5472), +(1109, 22, TIMESTAMP '2025-01-09 06:00:00', 'completed', 247738, 34917), +(1110, 15, TIMESTAMP '2025-11-21 22:21:00', 'completed', 38066, 4490), +(1111, 36, TIMESTAMP '2025-08-01 02:38:00', 'completed', 68703, 1629), +(1112, 13, TIMESTAMP '2025-01-29 09:39:00', 'refunded', 208015, 0), +(1113, 80, TIMESTAMP '2025-09-25 03:32:00', 'completed', 289576, 20056), +(1114, 1, TIMESTAMP '2025-12-23 00:33:00', 'completed', 91582, 24065), +(1115, 64, TIMESTAMP '2025-06-13 06:01:00', 'completed', 17938, 480), +(1116, 62, TIMESTAMP '2025-11-26 16:50:00', 'completed', 365124, 71074), +(1117, 21, TIMESTAMP '2025-01-18 01:37:00', 'completed', 93408, 7265), +(1118, 42, TIMESTAMP '2025-08-12 09:43:00', 'completed', 46341, 930), +(1119, 1, TIMESTAMP '2025-06-14 10:03:00', 'completed', 169067, 33392), +(1120, 9, TIMESTAMP '2025-01-08 23:59:00', 'completed', 28347, 2079), +(1121, 56, TIMESTAMP '2025-06-30 21:45:00', 'completed', 134905, 6748), +(1122, 68, TIMESTAMP '2025-09-29 20:46:00', 'completed', 31241, 4238), +(1123, 13, TIMESTAMP '2025-01-27 14:16:00', 'refunded', 186622, 0), +(1124, 17, TIMESTAMP '2025-06-21 01:11:00', 'refunded', 140422, 0), +(1125, 78, TIMESTAMP '2025-12-31 10:52:00', 'completed', 61648, 4011), +(1126, 49, TIMESTAMP '2025-07-21 09:01:00', 'completed', 75091, 8939), +(1127, 71, TIMESTAMP '2025-06-30 09:02:00', 'completed', 320408, 19215), +(1128, 76, TIMESTAMP '2025-08-13 01:44:00', 'pending', 221079, 0), +(1129, 78, TIMESTAMP '2025-05-03 09:13:00', 'completed', 281995, 16198), +(1130, 18, TIMESTAMP '2025-03-05 14:32:00', 'completed', 383272, 83003), +(1131, 8, TIMESTAMP '2025-10-13 13:31:00', 'completed', 209022, 20707), +(1132, 47, TIMESTAMP '2025-12-21 05:55:00', 'refunded', 18496, 0), +(1133, 52, TIMESTAMP '2025-01-26 02:14:00', 'completed', 167068, 12556), +(1134, 51, TIMESTAMP '2025-09-16 09:50:00', 'pending', 247512, 0), +(1135, 23, TIMESTAMP '2025-02-09 20:29:00', 'completed', 84161, 1793), +(1136, 44, TIMESTAMP '2025-12-17 01:57:00', 'completed', 282910, 2397), +(1137, 41, TIMESTAMP '2025-12-28 12:46:00', 'completed', 286560, 7205), +(1138, 44, TIMESTAMP '2025-01-08 20:02:00', 'completed', 297072, 15939), +(1139, 14, TIMESTAMP '2025-11-06 13:29:00', 'completed', 205535, 4424), +(1140, 9, TIMESTAMP '2025-05-31 05:35:00', 'completed', 57236, 5210), +(1141, 18, TIMESTAMP '2025-03-19 23:05:00', 'refunded', 59500, 0), +(1142, 21, TIMESTAMP '2025-04-06 07:10:00', 'completed', 3469, 324), +(1143, 66, TIMESTAMP '2025-11-03 17:32:00', 'completed', 158514, 3431), +(1144, 27, TIMESTAMP '2025-03-28 11:56:00', 'completed', 99085, 5152), +(1145, 80, TIMESTAMP '2025-02-24 13:11:00', 'completed', 273700, 11493), +(1146, 49, TIMESTAMP '2025-03-03 06:40:00', 'completed', 106900, 14178), +(1147, 45, TIMESTAMP '2025-10-20 08:19:00', 'completed', 204814, 96226), +(1148, 33, TIMESTAMP '2025-07-01 19:22:00', 'completed', 140455, 6168), +(1149, 54, TIMESTAMP '2025-04-29 13:57:00', 'completed', 164051, 2337), +(1150, 77, TIMESTAMP '2025-12-05 18:11:00', 'completed', 48750, 1218), +(1151, 67, TIMESTAMP '2025-10-17 17:05:00', 'completed', 138880, 2302), +(1152, 61, TIMESTAMP '2025-07-24 23:05:00', 'completed', 265591, 16667), +(1153, 15, TIMESTAMP '2025-05-16 23:05:00', 'completed', 142795, 4241), +(1154, 55, TIMESTAMP '2025-09-17 13:57:00', 'completed', 219510, 8772), +(1155, 50, TIMESTAMP '2025-06-04 14:22:00', 'completed', 140696, 432), +(1156, 62, TIMESTAMP '2025-01-17 08:01:00', 'refunded', 348854, 0), +(1157, 36, TIMESTAMP '2025-12-03 00:20:00', 'completed', 201428, 29761), +(1158, 9, TIMESTAMP '2025-09-18 07:44:00', 'completed', 137964, 4137), +(1159, 8, TIMESTAMP '2025-09-10 22:40:00', 'completed', 195481, 12937), +(1160, 9, TIMESTAMP '2025-04-27 08:51:00', 'completed', 17191, 2130), +(1161, 14, TIMESTAMP '2025-08-23 00:35:00', 'pending', 176075, 0), +(1162, 30, TIMESTAMP '2025-01-07 00:27:00', 'completed', 111697, 2371), +(1163, 28, TIMESTAMP '2025-07-14 05:07:00', 'completed', 247819, 24397), +(1164, 75, TIMESTAMP '2025-05-30 21:58:00', 'refunded', 311880, 0), +(1165, 42, TIMESTAMP '2025-09-07 20:56:00', 'completed', 64209, 9611), +(1166, 42, TIMESTAMP '2025-06-04 01:47:00', 'completed', 86673, 2037), +(1167, 54, TIMESTAMP '2025-03-17 01:04:00', 'completed', 57736, 4113), +(1168, 28, TIMESTAMP '2025-03-12 15:18:00', 'completed', 139659, 2400), +(1169, 53, TIMESTAMP '2025-08-11 14:02:00', 'completed', 2052, 136), +(1170, 52, TIMESTAMP '2025-10-18 00:23:00', 'completed', 193517, 2717), +(1171, 13, TIMESTAMP '2025-12-15 03:49:00', 'completed', 147259, 3958), +(1172, 60, TIMESTAMP '2025-08-18 20:22:00', 'completed', 189965, 23784), +(1173, 52, TIMESTAMP '2025-02-21 21:05:00', 'refunded', 267436, 0), +(1174, 70, TIMESTAMP '2025-03-05 21:28:00', 'completed', 274633, 14706), +(1175, 59, TIMESTAMP '2025-12-19 08:17:00', 'completed', 2854, 239), +(1176, 53, TIMESTAMP '2025-03-04 04:40:00', 'refunded', 42466, 0), +(1177, 13, TIMESTAMP '2025-04-29 16:36:00', 'completed', 146691, 17946), +(1178, 8, TIMESTAMP '2025-07-10 22:21:00', 'completed', 108217, 12729), +(1179, 42, TIMESTAMP '2025-09-25 03:22:00', 'completed', 188903, 20965), +(1180, 22, TIMESTAMP '2025-04-14 03:11:00', 'pending', 172120, 0), +(1181, 43, TIMESTAMP '2025-12-22 05:10:00', 'completed', 103649, 14002), +(1182, 20, TIMESTAMP '2025-09-21 00:47:00', 'completed', 251698, 14983), +(1183, 41, TIMESTAMP '2025-08-16 13:15:00', 'completed', 201907, 1793), +(1184, 2, TIMESTAMP '2025-10-11 17:57:00', 'completed', 389091, 46992), +(1185, 1, TIMESTAMP '2025-09-02 19:20:00', 'completed', 58312, 26404), +(1186, 80, TIMESTAMP '2025-06-22 23:43:00', 'completed', 14513, 903), +(1187, 48, TIMESTAMP '2025-06-13 06:47:00', 'completed', 187298, 5368), +(1188, 8, TIMESTAMP '2025-03-26 15:47:00', 'pending', 172706, 0), +(1189, 75, TIMESTAMP '2025-01-27 19:16:00', 'refunded', 5040, 0), +(1190, 54, TIMESTAMP '2025-06-02 10:01:00', 'completed', 105778, 11981), +(1191, 76, TIMESTAMP '2025-03-20 08:45:00', 'completed', 122495, 10224), +(1192, 13, TIMESTAMP '2025-06-01 11:10:00', 'completed', 35851, 2578), +(1193, 63, TIMESTAMP '2025-05-31 16:36:00', 'completed', 16095, 2125), +(1194, 24, TIMESTAMP '2025-12-16 06:14:00', 'completed', 139566, 19446), +(1195, 39, TIMESTAMP '2025-02-17 22:35:00', 'completed', 35864, 2379), +(1196, 60, TIMESTAMP '2025-04-01 07:45:00', 'completed', 43207, 491), +(1197, 50, TIMESTAMP '2025-03-31 13:24:00', 'completed', 88002, 5865), +(1198, 11, TIMESTAMP '2025-05-02 16:00:00', 'completed', 29535, 1804), +(1199, 34, TIMESTAMP '2025-02-25 21:35:00', 'completed', 34990, 1878), +(1200, 23, TIMESTAMP '2025-01-01 07:44:00', 'completed', 173887, 491), +(1201, 73, TIMESTAMP '2025-02-04 04:02:00', 'completed', 228138, 1834), +(1202, 39, TIMESTAMP '2025-02-27 08:34:00', 'refunded', 291598, 0), +(1203, 51, TIMESTAMP '2025-03-13 15:12:00', 'completed', 206109, 27576), +(1204, 32, TIMESTAMP '2025-06-26 02:01:00', 'completed', 188875, 629), +(1205, 40, TIMESTAMP '2025-08-28 10:23:00', 'completed', 195238, 28002), +(1206, 11, TIMESTAMP '2025-03-19 04:49:00', 'completed', 104265, 27), +(1207, 36, TIMESTAMP '2025-11-23 06:56:00', 'refunded', 208415, 0), +(1208, 73, TIMESTAMP '2025-06-13 02:57:00', 'completed', 65251, 2634), +(1209, 60, TIMESTAMP '2025-10-02 00:19:00', 'completed', 129024, 11715), +(1210, 75, TIMESTAMP '2025-03-25 14:53:00', 'completed', 216769, 60369), +(1211, 33, TIMESTAMP '2025-09-23 05:22:00', 'completed', 83886, 10755), +(1212, 9, TIMESTAMP '2025-07-20 10:24:00', 'completed', 199027, 728), +(1213, 58, TIMESTAMP '2025-03-06 13:19:00', 'completed', 89385, 6498), +(1214, 23, TIMESTAMP '2025-12-06 04:40:00', 'completed', 9585, 504), +(1215, 36, TIMESTAMP '2025-05-03 13:20:00', 'completed', 157594, 16544), +(1216, 51, TIMESTAMP '2025-06-03 01:04:00', 'completed', 31525, 166), +(1217, 62, TIMESTAMP '2025-05-13 02:13:00', 'refunded', 137446, 0), +(1218, 69, TIMESTAMP '2025-02-25 07:33:00', 'refunded', 57329, 0), +(1219, 61, TIMESTAMP '2025-02-12 09:26:00', 'completed', 244368, 207), +(1220, 73, TIMESTAMP '2025-04-05 12:36:00', 'completed', 233833, 16179), +(1221, 62, TIMESTAMP '2025-01-05 11:57:00', 'completed', 138728, 55385), +(1222, 13, TIMESTAMP '2025-04-19 23:19:00', 'completed', 228186, 16667), +(1223, 69, TIMESTAMP '2025-08-13 12:31:00', 'refunded', 195284, 0), +(1224, 60, TIMESTAMP '2025-01-31 06:15:00', 'completed', 198795, 2607), +(1225, 54, TIMESTAMP '2025-01-21 05:39:00', 'completed', 156557, 19167), +(1226, 47, TIMESTAMP '2025-03-10 22:42:00', 'completed', 284723, 17347), +(1227, 43, TIMESTAMP '2025-08-30 20:20:00', 'refunded', 315049, 0), +(1228, 16, TIMESTAMP '2025-12-12 02:47:00', 'completed', 230493, 30568), +(1229, 34, TIMESTAMP '2025-01-03 06:08:00', 'completed', 44653, 2007), +(1230, 30, TIMESTAMP '2025-06-30 05:48:00', 'pending', 212010, 0), +(1231, 62, TIMESTAMP '2025-12-16 10:12:00', 'completed', 184355, 71769), +(1232, 4, TIMESTAMP '2025-08-20 04:05:00', 'completed', 68460, 8629), +(1233, 27, TIMESTAMP '2025-04-10 04:54:00', 'completed', 101509, 5041), +(1234, 36, TIMESTAMP '2025-09-13 14:12:00', 'pending', 221638, 0), +(1235, 80, TIMESTAMP '2025-06-01 17:22:00', 'completed', 203706, 8116), +(1236, 11, TIMESTAMP '2025-02-14 04:35:00', 'completed', 75546, 5584), +(1237, 57, TIMESTAMP '2025-02-13 13:54:00', 'refunded', 11459, 0), +(1238, 13, TIMESTAMP '2025-09-03 07:51:00', 'completed', 213519, 22750), +(1239, 71, TIMESTAMP '2025-04-04 16:23:00', 'pending', 313110, 0), +(1240, 14, TIMESTAMP '2025-12-10 00:05:00', 'completed', 275257, 17005), +(1241, 68, TIMESTAMP '2025-02-23 08:07:00', 'refunded', 398824, 0), +(1242, 5, TIMESTAMP '2025-08-18 08:41:00', 'refunded', 39822, 0), +(1243, 1, TIMESTAMP '2025-09-16 05:52:00', 'refunded', 168449, 0), +(1244, 50, TIMESTAMP '2025-11-24 16:20:00', 'completed', 24033, 1554), +(1245, 66, TIMESTAMP '2025-12-06 01:38:00', 'completed', 186357, 25304), +(1246, 52, TIMESTAMP '2025-04-20 01:21:00', 'completed', 249266, 4165), +(1247, 77, TIMESTAMP '2025-03-03 03:25:00', 'completed', 43093, 617), +(1248, 78, TIMESTAMP '2025-07-19 20:04:00', 'completed', 67763, 2592), +(1249, 65, TIMESTAMP '2025-07-19 13:13:00', 'refunded', 59500, 0); +INSERT INTO memory.bench.orders VALUES +(1250, 5, TIMESTAMP '2025-11-23 07:12:00', 'completed', 111885, 7711), +(1251, 12, TIMESTAMP '2025-05-05 10:54:00', 'completed', 11163, 656), +(1252, 10, TIMESTAMP '2025-06-03 22:38:00', 'completed', 78173, 925), +(1253, 52, TIMESTAMP '2025-10-15 00:25:00', 'completed', 94693, 3029), +(1254, 15, TIMESTAMP '2025-05-04 23:28:00', 'completed', 185184, 9466), +(1255, 74, TIMESTAMP '2025-05-04 06:38:00', 'completed', 322766, 33015), +(1256, 48, TIMESTAMP '2025-07-01 02:37:00', 'completed', 75234, 9849), +(1257, 73, TIMESTAMP '2025-10-22 03:31:00', 'completed', 81675, 737), +(1258, 1, TIMESTAMP '2025-03-16 05:17:00', 'completed', 209644, 66284), +(1259, 70, TIMESTAMP '2025-12-06 09:35:00', 'completed', 274332, 19906), +(1260, 42, TIMESTAMP '2025-04-09 18:12:00', 'completed', 105353, 10711), +(1261, 56, TIMESTAMP '2025-05-09 18:47:00', 'completed', 300276, 4245), +(1262, 38, TIMESTAMP '2025-05-02 07:45:00', 'pending', 219139, 0), +(1263, 26, TIMESTAMP '2025-04-14 06:53:00', 'completed', 26970, 2111), +(1264, 21, TIMESTAMP '2025-06-21 23:55:00', 'refunded', 95392, 0), +(1265, 13, TIMESTAMP '2025-10-10 06:33:00', 'completed', 213539, 9566), +(1266, 64, TIMESTAMP '2025-03-16 21:42:00', 'completed', 127618, 8642), +(1267, 48, TIMESTAMP '2025-05-08 09:55:00', 'completed', 170514, 14042), +(1268, 69, TIMESTAMP '2025-11-15 10:07:00', 'completed', 167176, 54142), +(1269, 22, TIMESTAMP '2025-10-04 15:20:00', 'completed', 84860, 954), +(1270, 55, TIMESTAMP '2025-11-14 10:19:00', 'completed', 178262, 11470), +(1271, 19, TIMESTAMP '2025-11-17 02:25:00', 'completed', 176698, 9012), +(1272, 45, TIMESTAMP '2025-06-30 19:33:00', 'completed', 340897, 129642), +(1273, 42, TIMESTAMP '2025-03-16 09:34:00', 'pending', 146765, 0), +(1274, 30, TIMESTAMP '2025-05-07 23:11:00', 'completed', 180519, 13118), +(1275, 79, TIMESTAMP '2025-01-01 08:47:00', 'completed', 61416, 3583), +(1276, 4, TIMESTAMP '2025-09-21 00:21:00', 'completed', 78929, 1932), +(1277, 32, TIMESTAMP '2025-03-20 17:43:00', 'completed', 137401, 67472), +(1278, 65, TIMESTAMP '2025-04-06 05:54:00', 'completed', 117033, 27868), +(1279, 63, TIMESTAMP '2025-07-19 18:05:00', 'refunded', 239729, 0), +(1280, 27, TIMESTAMP '2025-10-20 02:20:00', 'refunded', 229793, 0), +(1281, 38, TIMESTAMP '2025-03-07 08:52:00', 'completed', 56589, 1585), +(1282, 47, TIMESTAMP '2025-04-11 19:39:00', 'refunded', 205744, 0), +(1283, 35, TIMESTAMP '2025-02-05 10:40:00', 'refunded', 20852, 0), +(1284, 68, TIMESTAMP '2025-10-26 09:35:00', 'completed', 353001, 73608), +(1285, 75, TIMESTAMP '2025-09-24 21:09:00', 'completed', 242193, 56435), +(1286, 76, TIMESTAMP '2025-08-24 12:43:00', 'completed', 146955, 18412), +(1287, 48, TIMESTAMP '2025-08-27 11:02:00', 'completed', 112442, 3629), +(1288, 49, TIMESTAMP '2025-05-24 04:06:00', 'completed', 150460, 14323), +(1289, 15, TIMESTAMP '2025-03-11 14:29:00', 'completed', 179282, 11605), +(1290, 67, TIMESTAMP '2025-01-07 23:09:00', 'pending', 135989, 0), +(1291, 78, TIMESTAMP '2025-11-14 06:11:00', 'completed', 161466, 3650), +(1292, 28, TIMESTAMP '2025-05-19 16:54:00', 'completed', 29128, 2643), +(1293, 32, TIMESTAMP '2025-05-08 20:34:00', 'completed', 170044, 61778), +(1294, 52, TIMESTAMP '2025-11-16 09:53:00', 'completed', 217873, 11152), +(1295, 7, TIMESTAMP '2025-08-08 05:43:00', 'completed', 46429, 1346), +(1296, 44, TIMESTAMP '2025-05-15 10:34:00', 'completed', 102126, 5484), +(1297, 60, TIMESTAMP '2025-06-19 19:30:00', 'completed', 101427, 12220), +(1298, 68, TIMESTAMP '2025-07-15 11:44:00', 'completed', 319307, 126178), +(1299, 20, TIMESTAMP '2025-05-27 17:12:00', 'completed', 201605, 8983), +(1300, 60, TIMESTAMP '2025-01-06 20:45:00', 'completed', 80617, 6136), +(1301, 57, TIMESTAMP '2025-04-24 03:15:00', 'refunded', 14790, 0), +(1302, 54, TIMESTAMP '2025-08-07 14:25:00', 'completed', 61898, 2293), +(1303, 14, TIMESTAMP '2025-10-23 01:06:00', 'completed', 48891, 2176), +(1304, 72, TIMESTAMP '2025-03-21 14:29:00', 'completed', 158337, 16732), +(1305, 4, TIMESTAMP '2025-04-22 16:43:00', 'completed', 125384, 157), +(1306, 4, TIMESTAMP '2025-09-09 08:54:00', 'completed', 36278, 4527), +(1307, 51, TIMESTAMP '2025-05-01 01:01:00', 'completed', 28983, 4179), +(1308, 29, TIMESTAMP '2025-04-16 04:11:00', 'refunded', 219801, 0), +(1309, 16, TIMESTAMP '2025-03-25 15:13:00', 'completed', 29393, 927), +(1310, 28, TIMESTAMP '2025-12-05 14:06:00', 'completed', 110993, 3164), +(1311, 26, TIMESTAMP '2025-05-10 17:47:00', 'completed', 123586, 6934), +(1312, 6, TIMESTAMP '2025-01-17 06:45:00', 'completed', 205961, 8015), +(1313, 45, TIMESTAMP '2025-03-09 20:40:00', 'refunded', 77668, 0), +(1314, 20, TIMESTAMP '2025-01-27 00:39:00', 'completed', 124882, 6231), +(1315, 47, TIMESTAMP '2025-04-06 05:44:00', 'refunded', 235990, 0), +(1316, 72, TIMESTAMP '2025-08-13 06:46:00', 'completed', 67038, 6236), +(1317, 46, TIMESTAMP '2025-07-27 06:19:00', 'completed', 48190, 3057), +(1318, 58, TIMESTAMP '2025-07-25 20:51:00', 'completed', 123307, 3040), +(1319, 64, TIMESTAMP '2025-12-28 14:08:00', 'pending', 143399, 0), +(1320, 13, TIMESTAMP '2025-11-21 03:48:00', 'completed', 236971, 4407), +(1321, 19, TIMESTAMP '2025-08-27 04:01:00', 'refunded', 287858, 0), +(1322, 4, TIMESTAMP '2025-11-27 05:08:00', 'completed', 235454, 15234), +(1323, 76, TIMESTAMP '2025-06-15 05:27:00', 'completed', 170326, 13471), +(1324, 50, TIMESTAMP '2025-10-26 21:12:00', 'completed', 8517, 1269), +(1325, 79, TIMESTAMP '2025-06-07 00:00:00', 'completed', 90465, 5583), +(1326, 7, TIMESTAMP '2025-11-29 12:55:00', 'completed', 112352, 12575), +(1327, 78, TIMESTAMP '2025-02-04 15:55:00', 'completed', 305091, 20493), +(1328, 43, TIMESTAMP '2025-03-13 04:20:00', 'completed', 118192, 43618), +(1329, 35, TIMESTAMP '2025-10-09 01:44:00', 'completed', 232496, 81998), +(1330, 51, TIMESTAMP '2025-05-27 00:47:00', 'refunded', 101952, 0), +(1331, 62, TIMESTAMP '2025-09-28 17:14:00', 'refunded', 18270, 0), +(1332, 25, TIMESTAMP '2025-01-29 05:03:00', 'completed', 9637, 491), +(1333, 64, TIMESTAMP '2025-09-02 18:10:00', 'completed', 12793, 399), +(1334, 23, TIMESTAMP '2025-11-18 03:29:00', 'completed', 96532, 6891), +(1335, 17, TIMESTAMP '2025-04-08 20:18:00', 'completed', 77078, 881), +(1336, 14, TIMESTAMP '2025-03-17 07:22:00', 'pending', 23008, 0), +(1337, 66, TIMESTAMP '2025-07-27 11:51:00', 'completed', 36121, 4670), +(1338, 3, TIMESTAMP '2025-12-04 01:15:00', 'completed', 18133, 1256), +(1339, 1, TIMESTAMP '2025-07-25 03:59:00', 'refunded', 44497, 0), +(1340, 77, TIMESTAMP '2025-07-12 05:15:00', 'completed', 199261, 4107), +(1341, 49, TIMESTAMP '2025-03-13 02:56:00', 'refunded', 105700, 0), +(1342, 2, TIMESTAMP '2025-12-02 10:04:00', 'completed', 265574, 30066), +(1343, 5, TIMESTAMP '2025-08-01 11:34:00', 'pending', 137364, 0), +(1344, 10, TIMESTAMP '2025-11-20 04:37:00', 'completed', 128921, 5161), +(1345, 79, TIMESTAMP '2025-04-05 02:22:00', 'completed', 228027, 5176), +(1346, 30, TIMESTAMP '2025-04-15 19:36:00', 'completed', 208548, 5034), +(1347, 7, TIMESTAMP '2025-12-20 22:49:00', 'completed', 206585, 13391), +(1348, 8, TIMESTAMP '2025-06-24 05:20:00', 'completed', 116841, 16828), +(1349, 12, TIMESTAMP '2025-08-29 19:44:00', 'completed', 195572, 5607), +(1350, 26, TIMESTAMP '2025-03-16 11:23:00', 'completed', 31061, 1102), +(1351, 3, TIMESTAMP '2025-02-08 03:23:00', 'completed', 124298, 10279), +(1352, 63, TIMESTAMP '2025-08-24 20:54:00', 'completed', 196242, 2905), +(1353, 15, TIMESTAMP '2025-06-18 08:55:00', 'completed', 27320, 1680), +(1354, 63, TIMESTAMP '2025-05-09 13:42:00', 'completed', 101262, 859), +(1355, 3, TIMESTAMP '2025-09-15 12:24:00', 'completed', 142912, 14458), +(1356, 6, TIMESTAMP '2025-05-29 23:20:00', 'completed', 245635, 2336), +(1357, 42, TIMESTAMP '2025-07-21 07:38:00', 'completed', 5480, 95), +(1358, 71, TIMESTAMP '2025-06-29 11:19:00', 'refunded', 53953, 0), +(1359, 58, TIMESTAMP '2025-03-26 18:30:00', 'completed', 115887, 810), +(1360, 40, TIMESTAMP '2025-04-22 03:53:00', 'completed', 140123, 18656), +(1361, 60, TIMESTAMP '2025-08-08 08:20:00', 'completed', 92828, 5294), +(1362, 50, TIMESTAMP '2025-07-31 10:20:00', 'pending', 37798, 0), +(1363, 55, TIMESTAMP '2025-05-11 04:14:00', 'completed', 213041, 6991), +(1364, 64, TIMESTAMP '2025-02-08 23:37:00', 'completed', 139165, 8699), +(1365, 73, TIMESTAMP '2025-06-29 01:22:00', 'completed', 211820, 12435), +(1366, 23, TIMESTAMP '2025-08-24 13:11:00', 'completed', 272271, 9900), +(1367, 70, TIMESTAMP '2025-04-16 03:16:00', 'pending', 15093, 0), +(1368, 65, TIMESTAMP '2025-08-04 03:23:00', 'refunded', 107878, 0), +(1369, 69, TIMESTAMP '2025-07-25 03:10:00', 'refunded', 284265, 0), +(1370, 45, TIMESTAMP '2025-03-04 16:36:00', 'completed', 55803, 20632), +(1371, 59, TIMESTAMP '2025-03-04 05:32:00', 'refunded', 232891, 0), +(1372, 36, TIMESTAMP '2025-08-25 18:24:00', 'refunded', 139200, 0), +(1373, 59, TIMESTAMP '2025-03-30 15:12:00', 'completed', 123110, 45167), +(1374, 39, TIMESTAMP '2025-08-01 13:27:00', 'completed', 166622, 24470), +(1375, 73, TIMESTAMP '2025-09-02 20:05:00', 'refunded', 169236, 0), +(1376, 44, TIMESTAMP '2025-08-28 06:49:00', 'completed', 6256, 170), +(1377, 17, TIMESTAMP '2025-05-11 12:23:00', 'completed', 70434, 6078), +(1378, 31, TIMESTAMP '2025-08-04 12:34:00', 'completed', 119662, 8000), +(1379, 60, TIMESTAMP '2025-09-19 09:03:00', 'completed', 202453, 25845), +(1380, 72, TIMESTAMP '2025-08-17 05:07:00', 'completed', 170116, 23401), +(1381, 59, TIMESTAMP '2025-09-05 10:01:00', 'completed', 8902, 3639), +(1382, 14, TIMESTAMP '2025-04-12 02:54:00', 'completed', 120716, 2744), +(1383, 33, TIMESTAMP '2025-12-22 15:08:00', 'completed', 37766, 2201), +(1384, 33, TIMESTAMP '2025-12-03 15:20:00', 'completed', 144596, 12146), +(1385, 21, TIMESTAMP '2025-11-18 08:52:00', 'completed', 230044, 2361), +(1386, 31, TIMESTAMP '2025-09-06 16:28:00', 'completed', 172707, 2946), +(1387, 71, TIMESTAMP '2025-12-14 07:23:00', 'refunded', 346075, 0), +(1388, 8, TIMESTAMP '2025-10-14 05:22:00', 'completed', 100134, 12795), +(1389, 3, TIMESTAMP '2025-08-04 21:52:00', 'completed', 32041, 2331), +(1390, 42, TIMESTAMP '2025-01-08 07:00:00', 'completed', 57239, 2104), +(1391, 23, TIMESTAMP '2025-08-01 06:07:00', 'completed', 159818, 9700), +(1392, 69, TIMESTAMP '2025-01-06 07:54:00', 'refunded', 338403, 0), +(1393, 24, TIMESTAMP '2025-03-03 15:21:00', 'completed', 143669, 9290), +(1394, 10, TIMESTAMP '2025-09-29 04:40:00', 'completed', 66300, 1210), +(1395, 13, TIMESTAMP '2025-03-03 07:09:00', 'completed', 190240, 26936), +(1396, 50, TIMESTAMP '2025-04-27 19:19:00', 'completed', 152152, 7051), +(1397, 52, TIMESTAMP '2025-05-25 15:35:00', 'completed', 65025, 322), +(1398, 67, TIMESTAMP '2025-11-04 13:06:00', 'completed', 246559, 2465), +(1399, 29, TIMESTAMP '2025-11-20 11:37:00', 'completed', 165987, 26401), +(1400, 46, TIMESTAMP '2025-07-22 11:07:00', 'completed', 225092, 4775), +(1401, 42, TIMESTAMP '2025-12-23 08:57:00', 'completed', 246844, 10501), +(1402, 51, TIMESTAMP '2025-01-14 01:43:00', 'completed', 134470, 2854), +(1403, 44, TIMESTAMP '2025-05-12 19:37:00', 'completed', 214818, 3720), +(1404, 30, TIMESTAMP '2025-07-21 07:58:00', 'completed', 156824, 17824), +(1405, 13, TIMESTAMP '2025-04-02 11:40:00', 'completed', 26762, 1458), +(1406, 80, TIMESTAMP '2025-09-27 18:02:00', 'completed', 15030, 479), +(1407, 39, TIMESTAMP '2025-11-26 14:52:00', 'completed', 177892, 29605), +(1408, 8, TIMESTAMP '2025-11-07 14:44:00', 'refunded', 28986, 0), +(1409, 77, TIMESTAMP '2025-10-18 19:41:00', 'completed', 289530, 6659), +(1410, 41, TIMESTAMP '2025-06-28 14:51:00', 'completed', 99542, 571), +(1411, 79, TIMESTAMP '2025-12-01 12:44:00', 'completed', 134361, 2607), +(1412, 63, TIMESTAMP '2025-06-11 15:37:00', 'completed', 222382, 32322), +(1413, 53, TIMESTAMP '2025-08-13 02:57:00', 'completed', 248274, 20926), +(1414, 55, TIMESTAMP '2025-04-17 12:41:00', 'completed', 241715, 9478), +(1415, 48, TIMESTAMP '2025-11-14 20:10:00', 'completed', 123458, 15130), +(1416, 50, TIMESTAMP '2025-08-06 07:04:00', 'completed', 210439, 2505), +(1417, 27, TIMESTAMP '2025-12-08 21:32:00', 'completed', 112242, 13187), +(1418, 53, TIMESTAMP '2025-03-14 11:44:00', 'completed', 166897, 14332), +(1419, 56, TIMESTAMP '2025-05-31 08:05:00', 'completed', 147237, 4073), +(1420, 34, TIMESTAMP '2025-03-27 00:24:00', 'completed', 86691, 1054), +(1421, 48, TIMESTAMP '2025-08-05 15:10:00', 'completed', 193418, 25101), +(1422, 48, TIMESTAMP '2025-03-20 22:08:00', 'completed', 39444, 5698), +(1423, 5, TIMESTAMP '2025-04-07 05:01:00', 'completed', 233619, 3032), +(1424, 24, TIMESTAMP '2025-05-08 13:00:00', 'completed', 90052, 952), +(1425, 40, TIMESTAMP '2025-05-30 20:36:00', 'completed', 88950, 1759), +(1426, 30, TIMESTAMP '2025-04-30 06:15:00', 'completed', 115739, 6950), +(1427, 67, TIMESTAMP '2025-12-15 18:34:00', 'completed', 213596, 18704), +(1428, 61, TIMESTAMP '2025-12-09 23:36:00', 'completed', 186402, 14741), +(1429, 71, TIMESTAMP '2025-08-15 16:57:00', 'completed', 308297, 60300), +(1430, 79, TIMESTAMP '2025-02-26 22:21:00', 'completed', 25301, 1777), +(1431, 50, TIMESTAMP '2025-04-08 04:00:00', 'completed', 172269, 11186), +(1432, 48, TIMESTAMP '2025-08-15 15:06:00', 'completed', 14923, 1837), +(1433, 40, TIMESTAMP '2025-03-13 22:50:00', 'completed', 227584, 16472), +(1434, 24, TIMESTAMP '2025-04-13 10:55:00', 'completed', 240619, 1519), +(1435, 62, TIMESTAMP '2025-05-19 06:07:00', 'refunded', 211972, 0), +(1436, 71, TIMESTAMP '2025-02-19 18:43:00', 'refunded', 115616, 0), +(1437, 60, TIMESTAMP '2025-04-05 04:00:00', 'completed', 156057, 6793), +(1438, 44, TIMESTAMP '2025-02-24 05:51:00', 'refunded', 270390, 0), +(1439, 33, TIMESTAMP '2025-12-11 17:29:00', 'refunded', 13225, 0), +(1440, 25, TIMESTAMP '2025-08-16 21:22:00', 'completed', 183418, 12677), +(1441, 61, TIMESTAMP '2025-02-21 14:37:00', 'completed', 261227, 8207), +(1442, 54, TIMESTAMP '2025-04-18 23:14:00', 'completed', 204704, 25592), +(1443, 36, TIMESTAMP '2025-08-04 19:47:00', 'completed', 226839, 11635), +(1444, 33, TIMESTAMP '2025-08-29 08:53:00', 'completed', 47722, 4566), +(1445, 78, TIMESTAMP '2025-09-07 12:54:00', 'completed', 103251, 96), +(1446, 32, TIMESTAMP '2025-02-18 08:40:00', 'pending', 72433, 0), +(1447, 41, TIMESTAMP '2025-10-15 15:30:00', 'completed', 88743, 3867), +(1448, 36, TIMESTAMP '2025-06-26 22:10:00', 'completed', 131855, 18438), +(1449, 57, TIMESTAMP '2025-10-27 12:10:00', 'completed', 361782, 27088), +(1450, 71, TIMESTAMP '2025-10-14 23:45:00', 'refunded', 381564, 0), +(1451, 43, TIMESTAMP '2025-01-07 07:57:00', 'refunded', 49128, 0), +(1452, 31, TIMESTAMP '2025-05-05 16:25:00', 'completed', 260635, 15877), +(1453, 48, TIMESTAMP '2025-01-15 21:08:00', 'completed', 235981, 19071), +(1454, 3, TIMESTAMP '2025-02-02 23:11:00', 'completed', 3709, 44), +(1455, 7, TIMESTAMP '2025-01-10 23:48:00', 'pending', 21703, 0), +(1456, 37, TIMESTAMP '2025-07-11 17:41:00', 'completed', 223060, 6947), +(1457, 24, TIMESTAMP '2025-05-08 14:41:00', 'completed', 171749, 23080), +(1458, 7, TIMESTAMP '2025-09-03 18:33:00', 'completed', 212487, 4299), +(1459, 13, TIMESTAMP '2025-12-15 07:03:00', 'completed', 84217, 1387), +(1460, 3, TIMESTAMP '2025-11-01 22:52:00', 'completed', 184296, 3278), +(1461, 80, TIMESTAMP '2025-10-16 16:53:00', 'completed', 226435, 11013), +(1462, 44, TIMESTAMP '2025-09-24 06:08:00', 'completed', 33250, 27), +(1463, 58, TIMESTAMP '2025-03-08 09:46:00', 'pending', 102836, 0), +(1464, 68, TIMESTAMP '2025-12-09 06:02:00', 'completed', 362395, 67184), +(1465, 43, TIMESTAMP '2025-11-12 17:24:00', 'completed', 326073, 148688), +(1466, 9, TIMESTAMP '2025-04-01 06:00:00', 'completed', 175577, 5713), +(1467, 16, TIMESTAMP '2025-09-16 10:14:00', 'pending', 131707, 0), +(1468, 30, TIMESTAMP '2025-10-25 19:06:00', 'pending', 48663, 0), +(1469, 47, TIMESTAMP '2025-04-02 01:04:00', 'refunded', 103363, 0), +(1470, 14, TIMESTAMP '2025-07-02 07:35:00', 'completed', 126765, 632), +(1471, 6, TIMESTAMP '2025-07-20 15:03:00', 'completed', 214248, 10995), +(1472, 29, TIMESTAMP '2025-02-27 15:46:00', 'completed', 71774, 35096), +(1473, 56, TIMESTAMP '2025-07-26 05:54:00', 'completed', 208780, 8543), +(1474, 34, TIMESTAMP '2025-06-01 02:53:00', 'completed', 239293, 14233), +(1475, 64, TIMESTAMP '2025-01-31 05:21:00', 'completed', 36207, 5056), +(1476, 34, TIMESTAMP '2025-11-03 17:20:00', 'completed', 177675, 11221), +(1477, 69, TIMESTAMP '2025-07-25 01:40:00', 'completed', 249606, 87786), +(1478, 62, TIMESTAMP '2025-05-03 04:07:00', 'refunded', 223457, 0), +(1479, 38, TIMESTAMP '2025-08-07 07:38:00', 'refunded', 72201, 0), +(1480, 38, TIMESTAMP '2025-10-24 00:41:00', 'pending', 93503, 0), +(1481, 24, TIMESTAMP '2025-09-11 20:44:00', 'completed', 130593, 15695), +(1482, 4, TIMESTAMP '2025-06-24 17:31:00', 'completed', 23868, 1676), +(1483, 63, TIMESTAMP '2025-07-16 22:11:00', 'completed', 85851, 4164), +(1484, 25, TIMESTAMP '2025-03-10 07:29:00', 'completed', 287591, 8747), +(1485, 56, TIMESTAMP '2025-04-13 23:42:00', 'completed', 260303, 10152), +(1486, 20, TIMESTAMP '2025-09-19 23:46:00', 'pending', 7566, 0), +(1487, 55, TIMESTAMP '2025-07-30 17:09:00', 'pending', 161783, 0), +(1488, 8, TIMESTAMP '2025-07-04 01:02:00', 'completed', 218705, 18347), +(1489, 60, TIMESTAMP '2025-01-14 12:02:00', 'refunded', 47673, 0), +(1490, 74, TIMESTAMP '2025-05-24 16:27:00', 'refunded', 1788, 0), +(1491, 41, TIMESTAMP '2025-08-27 05:13:00', 'completed', 127013, 3880), +(1492, 41, TIMESTAMP '2025-01-06 08:19:00', 'completed', 117022, 1738), +(1493, 45, TIMESTAMP '2025-11-27 19:53:00', 'completed', 396486, 41287), +(1494, 69, TIMESTAMP '2025-07-09 05:07:00', 'completed', 248553, 59805), +(1495, 37, TIMESTAMP '2025-07-30 02:46:00', 'completed', 53096, 25083), +(1496, 43, TIMESTAMP '2025-01-19 12:49:00', 'refunded', 151203, 0), +(1497, 56, TIMESTAMP '2025-01-01 19:14:00', 'completed', 219013, 8320), +(1498, 60, TIMESTAMP '2025-09-10 00:02:00', 'completed', 93811, 9643), +(1499, 8, TIMESTAMP '2025-06-25 10:50:00', 'completed', 49559, 1569); +INSERT INTO memory.bench.orders VALUES +(1500, 58, TIMESTAMP '2025-09-22 17:56:00', 'completed', 240355, 11254), +(1501, 80, TIMESTAMP '2025-04-28 03:17:00', 'completed', 81193, 1), +(1502, 6, TIMESTAMP '2025-11-26 02:46:00', 'completed', 244671, 19373), +(1503, 38, TIMESTAMP '2025-07-26 16:06:00', 'completed', 31276, 3494), +(1504, 22, TIMESTAMP '2025-11-21 20:42:00', 'completed', 158373, 2094), +(1505, 42, TIMESTAMP '2025-06-07 12:38:00', 'completed', 221637, 4046), +(1506, 76, TIMESTAMP '2025-05-31 22:21:00', 'completed', 112951, 13405), +(1507, 5, TIMESTAMP '2025-04-30 08:19:00', 'completed', 235804, 7303), +(1508, 43, TIMESTAMP '2025-01-26 12:28:00', 'pending', 217500, 0), +(1509, 38, TIMESTAMP '2025-04-02 14:19:00', 'completed', 89821, 7372), +(1510, 27, TIMESTAMP '2025-03-10 09:43:00', 'completed', 245318, 15063), +(1511, 62, TIMESTAMP '2025-10-09 13:05:00', 'completed', 124704, 12635), +(1512, 68, TIMESTAMP '2025-10-22 06:26:00', 'completed', 145974, 16776), +(1513, 50, TIMESTAMP '2025-06-19 02:41:00', 'completed', 232546, 19605), +(1514, 62, TIMESTAMP '2025-06-05 01:20:00', 'completed', 329275, 100732), +(1515, 4, TIMESTAMP '2025-10-02 12:09:00', 'completed', 93460, 4859), +(1516, 35, TIMESTAMP '2025-05-27 17:10:00', 'refunded', 150547, 0), +(1517, 48, TIMESTAMP '2025-06-26 08:08:00', 'pending', 177045, 0), +(1518, 65, TIMESTAMP '2025-10-28 23:02:00', 'refunded', 8009, 0), +(1519, 54, TIMESTAMP '2025-06-12 09:48:00', 'completed', 236595, 9424), +(1520, 14, TIMESTAMP '2025-01-22 22:02:00', 'completed', 22298, 1290), +(1521, 38, TIMESTAMP '2025-08-10 21:27:00', 'pending', 89684, 0), +(1522, 78, TIMESTAMP '2025-10-27 22:55:00', 'completed', 252955, 4775), +(1523, 36, TIMESTAMP '2025-02-25 10:20:00', 'completed', 202723, 14730), +(1524, 39, TIMESTAMP '2025-10-10 14:35:00', 'completed', 165676, 42511), +(1525, 8, TIMESTAMP '2025-03-30 05:37:00', 'completed', 210148, 28264), +(1526, 1, TIMESTAMP '2025-05-31 05:20:00', 'pending', 112169, 0), +(1527, 72, TIMESTAMP '2025-08-10 03:16:00', 'completed', 127691, 2120), +(1528, 22, TIMESTAMP '2025-01-04 09:39:00', 'completed', 207961, 24403), +(1529, 35, TIMESTAMP '2025-12-28 08:44:00', 'completed', 12947, 908), +(1530, 1, TIMESTAMP '2025-12-31 18:07:00', 'completed', 248753, 56475), +(1531, 67, TIMESTAMP '2025-10-30 13:25:00', 'completed', 219328, 17373), +(1532, 65, TIMESTAMP '2025-03-19 13:47:00', 'completed', 150475, 6033), +(1533, 4, TIMESTAMP '2025-05-18 00:10:00', 'completed', 23704, 3319), +(1534, 21, TIMESTAMP '2025-08-25 12:33:00', 'completed', 203764, 18204), +(1535, 25, TIMESTAMP '2025-12-07 12:18:00', 'completed', 188051, 14642), +(1536, 48, TIMESTAMP '2025-05-18 13:52:00', 'completed', 215796, 14455), +(1537, 30, TIMESTAMP '2025-11-19 18:31:00', 'pending', 1951, 0), +(1538, 46, TIMESTAMP '2025-10-26 03:39:00', 'completed', 222678, 22602), +(1539, 47, TIMESTAMP '2025-05-19 02:14:00', 'refunded', 320969, 0), +(1540, 16, TIMESTAMP '2025-02-12 14:12:00', 'refunded', 37509, 0), +(1541, 9, TIMESTAMP '2025-11-23 15:20:00', 'completed', 85186, 9482), +(1542, 51, TIMESTAMP '2025-02-08 18:36:00', 'completed', 243157, 5284), +(1543, 34, TIMESTAMP '2025-11-05 19:51:00', 'completed', 163138, 3121), +(1544, 43, TIMESTAMP '2025-12-23 13:49:00', 'refunded', 292000, 0), +(1545, 28, TIMESTAMP '2025-08-24 10:08:00', 'completed', 231371, 21430), +(1546, 38, TIMESTAMP '2025-05-15 07:41:00', 'completed', 123336, 4570), +(1547, 69, TIMESTAMP '2025-01-02 00:03:00', 'completed', 213764, 48283), +(1548, 69, TIMESTAMP '2025-09-06 13:04:00', 'refunded', 284284, 0), +(1549, 61, TIMESTAMP '2025-12-22 01:35:00', 'completed', 53907, 3005), +(1550, 69, TIMESTAMP '2025-05-20 22:24:00', 'pending', 114585, 0), +(1551, 5, TIMESTAMP '2025-01-12 13:27:00', 'completed', 9046, 341), +(1552, 41, TIMESTAMP '2025-09-14 18:59:00', 'completed', 256925, 12089), +(1553, 10, TIMESTAMP '2025-02-26 00:39:00', 'pending', 189261, 0), +(1554, 31, TIMESTAMP '2025-10-17 15:57:00', 'refunded', 296032, 0), +(1555, 28, TIMESTAMP '2025-06-04 20:19:00', 'completed', 224772, 24412), +(1556, 50, TIMESTAMP '2025-08-02 13:38:00', 'pending', 83675, 0), +(1557, 14, TIMESTAMP '2025-06-10 12:23:00', 'completed', 18846, 117), +(1558, 76, TIMESTAMP '2025-04-01 01:24:00', 'completed', 231769, 28811), +(1559, 7, TIMESTAMP '2025-02-19 14:20:00', 'completed', 578, 58), +(1560, 17, TIMESTAMP '2025-10-15 20:49:00', 'completed', 64031, 6441), +(1561, 45, TIMESTAMP '2025-03-10 00:57:00', 'refunded', 155078, 0), +(1562, 50, TIMESTAMP '2025-09-20 01:53:00', 'completed', 138314, 7861), +(1563, 63, TIMESTAMP '2025-05-08 06:38:00', 'completed', 120260, 7513), +(1564, 33, TIMESTAMP '2025-04-19 23:12:00', 'completed', 1450, 108), +(1565, 8, TIMESTAMP '2025-08-31 20:33:00', 'completed', 19716, 1896), +(1566, 72, TIMESTAMP '2025-09-10 17:33:00', 'completed', 102916, 6971), +(1567, 42, TIMESTAMP '2025-06-11 06:27:00', 'refunded', 236627, 0), +(1568, 36, TIMESTAMP '2025-02-03 18:24:00', 'completed', 179443, 22097), +(1569, 54, TIMESTAMP '2025-07-21 03:44:00', 'completed', 29741, 1629), +(1570, 69, TIMESTAMP '2025-03-25 13:24:00', 'completed', 126862, 11680), +(1571, 15, TIMESTAMP '2025-08-03 22:40:00', 'completed', 8254, 400), +(1572, 63, TIMESTAMP '2025-09-16 19:15:00', 'refunded', 130898, 0), +(1573, 10, TIMESTAMP '2025-07-23 22:02:00', 'completed', 187462, 14615), +(1574, 10, TIMESTAMP '2025-04-02 18:47:00', 'completed', 125818, 3633), +(1575, 39, TIMESTAMP '2025-03-06 21:48:00', 'completed', 26697, 11019), +(1576, 68, TIMESTAMP '2025-05-03 01:10:00', 'completed', 195822, 79716), +(1577, 45, TIMESTAMP '2025-04-12 21:32:00', 'refunded', 318052, 0), +(1578, 80, TIMESTAMP '2025-08-17 16:21:00', 'completed', 301953, 12735), +(1579, 34, TIMESTAMP '2025-12-04 22:40:00', 'completed', 279166, 16885), +(1580, 53, TIMESTAMP '2025-06-07 19:55:00', 'completed', 119592, 1662), +(1581, 42, TIMESTAMP '2025-06-02 08:27:00', 'completed', 63166, 6054), +(1582, 58, TIMESTAMP '2025-10-24 14:40:00', 'completed', 100797, 7868), +(1583, 73, TIMESTAMP '2025-11-17 13:30:00', 'completed', 95202, 2516), +(1584, 80, TIMESTAMP '2025-05-17 23:40:00', 'completed', 85735, 4059), +(1585, 9, TIMESTAMP '2025-02-18 14:04:00', 'completed', 241699, 2273), +(1586, 45, TIMESTAMP '2025-08-05 01:52:00', 'completed', 43505, 13154), +(1587, 71, TIMESTAMP '2025-04-20 04:38:00', 'completed', 325848, 117526), +(1588, 42, TIMESTAMP '2025-06-05 05:06:00', 'completed', 59005, 4111), +(1589, 20, TIMESTAMP '2025-10-07 12:21:00', 'completed', 129308, 3342), +(1590, 46, TIMESTAMP '2025-06-17 07:04:00', 'completed', 222841, 31058), +(1591, 55, TIMESTAMP '2025-11-09 22:01:00', 'completed', 208023, 2222), +(1592, 55, TIMESTAMP '2025-07-14 11:07:00', 'completed', 146472, 487), +(1593, 55, TIMESTAMP '2025-01-31 17:18:00', 'completed', 200163, 14280), +(1594, 35, TIMESTAMP '2025-03-22 20:30:00', 'refunded', 100784, 0), +(1595, 38, TIMESTAMP '2025-04-29 12:42:00', 'pending', 206499, 0), +(1596, 47, TIMESTAMP '2025-04-25 04:53:00', 'completed', 239068, 26496), +(1597, 39, TIMESTAMP '2025-01-19 10:50:00', 'completed', 257441, 71223), +(1598, 16, TIMESTAMP '2025-12-28 21:52:00', 'completed', 131691, 4722), +(1599, 26, TIMESTAMP '2025-03-01 03:21:00', 'completed', 168071, 11084), +(1600, 20, TIMESTAMP '2025-10-29 03:13:00', 'completed', 199807, 12706), +(1601, 29, TIMESTAMP '2025-12-06 11:37:00', 'completed', 295574, 82964), +(1602, 17, TIMESTAMP '2025-08-18 08:56:00', 'completed', 213253, 30631), +(1603, 79, TIMESTAMP '2025-01-11 20:53:00', 'completed', 18455, 2393), +(1604, 24, TIMESTAMP '2025-10-28 02:38:00', 'completed', 202932, 3255), +(1605, 59, TIMESTAMP '2025-04-01 21:46:00', 'completed', 15004, 2281), +(1606, 51, TIMESTAMP '2025-02-09 18:39:00', 'pending', 232624, 0), +(1607, 7, TIMESTAMP '2025-01-03 09:29:00', 'completed', 122192, 1049), +(1608, 42, TIMESTAMP '2025-03-11 17:16:00', 'completed', 141801, 13738), +(1609, 3, TIMESTAMP '2025-04-02 04:12:00', 'completed', 168258, 7087), +(1610, 32, TIMESTAMP '2025-10-24 02:38:00', 'pending', 189920, 0), +(1611, 31, TIMESTAMP '2025-01-05 04:06:00', 'completed', 150112, 8182), +(1612, 18, TIMESTAMP '2025-01-26 15:03:00', 'completed', 24120, 6594), +(1613, 71, TIMESTAMP '2025-04-23 22:50:00', 'completed', 87454, 33688), +(1614, 64, TIMESTAMP '2025-10-28 08:59:00', 'completed', 121839, 5326), +(1615, 75, TIMESTAMP '2025-02-17 14:11:00', 'refunded', 219118, 0), +(1616, 72, TIMESTAMP '2025-06-19 08:56:00', 'completed', 201016, 27990), +(1617, 50, TIMESTAMP '2025-11-21 19:40:00', 'completed', 192437, 15851), +(1618, 56, TIMESTAMP '2025-08-04 06:56:00', 'completed', 84942, 3007), +(1619, 78, TIMESTAMP '2025-09-02 10:48:00', 'completed', 264991, 19075), +(1620, 11, TIMESTAMP '2025-03-07 12:57:00', 'completed', 153371, 11419), +(1621, 17, TIMESTAMP '2025-08-18 07:57:00', 'completed', 168355, 7237), +(1622, 13, TIMESTAMP '2025-11-09 18:37:00', 'completed', 119658, 13622), +(1623, 15, TIMESTAMP '2025-09-12 09:27:00', 'completed', 236668, 34569), +(1624, 70, TIMESTAMP '2025-01-02 04:07:00', 'completed', 304228, 11612), +(1625, 57, TIMESTAMP '2025-01-15 12:46:00', 'pending', 363468, 0), +(1626, 58, TIMESTAMP '2025-04-06 11:41:00', 'completed', 11262, 789), +(1627, 57, TIMESTAMP '2025-03-18 15:21:00', 'refunded', 46641, 0), +(1628, 3, TIMESTAMP '2025-02-05 07:24:00', 'completed', 127117, 13684), +(1629, 56, TIMESTAMP '2025-11-15 20:36:00', 'completed', 34800, 520), +(1630, 79, TIMESTAMP '2025-05-20 10:31:00', 'completed', 223359, 11208), +(1631, 67, TIMESTAMP '2025-04-11 21:36:00', 'refunded', 115573, 0), +(1632, 54, TIMESTAMP '2025-10-02 07:01:00', 'completed', 237706, 10091), +(1633, 54, TIMESTAMP '2025-04-03 19:23:00', 'completed', 225813, 10270), +(1634, 17, TIMESTAMP '2025-11-30 15:40:00', 'completed', 201555, 17479), +(1635, 61, TIMESTAMP '2025-12-02 19:15:00', 'completed', 306942, 14028), +(1636, 10, TIMESTAMP '2025-10-27 20:27:00', 'completed', 264077, 16423), +(1637, 12, TIMESTAMP '2025-01-21 10:21:00', 'completed', 160292, 5747), +(1638, 7, TIMESTAMP '2025-11-16 22:27:00', 'completed', 55904, 7206), +(1639, 65, TIMESTAMP '2025-09-27 14:30:00', 'refunded', 399883, 0), +(1640, 50, TIMESTAMP '2025-07-18 20:06:00', 'completed', 82725, 11929), +(1641, 76, TIMESTAMP '2025-02-15 20:18:00', 'completed', 244437, 33981), +(1642, 47, TIMESTAMP '2025-02-19 07:01:00', 'refunded', 384566, 0), +(1643, 14, TIMESTAMP '2025-03-10 05:14:00', 'completed', 29293, 2105), +(1644, 13, TIMESTAMP '2025-06-26 00:23:00', 'completed', 153028, 269), +(1645, 26, TIMESTAMP '2025-11-27 14:39:00', 'completed', 282238, 12716), +(1646, 9, TIMESTAMP '2025-09-08 06:17:00', 'pending', 166985, 0), +(1647, 2, TIMESTAMP '2025-02-02 06:51:00', 'completed', 310588, 49200), +(1648, 71, TIMESTAMP '2025-12-28 23:45:00', 'refunded', 180209, 0), +(1649, 26, TIMESTAMP '2025-10-08 20:51:00', 'completed', 136102, 1179), +(1650, 3, TIMESTAMP '2025-09-12 22:07:00', 'completed', 224610, 26413), +(1651, 59, TIMESTAMP '2025-10-30 09:46:00', 'completed', 132732, 47630), +(1652, 14, TIMESTAMP '2025-08-30 03:04:00', 'completed', 92086, 5248), +(1653, 22, TIMESTAMP '2025-10-27 01:23:00', 'completed', 145181, 17097), +(1654, 54, TIMESTAMP '2025-09-29 15:31:00', 'completed', 171170, 12441), +(1655, 18, TIMESTAMP '2025-08-15 14:18:00', 'refunded', 218182, 0), +(1656, 57, TIMESTAMP '2025-02-10 20:42:00', 'completed', 241432, 5476), +(1657, 75, TIMESTAMP '2025-07-23 16:52:00', 'refunded', 218779, 0), +(1658, 58, TIMESTAMP '2025-11-21 19:54:00', 'completed', 83328, 3159), +(1659, 65, TIMESTAMP '2025-12-05 18:20:00', 'completed', 218793, 30389), +(1660, 16, TIMESTAMP '2025-06-11 21:45:00', 'completed', 106525, 9178), +(1661, 9, TIMESTAMP '2025-12-29 16:55:00', 'completed', 170284, 330), +(1662, 41, TIMESTAMP '2025-01-23 18:33:00', 'completed', 119495, 1784), +(1663, 40, TIMESTAMP '2025-12-04 11:16:00', 'completed', 15760, 617), +(1664, 25, TIMESTAMP '2025-06-02 21:11:00', 'completed', 155172, 2034), +(1665, 49, TIMESTAMP '2025-09-17 14:01:00', 'completed', 160061, 20261), +(1666, 66, TIMESTAMP '2025-02-04 19:03:00', 'completed', 185810, 1434), +(1667, 74, TIMESTAMP '2025-01-14 11:18:00', 'completed', 335680, 67957), +(1668, 55, TIMESTAMP '2025-06-21 21:44:00', 'completed', 57905, 2775), +(1669, 2, TIMESTAMP '2025-01-14 06:09:00', 'refunded', 253790, 0), +(1670, 49, TIMESTAMP '2025-01-26 12:25:00', 'completed', 214378, 10249), +(1671, 62, TIMESTAMP '2025-12-30 19:22:00', 'completed', 75080, 24419), +(1672, 39, TIMESTAMP '2025-12-10 10:13:00', 'refunded', 209436, 0), +(1673, 55, TIMESTAMP '2025-06-19 03:59:00', 'pending', 46126, 0), +(1674, 34, TIMESTAMP '2025-09-09 02:59:00', 'completed', 268878, 7731), +(1675, 4, TIMESTAMP '2025-10-03 05:49:00', 'completed', 39658, 4882), +(1676, 41, TIMESTAMP '2025-04-07 22:36:00', 'completed', 78861, 5665), +(1677, 76, TIMESTAMP '2025-09-22 18:02:00', 'completed', 33962, 3711), +(1678, 56, TIMESTAMP '2025-10-23 23:32:00', 'completed', 9408, 28), +(1679, 33, TIMESTAMP '2025-03-27 17:08:00', 'completed', 108079, 13942), +(1680, 45, TIMESTAMP '2025-09-23 14:26:00', 'completed', 357592, 54815), +(1681, 43, TIMESTAMP '2025-11-02 20:47:00', 'refunded', 374033, 0), +(1682, 9, TIMESTAMP '2025-09-14 21:32:00', 'completed', 223491, 10660), +(1683, 75, TIMESTAMP '2025-08-22 08:09:00', 'completed', 362329, 121235), +(1684, 7, TIMESTAMP '2025-09-06 14:37:00', 'pending', 207047, 0), +(1685, 41, TIMESTAMP '2025-06-16 19:37:00', 'completed', 158993, 4984), +(1686, 73, TIMESTAMP '2025-08-30 17:26:00', 'completed', 114213, 7836), +(1687, 76, TIMESTAMP '2025-03-26 16:40:00', 'completed', 33136, 4463), +(1688, 13, TIMESTAMP '2025-12-02 05:07:00', 'completed', 213591, 30001), +(1689, 40, TIMESTAMP '2025-02-22 19:18:00', 'completed', 104529, 15401), +(1690, 4, TIMESTAMP '2025-04-20 16:56:00', 'completed', 221725, 15210), +(1691, 5, TIMESTAMP '2025-06-27 14:11:00', 'completed', 59315, 3897), +(1692, 32, TIMESTAMP '2025-08-12 12:49:00', 'refunded', 106390, 0), +(1693, 4, TIMESTAMP '2025-10-29 06:55:00', 'completed', 160742, 5989), +(1694, 16, TIMESTAMP '2025-12-28 15:15:00', 'completed', 146706, 12492), +(1695, 9, TIMESTAMP '2025-06-01 12:07:00', 'completed', 88941, 9707), +(1696, 75, TIMESTAMP '2025-03-12 01:46:00', 'refunded', 310124, 0), +(1697, 17, TIMESTAMP '2025-02-18 18:47:00', 'completed', 90479, 3863), +(1698, 58, TIMESTAMP '2025-06-15 05:48:00', 'completed', 110233, 6968), +(1699, 17, TIMESTAMP '2025-07-10 16:22:00', 'completed', 186996, 17510), +(1700, 32, TIMESTAMP '2025-06-10 00:02:00', 'refunded', 293744, 0), +(1701, 51, TIMESTAMP '2025-11-19 20:04:00', 'completed', 132066, 16586), +(1702, 14, TIMESTAMP '2025-06-19 13:41:00', 'completed', 122725, 3704), +(1703, 5, TIMESTAMP '2025-10-26 08:33:00', 'completed', 115768, 16749), +(1704, 2, TIMESTAMP '2025-03-13 11:03:00', 'completed', 105699, 1357), +(1705, 17, TIMESTAMP '2025-07-20 15:46:00', 'completed', 97139, 6804), +(1706, 56, TIMESTAMP '2025-10-13 19:31:00', 'completed', 148077, 7404), +(1707, 18, TIMESTAMP '2025-06-19 13:47:00', 'completed', 53987, 20251), +(1708, 54, TIMESTAMP '2025-08-25 05:01:00', 'completed', 167231, 6862), +(1709, 48, TIMESTAMP '2025-09-12 08:41:00', 'completed', 79592, 9604), +(1710, 58, TIMESTAMP '2025-03-20 09:19:00', 'completed', 206372, 14681), +(1711, 5, TIMESTAMP '2025-02-10 01:01:00', 'completed', 125620, 5006), +(1712, 31, TIMESTAMP '2025-02-27 21:08:00', 'pending', 88696, 0), +(1713, 25, TIMESTAMP '2025-09-05 06:36:00', 'completed', 43501, 3086), +(1714, 45, TIMESTAMP '2025-05-09 16:40:00', 'completed', 257422, 82100), +(1715, 64, TIMESTAMP '2025-09-04 12:50:00', 'completed', 144301, 9441), +(1716, 22, TIMESTAMP '2025-04-19 22:25:00', 'completed', 231913, 24733), +(1717, 4, TIMESTAMP '2025-10-29 23:08:00', 'completed', 241517, 34157), +(1718, 71, TIMESTAMP '2025-01-15 14:11:00', 'refunded', 382281, 0), +(1719, 75, TIMESTAMP '2025-06-28 08:37:00', 'refunded', 241888, 0), +(1720, 22, TIMESTAMP '2025-10-29 14:11:00', 'completed', 153328, 17405), +(1721, 50, TIMESTAMP '2025-03-10 04:43:00', 'completed', 175260, 18993), +(1722, 41, TIMESTAMP '2025-01-15 05:59:00', 'completed', 93616, 2092), +(1723, 30, TIMESTAMP '2025-08-03 17:24:00', 'completed', 133821, 19095), +(1724, 77, TIMESTAMP '2025-01-08 05:58:00', 'completed', 264975, 17608), +(1725, 11, TIMESTAMP '2025-02-16 03:23:00', 'completed', 97091, 655), +(1726, 12, TIMESTAMP '2025-12-28 02:03:00', 'completed', 168016, 11808), +(1727, 44, TIMESTAMP '2025-02-13 16:55:00', 'completed', 14796, 515), +(1728, 64, TIMESTAMP '2025-01-19 12:07:00', 'completed', 181650, 8761), +(1729, 47, TIMESTAMP '2025-07-13 08:38:00', 'completed', 15424, 4551), +(1730, 56, TIMESTAMP '2025-06-06 06:07:00', 'completed', 123531, 4711), +(1731, 30, TIMESTAMP '2025-03-03 06:20:00', 'completed', 187062, 24338), +(1732, 33, TIMESTAMP '2025-03-10 23:23:00', 'completed', 55225, 4064), +(1733, 10, TIMESTAMP '2025-05-23 16:54:00', 'completed', 198363, 7126), +(1734, 64, TIMESTAMP '2025-04-03 22:09:00', 'completed', 241954, 612), +(1735, 64, TIMESTAMP '2025-08-17 21:37:00', 'completed', 212508, 14781), +(1736, 42, TIMESTAMP '2025-07-30 02:43:00', 'completed', 72617, 3243), +(1737, 80, TIMESTAMP '2025-04-16 12:02:00', 'completed', 273145, 2274), +(1738, 14, TIMESTAMP '2025-02-13 08:43:00', 'completed', 120712, 6370), +(1739, 62, TIMESTAMP '2025-11-03 19:20:00', 'completed', 372192, 157955), +(1740, 58, TIMESTAMP '2025-01-26 16:30:00', 'completed', 105728, 3216), +(1741, 17, TIMESTAMP '2025-03-26 03:23:00', 'completed', 109501, 8573), +(1742, 29, TIMESTAMP '2025-10-21 01:52:00', 'completed', 248592, 16673), +(1743, 46, TIMESTAMP '2025-01-19 14:39:00', 'completed', 107993, 11764), +(1744, 48, TIMESTAMP '2025-12-08 12:36:00', 'completed', 173494, 2445), +(1745, 18, TIMESTAMP '2025-06-28 00:16:00', 'refunded', 368499, 0), +(1746, 16, TIMESTAMP '2025-04-04 15:23:00', 'pending', 42820, 0), +(1747, 53, TIMESTAMP '2025-11-05 00:27:00', 'completed', 39768, 2023), +(1748, 2, TIMESTAMP '2025-10-21 14:55:00', 'completed', 121635, 37011), +(1749, 53, TIMESTAMP '2025-08-10 14:45:00', 'completed', 72581, 2115); +INSERT INTO memory.bench.orders VALUES +(1750, 31, TIMESTAMP '2025-08-13 20:11:00', 'completed', 276677, 8242), +(1751, 50, TIMESTAMP '2025-04-09 13:22:00', 'pending', 96740, 0), +(1752, 71, TIMESTAMP '2025-06-06 08:47:00', 'refunded', 304916, 0), +(1753, 4, TIMESTAMP '2025-06-10 11:03:00', 'completed', 91168, 4279), +(1754, 37, TIMESTAMP '2025-02-22 22:35:00', 'refunded', 146732, 0), +(1755, 16, TIMESTAMP '2025-07-04 06:16:00', 'refunded', 189640, 0), +(1756, 33, TIMESTAMP '2025-10-20 02:40:00', 'completed', 166114, 1555), +(1757, 43, TIMESTAMP '2025-02-20 19:50:00', 'completed', 298912, 107832), +(1758, 40, TIMESTAMP '2025-11-09 05:10:00', 'completed', 185790, 23764), +(1759, 24, TIMESTAMP '2025-10-18 14:58:00', 'refunded', 222294, 0), +(1760, 71, TIMESTAMP '2025-10-31 11:35:00', 'completed', 290716, 95266), +(1761, 61, TIMESTAMP '2025-08-03 16:18:00', 'completed', 304980, 8544), +(1762, 76, TIMESTAMP '2025-10-08 14:46:00', 'completed', 171556, 10792), +(1763, 2, TIMESTAMP '2025-07-07 19:24:00', 'completed', 344931, 129670), +(1764, 51, TIMESTAMP '2025-07-19 02:12:00', 'completed', 76657, 9590), +(1765, 19, TIMESTAMP '2025-08-06 20:42:00', 'completed', 147412, 1291), +(1766, 48, TIMESTAMP '2025-08-22 07:36:00', 'completed', 176601, 5243), +(1767, 76, TIMESTAMP '2025-09-12 00:30:00', 'completed', 208834, 18991), +(1768, 44, TIMESTAMP '2025-10-30 21:22:00', 'completed', 217833, 13575), +(1769, 75, TIMESTAMP '2025-07-16 22:13:00', 'completed', 142934, 8861), +(1770, 14, TIMESTAMP '2025-10-06 00:28:00', 'completed', 238495, 18003), +(1771, 12, TIMESTAMP '2025-12-14 20:30:00', 'completed', 134488, 3288), +(1772, 49, TIMESTAMP '2025-11-04 07:55:00', 'refunded', 214389, 0), +(1773, 39, TIMESTAMP '2025-08-26 03:26:00', 'refunded', 181368, 0), +(1774, 48, TIMESTAMP '2025-10-26 22:53:00', 'pending', 155635, 0), +(1775, 17, TIMESTAMP '2025-02-17 23:26:00', 'completed', 60661, 5820), +(1776, 55, TIMESTAMP '2025-11-17 15:24:00', 'pending', 309858, 0), +(1777, 49, TIMESTAMP '2025-07-29 04:39:00', 'completed', 193108, 13932), +(1778, 39, TIMESTAMP '2025-12-27 07:19:00', 'completed', 4593, 2071), +(1779, 30, TIMESTAMP '2025-04-02 17:56:00', 'refunded', 159413, 0), +(1780, 24, TIMESTAMP '2025-06-21 07:49:00', 'completed', 53064, 6080), +(1781, 74, TIMESTAMP '2025-12-20 04:20:00', 'refunded', 259441, 0), +(1782, 57, TIMESTAMP '2025-06-02 22:08:00', 'completed', 348289, 167586), +(1783, 38, TIMESTAMP '2025-07-26 14:43:00', 'refunded', 111833, 0), +(1784, 56, TIMESTAMP '2025-05-07 12:09:00', 'completed', 177745, 4062), +(1785, 1, TIMESTAMP '2025-03-12 06:39:00', 'completed', 53102, 23753), +(1786, 60, TIMESTAMP '2025-12-20 16:14:00', 'completed', 204819, 16185), +(1787, 16, TIMESTAMP '2025-08-14 11:40:00', 'completed', 161636, 12301), +(1788, 8, TIMESTAMP '2025-12-05 23:31:00', 'completed', 169955, 5704), +(1789, 40, TIMESTAMP '2025-01-10 09:44:00', 'completed', 240114, 22844), +(1790, 50, TIMESTAMP '2025-04-03 18:42:00', 'completed', 109779, 663), +(1791, 15, TIMESTAMP '2025-12-17 05:44:00', 'completed', 225698, 20762), +(1792, 77, TIMESTAMP '2025-03-23 23:12:00', 'completed', 58512, 1486), +(1793, 39, TIMESTAMP '2025-02-16 16:34:00', 'completed', 38912, 1871), +(1794, 47, TIMESTAMP '2025-11-03 03:21:00', 'pending', 354979, 0), +(1795, 65, TIMESTAMP '2025-02-21 09:34:00', 'refunded', 148657, 0), +(1796, 70, TIMESTAMP '2025-04-06 14:31:00', 'completed', 132116, 1031), +(1797, 67, TIMESTAMP '2025-04-24 10:15:00', 'completed', 68755, 2815), +(1798, 3, TIMESTAMP '2025-04-07 03:27:00', 'completed', 198904, 23703), +(1799, 30, TIMESTAMP '2025-10-07 12:13:00', 'completed', 86811, 11831), +(1800, 23, TIMESTAMP '2025-04-03 20:45:00', 'completed', 179448, 12057), +(1801, 26, TIMESTAMP '2025-07-09 14:42:00', 'completed', 166432, 3688), +(1802, 24, TIMESTAMP '2025-11-25 02:15:00', 'completed', 169318, 6871), +(1803, 30, TIMESTAMP '2025-07-07 14:53:00', 'refunded', 76962, 0), +(1804, 42, TIMESTAMP '2025-01-31 14:14:00', 'completed', 20903, 1822), +(1805, 47, TIMESTAMP '2025-08-19 03:56:00', 'completed', 394044, 137577), +(1806, 48, TIMESTAMP '2025-06-08 00:58:00', 'completed', 25644, 1539), +(1807, 75, TIMESTAMP '2025-03-06 11:00:00', 'completed', 313113, 35590), +(1808, 73, TIMESTAMP '2025-03-03 22:19:00', 'completed', 135567, 1277), +(1809, 72, TIMESTAMP '2025-12-31 21:24:00', 'completed', 102649, 7733), +(1810, 10, TIMESTAMP '2025-03-23 09:01:00', 'completed', 215751, 16454), +(1811, 17, TIMESTAMP '2025-08-08 18:12:00', 'completed', 12012, 1108), +(1812, 40, TIMESTAMP '2025-01-30 16:38:00', 'completed', 31011, 244), +(1813, 11, TIMESTAMP '2025-05-17 03:15:00', 'completed', 6288, 79), +(1814, 34, TIMESTAMP '2025-08-15 19:39:00', 'completed', 168808, 1637), +(1815, 3, TIMESTAMP '2025-11-16 04:13:00', 'completed', 231892, 4157), +(1816, 71, TIMESTAMP '2025-09-03 18:58:00', 'refunded', 367520, 0), +(1817, 15, TIMESTAMP '2025-03-17 23:55:00', 'completed', 94113, 1876), +(1818, 48, TIMESTAMP '2025-11-25 15:24:00', 'completed', 185033, 25903), +(1819, 41, TIMESTAMP '2025-09-16 15:16:00', 'pending', 132878, 0), +(1820, 49, TIMESTAMP '2025-02-05 00:22:00', 'completed', 193758, 15082), +(1821, 6, TIMESTAMP '2025-05-25 09:17:00', 'completed', 134996, 7482), +(1822, 27, TIMESTAMP '2025-10-15 22:29:00', 'completed', 66676, 3832), +(1823, 17, TIMESTAMP '2025-09-25 00:12:00', 'pending', 17081, 0), +(1824, 17, TIMESTAMP '2025-03-19 02:55:00', 'pending', 182972, 0), +(1825, 17, TIMESTAMP '2025-01-29 18:12:00', 'completed', 5439, 788), +(1826, 69, TIMESTAMP '2025-10-04 02:16:00', 'refunded', 306395, 0), +(1827, 73, TIMESTAMP '2025-07-19 15:30:00', 'completed', 152237, 4926), +(1828, 9, TIMESTAMP '2025-11-15 17:04:00', 'pending', 97688, 0), +(1829, 50, TIMESTAMP '2025-07-21 20:29:00', 'completed', 153631, 15691), +(1830, 17, TIMESTAMP '2025-06-09 22:38:00', 'completed', 88720, 3541), +(1831, 48, TIMESTAMP '2025-05-09 19:18:00', 'completed', 48930, 1553), +(1832, 6, TIMESTAMP '2025-01-11 22:16:00', 'completed', 158027, 2017), +(1833, 20, TIMESTAMP '2025-02-16 22:07:00', 'completed', 7506, 518), +(1834, 70, TIMESTAMP '2025-10-27 03:30:00', 'completed', 287523, 18898), +(1835, 71, TIMESTAMP '2025-09-23 01:42:00', 'refunded', 341113, 0), +(1836, 12, TIMESTAMP '2025-05-25 21:56:00', 'pending', 76310, 0), +(1837, 15, TIMESTAMP '2025-02-06 11:58:00', 'refunded', 61273, 0), +(1838, 33, TIMESTAMP '2025-09-23 00:50:00', 'completed', 129565, 15652), +(1839, 3, TIMESTAMP '2025-04-20 19:04:00', 'completed', 12767, 104), +(1840, 73, TIMESTAMP '2025-03-30 21:19:00', 'completed', 266423, 19466), +(1841, 53, TIMESTAMP '2025-08-08 04:22:00', 'refunded', 15937, 0), +(1842, 2, TIMESTAMP '2025-09-02 14:11:00', 'completed', 42648, 5731), +(1843, 2, TIMESTAMP '2025-08-25 14:23:00', 'completed', 336193, 98682), +(1844, 33, TIMESTAMP '2025-06-27 00:20:00', 'completed', 108278, 5980), +(1845, 42, TIMESTAMP '2025-04-18 09:03:00', 'completed', 130257, 1241), +(1846, 46, TIMESTAMP '2025-01-22 00:29:00', 'refunded', 3231, 0), +(1847, 59, TIMESTAMP '2025-06-18 14:18:00', 'refunded', 230036, 0), +(1848, 37, TIMESTAMP '2025-11-21 04:50:00', 'completed', 171601, 46200), +(1849, 60, TIMESTAMP '2025-05-31 01:08:00', 'pending', 85395, 0), +(1850, 13, TIMESTAMP '2025-09-19 14:05:00', 'completed', 165461, 16695), +(1851, 51, TIMESTAMP '2025-10-29 07:45:00', 'completed', 110872, 10384), +(1852, 79, TIMESTAMP '2025-07-07 00:38:00', 'refunded', 175811, 0), +(1853, 60, TIMESTAMP '2025-07-01 11:59:00', 'completed', 209213, 13722), +(1854, 37, TIMESTAMP '2025-07-14 05:01:00', 'refunded', 334374, 0), +(1855, 74, TIMESTAMP '2025-03-05 00:17:00', 'completed', 390089, 122753), +(1856, 68, TIMESTAMP '2025-04-25 15:35:00', 'refunded', 215532, 0), +(1857, 28, TIMESTAMP '2025-07-09 18:44:00', 'completed', 58804, 4399), +(1858, 30, TIMESTAMP '2025-09-07 02:53:00', 'completed', 204736, 9725), +(1859, 55, TIMESTAMP '2025-09-08 21:03:00', 'completed', 41326, 477), +(1860, 9, TIMESTAMP '2025-12-20 06:47:00', 'pending', 186896, 0), +(1861, 19, TIMESTAMP '2025-03-02 06:51:00', 'pending', 174438, 0), +(1862, 21, TIMESTAMP '2025-09-20 00:48:00', 'completed', 23488, 80), +(1863, 8, TIMESTAMP '2025-02-27 13:26:00', 'completed', 96036, 13977), +(1864, 62, TIMESTAMP '2025-02-26 15:57:00', 'refunded', 125830, 0), +(1865, 59, TIMESTAMP '2025-03-24 07:35:00', 'refunded', 397160, 0), +(1866, 9, TIMESTAMP '2025-04-21 20:39:00', 'completed', 139159, 16563), +(1867, 18, TIMESTAMP '2025-12-19 09:55:00', 'refunded', 222492, 0), +(1868, 43, TIMESTAMP '2025-02-18 10:09:00', 'completed', 47932, 16524), +(1869, 19, TIMESTAMP '2025-06-05 14:06:00', 'completed', 72503, 4562), +(1870, 30, TIMESTAMP '2025-03-26 17:23:00', 'completed', 192533, 13957), +(1871, 52, TIMESTAMP '2025-01-21 09:15:00', 'completed', 46343, 2382), +(1872, 29, TIMESTAMP '2025-07-23 03:02:00', 'refunded', 163931, 0), +(1873, 66, TIMESTAMP '2025-06-30 04:33:00', 'completed', 40275, 4191), +(1874, 39, TIMESTAMP '2025-06-15 14:49:00', 'refunded', 83278, 0), +(1875, 62, TIMESTAMP '2025-12-18 00:03:00', 'refunded', 10728, 0), +(1876, 63, TIMESTAMP '2025-10-18 09:15:00', 'completed', 145259, 17275), +(1877, 55, TIMESTAMP '2025-10-31 11:40:00', 'completed', 308326, 1552), +(1878, 61, TIMESTAMP '2025-10-17 20:58:00', 'completed', 120331, 6453), +(1879, 6, TIMESTAMP '2025-09-06 09:27:00', 'completed', 113776, 6946), +(1880, 50, TIMESTAMP '2025-01-22 09:00:00', 'pending', 116506, 0), +(1881, 33, TIMESTAMP '2025-05-02 03:34:00', 'completed', 220696, 24033), +(1882, 25, TIMESTAMP '2025-05-25 09:07:00', 'completed', 32205, 1257), +(1883, 53, TIMESTAMP '2025-04-29 13:51:00', 'completed', 237033, 17720), +(1884, 52, TIMESTAMP '2025-01-07 04:03:00', 'completed', 113661, 8342), +(1885, 72, TIMESTAMP '2025-08-28 08:35:00', 'pending', 219275, 0), +(1886, 72, TIMESTAMP '2025-09-08 11:21:00', 'pending', 34260, 0), +(1887, 1, TIMESTAMP '2025-12-18 09:59:00', 'completed', 393329, 38128), +(1888, 36, TIMESTAMP '2025-06-16 08:38:00', 'completed', 143660, 3816), +(1889, 11, TIMESTAMP '2025-05-02 14:20:00', 'completed', 42662, 1498), +(1890, 28, TIMESTAMP '2025-09-13 06:40:00', 'completed', 163713, 9082), +(1891, 32, TIMESTAMP '2025-04-28 11:32:00', 'completed', 44270, 6419), +(1892, 49, TIMESTAMP '2025-10-07 01:59:00', 'completed', 101969, 9691), +(1893, 17, TIMESTAMP '2025-03-23 18:49:00', 'completed', 141241, 10144), +(1894, 20, TIMESTAMP '2025-09-01 08:05:00', 'refunded', 64545, 0), +(1895, 29, TIMESTAMP '2025-10-27 23:17:00', 'refunded', 287825, 0), +(1896, 51, TIMESTAMP '2025-01-26 12:34:00', 'completed', 77920, 690), +(1897, 37, TIMESTAMP '2025-06-18 01:30:00', 'completed', 192708, 73616), +(1898, 14, TIMESTAMP '2025-07-08 02:36:00', 'completed', 141168, 8452), +(1899, 37, TIMESTAMP '2025-03-25 08:42:00', 'completed', 22139, 4030), +(1900, 5, TIMESTAMP '2025-10-22 18:36:00', 'completed', 119485, 14040), +(1901, 53, TIMESTAMP '2025-04-29 16:08:00', 'completed', 72211, 8474), +(1902, 6, TIMESTAMP '2025-12-14 01:13:00', 'completed', 106447, 5714), +(1903, 36, TIMESTAMP '2025-09-22 18:02:00', 'completed', 138339, 1582), +(1904, 63, TIMESTAMP '2025-06-27 17:29:00', 'completed', 20297, 1456), +(1905, 79, TIMESTAMP '2025-01-01 21:28:00', 'completed', 221135, 18871), +(1906, 15, TIMESTAMP '2025-03-03 18:03:00', 'completed', 17590, 234), +(1907, 68, TIMESTAMP '2025-01-20 13:28:00', 'completed', 365577, 89062), +(1908, 78, TIMESTAMP '2025-06-30 04:58:00', 'pending', 50270, 0), +(1909, 4, TIMESTAMP '2025-01-20 12:37:00', 'pending', 183133, 0), +(1910, 23, TIMESTAMP '2025-07-08 20:19:00', 'completed', 107198, 1660), +(1911, 16, TIMESTAMP '2025-08-19 08:38:00', 'refunded', 142922, 0), +(1912, 40, TIMESTAMP '2025-06-25 21:54:00', 'completed', 83411, 10653), +(1913, 75, TIMESTAMP '2025-05-19 14:57:00', 'completed', 164363, 18236), +(1914, 76, TIMESTAMP '2025-01-08 19:21:00', 'completed', 59508, 997), +(1915, 75, TIMESTAMP '2025-06-26 23:25:00', 'completed', 232192, 6959), +(1916, 59, TIMESTAMP '2025-02-27 15:57:00', 'completed', 200648, 28103), +(1917, 53, TIMESTAMP '2025-02-08 00:00:00', 'completed', 28637, 1077), +(1918, 14, TIMESTAMP '2025-04-13 00:57:00', 'completed', 157427, 4871), +(1919, 29, TIMESTAMP '2025-04-01 11:26:00', 'refunded', 304982, 0), +(1920, 65, TIMESTAMP '2025-04-15 16:36:00', 'refunded', 262881, 0), +(1921, 59, TIMESTAMP '2025-05-20 12:38:00', 'refunded', 188868, 0), +(1922, 53, TIMESTAMP '2025-11-20 02:53:00', 'completed', 65317, 8580), +(1923, 24, TIMESTAMP '2025-04-13 21:16:00', 'completed', 83532, 5223), +(1924, 52, TIMESTAMP '2025-04-10 16:31:00', 'completed', 23032, 251), +(1925, 72, TIMESTAMP '2025-12-19 16:48:00', 'completed', 107247, 9003), +(1926, 2, TIMESTAMP '2025-10-09 09:19:00', 'completed', 25112, 11045), +(1927, 72, TIMESTAMP '2025-03-08 22:42:00', 'pending', 59646, 0), +(1928, 12, TIMESTAMP '2025-11-05 17:09:00', 'completed', 188102, 11604), +(1929, 7, TIMESTAMP '2025-06-13 00:02:00', 'completed', 114565, 3498), +(1930, 51, TIMESTAMP '2025-02-20 09:35:00', 'completed', 43432, 5895), +(1931, 62, TIMESTAMP '2025-03-31 00:32:00', 'completed', 187148, 83282), +(1932, 52, TIMESTAMP '2025-11-10 17:07:00', 'completed', 189690, 1126), +(1933, 34, TIMESTAMP '2025-07-07 22:33:00', 'completed', 46903, 158), +(1934, 34, TIMESTAMP '2025-09-23 22:44:00', 'completed', 27918, 1471), +(1935, 74, TIMESTAMP '2025-05-20 19:14:00', 'completed', 210600, 62053), +(1936, 38, TIMESTAMP '2025-10-22 20:34:00', 'completed', 9203, 1000), +(1937, 37, TIMESTAMP '2025-05-22 18:21:00', 'completed', 199100, 12575), +(1938, 56, TIMESTAMP '2025-08-14 16:17:00', 'completed', 228662, 12892), +(1939, 65, TIMESTAMP '2025-11-19 17:05:00', 'completed', 275171, 41983), +(1940, 74, TIMESTAMP '2025-03-25 02:03:00', 'completed', 342193, 16647), +(1941, 14, TIMESTAMP '2025-06-12 19:02:00', 'completed', 151692, 5249), +(1942, 8, TIMESTAMP '2025-05-02 16:08:00', 'completed', 60837, 6008), +(1943, 57, TIMESTAMP '2025-10-29 01:57:00', 'completed', 261779, 59366), +(1944, 43, TIMESTAMP '2025-03-18 22:10:00', 'refunded', 355350, 0), +(1945, 31, TIMESTAMP '2025-09-10 16:57:00', 'pending', 129751, 0), +(1946, 44, TIMESTAMP '2025-01-13 09:15:00', 'completed', 116920, 6443), +(1947, 28, TIMESTAMP '2025-05-31 16:23:00', 'completed', 115797, 4467), +(1948, 45, TIMESTAMP '2025-10-07 09:41:00', 'refunded', 302401, 0), +(1949, 59, TIMESTAMP '2025-12-18 15:45:00', 'refunded', 64060, 0), +(1950, 78, TIMESTAMP '2025-12-26 12:52:00', 'completed', 158781, 4427), +(1951, 68, TIMESTAMP '2025-02-21 20:18:00', 'refunded', 116811, 0), +(1952, 27, TIMESTAMP '2025-07-21 23:30:00', 'completed', 117724, 6317), +(1953, 67, TIMESTAMP '2025-12-03 23:30:00', 'completed', 223245, 6412), +(1954, 48, TIMESTAMP '2025-05-03 06:38:00', 'completed', 84348, 5635), +(1955, 71, TIMESTAMP '2025-10-15 18:07:00', 'completed', 343604, 136896), +(1956, 13, TIMESTAMP '2025-12-25 17:55:00', 'pending', 118381, 0), +(1957, 72, TIMESTAMP '2025-06-26 09:55:00', 'refunded', 122019, 0), +(1958, 39, TIMESTAMP '2025-02-14 02:47:00', 'completed', 37187, 2554), +(1959, 5, TIMESTAMP '2025-03-09 21:51:00', 'completed', 12229, 1748), +(1960, 17, TIMESTAMP '2025-02-15 09:32:00', 'completed', 66856, 302), +(1961, 10, TIMESTAMP '2025-07-29 17:35:00', 'completed', 63938, 4966), +(1962, 73, TIMESTAMP '2025-12-29 11:43:00', 'completed', 262425, 6965), +(1963, 68, TIMESTAMP '2025-02-04 11:41:00', 'refunded', 143176, 0), +(1964, 45, TIMESTAMP '2025-04-25 06:50:00', 'refunded', 159094, 0), +(1965, 30, TIMESTAMP '2025-08-12 19:23:00', 'completed', 33908, 3693), +(1966, 70, TIMESTAMP '2025-09-30 08:39:00', 'completed', 18918, 1393), +(1967, 25, TIMESTAMP '2025-06-04 07:08:00', 'completed', 207172, 137), +(1968, 37, TIMESTAMP '2025-11-17 16:27:00', 'completed', 247840, 25364), +(1969, 54, TIMESTAMP '2025-11-23 08:53:00', 'completed', 239843, 17127), +(1970, 11, TIMESTAMP '2025-08-15 10:14:00', 'completed', 311710, 17038), +(1971, 72, TIMESTAMP '2025-03-24 02:00:00', 'completed', 235467, 16973), +(1972, 56, TIMESTAMP '2025-05-28 07:26:00', 'completed', 89347, 1474), +(1973, 58, TIMESTAMP '2025-07-12 05:41:00', 'completed', 258748, 12125), +(1974, 68, TIMESTAMP '2025-07-25 01:12:00', 'completed', 193675, 22642), +(1975, 30, TIMESTAMP '2025-11-21 00:29:00', 'completed', 13692, 1444), +(1976, 47, TIMESTAMP '2025-10-21 15:41:00', 'refunded', 80702, 0), +(1977, 50, TIMESTAMP '2025-12-03 15:36:00', 'completed', 175383, 6997), +(1978, 24, TIMESTAMP '2025-02-06 08:17:00', 'completed', 33505, 985), +(1979, 29, TIMESTAMP '2025-12-28 22:47:00', 'refunded', 168264, 0), +(1980, 42, TIMESTAMP '2025-10-17 21:54:00', 'completed', 131763, 10214), +(1981, 31, TIMESTAMP '2025-12-30 02:22:00', 'completed', 237490, 10902), +(1982, 61, TIMESTAMP '2025-05-12 06:33:00', 'completed', 36353, 2413), +(1983, 41, TIMESTAMP '2025-08-18 15:46:00', 'completed', 118446, 1701), +(1984, 61, TIMESTAMP '2025-07-27 10:36:00', 'completed', 10302, 20), +(1985, 76, TIMESTAMP '2025-12-18 00:23:00', 'completed', 196784, 14465), +(1986, 18, TIMESTAMP '2025-08-27 10:17:00', 'refunded', 43516, 0), +(1987, 44, TIMESTAMP '2025-09-03 17:43:00', 'completed', 60423, 1941), +(1988, 70, TIMESTAMP '2025-04-30 03:04:00', 'completed', 3082, 117), +(1989, 78, TIMESTAMP '2025-04-26 21:38:00', 'completed', 218578, 9674), +(1990, 80, TIMESTAMP '2025-09-25 00:07:00', 'completed', 9938, 655), +(1991, 14, TIMESTAMP '2025-07-04 17:11:00', 'completed', 176651, 6593), +(1992, 23, TIMESTAMP '2025-08-11 02:50:00', 'completed', 170105, 9039), +(1993, 46, TIMESTAMP '2025-04-12 16:55:00', 'refunded', 110714, 0), +(1994, 74, TIMESTAMP '2025-08-07 00:30:00', 'pending', 77396, 0), +(1995, 39, TIMESTAMP '2025-12-01 00:04:00', 'completed', 361496, 80791), +(1996, 66, TIMESTAMP '2025-03-25 03:48:00', 'completed', 152036, 13353), +(1997, 51, TIMESTAMP '2025-11-09 03:59:00', 'pending', 189666, 0), +(1998, 6, TIMESTAMP '2025-03-08 03:40:00', 'refunded', 186308, 0), +(1999, 62, TIMESTAMP '2025-10-23 09:04:00', 'completed', 49091, 17949); +INSERT INTO memory.bench.orders VALUES +(2000, 2, TIMESTAMP '2025-04-12 22:56:00', 'completed', 149225, 36691), +(2001, 56, TIMESTAMP '2025-04-05 14:53:00', 'completed', 108780, 4007), +(2002, 44, TIMESTAMP '2025-04-29 10:25:00', 'completed', 133652, 1738), +(2003, 40, TIMESTAMP '2025-08-17 08:35:00', 'completed', 63756, 2822), +(2004, 30, TIMESTAMP '2025-05-23 12:45:00', 'completed', 44348, 6332), +(2005, 10, TIMESTAMP '2025-10-01 21:02:00', 'completed', 129186, 3176), +(2006, 27, TIMESTAMP '2025-11-13 23:05:00', 'completed', 218219, 29364), +(2007, 79, TIMESTAMP '2025-04-13 19:46:00', 'pending', 243032, 0), +(2008, 37, TIMESTAMP '2025-06-22 23:53:00', 'completed', 180057, 78253), +(2009, 51, TIMESTAMP '2025-08-02 12:05:00', 'pending', 142988, 0), +(2010, 39, TIMESTAMP '2025-02-18 04:09:00', 'pending', 210270, 0), +(2011, 75, TIMESTAMP '2025-06-14 01:09:00', 'refunded', 205526, 0), +(2012, 1, TIMESTAMP '2025-02-02 15:49:00', 'completed', 136862, 20148), +(2013, 9, TIMESTAMP '2025-12-26 01:12:00', 'completed', 73215, 7810), +(2014, 72, TIMESTAMP '2025-05-25 19:37:00', 'pending', 153127, 0), +(2015, 60, TIMESTAMP '2025-02-11 11:01:00', 'completed', 150074, 16057), +(2016, 32, TIMESTAMP '2025-09-24 18:39:00', 'refunded', 54536, 0), +(2017, 5, TIMESTAMP '2025-06-04 09:21:00', 'completed', 26413, 3558), +(2018, 22, TIMESTAMP '2025-04-16 16:12:00', 'completed', 175437, 9840), +(2019, 35, TIMESTAMP '2025-11-10 06:00:00', 'refunded', 139136, 0), +(2020, 43, TIMESTAMP '2025-08-01 01:55:00', 'completed', 69816, 13276), +(2021, 48, TIMESTAMP '2025-02-15 17:45:00', 'completed', 46267, 5505), +(2022, 7, TIMESTAMP '2025-02-14 11:16:00', 'refunded', 70446, 0), +(2023, 23, TIMESTAMP '2025-06-15 22:36:00', 'completed', 97152, 5352), +(2024, 34, TIMESTAMP '2025-02-27 00:02:00', 'completed', 283405, 19239), +(2025, 15, TIMESTAMP '2025-09-20 13:27:00', 'completed', 158592, 1903), +(2026, 15, TIMESTAMP '2025-12-10 11:01:00', 'completed', 153521, 1658), +(2027, 3, TIMESTAMP '2025-07-28 08:47:00', 'pending', 207462, 0), +(2028, 67, TIMESTAMP '2025-02-22 02:35:00', 'completed', 76155, 7329), +(2029, 51, TIMESTAMP '2025-01-29 09:33:00', 'completed', 137972, 13482), +(2030, 57, TIMESTAMP '2025-03-23 16:13:00', 'completed', 364536, 179929), +(2031, 51, TIMESTAMP '2025-03-04 10:10:00', 'completed', 107113, 3265), +(2032, 65, TIMESTAMP '2025-11-01 21:00:00', 'completed', 107856, 53641), +(2033, 80, TIMESTAMP '2025-03-21 05:32:00', 'completed', 156827, 12497), +(2034, 11, TIMESTAMP '2025-03-07 05:51:00', 'completed', 186027, 5786), +(2035, 16, TIMESTAMP '2025-03-20 23:34:00', 'completed', 52074, 1175), +(2036, 37, TIMESTAMP '2025-07-29 22:19:00', 'completed', 104953, 15946), +(2037, 42, TIMESTAMP '2025-11-12 07:55:00', 'pending', 26377, 0), +(2038, 15, TIMESTAMP '2025-08-17 06:34:00', 'completed', 69311, 8928), +(2039, 32, TIMESTAMP '2025-09-05 08:07:00', 'completed', 79569, 4291), +(2040, 57, TIMESTAMP '2025-06-28 08:43:00', 'completed', 313588, 149339), +(2041, 8, TIMESTAMP '2025-01-06 00:30:00', 'completed', 65104, 7924), +(2042, 64, TIMESTAMP '2025-10-15 10:55:00', 'completed', 231844, 5362), +(2043, 74, TIMESTAMP '2025-10-18 09:34:00', 'completed', 211771, 7317), +(2044, 60, TIMESTAMP '2025-05-23 14:50:00', 'completed', 57374, 8590), +(2045, 23, TIMESTAMP '2025-12-10 23:40:00', 'completed', 133603, 7114), +(2046, 27, TIMESTAMP '2025-08-08 10:19:00', 'completed', 12833, 431), +(2047, 21, TIMESTAMP '2025-11-08 23:16:00', 'completed', 140362, 2099), +(2048, 36, TIMESTAMP '2025-08-06 15:58:00', 'completed', 19431, 2163), +(2049, 50, TIMESTAMP '2025-05-11 03:08:00', 'completed', 119484, 17865), +(2050, 77, TIMESTAMP '2025-03-05 07:31:00', 'pending', 257888, 0), +(2051, 72, TIMESTAMP '2025-07-28 03:52:00', 'completed', 26333, 3127), +(2052, 23, TIMESTAMP '2025-06-19 14:55:00', 'completed', 20981, 378), +(2053, 39, TIMESTAMP '2025-03-19 12:24:00', 'completed', 395937, 59063), +(2054, 60, TIMESTAMP '2025-11-19 04:11:00', 'completed', 195557, 27574), +(2055, 21, TIMESTAMP '2025-01-29 10:28:00', 'completed', 233568, 20923), +(2056, 8, TIMESTAMP '2025-07-23 01:23:00', 'completed', 151822, 1541), +(2057, 20, TIMESTAMP '2025-12-22 19:48:00', 'completed', 270958, 17833), +(2058, 54, TIMESTAMP '2025-08-28 15:26:00', 'completed', 193384, 3050), +(2059, 75, TIMESTAMP '2025-05-30 01:24:00', 'refunded', 342494, 0), +(2060, 74, TIMESTAMP '2025-02-21 07:08:00', 'completed', 161246, 27193), +(2061, 16, TIMESTAMP '2025-05-06 09:22:00', 'completed', 66184, 6063), +(2062, 3, TIMESTAMP '2025-02-18 16:17:00', 'completed', 197326, 26540), +(2063, 65, TIMESTAMP '2025-06-08 13:47:00', 'refunded', 363075, 0), +(2064, 60, TIMESTAMP '2025-05-31 03:50:00', 'completed', 189734, 26853), +(2065, 19, TIMESTAMP '2025-04-28 16:09:00', 'completed', 162588, 5670), +(2066, 69, TIMESTAMP '2025-12-04 13:14:00', 'completed', 226675, 83879), +(2067, 57, TIMESTAMP '2025-01-09 12:58:00', 'completed', 49348, 3691), +(2068, 16, TIMESTAMP '2025-01-26 17:51:00', 'completed', 72615, 8590), +(2069, 5, TIMESTAMP '2025-07-28 11:46:00', 'pending', 147768, 0), +(2070, 17, TIMESTAMP '2025-10-15 06:38:00', 'completed', 238646, 17390), +(2071, 57, TIMESTAMP '2025-09-14 16:58:00', 'completed', 274230, 12988), +(2072, 6, TIMESTAMP '2025-01-02 23:04:00', 'completed', 183353, 1407), +(2073, 76, TIMESTAMP '2025-07-20 02:30:00', 'completed', 100673, 1338), +(2074, 62, TIMESTAMP '2025-04-30 13:36:00', 'completed', 347248, 12148), +(2075, 58, TIMESTAMP '2025-04-11 02:58:00', 'completed', 67345, 4426), +(2076, 22, TIMESTAMP '2025-05-17 11:21:00', 'completed', 227423, 27459), +(2077, 67, TIMESTAMP '2025-07-30 18:45:00', 'completed', 212353, 3519), +(2078, 65, TIMESTAMP '2025-06-30 14:47:00', 'completed', 116166, 55769), +(2079, 18, TIMESTAMP '2025-04-02 16:01:00', 'refunded', 248070, 0), +(2080, 53, TIMESTAMP '2025-08-05 21:32:00', 'refunded', 16139, 0), +(2081, 3, TIMESTAMP '2025-03-18 12:39:00', 'completed', 226753, 16902), +(2082, 41, TIMESTAMP '2025-07-22 06:14:00', 'completed', 263432, 9926), +(2083, 36, TIMESTAMP '2025-09-26 07:26:00', 'completed', 65752, 2285), +(2084, 70, TIMESTAMP '2025-08-22 13:55:00', 'completed', 279362, 477), +(2085, 3, TIMESTAMP '2025-03-17 20:23:00', 'refunded', 66552, 0), +(2086, 43, TIMESTAMP '2025-03-20 09:28:00', 'completed', 212102, 74468), +(2087, 74, TIMESTAMP '2025-08-24 13:01:00', 'refunded', 196896, 0), +(2088, 40, TIMESTAMP '2025-02-27 18:15:00', 'completed', 119245, 120), +(2089, 55, TIMESTAMP '2025-11-21 16:14:00', 'completed', 112620, 6040), +(2090, 56, TIMESTAMP '2025-03-14 20:04:00', 'completed', 87198, 1699), +(2091, 16, TIMESTAMP '2025-04-15 02:02:00', 'completed', 203073, 22409), +(2092, 61, TIMESTAMP '2025-05-31 04:04:00', 'completed', 59772, 1901), +(2093, 53, TIMESTAMP '2025-03-07 00:40:00', 'completed', 100498, 982), +(2094, 50, TIMESTAMP '2025-02-22 01:42:00', 'completed', 220382, 11238), +(2095, 62, TIMESTAMP '2025-08-17 23:06:00', 'completed', 36156, 167), +(2096, 3, TIMESTAMP '2025-11-24 12:09:00', 'completed', 167210, 13036), +(2097, 6, TIMESTAMP '2025-08-09 00:22:00', 'completed', 309542, 10542), +(2098, 20, TIMESTAMP '2025-04-28 16:20:00', 'completed', 136903, 10699), +(2099, 66, TIMESTAMP '2025-09-20 23:01:00', 'completed', 78952, 8215), +(2100, 70, TIMESTAMP '2025-04-15 17:02:00', 'completed', 79041, 5604), +(2101, 56, TIMESTAMP '2025-12-16 17:49:00', 'completed', 77042, 5268), +(2102, 9, TIMESTAMP '2025-06-20 04:19:00', 'completed', 33787, 1456), +(2103, 73, TIMESTAMP '2025-03-05 08:43:00', 'completed', 272162, 1109), +(2104, 14, TIMESTAMP '2025-06-14 18:25:00', 'completed', 96323, 6044), +(2105, 17, TIMESTAMP '2025-01-23 02:36:00', 'completed', 20430, 1945), +(2106, 70, TIMESTAMP '2025-02-02 02:45:00', 'pending', 111675, 0), +(2107, 39, TIMESTAMP '2025-11-21 05:57:00', 'completed', 357590, 21824), +(2108, 56, TIMESTAMP '2025-06-28 02:50:00', 'completed', 173717, 9116), +(2109, 10, TIMESTAMP '2025-09-16 01:39:00', 'completed', 158222, 6368), +(2110, 44, TIMESTAMP '2025-12-05 13:04:00', 'completed', 155650, 5831), +(2111, 40, TIMESTAMP '2025-07-06 09:41:00', 'completed', 122576, 2065), +(2112, 13, TIMESTAMP '2025-02-18 17:15:00', 'completed', 10265, 158), +(2113, 48, TIMESTAMP '2025-04-26 00:06:00', 'completed', 146890, 17589), +(2114, 59, TIMESTAMP '2025-11-18 15:12:00', 'refunded', 328736, 0), +(2115, 54, TIMESTAMP '2025-10-17 02:11:00', 'refunded', 113944, 0), +(2116, 39, TIMESTAMP '2025-06-13 02:43:00', 'pending', 265187, 0), +(2117, 39, TIMESTAMP '2025-11-20 16:25:00', 'completed', 25032, 8056), +(2118, 68, TIMESTAMP '2025-04-05 08:57:00', 'refunded', 71484, 0), +(2119, 43, TIMESTAMP '2025-07-23 22:21:00', 'completed', 170996, 69723), +(2120, 70, TIMESTAMP '2025-07-05 12:40:00', 'completed', 55436, 981), +(2121, 65, TIMESTAMP '2025-03-29 03:57:00', 'completed', 200508, 8665), +(2122, 62, TIMESTAMP '2025-08-17 23:11:00', 'completed', 250851, 106726), +(2123, 32, TIMESTAMP '2025-12-26 13:02:00', 'completed', 383808, 3177), +(2124, 73, TIMESTAMP '2025-11-07 22:56:00', 'completed', 225923, 17525), +(2125, 17, TIMESTAMP '2025-08-24 06:04:00', 'completed', 139017, 13450), +(2126, 20, TIMESTAMP '2025-07-23 06:57:00', 'completed', 181240, 1791), +(2127, 32, TIMESTAMP '2025-02-18 02:27:00', 'completed', 244644, 50895), +(2128, 24, TIMESTAMP '2025-10-14 15:10:00', 'completed', 47380, 2154), +(2129, 14, TIMESTAMP '2025-02-16 16:28:00', 'completed', 6475, 259), +(2130, 50, TIMESTAMP '2025-05-30 06:40:00', 'completed', 247651, 1160), +(2131, 50, TIMESTAMP '2025-12-29 03:36:00', 'pending', 4777, 0), +(2132, 7, TIMESTAMP '2025-10-08 21:14:00', 'completed', 66320, 3714), +(2133, 13, TIMESTAMP '2025-11-16 07:40:00', 'completed', 167690, 12709), +(2134, 47, TIMESTAMP '2025-06-04 14:20:00', 'refunded', 219054, 0), +(2135, 51, TIMESTAMP '2025-01-16 03:34:00', 'completed', 138677, 13756), +(2136, 76, TIMESTAMP '2025-12-15 09:34:00', 'completed', 207290, 15683), +(2137, 32, TIMESTAMP '2025-05-18 11:47:00', 'completed', 194886, 6318), +(2138, 56, TIMESTAMP '2025-08-25 13:56:00', 'completed', 118926, 8258), +(2139, 14, TIMESTAMP '2025-06-04 13:48:00', 'completed', 164738, 116), +(2140, 70, TIMESTAMP '2025-12-11 16:35:00', 'completed', 55130, 1356), +(2141, 61, TIMESTAMP '2025-01-18 04:57:00', 'completed', 85071, 4505), +(2142, 24, TIMESTAMP '2025-09-24 17:58:00', 'refunded', 65787, 0), +(2143, 80, TIMESTAMP '2025-04-02 14:29:00', 'completed', 143202, 11372), +(2144, 55, TIMESTAMP '2025-08-07 14:59:00', 'completed', 221990, 3721), +(2145, 48, TIMESTAMP '2025-04-04 23:03:00', 'refunded', 4909, 0), +(2146, 65, TIMESTAMP '2025-10-15 09:00:00', 'completed', 39811, 4371), +(2147, 53, TIMESTAMP '2025-09-19 14:39:00', 'pending', 21002, 0), +(2148, 65, TIMESTAMP '2025-06-28 11:32:00', 'completed', 363556, 142769), +(2149, 20, TIMESTAMP '2025-05-13 02:04:00', 'completed', 215360, 3532), +(2150, 64, TIMESTAMP '2025-01-09 21:52:00', 'completed', 227297, 29902), +(2151, 38, TIMESTAMP '2025-05-29 09:54:00', 'completed', 51406, 3590), +(2152, 47, TIMESTAMP '2025-10-28 02:05:00', 'completed', 222750, 62830), +(2153, 24, TIMESTAMP '2025-10-26 00:49:00', 'completed', 229555, 19099), +(2154, 73, TIMESTAMP '2025-12-17 16:57:00', 'completed', 156580, 10156), +(2155, 46, TIMESTAMP '2025-07-07 02:12:00', 'refunded', 49857, 0), +(2156, 66, TIMESTAMP '2025-06-02 03:21:00', 'completed', 160861, 8953), +(2157, 51, TIMESTAMP '2025-04-05 07:19:00', 'completed', 160176, 894), +(2158, 49, TIMESTAMP '2025-02-26 13:08:00', 'completed', 124097, 178), +(2159, 78, TIMESTAMP '2025-04-24 01:11:00', 'completed', 121430, 4746), +(2160, 67, TIMESTAMP '2025-06-24 21:50:00', 'refunded', 56301, 0), +(2161, 25, TIMESTAMP '2025-06-07 17:35:00', 'refunded', 193377, 0), +(2162, 34, TIMESTAMP '2025-04-05 13:55:00', 'completed', 141830, 3), +(2163, 75, TIMESTAMP '2025-03-17 11:49:00', 'completed', 224616, 28382), +(2164, 78, TIMESTAMP '2025-05-29 23:00:00', 'completed', 177930, 5848), +(2165, 30, TIMESTAMP '2025-06-17 01:43:00', 'completed', 232018, 9934), +(2166, 30, TIMESTAMP '2025-01-13 05:44:00', 'completed', 71318, 3707), +(2167, 11, TIMESTAMP '2025-04-22 12:40:00', 'completed', 176276, 6638), +(2168, 62, TIMESTAMP '2025-07-07 14:39:00', 'refunded', 205528, 0), +(2169, 39, TIMESTAMP '2025-07-20 08:15:00', 'completed', 238518, 89609), +(2170, 2, TIMESTAMP '2025-03-05 08:47:00', 'refunded', 242104, 0), +(2171, 33, TIMESTAMP '2025-06-28 05:17:00', 'completed', 41542, 3020), +(2172, 12, TIMESTAMP '2025-08-14 10:23:00', 'completed', 259533, 6049), +(2173, 11, TIMESTAMP '2025-11-22 18:31:00', 'pending', 23696, 0), +(2174, 27, TIMESTAMP '2025-04-24 03:27:00', 'pending', 112481, 0), +(2175, 18, TIMESTAMP '2025-01-29 03:08:00', 'completed', 276283, 127904), +(2176, 32, TIMESTAMP '2025-09-09 18:43:00', 'refunded', 195297, 0), +(2177, 62, TIMESTAMP '2025-10-17 08:05:00', 'completed', 222816, 60521), +(2178, 6, TIMESTAMP '2025-08-30 08:36:00', 'completed', 251846, 12015), +(2179, 29, TIMESTAMP '2025-12-18 14:31:00', 'completed', 361467, 93119), +(2180, 17, TIMESTAMP '2025-04-01 23:14:00', 'pending', 82356, 0), +(2181, 52, TIMESTAMP '2025-07-27 19:46:00', 'completed', 29743, 510), +(2182, 47, TIMESTAMP '2025-09-22 11:34:00', 'completed', 88380, 687), +(2183, 5, TIMESTAMP '2025-10-29 17:25:00', 'completed', 72982, 8568), +(2184, 80, TIMESTAMP '2025-08-04 03:17:00', 'completed', 272286, 21274), +(2185, 44, TIMESTAMP '2025-08-08 14:18:00', 'completed', 258521, 20614), +(2186, 63, TIMESTAMP '2025-11-18 00:14:00', 'completed', 68794, 6476), +(2187, 21, TIMESTAMP '2025-09-01 16:49:00', 'completed', 73897, 5706), +(2188, 63, TIMESTAMP '2025-05-11 03:59:00', 'completed', 227474, 33374), +(2189, 7, TIMESTAMP '2025-05-28 21:40:00', 'completed', 213656, 28240), +(2190, 22, TIMESTAMP '2025-11-30 06:52:00', 'completed', 209604, 5477), +(2191, 51, TIMESTAMP '2025-10-17 00:30:00', 'completed', 145056, 86), +(2192, 61, TIMESTAMP '2025-11-13 09:15:00', 'completed', 286271, 3907), +(2193, 70, TIMESTAMP '2025-05-21 17:28:00', 'completed', 223208, 7236), +(2194, 10, TIMESTAMP '2025-01-15 21:54:00', 'completed', 232088, 16860), +(2195, 8, TIMESTAMP '2025-11-22 04:25:00', 'pending', 24258, 0), +(2196, 55, TIMESTAMP '2025-12-12 01:52:00', 'completed', 52011, 2933), +(2197, 28, TIMESTAMP '2025-09-20 22:37:00', 'completed', 125466, 10540), +(2198, 14, TIMESTAMP '2025-11-12 13:44:00', 'completed', 1447, 24), +(2199, 3, TIMESTAMP '2025-01-06 18:17:00', 'completed', 238038, 35206); + +DROP TABLE IF EXISTS memory.bench.legacy_orders; +CREATE TABLE memory.bench.legacy_orders ( + order_id INTEGER, + customer_id INTEGER, + order_date DATE, + total DOUBLE +); +INSERT INTO memory.bench.legacy_orders VALUES +(1000, 71, DATE '2025-11-11', 586.92), +(1001, 55, DATE '2025-11-07', 2525.86), +(1002, 41, DATE '2025-05-01', 2216.61), +(1003, 41, DATE '2025-07-15', 1767.12), +(1004, 3, DATE '2025-11-14', 129.79), +(1005, 54, DATE '2025-11-04', 1179.01), +(1006, 3, DATE '2025-09-25', 953.07), +(1007, 23, DATE '2025-03-21', 3003.95), +(1008, 50, DATE '2025-05-06', 2224.46), +(1009, 7, DATE '2025-12-28', 833.48), +(1010, 33, DATE '2025-01-01', 504.41), +(1011, 45, DATE '2025-03-16', 3984.81), +(1012, 7, DATE '2025-12-22', 396.74), +(1013, 78, DATE '2025-10-11', 2322.40), +(1014, 79, DATE '2025-06-11', 1911.14), +(1015, 36, DATE '2025-09-08', 1157.02), +(1016, 20, DATE '2025-01-29', 2407.60), +(1017, 5, DATE '2025-05-24', 2038.35), +(1018, 4, DATE '2025-06-15', 2466.72), +(1019, 16, DATE '2025-09-16', 1273.03), +(1020, 61, DATE '2025-02-18', 2002.88), +(1021, 11, DATE '2025-03-22', 905.46), +(1022, 71, DATE '2025-12-15', 1355.95), +(1023, 67, DATE '2025-03-22', 1017.42), +(1024, 39, DATE '2025-11-01', 3856.81), +(1025, 66, DATE '2025-02-05', 445.30), +(1026, 1, DATE '2025-08-26', 1603.95), +(1027, 31, DATE '2025-08-30', 2872.43), +(1028, 31, DATE '2025-06-10', 498.32), +(1029, 28, DATE '2025-04-12', 1351.11), +(1030, 61, DATE '2025-03-04', 1370.16), +(1031, 40, DATE '2025-10-23', 2000.98), +(1032, 10, DATE '2025-09-04', 1002.76), +(1033, 10, DATE '2025-04-12', 2879.51), +(1034, 7, DATE '2025-04-18', 2039.53), +(1035, 13, DATE '2025-08-25', 2201.98), +(1036, 22, DATE '2025-10-17', 2172.43), +(1037, 54, DATE '2025-09-07', 731.05), +(1038, 47, DATE '2025-03-24', 622.54), +(1039, 73, DATE '2025-01-12', 2462.21), +(1040, 76, DATE '2025-07-09', 2109.00), +(1041, 64, DATE '2025-03-19', 1193.08), +(1042, 35, DATE '2025-11-21', 2921.45), +(1043, 1, DATE '2025-07-15', 1121.26), +(1044, 77, DATE '2025-02-14', 2340.27), +(1045, 51, DATE '2025-07-01', 1550.33), +(1046, 45, DATE '2025-08-17', 3910.89), +(1047, 71, DATE '2025-02-05', 2754.84), +(1048, 10, DATE '2025-09-25', 2712.76), +(1049, 35, DATE '2025-09-15', 1601.80), +(1050, 27, DATE '2025-05-22', 822.14), +(1051, 43, DATE '2025-05-11', 2308.19), +(1052, 41, DATE '2025-01-23', 1904.70), +(1053, 56, DATE '2025-08-22', 1822.50), +(1054, 2, DATE '2025-11-01', 3565.79), +(1055, 24, DATE '2025-10-02', 1585.62), +(1056, 48, DATE '2025-07-22', 1209.37), +(1057, 70, DATE '2025-08-23', 1764.40), +(1058, 24, DATE '2025-05-20', 1259.14), +(1059, 22, DATE '2025-05-02', 1211.29); + +DROP TABLE IF EXISTS memory.bench.daily_region_revenue; +CREATE TABLE memory.bench.daily_region_revenue ( + day DATE, + region VARCHAR, + gross_revenue_usd DOUBLE +); +INSERT INTO memory.bench.daily_region_revenue VALUES +(DATE '2025-01-01', 'North', 2825.51), +(DATE '2025-01-01', 'South', 504.41), +(DATE '2025-01-01', 'West', 5453.71), +(DATE '2025-01-02', 'East', 2137.64), +(DATE '2025-01-02', 'West', 4875.81), +(DATE '2025-01-03', 'South', 1221.92), +(DATE '2025-01-03', 'West', 446.53), +(DATE '2025-01-04', 'South', 2079.61), +(DATE '2025-01-05', 'East', 1387.28), +(DATE '2025-01-05', 'West', 1501.12), +(DATE '2025-01-06', 'North', 651.04), +(DATE '2025-01-06', 'South', 3186.55), +(DATE '2025-01-06', 'West', 1170.22), +(DATE '2025-01-07', 'South', 1116.97), +(DATE '2025-01-07', 'West', 3798.12), +(DATE '2025-01-08', 'North', 1167.47), +(DATE '2025-01-08', 'South', 283.47), +(DATE '2025-01-08', 'West', 5620.47), +(DATE '2025-01-09', 'East', 493.48), +(DATE '2025-01-09', 'South', 4750.35), +(DATE '2025-01-10', 'South', 2401.14), +(DATE '2025-01-11', 'North', 184.55), +(DATE '2025-01-11', 'West', 1580.27), +(DATE '2025-01-12', 'South', 90.46), +(DATE '2025-01-12', 'West', 2462.21), +(DATE '2025-01-13', 'South', 713.18), +(DATE '2025-01-13', 'West', 1169.20), +(DATE '2025-01-14', 'East', 3356.80), +(DATE '2025-01-14', 'North', 1344.70), +(DATE '2025-01-15', 'North', 2359.81), +(DATE '2025-01-15', 'West', 3257.04), +(DATE '2025-01-16', 'North', 1386.77), +(DATE '2025-01-17', 'West', 2059.61), +(DATE '2025-01-18', 'South', 934.08), +(DATE '2025-01-18', 'West', 850.71), +(DATE '2025-01-19', 'East', 2574.41), +(DATE '2025-01-19', 'North', 1079.93), +(DATE '2025-01-19', 'South', 1816.50), +(DATE '2025-01-20', 'East', 3655.77), +(DATE '2025-01-21', 'South', 1565.57), +(DATE '2025-01-21', 'West', 2066.35), +(DATE '2025-01-22', 'West', 222.98), +(DATE '2025-01-23', 'South', 204.30), +(DATE '2025-01-23', 'West', 3099.65), +(DATE '2025-01-26', 'East', 241.20), +(DATE '2025-01-26', 'North', 3649.13), +(DATE '2025-01-26', 'West', 2727.96), +(DATE '2025-01-27', 'West', 3889.83), +(DATE '2025-01-29', 'East', 2762.83), +(DATE '2025-01-29', 'North', 1379.72), +(DATE '2025-01-29', 'South', 2390.07), +(DATE '2025-01-29', 'West', 2503.97), +(DATE '2025-01-30', 'South', 310.11), +(DATE '2025-01-31', 'North', 209.03), +(DATE '2025-01-31', 'South', 2350.02), +(DATE '2025-01-31', 'West', 2001.63), +(DATE '2025-02-02', 'East', 4474.50), +(DATE '2025-02-02', 'South', 37.09), +(DATE '2025-02-03', 'South', 1794.43), +(DATE '2025-02-04', 'South', 1858.10), +(DATE '2025-02-04', 'West', 5332.29), +(DATE '2025-02-05', 'North', 1937.58), +(DATE '2025-02-05', 'South', 1716.47), +(DATE '2025-02-06', 'North', 335.05), +(DATE '2025-02-07', 'East', 2836.40), +(DATE '2025-02-08', 'North', 2431.57), +(DATE '2025-02-08', 'South', 2921.00), +(DATE '2025-02-09', 'West', 841.61), +(DATE '2025-02-10', 'East', 2414.32), +(DATE '2025-02-10', 'South', 1256.20), +(DATE '2025-02-11', 'South', 1500.74), +(DATE '2025-02-12', 'West', 2443.68), +(DATE '2025-02-13', 'West', 1355.08), +(DATE '2025-02-14', 'East', 371.87), +(DATE '2025-02-14', 'West', 3095.73), +(DATE '2025-02-15', 'North', 2907.04), +(DATE '2025-02-15', 'South', 668.56), +(DATE '2025-02-16', 'East', 389.12), +(DATE '2025-02-16', 'West', 1110.72), +(DATE '2025-02-17', 'East', 4119.90), +(DATE '2025-02-17', 'South', 606.61), +(DATE '2025-02-18', 'East', 2925.76), +(DATE '2025-02-18', 'South', 5397.69), +(DATE '2025-02-18', 'West', 2002.88), +(DATE '2025-02-19', 'South', 5.78), +(DATE '2025-02-20', 'East', 2989.12), +(DATE '2025-02-20', 'North', 434.32), +(DATE '2025-02-21', 'East', 1612.46), +(DATE '2025-02-21', 'West', 2612.27), +(DATE '2025-02-22', 'South', 4010.66), +(DATE '2025-02-24', 'West', 2737.00), +(DATE '2025-02-25', 'South', 2027.23), +(DATE '2025-02-25', 'West', 3139.48), +(DATE '2025-02-26', 'North', 1493.98), +(DATE '2025-02-27', 'East', 2724.22), +(DATE '2025-02-27', 'North', 960.36), +(DATE '2025-02-27', 'South', 1192.45), +(DATE '2025-02-27', 'West', 2834.05), +(DATE '2025-03-01', 'West', 1680.71), +(DATE '2025-03-03', 'North', 2681.59), +(DATE '2025-03-03', 'South', 3773.02), +(DATE '2025-03-03', 'West', 1786.60), +(DATE '2025-03-04', 'East', 558.03), +(DATE '2025-03-04', 'North', 1071.13), +(DATE '2025-03-04', 'West', 1370.16), +(DATE '2025-03-05', 'East', 7733.61), +(DATE '2025-03-05', 'West', 5467.95), +(DATE '2025-03-06', 'East', 3398.10), +(DATE '2025-03-06', 'West', 893.85), +(DATE '2025-03-07', 'North', 565.89), +(DATE '2025-03-07', 'South', 1004.98), +(DATE '2025-03-07', 'West', 3393.98), +(DATE '2025-03-09', 'South', 122.29), +(DATE '2025-03-10', 'East', 2847.23), +(DATE '2025-03-10', 'North', 2453.18), +(DATE '2025-03-10', 'South', 2304.85), +(DATE '2025-03-10', 'West', 3168.84), +(DATE '2025-03-11', 'North', 3210.83), +(DATE '2025-03-12', 'East', 531.02), +(DATE '2025-03-12', 'North', 1396.59), +(DATE '2025-03-13', 'East', 2238.91), +(DATE '2025-03-13', 'North', 2061.09), +(DATE '2025-03-13', 'South', 2563.19), +(DATE '2025-03-14', 'South', 1668.97), +(DATE '2025-03-14', 'West', 871.98), +(DATE '2025-03-16', 'East', 6081.25), +(DATE '2025-03-16', 'South', 1276.18), +(DATE '2025-03-16', 'West', 310.61), +(DATE '2025-03-17', 'East', 2246.16), +(DATE '2025-03-17', 'North', 941.13), +(DATE '2025-03-17', 'South', 577.36), +(DATE '2025-03-18', 'South', 2267.53), +(DATE '2025-03-19', 'East', 5464.12), +(DATE '2025-03-19', 'West', 1042.65), +(DATE '2025-03-20', 'East', 3495.03), +(DATE '2025-03-20', 'North', 2140.13), +(DATE '2025-03-20', 'West', 2063.72), +(DATE '2025-03-21', 'North', 1583.37), +(DATE '2025-03-21', 'West', 4990.75), +(DATE '2025-03-22', 'West', 905.46), +(DATE '2025-03-23', 'East', 3645.36), +(DATE '2025-03-23', 'South', 1412.41), +(DATE '2025-03-23', 'West', 2742.63), +(DATE '2025-03-24', 'North', 2354.67), +(DATE '2025-03-25', 'East', 7079.63), +(DATE '2025-03-25', 'North', 293.93), +(DATE '2025-03-25', 'South', 1520.36), +(DATE '2025-03-26', 'North', 331.36), +(DATE '2025-03-26', 'South', 3020.34), +(DATE '2025-03-26', 'West', 1158.87), +(DATE '2025-03-27', 'South', 2788.70), +(DATE '2025-03-27', 'West', 866.91), +(DATE '2025-03-28', 'North', 990.85), +(DATE '2025-03-29', 'East', 2005.08), +(DATE '2025-03-30', 'East', 1231.10), +(DATE '2025-03-30', 'North', 2101.48), +(DATE '2025-03-30', 'West', 2664.23), +(DATE '2025-03-31', 'East', 1871.48), +(DATE '2025-03-31', 'South', 880.02), +(DATE '2025-04-01', 'East', 150.04), +(DATE '2025-04-01', 'North', 2317.69), +(DATE '2025-04-01', 'South', 2187.84), +(DATE '2025-04-01', 'West', 1920.43), +(DATE '2025-04-02', 'North', 898.21), +(DATE '2025-04-02', 'South', 1950.20), +(DATE '2025-04-02', 'West', 2690.20), +(DATE '2025-04-03', 'South', 5775.46), +(DATE '2025-04-03', 'West', 1794.48), +(DATE '2025-04-05', 'North', 3882.03), +(DATE '2025-04-05', 'South', 1560.57), +(DATE '2025-04-05', 'West', 4844.43), +(DATE '2025-04-06', 'East', 1170.33), +(DATE '2025-04-06', 'South', 34.69), +(DATE '2025-04-06', 'West', 1433.78), +(DATE '2025-04-07', 'South', 4325.23), +(DATE '2025-04-07', 'West', 788.61), +(DATE '2025-04-08', 'South', 2493.47), +(DATE '2025-04-09', 'North', 1053.53), +(DATE '2025-04-10', 'East', 2482.65), +(DATE '2025-04-10', 'North', 1015.09), +(DATE '2025-04-10', 'West', 230.32), +(DATE '2025-04-11', 'West', 673.45), +(DATE '2025-04-12', 'East', 1492.25), +(DATE '2025-04-12', 'North', 1351.11), +(DATE '2025-04-12', 'West', 4086.67), +(DATE '2025-04-13', 'North', 3241.51), +(DATE '2025-04-13', 'West', 4177.30), +(DATE '2025-04-14', 'West', 269.70), +(DATE '2025-04-15', 'North', 2030.73), +(DATE '2025-04-15', 'South', 2085.48), +(DATE '2025-04-15', 'West', 790.41), +(DATE '2025-04-16', 'South', 1754.37), +(DATE '2025-04-16', 'West', 2731.45), +(DATE '2025-04-17', 'West', 5025.42), +(DATE '2025-04-18', 'North', 1302.57), +(DATE '2025-04-18', 'South', 4086.57), +(DATE '2025-04-19', 'South', 4615.49), +(DATE '2025-04-20', 'East', 3258.48), +(DATE '2025-04-20', 'North', 2217.25), +(DATE '2025-04-20', 'South', 127.67), +(DATE '2025-04-20', 'West', 2492.66), +(DATE '2025-04-21', 'South', 1391.59), +(DATE '2025-04-22', 'North', 1253.84), +(DATE '2025-04-22', 'South', 1401.23), +(DATE '2025-04-22', 'West', 1762.76), +(DATE '2025-04-23', 'East', 874.54), +(DATE '2025-04-24', 'South', 687.55), +(DATE '2025-04-24', 'West', 1214.30), +(DATE '2025-04-25', 'East', 2390.68), +(DATE '2025-04-26', 'North', 1468.90), +(DATE '2025-04-26', 'West', 2185.78), +(DATE '2025-04-27', 'South', 1693.43), +(DATE '2025-04-28', 'East', 442.70), +(DATE '2025-04-28', 'West', 3806.84), +(DATE '2025-04-29', 'South', 6199.86), +(DATE '2025-04-29', 'West', 1336.52), +(DATE '2025-04-30', 'East', 3472.48), +(DATE '2025-04-30', 'South', 3515.43), +(DATE '2025-04-30', 'West', 30.82), +(DATE '2025-05-01', 'North', 289.83), +(DATE '2025-05-01', 'West', 2216.61), +(DATE '2025-05-02', 'North', 608.37), +(DATE '2025-05-02', 'South', 3418.25), +(DATE '2025-05-02', 'West', 721.97), +(DATE '2025-05-03', 'East', 1958.22), +(DATE '2025-05-03', 'North', 843.48), +(DATE '2025-05-03', 'South', 1575.94), +(DATE '2025-05-03', 'West', 2819.95), +(DATE '2025-05-04', 'East', 3227.66), +(DATE '2025-05-04', 'North', 1851.84), +(DATE '2025-05-05', 'West', 2717.98), +(DATE '2025-05-06', 'North', 661.84), +(DATE '2025-05-06', 'South', 2224.46), +(DATE '2025-05-07', 'South', 1805.19), +(DATE '2025-05-07', 'West', 1777.45), +(DATE '2025-05-08', 'East', 1700.44), +(DATE '2025-05-08', 'North', 5525.75), +(DATE '2025-05-09', 'East', 2574.22), +(DATE '2025-05-09', 'North', 1501.92), +(DATE '2025-05-09', 'West', 5743.43), +(DATE '2025-05-10', 'West', 1235.86), +(DATE '2025-05-11', 'East', 2308.19), +(DATE '2025-05-11', 'North', 2274.74), +(DATE '2025-05-11', 'South', 1899.18), +(DATE '2025-05-11', 'West', 2130.41), +(DATE '2025-05-12', 'South', 1562.62), +(DATE '2025-05-12', 'West', 2511.71), +(DATE '2025-05-13', 'West', 2153.60), +(DATE '2025-05-14', 'North', 87.07), +(DATE '2025-05-15', 'North', 1233.36); +INSERT INTO memory.bench.daily_region_revenue VALUES +(DATE '2025-05-15', 'West', 1021.26), +(DATE '2025-05-16', 'North', 1427.95), +(DATE '2025-05-17', 'South', 2274.23), +(DATE '2025-05-17', 'West', 920.23), +(DATE '2025-05-18', 'East', 1948.86), +(DATE '2025-05-18', 'North', 2395.00), +(DATE '2025-05-19', 'East', 1643.63), +(DATE '2025-05-19', 'North', 291.28), +(DATE '2025-05-20', 'East', 2106.00), +(DATE '2025-05-20', 'North', 2233.59), +(DATE '2025-05-21', 'West', 2232.08), +(DATE '2025-05-22', 'East', 1991.00), +(DATE '2025-05-22', 'North', 2009.38), +(DATE '2025-05-23', 'South', 1017.22), +(DATE '2025-05-23', 'West', 1983.63), +(DATE '2025-05-24', 'North', 1504.60), +(DATE '2025-05-25', 'West', 2322.26), +(DATE '2025-05-27', 'West', 2016.05), +(DATE '2025-05-28', 'South', 2136.56), +(DATE '2025-05-28', 'West', 893.47), +(DATE '2025-05-29', 'North', 514.06), +(DATE '2025-05-29', 'West', 4235.65), +(DATE '2025-05-30', 'South', 3366.01), +(DATE '2025-05-31', 'North', 2448.43), +(DATE '2025-05-31', 'South', 2469.70), +(DATE '2025-05-31', 'West', 2498.39), +(DATE '2025-06-01', 'South', 1247.92), +(DATE '2025-06-01', 'West', 4429.99), +(DATE '2025-06-02', 'East', 3482.89), +(DATE '2025-06-02', 'North', 631.66), +(DATE '2025-06-02', 'South', 2666.39), +(DATE '2025-06-02', 'West', 3026.75), +(DATE '2025-06-03', 'North', 315.25), +(DATE '2025-06-03', 'West', 781.73), +(DATE '2025-06-04', 'North', 3114.45), +(DATE '2025-06-04', 'South', 1671.09), +(DATE '2025-06-04', 'West', 3719.10), +(DATE '2025-06-05', 'East', 3292.75), +(DATE '2025-06-05', 'North', 590.05), +(DATE '2025-06-05', 'West', 725.03), +(DATE '2025-06-06', 'West', 2132.28), +(DATE '2025-06-07', 'North', 3121.02), +(DATE '2025-06-07', 'South', 1195.92), +(DATE '2025-06-08', 'North', 256.44), +(DATE '2025-06-09', 'South', 887.20), +(DATE '2025-06-10', 'North', 911.68), +(DATE '2025-06-10', 'West', 686.78), +(DATE '2025-06-11', 'North', 5200.21), +(DATE '2025-06-12', 'South', 2365.95), +(DATE '2025-06-12', 'West', 1516.92), +(DATE '2025-06-13', 'North', 1872.98), +(DATE '2025-06-13', 'South', 1325.03), +(DATE '2025-06-13', 'West', 652.51), +(DATE '2025-06-14', 'East', 1690.67), +(DATE '2025-06-14', 'West', 963.23), +(DATE '2025-06-15', 'North', 4169.98), +(DATE '2025-06-15', 'West', 2073.85), +(DATE '2025-06-16', 'South', 1436.60), +(DATE '2025-06-16', 'West', 1589.93), +(DATE '2025-06-17', 'North', 2228.41), +(DATE '2025-06-17', 'South', 2320.18), +(DATE '2025-06-18', 'East', 1927.08), +(DATE '2025-06-18', 'North', 273.20), +(DATE '2025-06-19', 'East', 539.87), +(DATE '2025-06-19', 'North', 2010.16), +(DATE '2025-06-19', 'South', 3339.73), +(DATE '2025-06-19', 'West', 1437.06), +(DATE '2025-06-20', 'South', 337.87), +(DATE '2025-06-21', 'North', 530.64), +(DATE '2025-06-21', 'West', 579.05), +(DATE '2025-06-22', 'East', 1800.57), +(DATE '2025-06-22', 'West', 145.13), +(DATE '2025-06-24', 'North', 1407.09), +(DATE '2025-06-24', 'South', 1087.23), +(DATE '2025-06-25', 'North', 495.59), +(DATE '2025-06-25', 'South', 834.11), +(DATE '2025-06-26', 'East', 4210.67), +(DATE '2025-06-26', 'South', 2848.83), +(DATE '2025-06-27', 'North', 202.97), +(DATE '2025-06-27', 'South', 1675.93), +(DATE '2025-06-28', 'East', 6771.44), +(DATE '2025-06-28', 'South', 415.42), +(DATE '2025-06-28', 'West', 2732.59), +(DATE '2025-06-29', 'West', 2118.20), +(DATE '2025-06-30', 'East', 7774.71), +(DATE '2025-06-30', 'South', 402.75), +(DATE '2025-06-30', 'West', 1349.05), +(DATE '2025-07-01', 'North', 2302.67), +(DATE '2025-07-01', 'South', 3496.68), +(DATE '2025-07-02', 'West', 1267.65), +(DATE '2025-07-04', 'North', 2187.05), +(DATE '2025-07-04', 'West', 1766.51), +(DATE '2025-07-05', 'West', 554.36), +(DATE '2025-07-06', 'South', 1225.76), +(DATE '2025-07-07', 'East', 3449.31), +(DATE '2025-07-07', 'West', 469.03), +(DATE '2025-07-08', 'West', 2483.66), +(DATE '2025-07-09', 'East', 2485.53), +(DATE '2025-07-09', 'North', 2697.04), +(DATE '2025-07-09', 'West', 1664.32), +(DATE '2025-07-10', 'North', 1082.17), +(DATE '2025-07-10', 'South', 1869.96), +(DATE '2025-07-11', 'East', 2230.60), +(DATE '2025-07-12', 'West', 4580.09), +(DATE '2025-07-13', 'East', 154.24), +(DATE '2025-07-13', 'West', 2038.35), +(DATE '2025-07-14', 'North', 3808.75), +(DATE '2025-07-14', 'West', 1464.72), +(DATE '2025-07-15', 'East', 3193.07), +(DATE '2025-07-15', 'West', 1767.12), +(DATE '2025-07-16', 'East', 1429.34), +(DATE '2025-07-16', 'North', 858.51), +(DATE '2025-07-18', 'South', 827.25), +(DATE '2025-07-19', 'North', 766.57), +(DATE '2025-07-19', 'West', 2200.00), +(DATE '2025-07-20', 'East', 2385.18), +(DATE '2025-07-20', 'North', 1006.73), +(DATE '2025-07-20', 'South', 2961.66), +(DATE '2025-07-20', 'West', 2142.48), +(DATE '2025-07-21', 'North', 1982.95), +(DATE '2025-07-21', 'South', 3401.96), +(DATE '2025-07-22', 'North', 3460.29), +(DATE '2025-07-22', 'West', 2634.32), +(DATE '2025-07-23', 'East', 1709.96), +(DATE '2025-07-23', 'North', 1518.22), +(DATE '2025-07-23', 'West', 3687.02), +(DATE '2025-07-24', 'West', 2655.91), +(DATE '2025-07-25', 'East', 4432.81), +(DATE '2025-07-25', 'West', 1233.07), +(DATE '2025-07-26', 'North', 312.76), +(DATE '2025-07-26', 'West', 2087.80), +(DATE '2025-07-27', 'North', 481.90), +(DATE '2025-07-27', 'South', 361.21), +(DATE '2025-07-27', 'West', 400.45), +(DATE '2025-07-28', 'North', 263.33), +(DATE '2025-07-29', 'East', 1049.53), +(DATE '2025-07-29', 'North', 1931.08), +(DATE '2025-07-29', 'West', 639.38), +(DATE '2025-07-30', 'East', 530.96), +(DATE '2025-07-30', 'North', 726.17), +(DATE '2025-07-30', 'South', 2123.53), +(DATE '2025-08-01', 'East', 2364.38), +(DATE '2025-08-01', 'South', 687.03), +(DATE '2025-08-01', 'West', 1598.18), +(DATE '2025-08-03', 'North', 82.54), +(DATE '2025-08-03', 'South', 1338.21), +(DATE '2025-08-03', 'West', 3049.80), +(DATE '2025-08-04', 'South', 2588.80), +(DATE '2025-08-04', 'West', 4768.90), +(DATE '2025-08-05', 'East', 435.05), +(DATE '2025-08-05', 'North', 1934.18), +(DATE '2025-08-06', 'South', 2298.70), +(DATE '2025-08-06', 'West', 2706.69), +(DATE '2025-08-07', 'South', 618.98), +(DATE '2025-08-07', 'West', 2219.90), +(DATE '2025-08-08', 'North', 128.33), +(DATE '2025-08-08', 'South', 1512.69), +(DATE '2025-08-08', 'West', 2585.21), +(DATE '2025-08-09', 'West', 3095.42), +(DATE '2025-08-10', 'North', 1276.91), +(DATE '2025-08-10', 'South', 725.81), +(DATE '2025-08-11', 'South', 20.52), +(DATE '2025-08-11', 'West', 1701.05), +(DATE '2025-08-12', 'North', 463.41), +(DATE '2025-08-12', 'South', 339.08), +(DATE '2025-08-13', 'North', 670.38), +(DATE '2025-08-13', 'South', 2482.74), +(DATE '2025-08-13', 'West', 2766.77), +(DATE '2025-08-14', 'North', 1616.36), +(DATE '2025-08-14', 'West', 4881.95), +(DATE '2025-08-15', 'East', 3082.97), +(DATE '2025-08-15', 'North', 149.23), +(DATE '2025-08-15', 'West', 4805.18), +(DATE '2025-08-16', 'West', 3853.25), +(DATE '2025-08-17', 'East', 6780.96), +(DATE '2025-08-17', 'North', 2394.27), +(DATE '2025-08-17', 'South', 2762.64), +(DATE '2025-08-17', 'West', 4365.81), +(DATE '2025-08-18', 'South', 5715.73), +(DATE '2025-08-18', 'West', 1184.46), +(DATE '2025-08-19', 'East', 3940.44), +(DATE '2025-08-20', 'North', 684.60), +(DATE '2025-08-22', 'East', 3623.29), +(DATE '2025-08-22', 'North', 1766.01), +(DATE '2025-08-22', 'West', 4616.12), +(DATE '2025-08-24', 'North', 5745.68), +(DATE '2025-08-24', 'South', 1390.17), +(DATE '2025-08-24', 'West', 2722.71), +(DATE '2025-08-25', 'East', 3361.93), +(DATE '2025-08-25', 'South', 3709.95), +(DATE '2025-08-25', 'West', 1189.26), +(DATE '2025-08-27', 'North', 1124.42), +(DATE '2025-08-27', 'West', 1270.13), +(DATE '2025-08-28', 'South', 3886.22), +(DATE '2025-08-28', 'West', 62.56), +(DATE '2025-08-29', 'North', 306.91), +(DATE '2025-08-29', 'South', 477.22), +(DATE '2025-08-29', 'West', 1955.72), +(DATE '2025-08-30', 'West', 7453.88), +(DATE '2025-08-31', 'North', 197.16), +(DATE '2025-09-01', 'South', 738.97), +(DATE '2025-09-02', 'East', 1009.60), +(DATE '2025-09-02', 'South', 127.93), +(DATE '2025-09-02', 'West', 2649.91), +(DATE '2025-09-03', 'South', 4260.06), +(DATE '2025-09-03', 'West', 1022.36), +(DATE '2025-09-04', 'South', 1443.01), +(DATE '2025-09-04', 'West', 1002.76), +(DATE '2025-09-05', 'East', 884.71), +(DATE '2025-09-05', 'West', 435.01), +(DATE '2025-09-06', 'West', 2864.83), +(DATE '2025-09-07', 'North', 642.09), +(DATE '2025-09-07', 'South', 2778.41), +(DATE '2025-09-07', 'West', 1032.51), +(DATE '2025-09-08', 'South', 1157.02), +(DATE '2025-09-08', 'West', 2776.21), +(DATE '2025-09-09', 'North', 362.78), +(DATE '2025-09-09', 'West', 2688.78), +(DATE '2025-09-10', 'North', 2983.97), +(DATE '2025-09-10', 'South', 938.11), +(DATE '2025-09-11', 'North', 1305.93), +(DATE '2025-09-11', 'South', 731.09), +(DATE '2025-09-12', 'North', 5250.94), +(DATE '2025-09-12', 'South', 2246.10), +(DATE '2025-09-13', 'North', 1637.13), +(DATE '2025-09-14', 'East', 2742.30), +(DATE '2025-09-14', 'South', 2234.91), +(DATE '2025-09-14', 'West', 2569.25), +(DATE '2025-09-15', 'East', 1601.80), +(DATE '2025-09-15', 'South', 1429.12), +(DATE '2025-09-16', 'North', 1273.03), +(DATE '2025-09-16', 'West', 1582.22), +(DATE '2025-09-17', 'North', 1600.61), +(DATE '2025-09-17', 'West', 2195.10), +(DATE '2025-09-18', 'South', 1379.64), +(DATE '2025-09-19', 'South', 3679.14), +(DATE '2025-09-20', 'North', 2840.58), +(DATE '2025-09-20', 'South', 2407.54), +(DATE '2025-09-21', 'North', 789.29), +(DATE '2025-09-21', 'West', 2516.98), +(DATE '2025-09-22', 'East', 883.80), +(DATE '2025-09-22', 'North', 339.62), +(DATE '2025-09-22', 'South', 1383.39), +(DATE '2025-09-22', 'West', 4628.37), +(DATE '2025-09-23', 'East', 3575.92), +(DATE '2025-09-23', 'South', 2134.51), +(DATE '2025-09-23', 'West', 279.18), +(DATE '2025-09-24', 'East', 2421.93), +(DATE '2025-09-24', 'West', 332.50), +(DATE '2025-09-25', 'North', 1889.03); +INSERT INTO memory.bench.daily_region_revenue VALUES +(DATE '2025-09-25', 'South', 953.07), +(DATE '2025-09-25', 'West', 5707.90), +(DATE '2025-09-26', 'South', 657.52), +(DATE '2025-09-27', 'West', 150.30), +(DATE '2025-09-29', 'East', 312.41), +(DATE '2025-09-29', 'South', 1711.70), +(DATE '2025-09-29', 'West', 663.00), +(DATE '2025-09-30', 'West', 189.18), +(DATE '2025-10-01', 'West', 1291.86), +(DATE '2025-10-02', 'North', 2520.22), +(DATE '2025-10-02', 'South', 3667.30), +(DATE '2025-10-03', 'North', 396.58), +(DATE '2025-10-04', 'North', 1513.07), +(DATE '2025-10-04', 'South', 848.60), +(DATE '2025-10-06', 'West', 2384.95), +(DATE '2025-10-07', 'North', 1019.69), +(DATE '2025-10-07', 'South', 868.11), +(DATE '2025-10-07', 'West', 1293.08), +(DATE '2025-10-08', 'North', 1715.56), +(DATE '2025-10-08', 'South', 663.20), +(DATE '2025-10-08', 'West', 1361.02), +(DATE '2025-10-09', 'East', 3823.12), +(DATE '2025-10-10', 'East', 1656.76), +(DATE '2025-10-10', 'South', 2135.39), +(DATE '2025-10-11', 'East', 3890.91), +(DATE '2025-10-11', 'West', 2322.40), +(DATE '2025-10-13', 'North', 2090.22), +(DATE '2025-10-13', 'West', 1480.77), +(DATE '2025-10-14', 'North', 1475.14), +(DATE '2025-10-15', 'East', 3834.15), +(DATE '2025-10-15', 'North', 666.76), +(DATE '2025-10-15', 'South', 5345.21), +(DATE '2025-10-15', 'West', 1834.36), +(DATE '2025-10-16', 'West', 2264.35), +(DATE '2025-10-17', 'East', 2228.16), +(DATE '2025-10-17', 'North', 2768.19), +(DATE '2025-10-17', 'South', 3561.23), +(DATE '2025-10-17', 'West', 1203.31), +(DATE '2025-10-18', 'East', 2117.71), +(DATE '2025-10-18', 'North', 1452.59), +(DATE '2025-10-18', 'West', 4830.47), +(DATE '2025-10-20', 'East', 2048.14), +(DATE '2025-10-20', 'South', 1661.14), +(DATE '2025-10-20', 'West', 277.42), +(DATE '2025-10-21', 'East', 3702.27), +(DATE '2025-10-22', 'East', 1459.74), +(DATE '2025-10-22', 'North', 92.03), +(DATE '2025-10-22', 'South', 1194.85), +(DATE '2025-10-22', 'West', 816.75), +(DATE '2025-10-23', 'East', 490.91), +(DATE '2025-10-23', 'South', 2000.98), +(DATE '2025-10-23', 'West', 582.99), +(DATE '2025-10-24', 'West', 1007.97), +(DATE '2025-10-26', 'East', 3530.01), +(DATE '2025-10-26', 'North', 4522.33), +(DATE '2025-10-26', 'South', 1242.85), +(DATE '2025-10-27', 'East', 3617.82), +(DATE '2025-10-27', 'South', 1451.81), +(DATE '2025-10-27', 'West', 8045.55), +(DATE '2025-10-28', 'East', 2227.50), +(DATE '2025-10-28', 'North', 2029.32), +(DATE '2025-10-28', 'South', 1218.39), +(DATE '2025-10-29', 'East', 2617.79), +(DATE '2025-10-29', 'North', 5131.31), +(DATE '2025-10-29', 'South', 2263.10), +(DATE '2025-10-29', 'West', 1998.07), +(DATE '2025-10-30', 'East', 1327.32), +(DATE '2025-10-30', 'South', 2193.28), +(DATE '2025-10-30', 'West', 2178.33), +(DATE '2025-10-31', 'East', 2907.16), +(DATE '2025-10-31', 'West', 3083.26), +(DATE '2025-11-01', 'East', 8501.16), +(DATE '2025-11-01', 'South', 1842.96), +(DATE '2025-11-02', 'West', 603.40), +(DATE '2025-11-03', 'East', 3721.92), +(DATE '2025-11-03', 'South', 1585.14), +(DATE '2025-11-03', 'West', 1776.75), +(DATE '2025-11-04', 'South', 3644.60), +(DATE '2025-11-05', 'South', 397.68), +(DATE '2025-11-05', 'West', 3512.40), +(DATE '2025-11-06', 'West', 2055.35), +(DATE '2025-11-07', 'West', 4785.09), +(DATE '2025-11-08', 'South', 1403.62), +(DATE '2025-11-08', 'West', 1950.85), +(DATE '2025-11-09', 'South', 3054.48), +(DATE '2025-11-09', 'West', 2080.23), +(DATE '2025-11-10', 'West', 3697.85), +(DATE '2025-11-11', 'East', 586.92), +(DATE '2025-11-12', 'East', 3260.73), +(DATE '2025-11-12', 'West', 14.47), +(DATE '2025-11-13', 'North', 2182.19), +(DATE '2025-11-13', 'West', 2862.71), +(DATE '2025-11-14', 'North', 1321.77), +(DATE '2025-11-14', 'South', 129.79), +(DATE '2025-11-14', 'West', 3397.28), +(DATE '2025-11-15', 'East', 1671.76), +(DATE '2025-11-15', 'West', 348.00), +(DATE '2025-11-16', 'South', 4554.86), +(DATE '2025-11-16', 'West', 2178.73), +(DATE '2025-11-17', 'East', 2478.40), +(DATE '2025-11-17', 'North', 415.95), +(DATE '2025-11-17', 'West', 2719.00), +(DATE '2025-11-18', 'North', 687.94), +(DATE '2025-11-18', 'South', 2300.44), +(DATE '2025-11-18', 'West', 965.32), +(DATE '2025-11-19', 'East', 2751.71), +(DATE '2025-11-19', 'North', 1320.66), +(DATE '2025-11-19', 'South', 1955.57), +(DATE '2025-11-20', 'East', 1910.19), +(DATE '2025-11-20', 'South', 653.17), +(DATE '2025-11-20', 'West', 1289.21), +(DATE '2025-11-21', 'East', 5291.91), +(DATE '2025-11-21', 'North', 380.66), +(DATE '2025-11-21', 'South', 6014.73), +(DATE '2025-11-21', 'West', 1959.48), +(DATE '2025-11-23', 'South', 4369.14), +(DATE '2025-11-24', 'South', 1912.43), +(DATE '2025-11-25', 'North', 3543.51), +(DATE '2025-11-26', 'East', 5430.16), +(DATE '2025-11-26', 'West', 2446.71), +(DATE '2025-11-27', 'East', 3964.86), +(DATE '2025-11-27', 'North', 2354.54), +(DATE '2025-11-27', 'West', 2822.38), +(DATE '2025-11-29', 'South', 1123.52), +(DATE '2025-11-30', 'South', 4111.59), +(DATE '2025-12-01', 'East', 3614.96), +(DATE '2025-12-01', 'North', 1343.61), +(DATE '2025-12-02', 'East', 2655.74), +(DATE '2025-12-02', 'North', 1350.88), +(DATE '2025-12-02', 'South', 2135.91), +(DATE '2025-12-02', 'West', 3069.42), +(DATE '2025-12-03', 'South', 7446.52), +(DATE '2025-12-04', 'East', 2266.75), +(DATE '2025-12-04', 'South', 338.93), +(DATE '2025-12-04', 'West', 2791.66), +(DATE '2025-12-05', 'East', 2187.93), +(DATE '2025-12-05', 'North', 2809.48), +(DATE '2025-12-05', 'West', 2044.00), +(DATE '2025-12-06', 'East', 2955.74), +(DATE '2025-12-06', 'South', 1863.57), +(DATE '2025-12-06', 'West', 2839.17), +(DATE '2025-12-07', 'West', 1880.51), +(DATE '2025-12-08', 'North', 2857.36), +(DATE '2025-12-09', 'East', 3623.95), +(DATE '2025-12-09', 'West', 1864.02), +(DATE '2025-12-10', 'North', 1535.21), +(DATE '2025-12-10', 'West', 4088.60), +(DATE '2025-12-11', 'West', 551.30), +(DATE '2025-12-12', 'North', 2304.93), +(DATE '2025-12-12', 'West', 520.11), +(DATE '2025-12-14', 'West', 2409.35), +(DATE '2025-12-15', 'East', 1355.95), +(DATE '2025-12-15', 'North', 2072.90), +(DATE '2025-12-15', 'South', 4450.72), +(DATE '2025-12-16', 'East', 1843.55), +(DATE '2025-12-16', 'North', 1395.66), +(DATE '2025-12-16', 'West', 770.42), +(DATE '2025-12-17', 'North', 2256.98), +(DATE '2025-12-17', 'West', 4394.90), +(DATE '2025-12-18', 'East', 7547.96), +(DATE '2025-12-18', 'North', 1967.84), +(DATE '2025-12-19', 'East', 28.54), +(DATE '2025-12-19', 'North', 1072.47), +(DATE '2025-12-20', 'South', 4114.04), +(DATE '2025-12-22', 'East', 1036.49), +(DATE '2025-12-22', 'South', 774.40), +(DATE '2025-12-22', 'West', 3248.65), +(DATE '2025-12-23', 'East', 915.82), +(DATE '2025-12-23', 'North', 2468.44), +(DATE '2025-12-26', 'East', 3838.08), +(DATE '2025-12-26', 'South', 2728.00), +(DATE '2025-12-26', 'West', 1587.81), +(DATE '2025-12-27', 'East', 45.93), +(DATE '2025-12-28', 'East', 129.47), +(DATE '2025-12-28', 'North', 2783.97), +(DATE '2025-12-28', 'South', 833.48), +(DATE '2025-12-28', 'West', 4545.76), +(DATE '2025-12-29', 'South', 2339.10), +(DATE '2025-12-29', 'West', 2624.25), +(DATE '2025-12-30', 'East', 750.80), +(DATE '2025-12-30', 'West', 2374.90), +(DATE '2025-12-31', 'East', 2487.53), +(DATE '2025-12-31', 'North', 1026.49), +(DATE '2025-12-31', 'West', 616.48); diff --git a/bench/seedgen/main.go b/bench/seedgen/main.go new file mode 100644 index 00000000..ebb07119 --- /dev/null +++ b/bench/seedgen/main.go @@ -0,0 +1,110 @@ +// Command seedgen regenerates the benchmark's committed seed artifacts and +// task set from the fixed-seed dataset model (bench/internal/gen). Ground +// truth is computed from the generated rows at generation time, never +// hand-typed; a determinism test diffs a fresh regeneration against the +// committed files. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/txn2/mcp-data-platform/bench/internal/gen" + "github.com/txn2/mcp-data-platform/bench/internal/task" +) + +func main() { + seedDir := flag.String("seed-dir", "seed", "output directory for seed artifacts") + tasksDir := flag.String("tasks-dir", "tasks", "output directory for task YAML and the smoke script") + flag.Parse() + if err := run(*seedDir, *tasksDir); err != nil { + fmt.Fprintln(os.Stderr, "seedgen:", err) + os.Exit(1) + } +} + +// run generates the dataset and writes every artifact. +func run(seedDir, tasksDir string) error { + ds := gen.Generate() + if err := writeSeedArtifacts(ds, seedDir); err != nil { + return err + } + tasks := ds.Tasks() + if err := writeTasks(tasks, tasksDir); err != nil { + return err + } + if err := writeSmokeScript(tasks, tasksDir); err != nil { + return err + } + fmt.Printf("seedgen: %d customers, %d orders, %d tasks, task-set hash %s\n", + len(ds.Customers), len(ds.Orders), len(tasks), task.Hash(tasks)) + return nil +} + +// writeSeedArtifacts emits the Trino SQL, DataHub MCEs, and knowledge-page SQL. +func writeSeedArtifacts(ds *gen.Dataset, dir string) error { + mces, err := ds.DataHubMCEs() + if err != nil { + return fmt.Errorf("render datahub mces: %w", err) + } + files := map[string][]byte{ + filepath.Join(dir, "trino", "setup.sql"): []byte(ds.TrinoSQL()), + filepath.Join(dir, "datahub", "bench_mces.json"): mces, + filepath.Join(dir, "postgres", "knowledge_pages.sql"): []byte(ds.KnowledgePagesSQL()), + } + for path, content := range files { + if err := writeFile(path, content); err != nil { + return err + } + } + return nil +} + +// writeTasks emits one YAML file per task. +func writeTasks(tasks []task.Task, dir string) error { + for _, t := range tasks { + raw, err := yaml.Marshal(t) + if err != nil { + return fmt.Errorf("marshal task %s: %w", t.ID, err) + } + header := "# Generated by bench/seedgen; ground truth is computed from the seeded dataset.\n# Do not edit; regenerate with `make bench-gen`.\n" + if err := writeFile(filepath.Join(dir, t.ID+".yaml"), append([]byte(header), raw...)); err != nil { + return err + } + } + return nil +} + +// writeSmokeScript emits the deterministic playback script for the scripted +// adapter. +func writeSmokeScript(tasks []task.Task, dir string) error { + raw, err := json.MarshalIndent(gen.ScriptedSmoke(tasks), "", " ") + if err != nil { + return fmt.Errorf("marshal smoke script: %w", err) + } + return writeFile(filepath.Join(dir, "scripted-smoke.json"), append(raw, '\n')) +} + +// writeFile writes content creating parent directories. Artifacts are +// world-readable (0644): they are committed repository sources (a git checkout +// materializes them 0644 regardless), and the Trino seed file is copied into a +// container whose CLI runs as a non-root user that must read it. +func writeFile(path string, content []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { // #nosec G301 -- committed source tree + return fmt.Errorf("mkdir for %s: %w", path, err) + } + if err := os.WriteFile(path, content, 0o644); err != nil { // #nosec G306 -- committed, non-secret artifacts + return fmt.Errorf("write %s: %w", path, err) + } + // WriteFile's mode applies only on creation; force it for pre-existing + // files so regeneration also repairs a wrong mode. + if err := os.Chmod(path, 0o644); err != nil { // #nosec G302 -- committed, non-secret artifacts + return fmt.Errorf("chmod %s: %w", path, err) + } + return nil +} diff --git a/bench/tasks/s1-account-created.yaml b/bench/tasks/s1-account-created.yaml new file mode 100644 index 00000000..20fd60ee --- /dev/null +++ b/bench/tasks/s1-account-created.yaml @@ -0,0 +1,14 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s1-account-created +suite: s1 +prompt: Which table in the bench warehouse would you use to look up when a customer's account was created? +arms: + - a0 + - a2 +budget_tool_calls: 30 +grading: + kind: entity + aliases: + - memory.bench.customers + - bench.customers diff --git a/bench/tasks/s1-current-orders.yaml b/bench/tasks/s1-current-orders.yaml new file mode 100644 index 00000000..1fa36247 --- /dev/null +++ b/bench/tasks/s1-current-orders.yaml @@ -0,0 +1,16 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s1-current-orders +suite: s1 +prompt: The bench warehouse contains more than one table of order data. Which one is the current, supported table for order analysis? +arms: + - a0 + - a2 +budget_tool_calls: 30 +grading: + kind: entity + aliases: + - memory.bench.orders + - bench.orders + wrong_aliases: + - legacy_orders diff --git a/bench/tasks/s1-customer-profile.yaml b/bench/tasks/s1-customer-profile.yaml new file mode 100644 index 00000000..0545c0f0 --- /dev/null +++ b/bench/tasks/s1-customer-profile.yaml @@ -0,0 +1,14 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s1-customer-profile +suite: s1 +prompt: Which table in the bench warehouse holds customer profile attributes such as tier and region? +arms: + - a0 + - a2 +budget_tool_calls: 30 +grading: + kind: entity + aliases: + - memory.bench.customers + - bench.customers diff --git a/bench/tasks/s1-daily-aggregate.yaml b/bench/tasks/s1-daily-aggregate.yaml new file mode 100644 index 00000000..77ea06a8 --- /dev/null +++ b/bench/tasks/s1-daily-aggregate.yaml @@ -0,0 +1,13 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s1-daily-aggregate +suite: s1 +prompt: Which table in the bench warehouse provides pre-aggregated daily revenue by region, so you do not have to aggregate raw orders yourself? +arms: + - a0 + - a2 +budget_tool_calls: 30 +grading: + kind: entity + aliases: + - daily_region_revenue diff --git a/bench/tasks/s1-order-transactions.yaml b/bench/tasks/s1-order-transactions.yaml new file mode 100644 index 00000000..f2ba87a2 --- /dev/null +++ b/bench/tasks/s1-order-transactions.yaml @@ -0,0 +1,16 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s1-order-transactions +suite: s1 +prompt: Which table in the bench warehouse would you query for individual customer order transactions (one row per order)? +arms: + - a0 + - a2 +budget_tool_calls: 30 +grading: + kind: entity + aliases: + - memory.bench.orders + - bench.orders + wrong_aliases: + - legacy_orders diff --git a/bench/tasks/s3-net-east-march.yaml b/bench/tasks/s3-net-east-march.yaml new file mode 100644 index 00000000..e97279c9 --- /dev/null +++ b/bench/tasks/s3-net-east-march.yaml @@ -0,0 +1,20 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s3-net-east-march +suite: s3 +prompt: Per the company revenue reporting policy, what was the revenue in USD for the East region in March 2025? Round to the nearest cent. +arms: + - a0 + - a2 +trap_classes: + - net_revenue + - units_cents +budget_tool_calls: 30 +expected_sql: SELECT ROUND(SUM(o.amount - o.discount) / 100e0, 2) AS revenue_usd FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE o.status = 'completed' AND c.region = 'East' AND o.order_ts >= TIMESTAMP '2025-03-01 00:00:00' AND o.order_ts < TIMESTAMP '2025-04-01 00:00:00' +grading: + kind: numeric + value: 38952.32 + abs_tolerance: 0.01 +rubric: + - id: caveat-policy + note: Answer should state that refunded/pending orders were excluded and discounts subtracted per policy. diff --git a/bench/tasks/s3-net-top-region.yaml b/bench/tasks/s3-net-top-region.yaml new file mode 100644 index 00000000..d0dc7839 --- /dev/null +++ b/bench/tasks/s3-net-top-region.yaml @@ -0,0 +1,23 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s3-net-top-region +suite: s3 +prompt: Per the company revenue reporting policy, which region had the highest revenue in calendar year 2025? Answer with the region name. +arms: + - a0 + - a2 +trap_classes: + - net_revenue +budget_tool_calls: 30 +expected_sql: SELECT c.region FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE o.status = 'completed' AND o.order_ts >= TIMESTAMP '2025-01-01 00:00:00' AND o.order_ts < TIMESTAMP '2026-01-01 00:00:00' GROUP BY c.region ORDER BY SUM(o.amount - o.discount) DESC LIMIT 1 +grading: + kind: entity + aliases: + - West + wrong_aliases: + - North + - South + - East +rubric: + - id: caveat-policy + note: Answer should state the ranking uses policy net revenue (completed orders, discounts subtracted). diff --git a/bench/tasks/s3-net-total-2025.yaml b/bench/tasks/s3-net-total-2025.yaml new file mode 100644 index 00000000..90d35677 --- /dev/null +++ b/bench/tasks/s3-net-total-2025.yaml @@ -0,0 +1,20 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s3-net-total-2025 +suite: s3 +prompt: Per the company revenue reporting policy, what was the company's total revenue in USD for calendar year 2025? Round to the nearest cent. +arms: + - a0 + - a2 +trap_classes: + - net_revenue + - units_cents +budget_tool_calls: 30 +expected_sql: SELECT ROUND(SUM(amount - discount) / 100e0, 2) AS revenue_usd FROM memory.bench.orders WHERE status = 'completed' AND order_ts >= TIMESTAMP '2025-01-01 00:00:00' AND order_ts < TIMESTAMP '2026-01-01 00:00:00' +grading: + kind: numeric + value: 1.28656119e+06 + abs_tolerance: 0.01 +rubric: + - id: caveat-policy + note: Answer should state that refunded/pending orders were excluded and discounts subtracted per policy. diff --git a/bench/tasks/s3-units-avg-enterprise.yaml b/bench/tasks/s3-units-avg-enterprise.yaml new file mode 100644 index 00000000..4a8229de --- /dev/null +++ b/bench/tasks/s3-units-avg-enterprise.yaml @@ -0,0 +1,19 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s3-units-avg-enterprise +suite: s3 +prompt: What is the average order amount in USD across all orders placed by enterprise-tier customers in the bench warehouse? Round to the nearest cent. +arms: + - a0 + - a2 +trap_classes: + - units_cents +budget_tool_calls: 30 +expected_sql: SELECT ROUND(AVG(o.amount) / 100e0, 2) AS avg_usd FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE c.tier = 'enterprise' +grading: + kind: numeric + value: 1614.57 + abs_tolerance: 0.01 +rubric: + - id: caveat-units + note: Answer should note that amounts are stored in cents and were converted to USD. diff --git a/bench/tasks/s3-units-q1-total.yaml b/bench/tasks/s3-units-q1-total.yaml new file mode 100644 index 00000000..47b421df --- /dev/null +++ b/bench/tasks/s3-units-q1-total.yaml @@ -0,0 +1,19 @@ +# Generated by bench/seedgen; ground truth is computed from the seeded dataset. +# Do not edit; regenerate with `make bench-gen`. +id: s3-units-q1-total +suite: s3 +prompt: Using the bench warehouse, what was the total order amount in USD across ALL orders (any status) placed in Q1 2025 (2025-01-01 through 2025-03-31)? Round to the nearest cent. +arms: + - a0 + - a2 +trap_classes: + - units_cents +budget_tool_calls: 30 +expected_sql: SELECT ROUND(SUM(amount) / 100e0, 2) AS total_usd FROM memory.bench.orders WHERE order_ts >= TIMESTAMP '2025-01-01 00:00:00' AND order_ts < TIMESTAMP '2025-04-01 00:00:00' +grading: + kind: numeric + value: 446819.28 + abs_tolerance: 0.01 +rubric: + - id: caveat-units + note: Answer should note that amounts are stored in cents and were converted to USD. diff --git a/bench/tasks/scripted-smoke.json b/bench/tasks/scripted-smoke.json new file mode 100644 index 00000000..4676d901 --- /dev/null +++ b/bench/tasks/scripted-smoke.json @@ -0,0 +1,217 @@ +{ + "s1-account-created": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Which table in the bench warehouse would you use to look up when a customer's account was created?" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: memory.bench.customers" + } + ], + "s1-current-orders": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "The bench warehouse contains more than one table of order data. Which one is the current, supported table for order analysis?" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: memory.bench.orders" + } + ], + "s1-customer-profile": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Which table in the bench warehouse holds customer profile attributes such as tier and region?" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: memory.bench.customers" + } + ], + "s1-daily-aggregate": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Which table in the bench warehouse provides pre-aggregated daily revenue by region, so you do not have to aggregate raw orders yourself?" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: daily_region_revenue" + } + ], + "s1-order-transactions": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Which table in the bench warehouse would you query for individual customer order transactions (one row per order)?" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: memory.bench.orders" + } + ], + "s3-net-east-march": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Per the company revenue reporting policy, what was the revenue in USD for the East region in March 2025? Round to the nearest cent." + } + } + ] + }, + { + "tool_calls": [ + { + "id": "", + "name": "trino_query", + "args": { + "sql": "SELECT ROUND(SUM(o.amount - o.discount) / 100e0, 2) AS revenue_usd FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE o.status = 'completed' AND c.region = 'East' AND o.order_ts \u003e= TIMESTAMP '2025-03-01 00:00:00' AND o.order_ts \u003c TIMESTAMP '2025-04-01 00:00:00'" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: {{last_result}}" + } + ], + "s3-net-top-region": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Per the company revenue reporting policy, which region had the highest revenue in calendar year 2025? Answer with the region name." + } + } + ] + }, + { + "tool_calls": [ + { + "id": "", + "name": "trino_query", + "args": { + "sql": "SELECT c.region FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE o.status = 'completed' AND o.order_ts \u003e= TIMESTAMP '2025-01-01 00:00:00' AND o.order_ts \u003c TIMESTAMP '2026-01-01 00:00:00' GROUP BY c.region ORDER BY SUM(o.amount - o.discount) DESC LIMIT 1" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: {{last_result}}" + } + ], + "s3-net-total-2025": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Per the company revenue reporting policy, what was the company's total revenue in USD for calendar year 2025? Round to the nearest cent." + } + } + ] + }, + { + "tool_calls": [ + { + "id": "", + "name": "trino_query", + "args": { + "sql": "SELECT ROUND(SUM(amount - discount) / 100e0, 2) AS revenue_usd FROM memory.bench.orders WHERE status = 'completed' AND order_ts \u003e= TIMESTAMP '2025-01-01 00:00:00' AND order_ts \u003c TIMESTAMP '2026-01-01 00:00:00'" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: {{last_result}}" + } + ], + "s3-units-avg-enterprise": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "What is the average order amount in USD across all orders placed by enterprise-tier customers in the bench warehouse? Round to the nearest cent." + } + } + ] + }, + { + "tool_calls": [ + { + "id": "", + "name": "trino_query", + "args": { + "sql": "SELECT ROUND(AVG(o.amount) / 100e0, 2) AS avg_usd FROM memory.bench.orders o JOIN memory.bench.customers c ON o.customer_id = c.customer_id WHERE c.tier = 'enterprise'" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: {{last_result}}" + } + ], + "s3-units-q1-total": [ + { + "tool_calls": [ + { + "id": "", + "name": "search", + "args": { + "intent": "Using the bench warehouse, what was the total order amount in USD across ALL orders (any status) placed in Q1 2025 (2025-01-01 through 2025-03-31)? Round to the nearest cent." + } + } + ] + }, + { + "tool_calls": [ + { + "id": "", + "name": "trino_query", + "args": { + "sql": "SELECT ROUND(SUM(amount) / 100e0, 2) AS total_usd FROM memory.bench.orders WHERE order_ts \u003e= TIMESTAMP '2025-01-01 00:00:00' AND order_ts \u003c TIMESTAMP '2025-04-01 00:00:00'" + } + } + ] + }, + { + "final_text": "FINAL ANSWER: {{last_result}}" + } + ] +} diff --git a/docs/llms-full.txt b/docs/llms-full.txt index e27e4752..3b76106b 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -1075,6 +1075,10 @@ Scenarios: `mcp-tool-call` (the primary hot path: `search` then `trino_query`), An opt-in pprof listener supports the harness: setting the `PPROF_ADDR` environment variable starts a dedicated `net/http/pprof` server on a private mux (off by default, never mounted on a client-facing listener, since it exposes process internals). Published single-replica throughput and latency ceilings, the saturation narrative (the async audit writer sheds first; CPU saturates on OAuth bcrypt, not the data path), and copy-pasteable reproduction commands are in `docs/reference/tuning-and-scaling.md` under "Measured limits". +## Agent-effectiveness benchmark + +A second measurement harness lives in `bench/` (also a separate Go module): the agent-effectiveness benchmark (issue #930). Where the load harness answers "how much" (throughput, latency, memory), the benchmark answers "how well": it holds the model, prompt scaffold, seed data, and task set constant and ablates only the platform configuration (arms as config profiles: `a0` raw toolkit tools vs `a2` the full semantic-first platform), measuring answer accuracy, pass^k reliability, and tool-call efficiency. Ground truth is generated from a fixed-seed dataset (Trino tables, DataHub metadata, knowledge pages) with the disambiguating facts for its knowledge-trap tasks (units-in-cents, gross-vs-net revenue policy) living only in the metadata and knowledge layers. Efficiency metrics are read back from the platform's own admin audit API (`audit.delivery: sync`), and a run fails loudly when a session's audit rows are incomplete. A deterministic scripted adapter validates the whole pipeline against a live stack with no model API key (`make bench-up`, `make bench-smoke`, `make bench-down`); real runs use a pinned Anthropic model. Like load testing, it is deliberately NOT part of `make verify`. See `bench/README.md`. + Environment-only configuration: `OTEL_TRACES_ENABLED` (default false), `OTEL_EXPORTER_OTLP_ENDPOINT` (OTLP/gRPC `host:port`, default `localhost:4317`), `OTEL_EXPORTER_OTLP_INSECURE` (default true — the common in-cluster topology; set false for a TLS remote collector), `OTEL_TRACES_SAMPLER_ARG` (head-based sampling ratio in [0,1] applied to root spans, default 0.1), and `OTEL_SERVICE_NAME` (service.name resource attribute, default `mcp-data-platform`). The OTLP exporter connects lazily: an unreachable or unconfigured collector never blocks or fails startup; spans are batched and dropped if undeliverable. Each tool call produces one trace. The ROOT span has the fixed, low-cardinality name `tool_call` (the specific tool is on the `mcp.tool` attribute, not the span name, so all tool calls share one queryable name) and is opened by the tracing middleware inner to auth, so it carries the request's identity. It holds the bounded attributes that mirror the metric labels (`mcp.tool`, `mcp.toolkit_kind`, `mcp.persona`, `status_category`) PLUS the high-cardinality fields deliberately kept off Prometheus labels — `mcp.user_id`, `mcp.user_email`, `mcp.session_id`, `mcp.request_id`, `mcp.connection`, `mcp.transport`, `mcp.source`, and the enrichment summary. CHILD spans nest under the root via context propagation: the cross-service `enrichment` fan-out (the original motivation for tracing — the deep, easily-misordered middleware chain), and one span per upstream call to Trino (`trino.`), DataHub (`datahub.`), and S3 (`s3.`). Those toolkit spans are emitted by the same instrumenting decorators that record the toolkit metrics, installed when EITHER metrics or tracing is enabled (the decorator's metric record is nil-safe and its span is a no-op outside an active trace, so whichever subsystem is off costs effectively nothing; when both are off the decorators are not installed at all). Span status is `Error` for any non-`ok` status_category, with the error recorded as a span event so error traces stand out. The inbound OAuth 2.1 server and the asynchronous audit write run outside a tool call's request context and so are not part of the tool-call trace. diff --git a/docs/llms.txt b/docs/llms.txt index 2b836e43..908fede0 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -91,7 +91,7 @@ mcp-data-platform is the orchestration layer for the txn2 MCP ecosystem. DataHub - [Configuration Reference](https://mcp-data-platform.txn2.com/reference/configuration/): Redirect page; the full YAML schema lives at [Configuration](https://mcp-data-platform.txn2.com/server/configuration/) - [Providers](https://mcp-data-platform.txn2.com/reference/providers/): Semantic, query, and storage provider interfaces - [Middleware](https://mcp-data-platform.txn2.com/reference/middleware/): Request processing chain: tool visibility, description overrides, the search-first gate, icon enrichment, client logging, progress notifications -- [Tuning and Scaling](https://mcp-data-platform.txn2.com/reference/tuning-and-scaling/): Resource limits, Go runtime tuning, horizontal scaling characteristics, connection pool sizing, autoscaling guidance, and measured single-replica throughput/latency limits from the load harness (test/load) +- [Tuning and Scaling](https://mcp-data-platform.txn2.com/reference/tuning-and-scaling/): Resource limits, Go runtime tuning, horizontal scaling characteristics, connection pool sizing, autoscaling guidance, and measured single-replica throughput/latency limits from the load harness (test/load); the sibling agent-effectiveness benchmark harness (bench/) measures arm-ablated accuracy and efficiency with audit-derived metrics ## Key Capabilities