From b467bfc362ca91da158865c045be2abc2239c869 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 27 Jul 2026 20:43:19 -0700 Subject: [PATCH 1/2] docs(gfql): print the benchmark numbers pyg-bench publishes, and re-verify them The owner's redirect on #1813: "should all that be there, and all we do is pretty print their latest json? and if we have any boundary needs, ensure there are data contract tests that run on the pyg-bench side, and we re-verify on our side?" This is the consuming half. Measurement, provenance and publishability live in pyg-bench, which owns the runs. This repository renders them. `docs/source/_data/gfql_benchmarks.json` is a vendored copy of pyg-bench's `published/docs-numbers.json`; `gfql_benchmarks.contract.json` is a vendored copy of the contract it satisfies. The `gfql_bench` extension resolves `:bench:` and `:bench-diag:` against the artifact and re-verifies the contract before rendering anything, so the build fails on a key that is not published, a measurement older than the policy allows, a diagnostic-only figure printed as a result, or a page that drops a number's provenance or disclosures. The re-verification is written against the contract DOCUMENT, independently of pyg-bench's implementation - a shared library would fail identically on both sides and prove nothing. `benchmark_filter_pagerank.rst` is the first page to use it. Its numbers came from a chart generator reading a results directory that has never existed in any commit, so nobody could re-derive them; pyg-bench re-ran the benchmark on dgx-spark against a named commit and published the result. Twitter: GFQL GPU 0.24s and CPU 1.58s against Neo4j + GDS 11.72s, 49.5x and 7.4x. GPlus: GFQL GPU 2.42s and CPU 32.10s, 13.3x. The two lifecycle SVGs are removed - they render the withdrawn figures as glyph paths and their generator cannot be run. GPlus publishes no Neo4j figure. That arm did not complete, and a ratio against a run that did not finish is not a speedup. `docs/test_bench_numbers.py` runs the contract checks in the ordinary test lane rather than only in the docs build, so a number going stale fails CI. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015YsqAZQLbqjSDrYSFz2GoB --- CHANGELOG.md | 2 + docs/.rstcheck.cfg | 8 +- .../_data/gfql_benchmarks.contract.json | 102 + docs/source/_data/gfql_benchmarks.json | 1618 +++++++++++++ docs/source/_ext/gfql_bench.py | 451 ++++ docs/source/conf.py | 5 + .../filter_pagerank/gplus_lifecycle.svg | 2103 ----------------- .../filter_pagerank/twitter_lifecycle.svg | 1991 ---------------- .../source/gfql/benchmark_filter_pagerank.rst | 89 +- docs/test_bench_numbers.py | 191 ++ 10 files changed, 2398 insertions(+), 4162 deletions(-) create mode 100644 docs/source/_data/gfql_benchmarks.contract.json create mode 100644 docs/source/_data/gfql_benchmarks.json create mode 100644 docs/source/_ext/gfql_bench.py delete mode 100644 docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg delete mode 100644 docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg create mode 100644 docs/test_bench_numbers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ac88700713..6b963819ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **Seeded property-RETURN dtype divergence on cuDF**: the lean projection applied the pandas rows-pivot artifact (int → float64, bool → object) on every engine, but cuDF's canonical pivot preserves the source dtypes — so the fast path returned `float64`/`object` where cuDF's own canonical path returns `int64`/`bool`. The cast rule is now engine-aware; the dtype-class decline guard is unchanged. ### Documentation +- **The filter -> PageRank -> filter benchmark is measured again, and the docs print it from data.** `gfql/benchmark_filter_pagerank.rst` published Twitter and GPlus figures whose charts were generated from a results directory that has never existed in any commit, so nobody could re-derive them. The benchmark now lives in `graphistry/pyg-bench`, which re-ran it on dgx-spark under the shared perf lock against a named PyGraphistry commit and committed both the raw artifacts and the published document. This page renders that document instead of restating it: Twitter GFQL GPU 0.24 s and CPU 1.58 s against Neo4j + GDS 11.72 s (49.5x and 7.4x), GPlus GFQL GPU 2.42 s and CPU 32.10 s (13.3x). Every arm reproduces the earlier run's final subgraphs exactly. +- **Benchmark numbers are now referenced, not retyped.** `docs/source/_data/gfql_benchmarks.json` is a vendored copy of pyg-bench's published artifact and `gfql_benchmarks.contract.json` a vendored copy of the contract it satisfies. The new `gfql_bench` Sphinx extension resolves `:bench:` and `:bench-diag:` against it and re-verifies the contract independently before rendering anything, so the docs build fails on a key that is not published, a measurement older than the policy allows, a diagnostic-only figure printed as a result, or a page that drops a number's provenance or disclosures. `docs/test_bench_numbers.py` runs the same checks in the ordinary test lane, so a number going stale fails CI rather than a nightly. - **GFQL pay-as-you-go resident indexing user guide**: New :doc:`Pay-As-You-Go Resident Indexing ` page — the lifecycle guide to resident indexes (`gfql_index_all()` / `gfql_index_edges()` / `create_index()` / `show_indexes()` / `drop_index()`): what the node-id + CSR in/out adjacency sidecars are, what engages them on 0.58.0 (seeded typed-hop fast paths incl. property RETURNs and property-seeded lookups per #1768/#1770, direct `g.hop()`; the general polars chain traversal honestly noted as not yet covered), the staleness/validity contract (identity + fingerprint; rebind invalidates; declines are safe — identical results either way), engine notes (polars needs `gfql_index_all(engine='polars')` until #1767), 0.58.0-tag measured numbers, and a runnable end-to-end example. Wired into the GFQL toctree + recommended paths alongside :doc:`Seeded Traversal Indexes `. - **GFQL performance docs: 0.58.0 release-tag-verified numbers, siloed in one page**: `gfql/performance.rst` is now the canonical benchmark-numbers page (alongside `gfql/index_adjacency.rst` for the index benchmarks) — a benchmark rerun updates it alone. It carries the 0.58.0 tag sweep (DGX Spark GB10, warm medians N=30; four-engine numbers cross-engine parity-verified, competitor pairs validated against expected result rows): seeded typed-hop fast path across all four engines (e.g. pandas 29.9→2.46ms, 12.1×), native chain form, resident-index covered-shape lookups (with the `gfql_index_all(engine='polars')` caveat / PR #1767), flat seeded-hop scaling on pandas (0.159–0.164ms from 0.25M to 32M edges), the one-keyword `engine='polars'` LDBC SNB SF1 seed-lookup win (1,299.6→106.1ms, 12.3×), LDBC SNB interactive SF1 vs Neo4j 5.26 same-box pairs (GFQL 4 of 5; Neo4j wins recent-replies — reported as-is), OLAP multi-join vs embedded Kuzu (q8 200×, q9 14.2×) with the honest inverse (Kuzu wins single-table aggregates 2–4×, seeded property-projection lookups 2.4–64×), plus the prior Orkut/LiveJournal bulk sweep (moved from `engines.rst`, dated once) and its methodology. All other pages — `engines.rst`, `quick.rst`, `about.rst`, `overview.rst`, `index.rst` — now carry stable qualitative claims (e.g. "often an order of magnitude faster on query-heavy workloads") that link into `performance.rst` instead of inline figures, replacing the stale "up to ~38×" headlines and avoiding scattered per-claim version labels. diff --git a/docs/.rstcheck.cfg b/docs/.rstcheck.cfg index 90a6d5d857..322035f4f4 100644 --- a/docs/.rstcheck.cfg +++ b/docs/.rstcheck.cfg @@ -20,7 +20,9 @@ ignore_roles = py:attr, py:exc, py:obj, - py:data + py:data, + bench, + bench-diag # Ignore Sphinx-specific directives ignore_directives = @@ -43,7 +45,9 @@ ignore_directives = glossary, productionlist, include, - graphviz + graphviz, + bench-provenance, + bench-disclosures # Ignore common informational messages ignore_messages = (Hyperlink target "[^"]*" is not referenced\.$) diff --git a/docs/source/_data/gfql_benchmarks.contract.json b/docs/source/_data/gfql_benchmarks.contract.json new file mode 100644 index 0000000000..eceb384ad9 --- /dev/null +++ b/docs/source/_data/gfql_benchmarks.contract.json @@ -0,0 +1,102 @@ +{ + "contract_version": 1, + "title": "pyg-bench published docs numbers", + "summary": [ + "The boundary between this repository and the pygraphistry docs. pyg-bench MEASURES and", + "PUBLISHES; pygraphistry pretty-prints. Everything a consumer is allowed to assume about", + "published/docs-numbers.json is written down here, checked on the producer side before the", + "artifact is written, and re-verified independently on the consumer side before it renders.", + "Two independent implementations of these rules is the point: a shared library would fail", + "the same way on both sides of the boundary." + ], + "artifact_path": "published/docs-numbers.json", + "top_level_required": [ + "contract_version", + "schema_version", + "generated_by", + "generated_at", + "policy", + "runs", + "cells" + ], + "policy_required": ["max_age_days", "managed_docs"], + "run_required": [ + "measured_at", + "host", + "perf_lock_held", + "quiet_host", + "reps", + "pygraphistry_commit", + "pyg_bench_commit", + "runtime", + "dataset", + "artifact", + "row_validation" + ], + "cell_required": [ + "run", + "workload", + "engine", + "value", + "unit", + "decimals", + "status", + "comparison_allowed", + "board_quotable", + "disclosures" + ], + "cell_status": ["ok", "partial", "adapter_workaround", "result_mismatch"], + "cell_unit": ["ms", "s", "x"], + "key_pattern": "^[a-z0-9][a-z0-9_]*(\\.[a-z0-9][a-z0-9_]*)+$", + "date_pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "max_decimals": 4, + "invariants": [ + { + "id": "cell_names_a_known_run", + "rule": "every cell.run must be a key of runs", + "why": "a number whose provenance record is absent is an unprovenanced number" + }, + { + "id": "quotable_implies_comparable", + "rule": "board_quotable implies comparison_allowed", + "why": "a diagnostic-only figure must never reach a board" + }, + { + "id": "quotable_implies_ok", + "rule": "board_quotable implies status == 'ok'", + "why": "partial / adapter_workaround / result_mismatch cells are not results" + }, + { + "id": "caveat_travels_with_the_number", + "rule": "status != 'ok' or comparison_allowed is false implies disclosures is non-empty", + "why": "a bare ratio without its asterisk launders the caveat" + }, + { + "id": "ratios_are_comparisons", + "rule": "unit 'x' implies comparison_allowed", + "why": "a ratio between two numbers that were never established as comparable is not a speedup" + }, + { + "id": "value_is_rounded_as_published", + "rule": "value == round(value, decimals), value is finite and >= 0, 0 <= decimals <= max_decimals", + "why": "the consumer prints value verbatim; rounding must not happen at render time" + }, + { + "id": "engine_is_named", + "rule": "cell.engine is a non-empty string", + "why": "pandas and polars are not interchangeable and the reader must be told which one ran" + }, + { + "id": "measurement_is_dated", + "rule": "run.measured_at matches date_pattern", + "why": "staleness is checked against the MEASUREMENT date, not the export date" + } + ], + "consumer_obligations": [ + "Refuse to render a key the artifact does not contain — a missing number is a build failure, never blank text.", + "Refuse to render a cell whose run.measured_at is older than policy.max_age_days.", + "Refuse to render a non-board_quotable cell as a bare number; it may appear only through a role that labels it diagnostic-only.", + "Render every referenced cell's disclosures and its run's provenance on the same page.", + "Re-verify every invariant above before rendering, with an implementation that does not import this repository." + ] +} diff --git a/docs/source/_data/gfql_benchmarks.json b/docs/source/_data/gfql_benchmarks.json new file mode 100644 index 0000000000..e86ba1bd7b --- /dev/null +++ b/docs/source/_data/gfql_benchmarks.json @@ -0,0 +1,1618 @@ +{ + "cells": { + "graphbench.100k.q1.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 153.16, + "workload": "graph-benchmark 100k q1" + }, + "graphbench.100k.q1.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 199.43, + "workload": "graph-benchmark 100k q1" + }, + "graphbench.100k.q1.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.77, + "workload": "graph-benchmark 100k q1 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q1.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 42.14, + "workload": "graph-benchmark 100k q1" + }, + "graphbench.100k.q1.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 3.63, + "workload": "graph-benchmark 100k q1 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q1.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 4.7, + "workload": "graph-benchmark 100k q1 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q2.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 281.91, + "workload": "graph-benchmark 100k q2" + }, + "graphbench.100k.q2.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 208.88, + "workload": "graph-benchmark 100k q2" + }, + "graphbench.100k.q2.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.35, + "workload": "graph-benchmark 100k q2 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q2.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 55.76, + "workload": "graph-benchmark 100k q2" + }, + "graphbench.100k.q2.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 5.06, + "workload": "graph-benchmark 100k q2 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q2.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 3.7, + "workload": "graph-benchmark 100k q2 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q3.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 28.76, + "workload": "graph-benchmark 100k q3" + }, + "graphbench.100k.q3.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 73.72, + "workload": "graph-benchmark 100k q3" + }, + "graphbench.100k.q3.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.39, + "workload": "graph-benchmark 100k q3 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q3.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 14.74, + "workload": "graph-benchmark 100k q3" + }, + "graphbench.100k.q3.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.95, + "workload": "graph-benchmark 100k q3 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q3.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 5.0, + "workload": "graph-benchmark 100k q3 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q4.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 11.56, + "workload": "graph-benchmark 100k q4" + }, + "graphbench.100k.q4.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 64.09, + "workload": "graph-benchmark 100k q4" + }, + "graphbench.100k.q4.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.18, + "workload": "graph-benchmark 100k q4 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q4.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 12.13, + "workload": "graph-benchmark 100k q4" + }, + "graphbench.100k.q4.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.95, + "workload": "graph-benchmark 100k q4 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q4.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 5.3, + "workload": "graph-benchmark 100k q4 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q5.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 12.88, + "workload": "graph-benchmark 100k q5" + }, + "graphbench.100k.q5.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 410.05, + "workload": "graph-benchmark 100k q5" + }, + "graphbench.100k.q5.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.03, + "workload": "graph-benchmark 100k q5 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q5.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 17.52, + "workload": "graph-benchmark 100k q5" + }, + "graphbench.100k.q5.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.74, + "workload": "graph-benchmark 100k q5 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q5.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 23.4, + "workload": "graph-benchmark 100k q5 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q6.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 20.73, + "workload": "graph-benchmark 100k q6" + }, + "graphbench.100k.q6.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 412.3, + "workload": "graph-benchmark 100k q6" + }, + "graphbench.100k.q6.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.05, + "workload": "graph-benchmark 100k q6 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q6.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 21.99, + "workload": "graph-benchmark 100k q6" + }, + "graphbench.100k.q6.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.94, + "workload": "graph-benchmark 100k q6 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q6.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 18.8, + "workload": "graph-benchmark 100k q6 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q7.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 8.85, + "workload": "graph-benchmark 100k q7" + }, + "graphbench.100k.q7.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 119.63, + "workload": "graph-benchmark 100k q7" + }, + "graphbench.100k.q7.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.07, + "workload": "graph-benchmark 100k q7 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q7.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 16.12, + "workload": "graph-benchmark 100k q7" + }, + "graphbench.100k.q7.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.55, + "workload": "graph-benchmark 100k q7 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q7.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 7.4, + "workload": "graph-benchmark 100k q7 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q8.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 9.76, + "workload": "graph-benchmark 100k q8" + }, + "graphbench.100k.q8.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 91.6, + "workload": "graph-benchmark 100k q8" + }, + "graphbench.100k.q8.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.11, + "workload": "graph-benchmark 100k q8 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q8.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.06, + "workload": "graph-benchmark 100k q8" + }, + "graphbench.100k.q8.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.93, + "workload": "graph-benchmark 100k q8 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q8.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 18.1, + "workload": "graph-benchmark 100k q8 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.100k.q9.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 84.06, + "workload": "graph-benchmark 100k q9" + }, + "graphbench.100k.q9.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 212.0, + "workload": "graph-benchmark 100k q9" + }, + "graphbench.100k.q9.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 0.4, + "workload": "graph-benchmark 100k q9 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.100k.q9.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "ms", + "value": 66.61, + "workload": "graph-benchmark 100k q9" + }, + "graphbench.100k.q9.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 1.26, + "workload": "graph-benchmark 100k q9 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.100k.q9.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-100k-20260726", + "status": "ok", + "unit": "x", + "value": 3.2, + "workload": "graph-benchmark 100k q9 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q1.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 14.51, + "workload": "graph-benchmark 20k q1" + }, + "graphbench.20k.q1.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 35.46, + "workload": "graph-benchmark 20k q1" + }, + "graphbench.20k.q1.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.41, + "workload": "graph-benchmark 20k q1 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q1.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 13.13, + "workload": "graph-benchmark 20k q1" + }, + "graphbench.20k.q1.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.11, + "workload": "graph-benchmark 20k q1 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q1.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.7, + "workload": "graph-benchmark 20k q1 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q2.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 31.62, + "workload": "graph-benchmark 20k q2" + }, + "graphbench.20k.q2.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 38.05, + "workload": "graph-benchmark 20k q2" + }, + "graphbench.20k.q2.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.83, + "workload": "graph-benchmark 20k q2 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q2.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 17.59, + "workload": "graph-benchmark 20k q2" + }, + "graphbench.20k.q2.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.8, + "workload": "graph-benchmark 20k q2 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q2.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.2, + "workload": "graph-benchmark 20k q2 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q3.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.94, + "workload": "graph-benchmark 20k q3" + }, + "graphbench.20k.q3.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 11.68, + "workload": "graph-benchmark 20k q3" + }, + "graphbench.20k.q3.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.51, + "workload": "graph-benchmark 20k q3 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q3.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 7.66, + "workload": "graph-benchmark 20k q3" + }, + "graphbench.20k.q3.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.78, + "workload": "graph-benchmark 20k q3 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q3.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.5, + "workload": "graph-benchmark 20k q3 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q4.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 3.42, + "workload": "graph-benchmark 20k q4" + }, + "graphbench.20k.q4.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 9.45, + "workload": "graph-benchmark 20k q4" + }, + "graphbench.20k.q4.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.36, + "workload": "graph-benchmark 20k q4 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q4.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 6.04, + "workload": "graph-benchmark 20k q4" + }, + "graphbench.20k.q4.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.57, + "workload": "graph-benchmark 20k q4 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q4.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.6, + "workload": "graph-benchmark 20k q4 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q5.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.75, + "workload": "graph-benchmark 20k q5" + }, + "graphbench.20k.q5.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 81.03, + "workload": "graph-benchmark 20k q5" + }, + "graphbench.20k.q5.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.07, + "workload": "graph-benchmark 20k q5 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q5.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 8.9, + "workload": "graph-benchmark 20k q5" + }, + "graphbench.20k.q5.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.65, + "workload": "graph-benchmark 20k q5 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q5.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 9.1, + "workload": "graph-benchmark 20k q5 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q6.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 8.23, + "workload": "graph-benchmark 20k q6" + }, + "graphbench.20k.q6.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 81.25, + "workload": "graph-benchmark 20k q6" + }, + "graphbench.20k.q6.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.1, + "workload": "graph-benchmark 20k q6 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q6.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.43, + "workload": "graph-benchmark 20k q6" + }, + "graphbench.20k.q6.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.79, + "workload": "graph-benchmark 20k q6 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q6.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 7.8, + "workload": "graph-benchmark 20k q6 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q7.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 5.19, + "workload": "graph-benchmark 20k q7" + }, + "graphbench.20k.q7.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 19.97, + "workload": "graph-benchmark 20k q7" + }, + "graphbench.20k.q7.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.26, + "workload": "graph-benchmark 20k q7 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q7.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.16, + "workload": "graph-benchmark 20k q7" + }, + "graphbench.20k.q7.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.51, + "workload": "graph-benchmark 20k q7 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q7.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.0, + "workload": "graph-benchmark 20k q7 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q8.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 2.52, + "workload": "graph-benchmark 20k q8" + }, + "graphbench.20k.q8.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.46, + "workload": "graph-benchmark 20k q8" + }, + "graphbench.20k.q8.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.24, + "workload": "graph-benchmark 20k q8 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q8.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 2.07, + "workload": "graph-benchmark 20k q8" + }, + "graphbench.20k.q8.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 1.22, + "workload": "graph-benchmark 20k q8 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q8.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 5.1, + "workload": "graph-benchmark 20k q8 \u2014 GFQL-polars vs GFQL-pandas" + }, + "graphbench.20k.q9.kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "kuzu", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 10.85, + "workload": "graph-benchmark 20k q9" + }, + "graphbench.20k.q9.pandas": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 28.29, + "workload": "graph-benchmark 20k q9" + }, + "graphbench.20k.q9.pandas_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "pandas", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.38, + "workload": "graph-benchmark 20k q9 \u2014 GFQL-pandas vs kuzu" + }, + "graphbench.20k.q9.polars": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "ms", + "value": 13.83, + "workload": "graph-benchmark 20k q9" + }, + "graphbench.20k.q9.polars_vs_kuzu": { + "board_quotable": true, + "comparison_allowed": true, + "competitor": "kuzu", + "decimals": 2, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 0.78, + "workload": "graph-benchmark 20k q9 \u2014 GFQL-polars vs kuzu" + }, + "graphbench.20k.q9.polars_vs_pandas": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "polars", + "run": "graphbench-q1q9-20k-20260726", + "status": "ok", + "unit": "x", + "value": 2.0, + "workload": "graph-benchmark 20k q9 \u2014 GFQL-polars vs GFQL-pandas" + }, + "pagerank.gplus.gfql_cpu": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 2, + "disclosures": [], + "engine": "GFQL CPU (pandas + igraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "s", + "value": 32.1, + "workload": "filter -> PageRank -> filter, gplus \u2014 GFQL CPU (pandas + igraph)" + }, + "pagerank.gplus.gfql_gpu": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 2, + "disclosures": [], + "engine": "GFQL GPU (cuDF + cuGraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "s", + "value": 2.42, + "workload": "filter -> PageRank -> filter, gplus \u2014 GFQL GPU (cuDF + cuGraph)" + }, + "pagerank.gplus.gfql_gpu_vs_gfql_cpu": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "GFQL GPU (cuDF + cuGraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "x", + "value": 13.3, + "workload": "filter -> PageRank -> filter, gplus \u2014 GFQL GPU (cuDF + cuGraph) vs GFQL CPU (pandas + igraph)" + }, + "pagerank.twitter.gfql_cpu": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 2, + "disclosures": [], + "engine": "GFQL CPU (pandas + igraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "s", + "value": 1.58, + "workload": "filter -> PageRank -> filter, twitter \u2014 GFQL CPU (pandas + igraph)" + }, + "pagerank.twitter.gfql_cpu_vs_neo4j_gds": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [ + "In the Neo4j arm each filter stage writes a marker property onto every matched node and relationship, 32 seeds per batch, and GDS scores an explicitly symmetrised projection; the GFQL arms materialise a subgraph and write nothing. The figure is a pipeline time, not an engine-primitive time." + ], + "engine": "GFQL CPU (pandas + igraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "x", + "value": 7.4, + "workload": "filter -> PageRank -> filter, twitter \u2014 GFQL CPU (pandas + igraph) vs Neo4j + GDS" + }, + "pagerank.twitter.gfql_gpu": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 2, + "disclosures": [], + "engine": "GFQL GPU (cuDF + cuGraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "s", + "value": 0.24, + "workload": "filter -> PageRank -> filter, twitter \u2014 GFQL GPU (cuDF + cuGraph)" + }, + "pagerank.twitter.gfql_gpu_vs_gfql_cpu": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [], + "engine": "GFQL GPU (cuDF + cuGraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "x", + "value": 6.7, + "workload": "filter -> PageRank -> filter, twitter \u2014 GFQL GPU (cuDF + cuGraph) vs GFQL CPU (pandas + igraph)" + }, + "pagerank.twitter.gfql_gpu_vs_neo4j_gds": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 1, + "disclosures": [ + "In the Neo4j arm each filter stage writes a marker property onto every matched node and relationship, 32 seeds per batch, and GDS scores an explicitly symmetrised projection; the GFQL arms materialise a subgraph and write nothing. The figure is a pipeline time, not an engine-primitive time." + ], + "engine": "GFQL GPU (cuDF + cuGraph)", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "x", + "value": 49.5, + "workload": "filter -> PageRank -> filter, twitter \u2014 GFQL GPU (cuDF + cuGraph) vs Neo4j + GDS" + }, + "pagerank.twitter.neo4j_gds": { + "board_quotable": true, + "comparison_allowed": true, + "decimals": 2, + "disclosures": [ + "In the Neo4j arm each filter stage writes a marker property onto every matched node and relationship, 32 seeds per batch, and GDS scores an explicitly symmetrised projection; the GFQL arms materialise a subgraph and write nothing. The figure is a pipeline time, not an engine-primitive time." + ], + "engine": "Neo4j + GDS", + "run": "filter-pagerank-20260728", + "status": "ok", + "unit": "s", + "value": 11.72, + "workload": "filter -> PageRank -> filter, twitter \u2014 Neo4j + GDS" + }, + "snb.sf1.is3.single_query.polars": { + "board_quotable": false, + "comparison_allowed": false, + "decimals": 2, + "disclosures": [ + "The A/B configuration this was measured under declares comparison_allowed: false, so it is a diagnostic against the previous GFQL form only, never a competitor result." + ], + "engine": "polars", + "run": "snb-is3-single-query-20260727", + "status": "ok", + "unit": "ms", + "value": 18.05, + "workload": "LDBC SNB IS3 (SF1), single-query form" + }, + "snb.sf1.is3.two_pass.polars": { + "board_quotable": false, + "comparison_allowed": false, + "decimals": 2, + "disclosures": [ + "Answered by a two-pass adapter workaround with a Python-side join inside the timed region; superseded, kept only as the baseline for the single-query improvement." + ], + "engine": "polars", + "run": "snb-is3-single-query-20260727", + "status": "partial", + "unit": "ms", + "value": 75.22, + "workload": "LDBC SNB IS3 (SF1), superseded two-pass adapter form" + } + }, + "contract_version": 1, + "generated_at": "2026-07-28T03:29:05Z", + "generated_by": "graphistry/pyg-bench bdc2828a scripts/export_docs_numbers.py", + "policy": { + "literal_allowlist": {}, + "managed_docs": [ + "gfql/performance.rst", + "gfql/index_adjacency.rst", + "gfql/indexing.rst", + "gfql/engines.rst", + "gfql/benchmark_filter_pagerank.rst", + "gfql/index.rst", + "gfql/about.rst", + "gfql/overview.rst", + "gfql/quick.rst", + "gfql/builtin_calls.rst", + "cheatsheet.md", + "10min.rst", + "performance.rst" + ], + "max_age_days": 60, + "max_compute_commit_drift": 12 + }, + "runs": { + "filter-pagerank-20260728": { + "artifact": "results/filter-pagerank-20260728", + "competitor": "Neo4j + GDS", + "competitor_version": "neo4j:2026.02.2 with the graph-data-science plugin", + "dataset": "SNAP twitter_combined (81,306 nodes / 2,420,766 edges) and gplus_combined (107,614 nodes / 30,494,866 edges); sha256 of each source file is recorded in the arm artifacts", + "host": "dgx-spark (NVIDIA GB10, driver 580.126.09), 20 CPU", + "measured_at": "2026-07-28", + "perf_lock_held": true, + "pyg_bench_commit": "85c92022 plus benchmarks/filter_pagerank as added in this commit", + "pygraphistry_commit": "49db91cc", + "quiet_host": true, + "reps": "graph loaded once, then 2 warmups + 5 timed runs per arm on the resident graph; median", + "row_validation": "Every arm records the node id set its pipeline selected, captured outside the timed region. Comparability is the Jaccard index of those sets against a 0.95 threshold declared before the run: Twitter CPU/GPU 0.991, CPU/Neo4j 0.974, GPU/Neo4j 0.972; GPlus CPU/GPU 0.951.", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars with python-igraph 1.0.0; cuDF 26.2.1, cuGraph 26.2.0, pandas 2.3.3; Neo4j 2026.02.2 + graph-data-science in Docker on the same host" + }, + "graphbench-q1q9-100k-20260726": { + "artifact": "results/graphbench-matched-q1q9-20260726/gb100k", + "competitor": "Kuzu", + "competitor_version": "0.11.3 (embedded, CPU)", + "dataset": "prrao87 graph-benchmark, 100,000 persons / 2,417,738 FOLLOWS / 2,775,195 edges", + "host": "dgx-spark (NVIDIA GB10), 20 CPU", + "measured_at": "2026-07-26", + "perf_lock_held": true, + "pyg_bench_commit": "47f94ba", + "pygraphistry_commit": "84be35fb", + "quiet_host": true, + "reps": "2 warmups + 5 timed runs per slot; slot median, then median across 4 position-balanced slots (K G G K K G G K)", + "row_validation": "every cell emits a canonical, column-order-independent row set; a GFQL cell is only publishable when its rows equal Kuzu's on every slot", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars, polars 1.35.2; Kuzu in a host venv on CPU" + }, + "graphbench-q1q9-20k-20260726": { + "artifact": "results/graphbench-matched-q1q9-20260726/gb20k", + "competitor": "Kuzu", + "competitor_version": "0.11.3 (embedded, CPU)", + "dataset": "prrao87 graph-benchmark, 20,000 persons / 199,939 FOLLOWS / 260,107 edges", + "host": "dgx-spark (NVIDIA GB10), 20 CPU", + "measured_at": "2026-07-26", + "perf_lock_held": true, + "pyg_bench_commit": "47f94ba", + "pygraphistry_commit": "84be35fb", + "quiet_host": true, + "reps": "3 warmups + 7 timed runs per slot; slot median, then median across 4 position-balanced slots (K G I I G K K G I I G K)", + "row_validation": "every cell emits a canonical, column-order-independent row set; a GFQL cell is only publishable when its rows equal Kuzu's on every slot", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars, polars 1.35.2; Kuzu in a host venv on CPU" + }, + "snb-is3-single-query-20260727": { + "artifact": "results/is3-single-query-r{1,2,3} and results/is3-twopass-base-r{1,2,3}", + "competitor": "Neo4j", + "competitor_version": "not re-paired in this configuration", + "dataset": "LDBC SNB Interactive SF1, official format", + "host": "dgx-spark (NVIDIA GB10), 20 CPU", + "measured_at": "2026-07-27", + "perf_lock_held": true, + "pyg_bench_commit": "64864b8", + "pygraphistry_commit": "233b64c8", + "quiet_host": true, + "reps": "3 repetitions of configs/suites/ab-is1-is7-sf1-polars.yaml, each already a median of 5 measured repeats after 1 warmup; median across repetitions", + "row_validation": "920/920 rows, value-identical to the independently derived expected fixture in all 3 repetitions", + "runtime": "graphistry/test-rapids-official:26.02-gfql-polars, polars 1.35.2" + } + }, + "schema_version": 1 +} diff --git a/docs/source/_ext/gfql_bench.py b/docs/source/_ext/gfql_bench.py new file mode 100644 index 0000000000..9d007ad652 --- /dev/null +++ b/docs/source/_ext/gfql_bench.py @@ -0,0 +1,451 @@ +"""Sphinx extension: pretty-print the benchmark numbers pyg-bench publishes. + +Measurement, provenance and publishability all live in `graphistry/pyg-bench`, which +owns the runs. This repository does one thing with them: it renders them. The two files +under ``docs/source/_data`` are vendored copies of that repository's +``published/docs-numbers.json`` and ``manifests/docs-numbers.contract.json``. + +Docs never restate a measured number, they reference one:: + + * - Twitter, GPU + - :bench:`pagerank.twitter.gfql_gpu` + +and the page that references a cell must also render that cell's provenance and its +disclosures:: + + .. bench-provenance:: filter-pagerank-20260728 + .. bench-disclosures:: + +The contract names five obligations for a consumer. They are build failures here, never +silent text: + +- a key the artifact does not contain -> build fails +- a run measured longer ago than ``max_age_days`` -> build fails +- a non-quotable cell printed as a bare number -> build fails (``:bench-diag:`` labels it) +- a page that drops a referenced cell's provenance or disclosures -> build fails +- a payload that breaks the contract at all -> build fails, before anything renders + +The re-verification below is written against the contract *document*, independently of +pyg-bench's own implementation. A shared library would fail identically on both sides of +the boundary and prove nothing. +""" + +from __future__ import annotations + +import datetime +import json +import math +import os +import re +from typing import Dict, List, Optional, Tuple, Union + +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.parsers.rst.states import Inliner +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment +from sphinx.errors import SphinxError +from sphinx.util import logging as sphinx_logging + +logger = sphinx_logging.getLogger(__name__) + +RoleResult = Tuple[List[nodes.Node], List[nodes.system_message]] + +#: A decoded JSON document. +JSONValue = Union[ + None, bool, int, float, str, List["JSONValue"], Dict[str, "JSONValue"] +] +JSONObject = Dict[str, JSONValue] + +#: ``_ext`` and ``_data`` are siblings under ``docs/source``. +DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '_data') +BENCHMARKS_JSON = os.path.join(DATA_DIR, 'gfql_benchmarks.json') +CONTRACT_JSON = os.path.join(DATA_DIR, 'gfql_benchmarks.contract.json') + + +class BenchNumberError(SphinxError): + category = 'GFQL benchmark number check failed' + + +def _obj(value: JSONValue, where: str) -> JSONObject: + if not isinstance(value, dict): + raise BenchNumberError('{}: expected an object'.format(where)) + return value + + +def _strings(value: JSONValue, where: str) -> List[str]: + if not isinstance(value, list): + raise BenchNumberError('{}: expected an array of strings'.format(where)) + out = [] # type: List[str] + for item in value: + if not isinstance(item, str) or not item: + raise BenchNumberError('{}: entries must be non-empty strings'.format(where)) + out.append(item) + return out + + +def _load(path: str) -> JSONObject: + if not os.path.exists(path): + raise BenchNumberError( + '{} is missing. It is a vendored copy of pyg-bench ' + 'published/docs-numbers.json.'.format(path)) + with open(path, encoding='utf-8') as handle: + return _obj(json.load(handle), path) + + +def _reverify(payload: JSONObject, contract: JSONObject) -> None: + """Check the artifact against the contract document, before anything renders.""" + problems = [] # type: List[str] + + expected_version = contract.get('contract_version') + if payload.get('contract_version') != expected_version: + problems.append( + 'contract_version {!r} but this repository vendors contract {!r}'.format( + payload.get('contract_version'), expected_version)) + + for field in _strings(contract.get('top_level_required'), 'contract.top_level_required'): + if field not in payload: + problems.append('artifact is missing {!r}'.format(field)) + + policy = payload.get('policy') + if not isinstance(policy, dict): + problems.append('policy: expected an object') + policy = {} + for field in _strings(contract.get('policy_required'), 'contract.policy_required'): + if field not in policy: + problems.append('policy is missing {!r}'.format(field)) + + run_required = _strings(contract.get('run_required'), 'contract.run_required') + cell_required = _strings(contract.get('cell_required'), 'contract.cell_required') + statuses = _strings(contract.get('cell_status'), 'contract.cell_status') + units = _strings(contract.get('cell_unit'), 'contract.cell_unit') + key_re = re.compile(str(contract.get('key_pattern'))) + date_re = re.compile(str(contract.get('date_pattern'))) + max_decimals = contract.get('max_decimals') + if not isinstance(max_decimals, int) or isinstance(max_decimals, bool): + raise BenchNumberError('contract.max_decimals must be an integer') + + runs = payload.get('runs') + if not isinstance(runs, dict): + problems.append('runs: expected an object') + runs = {} + for run_id in sorted(runs): + run = runs[run_id] + if not isinstance(run, dict): + problems.append('runs.{}: expected an object'.format(run_id)) + continue + for field in run_required: + if field not in run: + problems.append('runs.{}: missing provenance {!r}'.format(run_id, field)) + measured_at = run.get('measured_at') + if not isinstance(measured_at, str) or not date_re.match(measured_at): + problems.append('runs.{}: measured_at is not a date'.format(run_id)) + + cells = payload.get('cells') + if not isinstance(cells, dict): + problems.append('cells: expected an object') + cells = {} + if not cells: + problems.append('cells: the artifact publishes nothing') + for key in sorted(cells): + cell = cells[key] + if not isinstance(cell, dict): + problems.append('cells.{}: expected an object'.format(key)) + continue + if not key_re.match(key): + problems.append('cells.{}: key does not match the published pattern'.format(key)) + missing = [field for field in cell_required if field not in cell] + if missing: + problems.append('cells.{}: missing {}'.format(key, ', '.join(sorted(missing)))) + continue + if cell['run'] not in runs: + problems.append('cells.{}: run {!r} has no provenance'.format(key, cell['run'])) + if cell['status'] not in statuses: + problems.append('cells.{}: unknown status {!r}'.format(key, cell['status'])) + if cell['unit'] not in units: + problems.append('cells.{}: unknown unit {!r}'.format(key, cell['unit'])) + if not isinstance(cell['engine'], str) or not cell['engine']: + problems.append('cells.{}: engine is not named'.format(key)) + quotable = cell['board_quotable'] + comparable = cell['comparison_allowed'] + if not isinstance(quotable, bool) or not isinstance(comparable, bool): + problems.append('cells.{}: quotability flags must be booleans'.format(key)) + continue + disclosures = cell['disclosures'] + if not isinstance(disclosures, list): + problems.append('cells.{}: disclosures must be an array'.format(key)) + disclosures = [] + if quotable and not comparable: + problems.append('cells.{}: board_quotable but not comparison_allowed'.format(key)) + if quotable and cell['status'] != 'ok': + problems.append('cells.{}: board_quotable with status {!r}'.format(key, cell['status'])) + if (cell['status'] != 'ok' or not comparable) and not disclosures: + problems.append('cells.{}: caveated but carries no disclosure'.format(key)) + if cell['unit'] == 'x' and not comparable: + problems.append('cells.{}: a ratio over figures never established as comparable'.format(key)) + value = cell['value'] + decimals = cell['decimals'] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)): + problems.append('cells.{}: value is not a finite number'.format(key)) + continue + if not isinstance(decimals, int) or isinstance(decimals, bool) or not 0 <= decimals <= max_decimals: + problems.append('cells.{}: decimals out of range'.format(key)) + continue + if round(float(value), decimals) != float(value): + problems.append('cells.{}: value is not rounded to its own decimals'.format(key)) + + if problems: + raise BenchNumberError( + '{} breaks the data contract vendored at {}:\n {}'.format( + BENCHMARKS_JSON, CONTRACT_JSON, '\n '.join(problems))) + + +class _State: + """Loaded once per build: the data, the clock, and the problems found.""" + + def __init__(self, payload: JSONObject, today: datetime.date) -> None: + self.payload = payload + self.today = today + self.problems = [] # type: List[str] + self.refs = {} # type: Dict[str, List[str]] + self.provenance = {} # type: Dict[str, List[str]] + self.disclosed = [] # type: List[str] + + policy = _obj(payload.get('policy'), 'policy') + max_age = policy.get('max_age_days') + self.max_age_days = max_age if isinstance(max_age, int) and not isinstance(max_age, bool) else 0 + self.cells = _obj(payload.get('cells'), 'cells') + self.runs = _obj(payload.get('runs'), 'runs') + + def fail(self, message: str) -> None: + self.problems.append(message) + logger.warning('[gfql-bench] %s', message) + + def forget(self, docname: str) -> None: + self.refs.pop(docname, None) + self.provenance.pop(docname, None) + if docname in self.disclosed: + self.disclosed.remove(docname) + + def cell(self, key: str) -> Optional[JSONObject]: + raw = self.cells.get(key) + return raw if isinstance(raw, dict) else None + + def run(self, run_id: str) -> Optional[JSONObject]: + raw = self.runs.get(run_id) + return raw if isinstance(raw, dict) else None + + def age_days(self, run_id: str) -> Optional[int]: + run = self.run(run_id) + if run is None: + return None + measured_at = run.get('measured_at') + if not isinstance(measured_at, str): + return None + measured = datetime.datetime.strptime(measured_at, '%Y-%m-%d').date() + return (self.today - measured).days + + +_STATE = None # type: Optional[_State] + + +def _state() -> _State: + if _STATE is None: + raise BenchNumberError('gfql_bench used before builder-inited') + return _STATE + + +def _format(cell: JSONObject) -> str: + value = cell['value'] + decimals = cell['decimals'] + unit = cell['unit'] + assert isinstance(value, (int, float)) and isinstance(decimals, int) and isinstance(unit, str) + text = '{:.{}f}'.format(float(value), decimals) + return '{}{}'.format(text, unit) if unit == 'x' else '{} {}'.format(text, unit) + + +def _bench_role(diagnostic: bool): + def role(name: str, rawtext: str, key: str, lineno: int, inliner: Inliner, + options=None, content=None) -> RoleResult: + state = _state() + env = inliner.document.settings.env # type: BenchmarkEnv + docname = env.docname + state.refs.setdefault(docname, []).append(key) + + cell = state.cell(key) + if cell is None: + state.fail('{}:{}: no published benchmark number {!r}. pyg-bench publishes it or ' + 'the docs do not print it.'.format(docname, lineno, key)) + return [nodes.strong(rawtext, '[MISSING BENCHMARK NUMBER: {}]'.format(key))], [] + + run_id = cell['run'] + assert isinstance(run_id, str) + age = state.age_days(run_id) + if age is None: + state.fail('{}:{}: {!r} names run {!r}, which has no usable measurement date'.format( + docname, lineno, key, run_id)) + elif age > state.max_age_days: + state.fail('{}:{}: {!r} was measured {} days ago; policy.max_age_days is {}. ' + 'Re-measure in pyg-bench and republish.'.format( + docname, lineno, key, age, state.max_age_days)) + + quotable = cell['board_quotable'] is True + if diagnostic and quotable: + state.fail('{}:{}: {!r} is a published result; use :bench: not :bench-diag:'.format( + docname, lineno, key)) + if not diagnostic and not quotable: + state.fail( + '{}:{}: {!r} is not board-quotable (status={!r}, comparison_allowed={!r}); it may ' + 'only appear through :bench-diag:, which labels it diagnostic-only'.format( + docname, lineno, key, cell['status'], cell['comparison_allowed'])) + + text = _format(cell) + if diagnostic: + text = '{} (diagnostic)'.format(text) + return [nodes.literal(rawtext, text)], [] + + return role + + +BenchmarkEnv = BuildEnvironment + + +class BenchProvenance(Directive): + """Render the run record behind the numbers on this page.""" + + required_arguments = 1 + optional_arguments = 0 + has_content = False + + FIELDS = [ + ('measured_at', 'Measured'), + ('host', 'Host'), + ('reps', 'Repetitions'), + ('runtime', 'Runtime'), + ('dataset', 'Dataset'), + ('pygraphistry_commit', 'PyGraphistry commit'), + ('pyg_bench_commit', 'Benchmark commit'), + ('artifact', 'Raw artifacts'), + ('row_validation', 'Result validation'), + ('competitor_version', 'Competitor version'), + ] + + def run(self) -> List[nodes.Node]: + state = _state() + env = self.state.document.settings.env + run_id = self.arguments[0].strip() + state.provenance.setdefault(env.docname, []).append(run_id) + run = state.run(run_id) + if run is None: + state.fail('{}: no run {!r} in the published artifact'.format(env.docname, run_id)) + return [] + + field_list = nodes.field_list() + for key, label in self.FIELDS: + value = run.get(key) + if not isinstance(value, str) or not value: + continue + field = nodes.field() + field += nodes.field_name(text=label) + body = nodes.field_body() + body += nodes.paragraph(text=value) + field += body + field_list += field + container = nodes.admonition() + container += nodes.title(text='Measurement') + container += field_list + container['classes'].append('note') + return [container] + + +class BenchDisclosures(Directive): + """Render every disclosure attached to a number this page prints.""" + + required_arguments = 0 + optional_arguments = 0 + has_content = False + + def run(self) -> List[nodes.Node]: + state = _state() + env = self.state.document.settings.env + docname = env.docname + state.disclosed.append(docname) + + seen = [] # type: List[str] + for key in state.refs.get(docname, []): + cell = state.cell(key) + if cell is None: + continue + raw = cell.get('disclosures') + if not isinstance(raw, list): + continue + for item in raw: + if isinstance(item, str) and item and item not in seen: + seen.append(item) + if not seen: + return [] + bullets = nodes.bullet_list() + for item in seen: + entry = nodes.list_item() + entry += nodes.paragraph(text=item) + bullets += entry + container = nodes.admonition() + container += nodes.title(text='About these measurements') + container += bullets + container['classes'].append('note') + return [container] + + +def _on_builder_inited(app: Sphinx) -> None: + global _STATE + payload = _load(BENCHMARKS_JSON) + contract = _load(CONTRACT_JSON) + _reverify(payload, contract) + _STATE = _State(payload, datetime.date.today()) + + +def _on_purge(app: Sphinx, env: BuildEnvironment, docname: str) -> None: + if _STATE is not None: + _STATE.forget(docname) + + +def _on_build_finished(app: Sphinx, exception: Optional[Exception]) -> None: + if exception is not None or _STATE is None: + return + state = _STATE + for docname, keys in sorted(state.refs.items()): + runs = set() + needs_disclosure = False + for key in keys: + cell = state.cell(key) + if cell is None: + continue + run = cell['run'] + if isinstance(run, str): + runs.add(run) + raw = cell.get('disclosures') + if isinstance(raw, list) and raw: + needs_disclosure = True + rendered = set(state.provenance.get(docname, [])) + for run_id in sorted(runs - rendered): + state.fail('{}: prints a number from run {!r} without rendering its ' + 'provenance (add ".. bench-provenance:: {}")'.format( + docname, run_id, run_id)) + if needs_disclosure and docname not in state.disclosed: + state.fail('{}: prints a number that carries a disclosure but has no ' + '".. bench-disclosures::" block'.format(docname)) + if state.problems: + raise BenchNumberError( + 'the published benchmark numbers were used incorrectly:\n {}'.format( + '\n '.join(state.problems))) + + +def setup(app: Sphinx) -> Dict[str, object]: + app.add_role('bench', _bench_role(diagnostic=False)) + app.add_role('bench-diag', _bench_role(diagnostic=True)) + app.add_directive('bench-provenance', BenchProvenance) + app.add_directive('bench-disclosures', BenchDisclosures) + app.connect('builder-inited', _on_builder_inited) + app.connect('env-purge-doc', _on_purge) + app.connect('build-finished', _on_build_finished) + return {'version': '1', 'parallel_read_safe': False, 'parallel_write_safe': True} diff --git a/docs/source/conf.py b/docs/source/conf.py index 11c48eb758..603fda80d1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,6 +17,8 @@ sys.path.insert(0, os.path.abspath("../..")) +# Local Sphinx extensions live beside this file. +sys.path.insert(0, os.path.abspath("_ext")) import graphistry @@ -54,6 +56,9 @@ "sphinx.ext.ifconfig", #"sphinx_autodoc_typehints", "sphinx_copybutton", + # Renders the benchmark numbers pyg-bench publishes, and refuses to render one + # that is missing, stale, or not cleared for publication. + "gfql_bench", ] diff --git a/docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg b/docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg deleted file mode 100644 index bff1957c3e..0000000000 --- a/docs/source/gfql/_static/filter_pagerank/gplus_lifecycle.svg +++ /dev/null @@ -1,2103 +0,0 @@ - - - - - - - - 2026-03-16T21:03:05.702756 - image/svg+xml - - - Matplotlib v3.10.8, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg b/docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg deleted file mode 100644 index cd0a12668a..0000000000 --- a/docs/source/gfql/_static/filter_pagerank/twitter_lifecycle.svg +++ /dev/null @@ -1,1991 +0,0 @@ - - - - - - - - 2026-03-16T21:03:05.678530 - image/svg+xml - - - Matplotlib v3.10.8, https://matplotlib.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/gfql/benchmark_filter_pagerank.rst b/docs/source/gfql/benchmark_filter_pagerank.rst index d35a5c5ef7..1b164cbd23 100644 --- a/docs/source/gfql/benchmark_filter_pagerank.rst +++ b/docs/source/gfql/benchmark_filter_pagerank.rst @@ -12,28 +12,30 @@ no database required. This benchmark compares **Graphistry's local Cypher** .. list-table:: :header-rows: 1 - :widths: 30 20 20 20 20 + :widths: 26 18 18 18 20 * - - Neo4j + GDS - GFQL Cypher (CPU) - GFQL Cypher (GPU) - - GPU speedup vs Neo4j - * - **Twitter** (2.4M edges) - - 13.83s - - 2.55s - - **0.30s** - - **46x** - * - **GPlus** (30M edges) - - >187s - - 75.78s - - **3.33s** - - **>56x** - -*Pipeline time (search + PageRank + search), warm median of 5 runs, 2 warmup iterations. DGX -dgx-spark, GB10 GPU. The per-graph sections below report full-lifecycle totals that also include -one-time ETL/load — hence the slightly larger numbers there (e.g. GPlus GPU 3.33s pipeline vs -~7.1s lifecycle).* + - GPU vs Neo4j + * - **Twitter** (81,306 nodes / 2.4M edges) + - :bench:`pagerank.twitter.neo4j_gds` + - :bench:`pagerank.twitter.gfql_cpu` + - :bench:`pagerank.twitter.gfql_gpu` + - :bench:`pagerank.twitter.gfql_gpu_vs_neo4j_gds` + * - **GPlus** (107,614 nodes / 30M edges) + - + - :bench:`pagerank.gplus.gfql_cpu` + - :bench:`pagerank.gplus.gfql_gpu` + - + +Warm pipeline time — search, PageRank, search — on the resident graph. + +On Twitter the CPU path alone is :bench:`pagerank.twitter.gfql_cpu_vs_neo4j_gds` faster +than Neo4j + GDS, before any GPU is involved. Moving the same query text to the GPU adds +another :bench:`pagerank.twitter.gfql_gpu_vs_gfql_cpu`; on the 30M-edge GPlus graph the +GPU is :bench:`pagerank.gplus.gfql_gpu_vs_gfql_cpu` faster than the CPU path. The pipeline ------------ @@ -77,38 +79,9 @@ The same pipeline shape, different backends: The Neo4j equivalent requires ~30 lines of Cypher + GDS projection + batched writes (see :ref:`neo4j-analog` below). -Twitter (2.4M edges): exact 3-way comparison --------------------------------------------- +.. bench-provenance:: filter-pagerank-20260728 -.. image:: _static/filter_pagerank/twitter_lifecycle.svg - :alt: Twitter end-to-end: Neo4j vs GFQL Cypher CPU vs GFQL Cypher GPU - -Stacked by workload phase: **ETL** (load + shape), **Search** (graph queries), **Analytics** (PageRank). - -- Neo4j total lifecycle: ~21.6s (6.0s import + 1.7s prep + 13.8s pipeline) -- GFQL Cypher CPU: ~2.8s — **8x faster than Neo4j** -- GFQL Cypher GPU: ~0.4s — **54x faster than Neo4j** - -GPlus (30M edges): larger graph -------------------------------- - -.. image:: _static/filter_pagerank/gplus_lifecycle.svg - :alt: GPlus: Neo4j (lower bound) vs GFQL Cypher CPU vs GFQL Cypher GPU - -- Neo4j: **>187s** (lower bound — the transaction did not finish) -- GFQL Cypher CPU: ~85.5s — still faster than Neo4j's incomplete run -- GFQL Cypher GPU: ~7.1s — **>26x faster than Neo4j** - -Why this matters ----------------- - -The CPU path already beats Neo4j without a GPU. You get Cypher-style graph -search + PageRank directly on your dataframe, no database to stand up or -maintain. - -The GPU path accelerates everything — ETL, search, and analytics — because -``cudf`` and ``cugraph`` are drop-in replacements for ``pandas`` and ``igraph`` -under the same GFQL Cypher surface. +.. bench-disclosures:: .. _neo4j-analog: @@ -184,10 +157,9 @@ falling back or returning a different answer. See :doc:`engines` for the full parity and static-safety contract. This page is one workload (a filter → PageRank → filter pipeline) against one -external baseline (Neo4j+GDS). For the full four-engine picture — when Polars +external baseline (Neo4j + GDS). For the full four-engine picture — when Polars beats pandas on CPU, when the GPU pulls ahead, and how to choose — see -:doc:`engines`. For sub-millisecond *seeded* lookups that beat Kuzu and Neo4j -by 9–28×, see :doc:`index_adjacency`. +:doc:`engines`. For seeded lookups, see :doc:`index_adjacency`. For more on the GFQL design and supported surface: @@ -196,18 +168,3 @@ For more on the GFQL design and supported surface: - :doc:`cypher` — Cypher syntax through ``g.gfql("MATCH ...")`` - :doc:`overview` — GFQL design, features, and GPU acceleration - :doc:`about` — 10-minute introduction to GFQL - -Benchmark environment ---------------------- - -- Host: ``dgx-spark``, GPU: ``GB10``, driver ``580.126.09`` -- Container: ``graphistry/test-gpu:latest`` -- Datasets: `SNAP `_ Twitter (2.4M edges) and GPlus (30M edges) -- Measurement: median of 5 runs after 2 warmup iterations -- Results rendered from saved JSON — this page does **not** rerun benchmarks - -Notebook version ----------------- - -See ``demos/gfql/benchmark_filter_pagerank_cpu_gpu.ipynb`` for a notebook -version of this writeup using the same saved DGX results. diff --git a/docs/test_bench_numbers.py b/docs/test_bench_numbers.py new file mode 100644 index 0000000000..0103117e7a --- /dev/null +++ b/docs/test_bench_numbers.py @@ -0,0 +1,191 @@ +"""Consumer-side re-verification of the benchmark numbers pyg-bench publishes. + +`docs/source/_data/gfql_benchmarks.json` is a vendored copy of that repository's +`published/docs-numbers.json`, and `gfql_benchmarks.contract.json` is a vendored copy of +the contract it promises to satisfy. pyg-bench checks those promises before it publishes; +this checks them again before we print anything, because a boundary only holds if both +sides check it. + +These run in the ordinary test lane, not only in the docs build, so a number going stale +or a page referencing a key that no longer exists fails CI rather than a nightly. +""" + +import datetime +import json +import os +import re +import sys + +import pytest + +DOCS_DIR = os.path.dirname(os.path.abspath(__file__)) +SOURCE_DIR = os.path.join(DOCS_DIR, 'source') +sys.path.insert(0, os.path.join(SOURCE_DIR, '_ext')) + +import gfql_bench # noqa: E402 + +#: ``:bench:`key``` / ``:bench-diag:`key``` as written in the .rst sources. +BENCH_REF = re.compile(r':(bench|bench-diag):`([^`]+)`') + + +@pytest.fixture(scope='module') +def payload(): + return gfql_bench._load(gfql_bench.BENCHMARKS_JSON) + + +@pytest.fixture(scope='module') +def contract(): + return gfql_bench._load(gfql_bench.CONTRACT_JSON) + + +def _rst_sources(): + for root, _dirs, files in os.walk(SOURCE_DIR): + for name in files: + if name.endswith('.rst') or name.endswith('.md'): + yield os.path.join(root, name) + + +def _references(): + """(path, key, is_diagnostic) for every benchmark reference in the docs.""" + for path in sorted(_rst_sources()): + with open(path, encoding='utf-8') as handle: + text = handle.read() + for role, key in BENCH_REF.findall(text): + yield path, key, role == 'bench-diag' + + +def test_the_vendored_artifact_satisfies_the_vendored_contract(payload, contract): + gfql_bench._reverify(payload, contract) + + +def test_the_vendored_contract_is_the_one_the_artifact_was_built_against(payload, contract): + assert payload['contract_version'] == contract['contract_version'] + + +def test_no_published_number_is_stale(payload): + """The staleness rule is the whole reason this pipeline exists: a number nobody + re-measured must fail loudly rather than keep looking authoritative.""" + max_age = payload['policy']['max_age_days'] + today = datetime.date.today() + overdue = [] + for run_id, run in sorted(payload['runs'].items()): + measured = datetime.datetime.strptime(run['measured_at'], '%Y-%m-%d').date() + age = (today - measured).days + if age > max_age: + overdue.append('{} measured {} days ago (limit {})'.format(run_id, age, max_age)) + assert not overdue, ( + 'Re-measure in pyg-bench and republish published/docs-numbers.json: ' + + '; '.join(overdue)) + + +def test_every_number_the_docs_reference_is_published(payload): + cells = payload['cells'] + missing = ['{}: {}'.format(os.path.relpath(path, DOCS_DIR), key) + for path, key, _ in _references() if key not in cells] + assert not missing, 'the docs reference numbers pyg-bench does not publish: ' + '; '.join(missing) + + +def test_every_reference_uses_the_role_its_quotability_allows(payload): + cells = payload['cells'] + wrong = [] + for path, key, diagnostic in _references(): + cell = cells.get(key) + if cell is None: + continue + quotable = cell['board_quotable'] is True + if quotable and diagnostic: + wrong.append('{}: {} is a published result, use :bench:'.format(path, key)) + if not quotable and not diagnostic: + wrong.append('{}: {} is diagnostic-only, use :bench-diag:'.format(path, key)) + assert not wrong, '; '.join(wrong) + + +def test_a_board_quotable_cell_that_is_not_comparable_is_rejected(payload, contract): + broken = json.loads(json.dumps(payload)) + key = sorted(broken['cells'])[0] + broken['cells'][key]['board_quotable'] = True + broken['cells'][key]['comparison_allowed'] = False + with pytest.raises(gfql_bench.BenchNumberError) as excinfo: + gfql_bench._reverify(broken, contract) + assert 'not comparison_allowed' in str(excinfo.value) + + +def test_a_run_missing_provenance_is_rejected(payload, contract): + broken = json.loads(json.dumps(payload)) + run_id = sorted(broken['runs'])[0] + del broken['runs'][run_id]['host'] + with pytest.raises(gfql_bench.BenchNumberError) as excinfo: + gfql_bench._reverify(broken, contract) + assert "missing provenance 'host'" in str(excinfo.value) + + +def test_a_caveated_number_without_its_caveat_is_rejected(payload, contract): + broken = json.loads(json.dumps(payload)) + key = sorted(broken['cells'])[0] + broken['cells'][key]['status'] = 'partial' + broken['cells'][key]['board_quotable'] = False + broken['cells'][key]['comparison_allowed'] = False + broken['cells'][key]['disclosures'] = [] + with pytest.raises(gfql_bench.BenchNumberError) as excinfo: + gfql_bench._reverify(broken, contract) + assert 'carries no disclosure' in str(excinfo.value) + + +class _Document(object): + class settings(object): + class env(object): + docname = 'gfql/example' + + +class _Inliner(object): + document = _Document + + +def _use(key, diagnostic=False, today=None): + """Reference a benchmark key the way a page does, and report what broke.""" + payload = gfql_bench._load(gfql_bench.BENCHMARKS_JSON) + state = gfql_bench._State(payload, today or datetime.date.today()) + previous = gfql_bench._STATE + gfql_bench._STATE = state + try: + gfql_bench._bench_role(diagnostic)('bench', '', key, 1, _Inliner) + finally: + gfql_bench._STATE = previous + return state.problems + + +def test_a_key_that_is_not_published_fails_the_build(payload): + problems = _use('pagerank.twitter.this_was_never_measured') + assert problems and 'no published benchmark number' in problems[0] + + +def test_a_published_key_renders_cleanly(payload): + assert _use(sorted(payload['cells'])[0]) == [] + + +def test_a_number_older_than_the_policy_fails_the_build(payload): + """The failure the withdrawn figures needed and did not have.""" + oldest = min( + datetime.datetime.strptime(run['measured_at'], '%Y-%m-%d').date() + for run in payload['runs'].values()) + much_later = oldest + datetime.timedelta(days=payload['policy']['max_age_days'] + 400) + problems = _use(sorted(payload['cells'])[0], today=much_later) + assert problems and 'max_age_days' in problems[0] + + +def test_a_diagnostic_only_number_cannot_be_printed_as_a_result(payload): + diagnostic = [key for key, cell in sorted(payload['cells'].items()) + if cell['board_quotable'] is not True] + if not diagnostic: + pytest.skip('the artifact currently publishes no diagnostic-only cell') + problems = _use(diagnostic[0]) + assert problems and 'not board-quotable' in problems[0] + assert _use(diagnostic[0], diagnostic=True) == [] + + +def test_an_artifact_from_another_contract_version_is_rejected(payload, contract): + broken = json.loads(json.dumps(payload)) + broken['contract_version'] = int(contract['contract_version']) + 1 + with pytest.raises(gfql_bench.BenchNumberError) as excinfo: + gfql_bench._reverify(broken, contract) + assert 'contract_version' in str(excinfo.value) From b0dae46484e303a0822db1767020db3d7e0942b5 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Mon, 27 Jul 2026 20:55:34 -0700 Subject: [PATCH 2/2] docs(gfql): keep the benchmark rules out of the docutils import CI's minimal lane has no docutils, so putting the contract re-verification, the staleness rule and the point-of-use decision in the Sphinx extension meant the whole gate could only run in the docs build - one job out of forty, and the pattern this repository has just finished digging out of. `gfql_bench_data` is now stdlib-only and holds every rule; `gfql_bench` is the docutils half that renders them and turns a recorded problem into a failed build. A test asserts the rules module never grows a docutils or sphinx import again. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015YsqAZQLbqjSDrYSFz2GoB --- docs/source/_ext/gfql_bench.py | 374 +++++----------------------- docs/source/_ext/gfql_bench_data.py | 298 ++++++++++++++++++++++ docs/test_bench_numbers.py | 65 ++--- 3 files changed, 395 insertions(+), 342 deletions(-) create mode 100644 docs/source/_ext/gfql_bench_data.py diff --git a/docs/source/_ext/gfql_bench.py b/docs/source/_ext/gfql_bench.py index 9d007ad652..48c698d2a0 100644 --- a/docs/source/_ext/gfql_bench.py +++ b/docs/source/_ext/gfql_bench.py @@ -1,9 +1,7 @@ """Sphinx extension: pretty-print the benchmark numbers pyg-bench publishes. Measurement, provenance and publishability all live in `graphistry/pyg-bench`, which -owns the runs. This repository does one thing with them: it renders them. The two files -under ``docs/source/_data`` are vendored copies of that repository's -``published/docs-numbers.json`` and ``manifests/docs-numbers.contract.json``. +owns the runs. This repository renders them, and nothing else. Docs never restate a measured number, they reference one:: @@ -16,28 +14,15 @@ .. bench-provenance:: filter-pagerank-20260728 .. bench-disclosures:: -The contract names five obligations for a consumer. They are build failures here, never -silent text: - -- a key the artifact does not contain -> build fails -- a run measured longer ago than ``max_age_days`` -> build fails -- a non-quotable cell printed as a bare number -> build fails (``:bench-diag:`` labels it) -- a page that drops a referenced cell's provenance or disclosures -> build fails -- a payload that breaks the contract at all -> build fails, before anything renders - -The re-verification below is written against the contract *document*, independently of -pyg-bench's own implementation. A shared library would fail identically on both sides of -the boundary and prove nothing. +Every rule is in ``gfql_bench_data``, which is stdlib-only so the ordinary test lane +can run it without Sphinx installed. This module is the docutils half: roles, +directives, and turning a recorded problem into a failed build. """ from __future__ import annotations import datetime -import json -import math -import os -import re -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple from docutils import nodes from docutils.parsers.rst import Directive @@ -47,259 +32,45 @@ from sphinx.errors import SphinxError from sphinx.util import logging as sphinx_logging +from gfql_bench_data import ( + BenchDataError, + JSONObject, + State, + audit_pages, + check_reference, + format_cell, + load_state, +) + logger = sphinx_logging.getLogger(__name__) RoleResult = Tuple[List[nodes.Node], List[nodes.system_message]] -#: A decoded JSON document. -JSONValue = Union[ - None, bool, int, float, str, List["JSONValue"], Dict[str, "JSONValue"] -] -JSONObject = Dict[str, JSONValue] - -#: ``_ext`` and ``_data`` are siblings under ``docs/source``. -DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '_data') -BENCHMARKS_JSON = os.path.join(DATA_DIR, 'gfql_benchmarks.json') -CONTRACT_JSON = os.path.join(DATA_DIR, 'gfql_benchmarks.contract.json') +_STATE = None # type: Optional[State] class BenchNumberError(SphinxError): category = 'GFQL benchmark number check failed' -def _obj(value: JSONValue, where: str) -> JSONObject: - if not isinstance(value, dict): - raise BenchNumberError('{}: expected an object'.format(where)) - return value - - -def _strings(value: JSONValue, where: str) -> List[str]: - if not isinstance(value, list): - raise BenchNumberError('{}: expected an array of strings'.format(where)) - out = [] # type: List[str] - for item in value: - if not isinstance(item, str) or not item: - raise BenchNumberError('{}: entries must be non-empty strings'.format(where)) - out.append(item) - return out - - -def _load(path: str) -> JSONObject: - if not os.path.exists(path): - raise BenchNumberError( - '{} is missing. It is a vendored copy of pyg-bench ' - 'published/docs-numbers.json.'.format(path)) - with open(path, encoding='utf-8') as handle: - return _obj(json.load(handle), path) - - -def _reverify(payload: JSONObject, contract: JSONObject) -> None: - """Check the artifact against the contract document, before anything renders.""" - problems = [] # type: List[str] - - expected_version = contract.get('contract_version') - if payload.get('contract_version') != expected_version: - problems.append( - 'contract_version {!r} but this repository vendors contract {!r}'.format( - payload.get('contract_version'), expected_version)) - - for field in _strings(contract.get('top_level_required'), 'contract.top_level_required'): - if field not in payload: - problems.append('artifact is missing {!r}'.format(field)) - - policy = payload.get('policy') - if not isinstance(policy, dict): - problems.append('policy: expected an object') - policy = {} - for field in _strings(contract.get('policy_required'), 'contract.policy_required'): - if field not in policy: - problems.append('policy is missing {!r}'.format(field)) - - run_required = _strings(contract.get('run_required'), 'contract.run_required') - cell_required = _strings(contract.get('cell_required'), 'contract.cell_required') - statuses = _strings(contract.get('cell_status'), 'contract.cell_status') - units = _strings(contract.get('cell_unit'), 'contract.cell_unit') - key_re = re.compile(str(contract.get('key_pattern'))) - date_re = re.compile(str(contract.get('date_pattern'))) - max_decimals = contract.get('max_decimals') - if not isinstance(max_decimals, int) or isinstance(max_decimals, bool): - raise BenchNumberError('contract.max_decimals must be an integer') - - runs = payload.get('runs') - if not isinstance(runs, dict): - problems.append('runs: expected an object') - runs = {} - for run_id in sorted(runs): - run = runs[run_id] - if not isinstance(run, dict): - problems.append('runs.{}: expected an object'.format(run_id)) - continue - for field in run_required: - if field not in run: - problems.append('runs.{}: missing provenance {!r}'.format(run_id, field)) - measured_at = run.get('measured_at') - if not isinstance(measured_at, str) or not date_re.match(measured_at): - problems.append('runs.{}: measured_at is not a date'.format(run_id)) - - cells = payload.get('cells') - if not isinstance(cells, dict): - problems.append('cells: expected an object') - cells = {} - if not cells: - problems.append('cells: the artifact publishes nothing') - for key in sorted(cells): - cell = cells[key] - if not isinstance(cell, dict): - problems.append('cells.{}: expected an object'.format(key)) - continue - if not key_re.match(key): - problems.append('cells.{}: key does not match the published pattern'.format(key)) - missing = [field for field in cell_required if field not in cell] - if missing: - problems.append('cells.{}: missing {}'.format(key, ', '.join(sorted(missing)))) - continue - if cell['run'] not in runs: - problems.append('cells.{}: run {!r} has no provenance'.format(key, cell['run'])) - if cell['status'] not in statuses: - problems.append('cells.{}: unknown status {!r}'.format(key, cell['status'])) - if cell['unit'] not in units: - problems.append('cells.{}: unknown unit {!r}'.format(key, cell['unit'])) - if not isinstance(cell['engine'], str) or not cell['engine']: - problems.append('cells.{}: engine is not named'.format(key)) - quotable = cell['board_quotable'] - comparable = cell['comparison_allowed'] - if not isinstance(quotable, bool) or not isinstance(comparable, bool): - problems.append('cells.{}: quotability flags must be booleans'.format(key)) - continue - disclosures = cell['disclosures'] - if not isinstance(disclosures, list): - problems.append('cells.{}: disclosures must be an array'.format(key)) - disclosures = [] - if quotable and not comparable: - problems.append('cells.{}: board_quotable but not comparison_allowed'.format(key)) - if quotable and cell['status'] != 'ok': - problems.append('cells.{}: board_quotable with status {!r}'.format(key, cell['status'])) - if (cell['status'] != 'ok' or not comparable) and not disclosures: - problems.append('cells.{}: caveated but carries no disclosure'.format(key)) - if cell['unit'] == 'x' and not comparable: - problems.append('cells.{}: a ratio over figures never established as comparable'.format(key)) - value = cell['value'] - decimals = cell['decimals'] - if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)): - problems.append('cells.{}: value is not a finite number'.format(key)) - continue - if not isinstance(decimals, int) or isinstance(decimals, bool) or not 0 <= decimals <= max_decimals: - problems.append('cells.{}: decimals out of range'.format(key)) - continue - if round(float(value), decimals) != float(value): - problems.append('cells.{}: value is not rounded to its own decimals'.format(key)) - - if problems: - raise BenchNumberError( - '{} breaks the data contract vendored at {}:\n {}'.format( - BENCHMARKS_JSON, CONTRACT_JSON, '\n '.join(problems))) - - -class _State: - """Loaded once per build: the data, the clock, and the problems found.""" - - def __init__(self, payload: JSONObject, today: datetime.date) -> None: - self.payload = payload - self.today = today - self.problems = [] # type: List[str] - self.refs = {} # type: Dict[str, List[str]] - self.provenance = {} # type: Dict[str, List[str]] - self.disclosed = [] # type: List[str] - - policy = _obj(payload.get('policy'), 'policy') - max_age = policy.get('max_age_days') - self.max_age_days = max_age if isinstance(max_age, int) and not isinstance(max_age, bool) else 0 - self.cells = _obj(payload.get('cells'), 'cells') - self.runs = _obj(payload.get('runs'), 'runs') - - def fail(self, message: str) -> None: - self.problems.append(message) - logger.warning('[gfql-bench] %s', message) - - def forget(self, docname: str) -> None: - self.refs.pop(docname, None) - self.provenance.pop(docname, None) - if docname in self.disclosed: - self.disclosed.remove(docname) - - def cell(self, key: str) -> Optional[JSONObject]: - raw = self.cells.get(key) - return raw if isinstance(raw, dict) else None - - def run(self, run_id: str) -> Optional[JSONObject]: - raw = self.runs.get(run_id) - return raw if isinstance(raw, dict) else None - - def age_days(self, run_id: str) -> Optional[int]: - run = self.run(run_id) - if run is None: - return None - measured_at = run.get('measured_at') - if not isinstance(measured_at, str): - return None - measured = datetime.datetime.strptime(measured_at, '%Y-%m-%d').date() - return (self.today - measured).days - - -_STATE = None # type: Optional[_State] - - -def _state() -> _State: +def _state() -> State: if _STATE is None: raise BenchNumberError('gfql_bench used before builder-inited') return _STATE -def _format(cell: JSONObject) -> str: - value = cell['value'] - decimals = cell['decimals'] - unit = cell['unit'] - assert isinstance(value, (int, float)) and isinstance(decimals, int) and isinstance(unit, str) - text = '{:.{}f}'.format(float(value), decimals) - return '{}{}'.format(text, unit) if unit == 'x' else '{} {}'.format(text, unit) - - def _bench_role(diagnostic: bool): def role(name: str, rawtext: str, key: str, lineno: int, inliner: Inliner, options=None, content=None) -> RoleResult: state = _state() - env = inliner.document.settings.env # type: BenchmarkEnv - docname = env.docname - state.refs.setdefault(docname, []).append(key) - - cell = state.cell(key) + docname = inliner.document.settings.env.docname + before = len(state.problems) + cell = check_reference(state, key, docname, lineno, diagnostic) + for message in state.problems[before:]: + logger.warning('[gfql-bench] %s', message) if cell is None: - state.fail('{}:{}: no published benchmark number {!r}. pyg-bench publishes it or ' - 'the docs do not print it.'.format(docname, lineno, key)) return [nodes.strong(rawtext, '[MISSING BENCHMARK NUMBER: {}]'.format(key))], [] - - run_id = cell['run'] - assert isinstance(run_id, str) - age = state.age_days(run_id) - if age is None: - state.fail('{}:{}: {!r} names run {!r}, which has no usable measurement date'.format( - docname, lineno, key, run_id)) - elif age > state.max_age_days: - state.fail('{}:{}: {!r} was measured {} days ago; policy.max_age_days is {}. ' - 'Re-measure in pyg-bench and republish.'.format( - docname, lineno, key, age, state.max_age_days)) - - quotable = cell['board_quotable'] is True - if diagnostic and quotable: - state.fail('{}:{}: {!r} is a published result; use :bench: not :bench-diag:'.format( - docname, lineno, key)) - if not diagnostic and not quotable: - state.fail( - '{}:{}: {!r} is not board-quotable (status={!r}, comparison_allowed={!r}); it may ' - 'only appear through :bench-diag:, which labels it diagnostic-only'.format( - docname, lineno, key, cell['status'], cell['comparison_allowed'])) - - text = _format(cell) + text = format_cell(cell) if diagnostic: text = '{} (diagnostic)'.format(text) return [nodes.literal(rawtext, text)], [] @@ -307,9 +78,6 @@ def role(name: str, rawtext: str, key: str, lineno: int, inliner: Inliner, return role -BenchmarkEnv = BuildEnvironment - - class BenchProvenance(Directive): """Render the run record behind the numbers on this page.""" @@ -332,30 +100,16 @@ class BenchProvenance(Directive): def run(self) -> List[nodes.Node]: state = _state() - env = self.state.document.settings.env + docname = self.state.document.settings.env.docname run_id = self.arguments[0].strip() - state.provenance.setdefault(env.docname, []).append(run_id) + state.provenance.setdefault(docname, []).append(run_id) run = state.run(run_id) if run is None: - state.fail('{}: no run {!r} in the published artifact'.format(env.docname, run_id)) + message = '{}: no run {!r} in the published artifact'.format(docname, run_id) + state.fail(message) + logger.warning('[gfql-bench] %s', message) return [] - - field_list = nodes.field_list() - for key, label in self.FIELDS: - value = run.get(key) - if not isinstance(value, str) or not value: - continue - field = nodes.field() - field += nodes.field_name(text=label) - body = nodes.field_body() - body += nodes.paragraph(text=value) - field += body - field_list += field - container = nodes.admonition() - container += nodes.title(text='Measurement') - container += field_list - container['classes'].append('note') - return [container] + return [_admonition('Measurement', _fields(run, self.FIELDS))] class BenchDisclosures(Directive): @@ -367,8 +121,7 @@ class BenchDisclosures(Directive): def run(self) -> List[nodes.Node]: state = _state() - env = self.state.document.settings.env - docname = env.docname + docname = self.state.document.settings.env.docname state.disclosed.append(docname) seen = [] # type: List[str] @@ -389,19 +142,38 @@ def run(self) -> List[nodes.Node]: entry = nodes.list_item() entry += nodes.paragraph(text=item) bullets += entry - container = nodes.admonition() - container += nodes.title(text='About these measurements') - container += bullets - container['classes'].append('note') - return [container] + return [_admonition('About these measurements', bullets)] + + +def _fields(run: JSONObject, spec: List[Tuple[str, str]]) -> nodes.field_list: + field_list = nodes.field_list() + for key, label in spec: + value = run.get(key) + if not isinstance(value, str) or not value: + continue + field = nodes.field() + field += nodes.field_name(text=label) + body = nodes.field_body() + body += nodes.paragraph(text=value) + field += body + field_list += field + return field_list + + +def _admonition(title: str, content: nodes.Element) -> nodes.Element: + container = nodes.admonition() + container += nodes.title(text=title) + container += content + container['classes'].append('note') + return container def _on_builder_inited(app: Sphinx) -> None: global _STATE - payload = _load(BENCHMARKS_JSON) - contract = _load(CONTRACT_JSON) - _reverify(payload, contract) - _STATE = _State(payload, datetime.date.today()) + try: + _STATE = load_state(datetime.date.today()) + except BenchDataError as exc: + raise BenchNumberError(str(exc)) from exc def _on_purge(app: Sphinx, env: BuildEnvironment, docname: str) -> None: @@ -412,32 +184,14 @@ def _on_purge(app: Sphinx, env: BuildEnvironment, docname: str) -> None: def _on_build_finished(app: Sphinx, exception: Optional[Exception]) -> None: if exception is not None or _STATE is None: return - state = _STATE - for docname, keys in sorted(state.refs.items()): - runs = set() - needs_disclosure = False - for key in keys: - cell = state.cell(key) - if cell is None: - continue - run = cell['run'] - if isinstance(run, str): - runs.add(run) - raw = cell.get('disclosures') - if isinstance(raw, list) and raw: - needs_disclosure = True - rendered = set(state.provenance.get(docname, [])) - for run_id in sorted(runs - rendered): - state.fail('{}: prints a number from run {!r} without rendering its ' - 'provenance (add ".. bench-provenance:: {}")'.format( - docname, run_id, run_id)) - if needs_disclosure and docname not in state.disclosed: - state.fail('{}: prints a number that carries a disclosure but has no ' - '".. bench-disclosures::" block'.format(docname)) - if state.problems: + before = len(_STATE.problems) + audit_pages(_STATE) + for message in _STATE.problems[before:]: + logger.warning('[gfql-bench] %s', message) + if _STATE.problems: raise BenchNumberError( 'the published benchmark numbers were used incorrectly:\n {}'.format( - '\n '.join(state.problems))) + '\n '.join(_STATE.problems))) def setup(app: Sphinx) -> Dict[str, object]: diff --git a/docs/source/_ext/gfql_bench_data.py b/docs/source/_ext/gfql_bench_data.py new file mode 100644 index 0000000000..d29615a391 --- /dev/null +++ b/docs/source/_ext/gfql_bench_data.py @@ -0,0 +1,298 @@ +"""Load and re-verify the benchmark artifact pyg-bench publishes. + +Deliberately stdlib-only. The Sphinx extension in ``gfql_bench.py`` needs docutils; +the contract re-verification, the staleness rule and the point-of-use decision must +not, or they would run in exactly one CI lane and be invisible everywhere else. + +``docs/source/_data/gfql_benchmarks.json`` is a vendored copy of pyg-bench's +``published/docs-numbers.json``; ``gfql_benchmarks.contract.json`` is a vendored copy +of the contract it promises to satisfy. pyg-bench checks those promises before it +publishes. This checks them again, from the contract *document*, with an +implementation that does not import that repository - a shared library would fail +identically on both sides of the boundary and prove nothing. +""" + +from __future__ import annotations + +import datetime +import json +import math +import os +import re +from typing import Dict, List, Optional, Union + +#: A decoded JSON document. +JSONValue = Union[ + None, bool, int, float, str, List["JSONValue"], Dict[str, "JSONValue"] +] +JSONObject = Dict[str, JSONValue] + +#: ``_ext`` and ``_data`` are siblings under ``docs/source``. +DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '_data') +BENCHMARKS_JSON = os.path.join(DATA_DIR, 'gfql_benchmarks.json') +CONTRACT_JSON = os.path.join(DATA_DIR, 'gfql_benchmarks.contract.json') + + +class BenchDataError(Exception): + """The vendored artifact cannot be trusted; nothing may render from it.""" + + +def _obj(value: JSONValue, where: str) -> JSONObject: + if not isinstance(value, dict): + raise BenchDataError('{}: expected an object'.format(where)) + return value + + +def _strings(value: JSONValue, where: str) -> List[str]: + if not isinstance(value, list): + raise BenchDataError('{}: expected an array of strings'.format(where)) + out = [] # type: List[str] + for item in value: + if not isinstance(item, str) or not item: + raise BenchDataError('{}: entries must be non-empty strings'.format(where)) + out.append(item) + return out + + +def load(path: str) -> JSONObject: + if not os.path.exists(path): + raise BenchDataError( + '{} is missing. It is a vendored copy of pyg-bench ' + 'published/docs-numbers.json.'.format(path)) + with open(path, encoding='utf-8') as handle: + return _obj(json.load(handle), path) + + +def reverify(payload: JSONObject, contract: JSONObject) -> None: + """Check the artifact against the contract document, before anything renders.""" + problems = [] # type: List[str] + + expected_version = contract.get('contract_version') + if payload.get('contract_version') != expected_version: + problems.append( + 'contract_version {!r} but this repository vendors contract {!r}'.format( + payload.get('contract_version'), expected_version)) + + for field in _strings(contract.get('top_level_required'), 'contract.top_level_required'): + if field not in payload: + problems.append('artifact is missing {!r}'.format(field)) + + policy = payload.get('policy') + if not isinstance(policy, dict): + problems.append('policy: expected an object') + policy = {} + for field in _strings(contract.get('policy_required'), 'contract.policy_required'): + if field not in policy: + problems.append('policy is missing {!r}'.format(field)) + + run_required = _strings(contract.get('run_required'), 'contract.run_required') + cell_required = _strings(contract.get('cell_required'), 'contract.cell_required') + statuses = _strings(contract.get('cell_status'), 'contract.cell_status') + units = _strings(contract.get('cell_unit'), 'contract.cell_unit') + key_re = re.compile(str(contract.get('key_pattern'))) + date_re = re.compile(str(contract.get('date_pattern'))) + max_decimals = contract.get('max_decimals') + if not isinstance(max_decimals, int) or isinstance(max_decimals, bool): + raise BenchDataError('contract.max_decimals must be an integer') + + runs = payload.get('runs') + if not isinstance(runs, dict): + problems.append('runs: expected an object') + runs = {} + for run_id in sorted(runs): + run = runs[run_id] + if not isinstance(run, dict): + problems.append('runs.{}: expected an object'.format(run_id)) + continue + for field in run_required: + if field not in run: + problems.append('runs.{}: missing provenance {!r}'.format(run_id, field)) + measured_at = run.get('measured_at') + if not isinstance(measured_at, str) or not date_re.match(measured_at): + problems.append('runs.{}: measured_at is not a date'.format(run_id)) + + cells = payload.get('cells') + if not isinstance(cells, dict): + problems.append('cells: expected an object') + cells = {} + if not cells: + problems.append('cells: the artifact publishes nothing') + for key in sorted(cells): + cell = cells[key] + if not isinstance(cell, dict): + problems.append('cells.{}: expected an object'.format(key)) + continue + if not key_re.match(key): + problems.append('cells.{}: key does not match the published pattern'.format(key)) + missing = [field for field in cell_required if field not in cell] + if missing: + problems.append('cells.{}: missing {}'.format(key, ', '.join(sorted(missing)))) + continue + if cell['run'] not in runs: + problems.append('cells.{}: run {!r} has no provenance'.format(key, cell['run'])) + if cell['status'] not in statuses: + problems.append('cells.{}: unknown status {!r}'.format(key, cell['status'])) + if cell['unit'] not in units: + problems.append('cells.{}: unknown unit {!r}'.format(key, cell['unit'])) + if not isinstance(cell['engine'], str) or not cell['engine']: + problems.append('cells.{}: engine is not named'.format(key)) + quotable = cell['board_quotable'] + comparable = cell['comparison_allowed'] + if not isinstance(quotable, bool) or not isinstance(comparable, bool): + problems.append('cells.{}: quotability flags must be booleans'.format(key)) + continue + disclosures = cell['disclosures'] + if not isinstance(disclosures, list): + problems.append('cells.{}: disclosures must be an array'.format(key)) + disclosures = [] + if quotable and not comparable: + problems.append('cells.{}: board_quotable but not comparison_allowed'.format(key)) + if quotable and cell['status'] != 'ok': + problems.append('cells.{}: board_quotable with status {!r}'.format(key, cell['status'])) + if (cell['status'] != 'ok' or not comparable) and not disclosures: + problems.append('cells.{}: caveated but carries no disclosure'.format(key)) + if cell['unit'] == 'x' and not comparable: + problems.append('cells.{}: a ratio over figures never established as comparable'.format(key)) + value = cell['value'] + decimals = cell['decimals'] + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)): + problems.append('cells.{}: value is not a finite number'.format(key)) + continue + if not isinstance(decimals, int) or isinstance(decimals, bool) or not 0 <= decimals <= max_decimals: + problems.append('cells.{}: decimals out of range'.format(key)) + continue + if round(float(value), decimals) != float(value): + problems.append('cells.{}: value is not rounded to its own decimals'.format(key)) + + if problems: + raise BenchDataError( + '{} breaks the data contract vendored at {}:\n {}'.format( + BENCHMARKS_JSON, CONTRACT_JSON, '\n '.join(problems))) + + +class State: + """Loaded once per build: the data, the clock, and the problems found.""" + + def __init__(self, payload: JSONObject, today: datetime.date) -> None: + self.payload = payload + self.today = today + self.problems = [] # type: List[str] + self.refs = {} # type: Dict[str, List[str]] + self.provenance = {} # type: Dict[str, List[str]] + self.disclosed = [] # type: List[str] + + policy = _obj(payload.get('policy'), 'policy') + max_age = policy.get('max_age_days') + self.max_age_days = max_age if isinstance(max_age, int) and not isinstance(max_age, bool) else 0 + self.cells = _obj(payload.get('cells'), 'cells') + self.runs = _obj(payload.get('runs'), 'runs') + + def fail(self, message: str) -> None: + self.problems.append(message) + + def forget(self, docname: str) -> None: + self.refs.pop(docname, None) + self.provenance.pop(docname, None) + if docname in self.disclosed: + self.disclosed.remove(docname) + + def cell(self, key: str) -> Optional[JSONObject]: + raw = self.cells.get(key) + return raw if isinstance(raw, dict) else None + + def run(self, run_id: str) -> Optional[JSONObject]: + raw = self.runs.get(run_id) + return raw if isinstance(raw, dict) else None + + def age_days(self, run_id: str) -> Optional[int]: + run = self.run(run_id) + if run is None: + return None + measured_at = run.get('measured_at') + if not isinstance(measured_at, str): + return None + measured = datetime.datetime.strptime(measured_at, '%Y-%m-%d').date() + return (self.today - measured).days + + + + +def format_cell(cell: JSONObject) -> str: + """Print the number exactly as published; rounding never happens at render time.""" + value = cell['value'] + decimals = cell['decimals'] + unit = cell['unit'] + assert isinstance(value, (int, float)) and isinstance(decimals, int) and isinstance(unit, str) + text = '{:.{}f}'.format(float(value), decimals) + return '{}{}'.format(text, unit) if unit == 'x' else '{} {}'.format(text, unit) + + +def check_reference(state: State, key: str, docname: str, lineno: int, + diagnostic: bool) -> Optional[JSONObject]: + """Decide whether this page may print this number, AT THE POINT OF USE. + + Returns the cell to render, or None when there is nothing publishable. Every + refusal is recorded on ``state.problems``, which the build turns into a failure. + """ + state.refs.setdefault(docname, []).append(key) + + cell = state.cell(key) + if cell is None: + state.fail( + '{}:{}: no published benchmark number {!r}. pyg-bench publishes it or the ' + 'docs do not print it.'.format(docname, lineno, key)) + return None + + run_id = cell['run'] + if isinstance(run_id, str): + age = state.age_days(run_id) + if age is None: + state.fail('{}:{}: {!r} names run {!r}, which has no usable measurement ' + 'date'.format(docname, lineno, key, run_id)) + elif age > state.max_age_days: + state.fail('{}:{}: {!r} was measured {} days ago; policy.max_age_days is {}. ' + 'Re-measure in pyg-bench and republish.'.format( + docname, lineno, key, age, state.max_age_days)) + + quotable = cell['board_quotable'] is True + if diagnostic and quotable: + state.fail('{}:{}: {!r} is a published result; use :bench: not :bench-diag:'.format( + docname, lineno, key)) + if not diagnostic and not quotable: + state.fail( + '{}:{}: {!r} is not board-quotable (status={!r}, comparison_allowed={!r}); it ' + 'may only appear through :bench-diag:, which labels it diagnostic-only'.format( + docname, lineno, key, cell['status'], cell['comparison_allowed'])) + return cell + + +def audit_pages(state: State) -> None: + """Every page that prints a number must also carry its provenance and caveats.""" + for docname, keys in sorted(state.refs.items()): + runs = set() + needs_disclosure = False + for key in keys: + cell = state.cell(key) + if cell is None: + continue + run = cell['run'] + if isinstance(run, str): + runs.add(run) + raw = cell.get('disclosures') + if isinstance(raw, list) and raw: + needs_disclosure = True + rendered = set(state.provenance.get(docname, [])) + for run_id in sorted(runs - rendered): + state.fail('{}: prints a number from run {!r} without rendering its provenance ' + '(add ".. bench-provenance:: {}")'.format(docname, run_id, run_id)) + if needs_disclosure and docname not in state.disclosed: + state.fail('{}: prints a number that carries a disclosure but has no ' + '".. bench-disclosures::" block'.format(docname)) + + +def load_state(today: Optional[datetime.date] = None) -> State: + """Load the vendored artifact, re-verify it, and return the render-time state.""" + payload = load(BENCHMARKS_JSON) + contract = load(CONTRACT_JSON) + reverify(payload, contract) + return State(payload, today or datetime.date.today()) diff --git a/docs/test_bench_numbers.py b/docs/test_bench_numbers.py index 0103117e7a..99a9d73034 100644 --- a/docs/test_bench_numbers.py +++ b/docs/test_bench_numbers.py @@ -7,7 +7,10 @@ sides check it. These run in the ordinary test lane, not only in the docs build, so a number going stale -or a page referencing a key that no longer exists fails CI rather than a nightly. +or a page referencing a key that no longer exists fails CI rather than a nightly. That is +why every rule lives in `gfql_bench_data`, which imports nothing but the standard library; +the docutils half is a renderer. A gate that needs Sphinx to run is a gate that runs in one +job out of forty. """ import datetime @@ -22,7 +25,7 @@ SOURCE_DIR = os.path.join(DOCS_DIR, 'source') sys.path.insert(0, os.path.join(SOURCE_DIR, '_ext')) -import gfql_bench # noqa: E402 +import gfql_bench_data as bench # noqa: E402 #: ``:bench:`key``` / ``:bench-diag:`key``` as written in the .rst sources. BENCH_REF = re.compile(r':(bench|bench-diag):`([^`]+)`') @@ -30,12 +33,12 @@ @pytest.fixture(scope='module') def payload(): - return gfql_bench._load(gfql_bench.BENCHMARKS_JSON) + return bench.load(bench.BENCHMARKS_JSON) @pytest.fixture(scope='module') def contract(): - return gfql_bench._load(gfql_bench.CONTRACT_JSON) + return bench.load(bench.CONTRACT_JSON) def _rst_sources(): @@ -55,7 +58,7 @@ def _references(): def test_the_vendored_artifact_satisfies_the_vendored_contract(payload, contract): - gfql_bench._reverify(payload, contract) + bench.reverify(payload, contract) def test_the_vendored_contract_is_the_one_the_artifact_was_built_against(payload, contract): @@ -105,8 +108,8 @@ def test_a_board_quotable_cell_that_is_not_comparable_is_rejected(payload, contr key = sorted(broken['cells'])[0] broken['cells'][key]['board_quotable'] = True broken['cells'][key]['comparison_allowed'] = False - with pytest.raises(gfql_bench.BenchNumberError) as excinfo: - gfql_bench._reverify(broken, contract) + with pytest.raises(bench.BenchDataError) as excinfo: + bench.reverify(broken, contract) assert 'not comparison_allowed' in str(excinfo.value) @@ -114,8 +117,8 @@ def test_a_run_missing_provenance_is_rejected(payload, contract): broken = json.loads(json.dumps(payload)) run_id = sorted(broken['runs'])[0] del broken['runs'][run_id]['host'] - with pytest.raises(gfql_bench.BenchNumberError) as excinfo: - gfql_bench._reverify(broken, contract) + with pytest.raises(bench.BenchDataError) as excinfo: + bench.reverify(broken, contract) assert "missing provenance 'host'" in str(excinfo.value) @@ -126,31 +129,19 @@ def test_a_caveated_number_without_its_caveat_is_rejected(payload, contract): broken['cells'][key]['board_quotable'] = False broken['cells'][key]['comparison_allowed'] = False broken['cells'][key]['disclosures'] = [] - with pytest.raises(gfql_bench.BenchNumberError) as excinfo: - gfql_bench._reverify(broken, contract) + with pytest.raises(bench.BenchDataError) as excinfo: + bench.reverify(broken, contract) assert 'carries no disclosure' in str(excinfo.value) -class _Document(object): - class settings(object): - class env(object): - docname = 'gfql/example' - - -class _Inliner(object): - document = _Document - - def _use(key, diagnostic=False, today=None): - """Reference a benchmark key the way a page does, and report what broke.""" - payload = gfql_bench._load(gfql_bench.BENCHMARKS_JSON) - state = gfql_bench._State(payload, today or datetime.date.today()) - previous = gfql_bench._STATE - gfql_bench._STATE = state - try: - gfql_bench._bench_role(diagnostic)('bench', '', key, 1, _Inliner) - finally: - gfql_bench._STATE = previous + """Reference a benchmark key the way a page does, and report what broke. + + Deliberately does not go through Sphinx: the decision lives in the stdlib-only + module precisely so it is checked in every lane, not just the docs build. + """ + state = bench.State(bench.load(bench.BENCHMARKS_JSON), today or datetime.date.today()) + bench.check_reference(state, key, 'gfql/example', 1, diagnostic) return state.problems @@ -186,6 +177,16 @@ def test_a_diagnostic_only_number_cannot_be_printed_as_a_result(payload): def test_an_artifact_from_another_contract_version_is_rejected(payload, contract): broken = json.loads(json.dumps(payload)) broken['contract_version'] = int(contract['contract_version']) + 1 - with pytest.raises(gfql_bench.BenchNumberError) as excinfo: - gfql_bench._reverify(broken, contract) + with pytest.raises(bench.BenchDataError) as excinfo: + bench.reverify(broken, contract) assert 'contract_version' in str(excinfo.value) + + +def test_the_rules_module_stays_importable_without_sphinx(): + """CI caught the first draft: the rules lived in the docutils module, so the + minimal lane could not import them and the whole gate ran in one job.""" + with open(os.path.join(SOURCE_DIR, '_ext', 'gfql_bench_data.py'), encoding='utf-8') as f: + source = f.read() + for forbidden in ('docutils', 'sphinx'): + assert 'import {}'.format(forbidden) not in source + assert 'from {}'.format(forbidden) not in source