Skip to content

Commit d47f061

Browse files
committed
Fix #1454: add ranking regression gate with Kendall-tau snapshots (#1505)
1 parent 1ab0763 commit d47f061

10 files changed

Lines changed: 1000 additions & 3 deletions

File tree

.github/PULL_REQUEST_TEMPLATE.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
## Summary
2+
3+
<!-- Describe what this PR does and why -->
4+
5+
## Test plan
6+
7+
- [ ] `cargo test --workspace` passes locally
8+
- [ ] `cargo fmt --all -- --check` passes
9+
- [ ] `cargo clippy --workspace --all-targets` clean
10+
- [ ] Relevant integration tests verified
11+
12+
## Ranking change ACK
13+
14+
<!-- Complete this section ONLY if ranking snapshot files were updated -->
15+
16+
- [ ] **Ranking change ACK** -- I ran `UPDATE_RANKING_SNAPSHOTS=1 cargo test -p terraphim_service ranking_regression` locally, reviewed the snapshot diff, and confirm this is an intentional ranking change
17+
18+
<!-- If ranking snapshots are unchanged, leave the checkbox unchecked and delete this note -->
19+
20+
## Related issues
21+
22+
<!-- Refs #ISSUE or Fixes #ISSUE -->

.github/workflows/ci-pr.yml

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,73 @@ jobs:
374374
if: always()
375375
run: /home/alex/.local/bin/sccache --show-stats
376376

377+
# Ranking regression gate (WIG-1 lead measure)
378+
ranking-regression-gate:
379+
name: Ranking Regression Gate
380+
runs-on: [self-hosted, bigbox]
381+
timeout-minutes: 5
382+
needs: changes
383+
if: needs.changes.outputs.rust-changed == 'true'
384+
env:
385+
RUSTC_WRAPPER: /home/alex/.local/bin/sccache
386+
SCCACHE_BUCKET: rust-cache
387+
SCCACHE_SERVER_PORT: "4231"
388+
SCCACHE_ENDPOINT: http://172.26.0.1:8333
389+
SCCACHE_S3_USE_SSL: "false"
390+
SCCACHE_REGION: us-east-1
391+
SCCACHE_S3_KEY_PREFIX: terraphim-ai
392+
AWS_ACCESS_KEY_ID: any
393+
AWS_SECRET_ACCESS_KEY: any
394+
395+
steps:
396+
- name: Fix workspace permissions
397+
run: |
398+
WORKDIR="${GITHUB_WORKSPACE:-$PWD}"
399+
sudo chown -R $(id -u):$(id -g) "${WORKDIR}" 2>/dev/null || true
400+
sudo chmod -R u+rw "${WORKDIR}" 2>/dev/null || true
401+
402+
- name: Pre-checkout cleanup
403+
run: |
404+
WORKDIR="${GITHUB_WORKSPACE:-$PWD}"
405+
sudo rm -rf "${WORKDIR}/target" || true
406+
407+
- name: Checkout
408+
uses: actions/checkout@v6
409+
410+
- name: Install Rust toolchain
411+
uses: dtolnay/rust-toolchain@stable
412+
with:
413+
toolchain: "1.94.0"
414+
415+
- name: sccache start and zero stats
416+
run: |
417+
/home/alex/.local/bin/sccache --start-server || true
418+
/home/alex/.local/bin/sccache --zero-stats
419+
420+
- name: Cache Cargo registry and index
421+
uses: actions/cache@v4
422+
with:
423+
path: |
424+
~/.cargo/registry/index
425+
~/.cargo/registry/cache
426+
~/.cargo/git/db
427+
target
428+
key: ${{ runner.os }}-cargo-ranking-${{ hashFiles('**/Cargo.lock') }}
429+
restore-keys: |
430+
${{ runner.os }}-cargo-ranking-
431+
${{ runner.os }}-cargo-
432+
433+
- name: Run ranking regression gate
434+
run: |
435+
# Fails if Kendall-tau < 0.95 against committed snapshots.
436+
# To accept an intentional ranking change, run locally with
437+
# UPDATE_RANKING_SNAPSHOTS=1 and add "Ranking change ACK" to the PR.
438+
cargo test -p terraphim_service ranking_regression
439+
440+
- name: sccache stats
441+
if: always()
442+
run: /home/alex/.local/bin/sccache --show-stats
443+
377444
# WASM build verification
378445
wasm-build:
379446
name: WASM Build Check
@@ -491,7 +558,7 @@ jobs:
491558
name: PR Validation Summary
492559
runs-on: [self-hosted, bigbox]
493560
timeout-minutes: 1
494-
needs: [changes, build-frontend, rust-format, rust-clippy, rust-compile, rust-tests, frontend-check, wasm-build]
561+
needs: [changes, build-frontend, rust-format, rust-clippy, rust-compile, rust-tests, frontend-check, wasm-build, ranking-regression-gate]
495562
if: always()
496563

497564
steps:
@@ -515,12 +582,14 @@ jobs:
515582
echo "| Rust Tests | ${{ needs.rust-tests.result || 'skipped' }} | Unit test execution |" >> $GITHUB_STEP_SUMMARY
516583
echo "| Frontend Check | ${{ needs.frontend-check.result || 'skipped' }} | Frontend linting and types |" >> $GITHUB_STEP_SUMMARY
517584
echo "| WASM Build | ${{ needs.wasm-build.result || 'skipped' }} | WebAssembly compilation |" >> $GITHUB_STEP_SUMMARY
585+
echo "| Ranking Regression Gate | ${{ needs.ranking-regression-gate.result || 'skipped' }} | Kendall-tau snapshot gate (WIG-1) |" >> $GITHUB_STEP_SUMMARY
518586
echo "" >> $GITHUB_STEP_SUMMARY
519587
520588
if [[ "${{ needs.rust-format.result }}" == "failure" ]] || \
521589
[[ "${{ needs.rust-clippy.result }}" == "failure" ]] || \
522590
[[ "${{ needs.rust-compile.result }}" == "failure" ]] || \
523-
[[ "${{ needs.rust-tests.result }}" == "failure" ]]; then
591+
[[ "${{ needs.rust-tests.result }}" == "failure" ]] || \
592+
[[ "${{ needs.ranking-regression-gate.result }}" == "failure" ]]; then
524593
echo "❌ **PR Validation Failed** - Please fix the failing checks before merging." >> $GITHUB_STEP_SUMMARY
525594
exit 1
526595
else

crates/terraphim_multi_agent/examples/specialized_agents.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
//! that leverage the generic LLM interface instead of OpenRouter-specific code.
55
66
use terraphim_multi_agent::{
7-
test_utils, ChatAgent, ChatConfig, SummarizationAgent, SummarizationConfig, SummaryStyle,
7+
ChatAgent, ChatConfig, SummarizationAgent, SummarizationConfig, SummaryStyle, test_utils,
88
};
99

1010
#[tokio::main]
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
{
2+
"role": "Default",
3+
"scorer": "BM25",
4+
"description": "20-document corpus for default role BM25 regression gate",
5+
"documents": [
6+
{
7+
"id": "doc_001",
8+
"title": "Rust Programming Language",
9+
"body": "Rust is a systems programming language focused on safety concurrency and performance. It prevents segfaults and guarantees thread safety.",
10+
"description": "Official guide to the Rust programming language"
11+
},
12+
{
13+
"id": "doc_002",
14+
"title": "Knowledge Graph Systems",
15+
"body": "Knowledge graphs represent entities and their relationships in a structured graph format. They enable semantic reasoning over connected data.",
16+
"description": "Introduction to knowledge graph design and querying"
17+
},
18+
{
19+
"id": "doc_003",
20+
"title": "Search Engine Design",
21+
"body": "Modern search engines use inverted indexes BM25 ranking and neural reranking to deliver relevant results to user queries.",
22+
"description": "Architecture and algorithms behind web search engines"
23+
},
24+
{
25+
"id": "doc_004",
26+
"title": "Automata Theory Fundamentals",
27+
"body": "Finite automata recognise regular languages. Pushdown automata handle context-free grammars. Turing machines compute all computable functions.",
28+
"description": "Theoretical foundations of computation and automata"
29+
},
30+
{
31+
"id": "doc_005",
32+
"title": "Graph Algorithms and Data Structures",
33+
"body": "Graph traversal algorithms include depth-first search and breadth-first search. Shortest path algorithms include Dijkstra and A-star.",
34+
"description": "Algorithmic techniques for graph problems"
35+
},
36+
{
37+
"id": "doc_006",
38+
"title": "Async Rust with Tokio",
39+
"body": "Tokio provides an asynchronous runtime for Rust. Async await syntax enables non-blocking IO operations without callback hell.",
40+
"description": "Asynchronous programming patterns in Rust using Tokio"
41+
},
42+
{
43+
"id": "doc_007",
44+
"title": "Semantic Search with Embeddings",
45+
"body": "Semantic search uses dense vector embeddings to find conceptually similar documents. Vector databases store and query these embeddings efficiently.",
46+
"description": "Embedding-based semantic search systems"
47+
},
48+
{
49+
"id": "doc_008",
50+
"title": "Finite State Machines",
51+
"body": "A finite state machine consists of states transitions and actions. Automata theory provides formal models for sequential logic systems.",
52+
"description": "Design and implementation of finite state machines"
53+
},
54+
{
55+
"id": "doc_009",
56+
"title": "Rust Web Frameworks",
57+
"body": "Axum Actix-web and Salvo are popular Rust web frameworks. They offer async request handling middleware and routing capabilities.",
58+
"description": "Comparison of Rust web development frameworks"
59+
},
60+
{
61+
"id": "doc_010",
62+
"title": "Graph Database Systems",
63+
"body": "Graph databases like Neo4j store data as nodes and edges. Property graphs support rich metadata on both nodes and relationships.",
64+
"description": "Survey of graph database technologies"
65+
},
66+
{
67+
"id": "doc_011",
68+
"title": "Machine Learning Fundamentals",
69+
"body": "Neural networks learn representations through backpropagation. Gradient descent optimises the loss function over training examples.",
70+
"description": "Core concepts in machine learning and deep learning"
71+
},
72+
{
73+
"id": "doc_012",
74+
"title": "Distributed Systems Design",
75+
"body": "The CAP theorem states distributed systems cannot simultaneously guarantee consistency availability and partition tolerance.",
76+
"description": "Principles and patterns for distributed system architecture"
77+
},
78+
{
79+
"id": "doc_013",
80+
"title": "Python for Data Science",
81+
"body": "Python libraries NumPy Pandas and Matplotlib are standard tools for data analysis visualisation and scientific computing.",
82+
"description": "Python ecosystem for data science workflows"
83+
},
84+
{
85+
"id": "doc_014",
86+
"title": "Database Indexing Strategies",
87+
"body": "B-tree indexes support range queries efficiently. Hash indexes excel at equality lookups. Covering indexes eliminate table scans.",
88+
"description": "Indexing techniques for relational and NoSQL databases"
89+
},
90+
{
91+
"id": "doc_015",
92+
"title": "Compiler Design Principles",
93+
"body": "Compilers consist of lexers parsers semantic analysers and code generators. LLVM provides a widely used compiler backend infrastructure.",
94+
"description": "Phases and techniques in compiler construction"
95+
},
96+
{
97+
"id": "doc_016",
98+
"title": "Rust Memory Safety",
99+
"body": "Rust ownership borrow checker and lifetimes enforce memory safety at compile time. No garbage collector means deterministic performance.",
100+
"description": "How Rust achieves memory safety without runtime overhead"
101+
},
102+
{
103+
"id": "doc_017",
104+
"title": "Information Retrieval Methods",
105+
"body": "Information retrieval systems rank documents by relevance to queries using BM25 TF-IDF and learned ranking models.",
106+
"description": "Classic and modern information retrieval techniques"
107+
},
108+
{
109+
"id": "doc_018",
110+
"title": "Aho-Corasick String Matching",
111+
"body": "The Aho-Corasick automaton enables simultaneous multi-pattern string matching in linear time. It is used in network intrusion detection and text search.",
112+
"description": "Efficient multi-pattern string matching with automata"
113+
},
114+
{
115+
"id": "doc_019",
116+
"title": "Rust Error Handling",
117+
"body": "Rust uses Result and Option types for explicit error handling. The question mark operator propagates errors ergonomically through call chains.",
118+
"description": "Idiomatic error handling patterns in Rust"
119+
},
120+
{
121+
"id": "doc_020",
122+
"title": "Knowledge Representation Languages",
123+
"body": "OWL RDF and SPARQL are W3C standards for knowledge representation and querying. They underpin the semantic web and linked data initiatives.",
124+
"description": "Formal languages for knowledge representation"
125+
}
126+
]
127+
}
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"corpus": "corpus_default",
3+
"snapshots": [
4+
{
5+
"query": "rust",
6+
"scorer": "BM25",
7+
"top_n": 5,
8+
"expected_ids": [
9+
"doc_009",
10+
"doc_016",
11+
"doc_001",
12+
"doc_019",
13+
"doc_002"
14+
]
15+
},
16+
{
17+
"query": "search engine",
18+
"scorer": "BM25",
19+
"top_n": 5,
20+
"expected_ids": [
21+
"doc_005",
22+
"doc_003",
23+
"doc_007",
24+
"doc_001",
25+
"doc_002"
26+
]
27+
},
28+
{
29+
"query": "automata theory",
30+
"scorer": "BM25",
31+
"top_n": 5,
32+
"expected_ids": [
33+
"doc_008",
34+
"doc_004",
35+
"doc_001",
36+
"doc_002",
37+
"doc_003"
38+
]
39+
},
40+
{
41+
"query": "graph database",
42+
"scorer": "BM25",
43+
"top_n": 5,
44+
"expected_ids": [
45+
"doc_005",
46+
"doc_002",
47+
"doc_010",
48+
"doc_001",
49+
"doc_003"
50+
]
51+
},
52+
{
53+
"query": "knowledge representation",
54+
"scorer": "BM25",
55+
"top_n": 5,
56+
"expected_ids": [
57+
"doc_020",
58+
"doc_002",
59+
"doc_001",
60+
"doc_003",
61+
"doc_004"
62+
]
63+
}
64+
]
65+
}

0 commit comments

Comments
 (0)