Skip to content

Commit d9be031

Browse files
AlexMikhalevAlexMikhalev
committed
Merge pull request 'feat(terraphim_grep): failover to fff-search enhanced grep when KG is not configured' (#56) from task/55-grep-kg-failover into main
2 parents 3a46fea + 032a686 commit d9be031

11 files changed

Lines changed: 498 additions & 25 deletions

File tree

‎.cargo/config.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
[registry]
22
global-credential-providers = ["cargo:token"]
33

4+
[registries.terraphim]
5+
index = "sparse+https://git.terraphim.cloud/api/packages/terraphim/cargo/"
6+

‎.github/workflows/publish-crates.yml‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,36 @@ jobs:
2525
- uses: actions/checkout@v4
2626
- uses: dtolnay/rust-toolchain@stable
2727
- uses: Swatinem/rust-cache@v2
28+
- name: Strip private-registry refs for crates.io publish
29+
run: |
30+
python3 - <<'PY'
31+
import re, pathlib
32+
# The Gitea `terraphim` registry mirrors crates.io for these deps.
33+
# crates.io rejects manifests whose deps pin a non-default registry,
34+
# so drop the `registry = "terraphim"` key everywhere and remove any
35+
# [patch.crates-io] entry that pins to it (a patch needs a source).
36+
root = pathlib.Path('Cargo.toml')
37+
lines = root.read_text().splitlines()
38+
out, inp = [], False
39+
for ln in lines:
40+
if ln.strip() == '[patch.crates-io]':
41+
inp = True; out.append(ln); continue
42+
if ln.startswith('['):
43+
inp = False
44+
if inp and 'registry = "terraphim"' in ln:
45+
continue
46+
out.append(ln)
47+
root.write_text("\n".join(out) + "\n")
48+
for f in pathlib.Path('.').rglob('Cargo.toml'):
49+
s = f.read_text()
50+
s = re.sub(r',\s*registry = "terraphim"', '', s)
51+
s = re.sub(r'registry = "terraphim",\s*', '', s)
52+
f.write_text(s)
53+
leftover = [str(p) for p in pathlib.Path('.').rglob('Cargo.toml')
54+
if 'registry = "terraphim"' in p.read_text()]
55+
assert not leftover, f"registry refs remain: {leftover}"
56+
print("Stripped terraphim-registry refs")
57+
PY
2858
- name: Publish crates in dependency order
2959
env:
3060
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
@@ -38,17 +68,26 @@ jobs:
3868
echo "Missing manifest for $crate at $manifest" >&2
3969
exit 1
4070
fi
41-
version=$(grep -m1 '^version' "$manifest" | sed 's/.*"\(.*\)".*/\1/')
71+
# Resolve the real version via cargo metadata so that
72+
# `version.workspace = true` is expanded to the workspace value.
73+
# (A naive `grep '^version'` yields the literal "true".)
74+
version=$(cargo metadata --no-deps --format-version=1 \
75+
| jq -r --arg manifest "$manifest" \
76+
'.packages[] | select(.manifest_path | endswith($manifest)) | .version')
77+
if [ -z "$version" ]; then
78+
echo "ERROR: could not resolve version for $crate from $manifest" >&2
79+
exit 1
80+
fi
4281
# Check crates.io with auth to avoid rate limits
4382
if curl -fsS -H "Authorization: ${CARGO_REGISTRY_TOKEN}" "https://crates.io/api/v1/crates/$crate/$version" >/dev/null 2>&1; then
4483
echo "$crate $version already exists on crates.io; skipping"
4584
continue
4685
fi
4786
if [ "$DRY_RUN" = "true" ]; then
48-
cargo publish --manifest-path "$manifest" --dry-run
87+
cargo publish --manifest-path "$manifest" --dry-run --allow-dirty
4988
else
5089
set +e
51-
output=$(cargo publish --manifest-path "$manifest" 2>&1)
90+
output=$(cargo publish --manifest-path "$manifest" --allow-dirty 2>&1)
5291
exit_code=$?
5392
set -e
5493
if [ "$exit_code" -ne 0 ]; then

‎.github/workflows/release-binaries.yml‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
name: Release Client Binaries
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
version:
7+
description: 'Release version without v prefix (e.g. 1.20.5)'
8+
required: true
9+
type: string
10+
release_tag:
11+
description: 'GitHub release tag (e.g. v1.20.5)'
12+
required: true
13+
type: string
14+
target_repo:
15+
description: 'GitHub repo to attach binaries to'
16+
required: false
17+
default: terraphim-ai
18+
type: string
19+
20+
permissions:
21+
contents: write
22+
23+
env:
24+
CARGO_TERM_COLOR: always
25+
26+
jobs:
27+
build-binaries:
28+
name: Build client binaries for ${{ matrix.target }}
29+
strategy:
30+
fail-fast: false
31+
matrix:
32+
include:
33+
# GitHub-hosted Linux: terraphim-ai self-hosted runners are repo-scoped.
34+
- os: ubuntu-22.04
35+
target: x86_64-unknown-linux-gnu
36+
use_cross: false
37+
- os: ubuntu-22.04
38+
target: x86_64-unknown-linux-musl
39+
use_cross: true
40+
- os: ubuntu-22.04
41+
target: aarch64-unknown-linux-musl
42+
use_cross: true
43+
- os: macos-latest
44+
target: x86_64-apple-darwin
45+
use_cross: false
46+
- os: macos-latest
47+
target: aarch64-apple-darwin
48+
use_cross: false
49+
- os: windows-latest
50+
target: x86_64-pc-windows-msvc
51+
use_cross: false
52+
runs-on: ${{ matrix.os }}
53+
env:
54+
CARGO_REGISTRIES_TERRAPHIM_TOKEN: ${{ secrets.CARGO_REGISTRIES_TERRAPHIM_TOKEN }}
55+
steps:
56+
- uses: actions/checkout@v4
57+
- uses: dtolnay/rust-toolchain@stable
58+
with:
59+
targets: ${{ matrix.target }}
60+
- name: Install zig
61+
if: contains(matrix.target, 'apple-darwin') || contains(matrix.target, 'windows')
62+
shell: bash
63+
run: |
64+
if command -v zig &>/dev/null; then exit 0; fi
65+
if command -v brew &>/dev/null; then brew install zig; fi
66+
if command -v choco &>/dev/null; then choco install zig -y; fi
67+
- name: Install cross
68+
if: matrix.use_cross
69+
run: |
70+
if command -v cross &>/dev/null; then
71+
cross --version
72+
exit 0
73+
fi
74+
rustup run stable cargo install cross --locked --git https://github.com/cross-rs/cross
75+
- uses: Swatinem/rust-cache@v2
76+
if: matrix.target != 'x86_64-unknown-linux-gnu'
77+
with:
78+
key: clients-${{ matrix.target }}
79+
- name: Build client binaries
80+
shell: bash
81+
run: |
82+
if [ "${{ matrix.use_cross }}" = "true" ]; then
83+
BUILD="rustup run stable cross"
84+
else
85+
BUILD="rustup run stable cargo"
86+
fi
87+
$BUILD build --release --target ${{ matrix.target }} -p terraphim_agent --bin terraphim-agent
88+
$BUILD build --release --target ${{ matrix.target }} -p terraphim-cli --bin terraphim-cli
89+
$BUILD build --release --target ${{ matrix.target }} -p terraphim_grep --bin terraphim-grep --features "code-search openrouter"
90+
- name: Package artifacts (Unix)
91+
if: matrix.os != 'windows-latest'
92+
env:
93+
VERSION: ${{ inputs.version }}
94+
run: |
95+
mkdir -p artifacts
96+
tar -czf "artifacts/terraphim-agent-${VERSION}-${{ matrix.target }}.tar.gz" -C "target/${{ matrix.target }}/release" terraphim-agent
97+
tar -czf "artifacts/terraphim-cli-${VERSION}-${{ matrix.target }}.tar.gz" -C "target/${{ matrix.target }}/release" terraphim-cli
98+
tar -czf "artifacts/terraphim-grep-${VERSION}-${{ matrix.target }}.tar.gz" -C "target/${{ matrix.target }}/release" terraphim-grep
99+
cp target/${{ matrix.target }}/release/terraphim-agent artifacts/terraphim-agent-${{ matrix.target }}
100+
cp target/${{ matrix.target }}/release/terraphim-cli artifacts/terraphim-cli-${{ matrix.target }}
101+
cp target/${{ matrix.target }}/release/terraphim-grep artifacts/terraphim-grep-${{ matrix.target }}
102+
chmod +x artifacts/*
103+
- name: Package artifacts (Windows)
104+
if: matrix.os == 'windows-latest'
105+
shell: bash
106+
env:
107+
VERSION: ${{ inputs.version }}
108+
run: |
109+
mkdir -p artifacts
110+
cd "target/${{ matrix.target }}/release"
111+
7z a -tzip "../../../artifacts/terraphim-agent-${VERSION}-${{ matrix.target }}.zip" terraphim-agent.exe
112+
7z a -tzip "../../../artifacts/terraphim-cli-${VERSION}-${{ matrix.target }}.zip" terraphim-cli.exe
113+
7z a -tzip "../../../artifacts/terraphim-grep-${VERSION}-${{ matrix.target }}.zip" terraphim-grep.exe
114+
cd -
115+
cp target/${{ matrix.target }}/release/terraphim-agent.exe artifacts/terraphim-agent-${{ matrix.target }}.exe
116+
cp target/${{ matrix.target }}/release/terraphim-cli.exe artifacts/terraphim-cli-${{ matrix.target }}.exe
117+
cp target/${{ matrix.target }}/release/terraphim-grep.exe artifacts/terraphim-grep-${{ matrix.target }}.exe
118+
- uses: actions/upload-artifact@v4
119+
with:
120+
name: client-binaries-${{ matrix.target }}
121+
path: artifacts/*
122+
123+
create-universal-macos:
124+
name: Create macOS universal client binaries
125+
needs: build-binaries
126+
if: always() && needs.build-binaries.result != 'cancelled'
127+
runs-on: macos-latest
128+
steps:
129+
- uses: actions/download-artifact@v4
130+
with:
131+
name: client-binaries-x86_64-apple-darwin
132+
path: x86_64
133+
- uses: actions/download-artifact@v4
134+
with:
135+
name: client-binaries-aarch64-apple-darwin
136+
path: aarch64
137+
- run: |
138+
mkdir -p universal
139+
lipo -create x86_64/terraphim-agent-x86_64-apple-darwin aarch64/terraphim-agent-aarch64-apple-darwin -output universal/terraphim-agent-universal-apple-darwin
140+
lipo -create x86_64/terraphim-grep-x86_64-apple-darwin aarch64/terraphim-grep-aarch64-apple-darwin -output universal/terraphim-grep-universal-apple-darwin
141+
chmod +x universal/*
142+
- uses: actions/upload-artifact@v4
143+
with:
144+
name: client-binaries-universal-apple-darwin
145+
path: universal/*
146+
147+
sign-and-notarize-macos:
148+
name: Sign and notarize macOS client binaries
149+
needs: create-universal-macos
150+
if: always() && needs.create-universal-macos.result == 'success'
151+
runs-on: macos-latest
152+
steps:
153+
- uses: actions/checkout@v4
154+
- uses: actions/download-artifact@v4
155+
with:
156+
name: client-binaries-universal-apple-darwin
157+
path: universal
158+
- uses: 1password/install-cli-action@v2
159+
- name: Load signing credentials
160+
env:
161+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
162+
run: |
163+
echo "APPLE_ID=$(op read 'op://TerraphimPlatform/apple.developer.credentials/username' --no-newline)" >> $GITHUB_ENV
164+
echo "APPLE_TEAM_ID=$(op read 'op://TerraphimPlatform/apple.developer.credentials/APPLE_TEAM_ID' --no-newline)" >> $GITHUB_ENV
165+
echo "APPLE_APP_PASSWORD=$(op read 'op://TerraphimPlatform/apple.developer.credentials/APPLE_APP_SPECIFIC_PASSWORD' --no-newline)" >> $GITHUB_ENV
166+
echo "CERT_BASE64=$(op read 'op://TerraphimPlatform/apple.developer.certificate/base64' --no-newline)" >> $GITHUB_ENV
167+
echo "CERT_PASSWORD=$(op read 'op://TerraphimPlatform/apple.developer.certificate/password' --no-newline)" >> $GITHUB_ENV
168+
- name: Sign and notarize agent and grep
169+
env:
170+
RUNNER_TEMP: ${{ runner.temp }}
171+
run: |
172+
chmod +x scripts/sign-macos-binary.sh
173+
./scripts/sign-macos-binary.sh universal/terraphim-agent-universal-apple-darwin "$APPLE_ID" "$APPLE_TEAM_ID" "$APPLE_APP_PASSWORD" "$CERT_BASE64" "$CERT_PASSWORD"
174+
./scripts/sign-macos-binary.sh universal/terraphim-grep-universal-apple-darwin "$APPLE_ID" "$APPLE_TEAM_ID" "$APPLE_APP_PASSWORD" "$CERT_BASE64" "$CERT_PASSWORD"
175+
- uses: actions/upload-artifact@v4
176+
with:
177+
name: client-binaries-signed-universal-apple-darwin
178+
path: universal/*
179+
180+
upload-to-target-release:
181+
name: Attach client binaries to terraphim-ai release
182+
needs: [build-binaries, sign-and-notarize-macos]
183+
# Attach when macOS sign succeeded; do not require full matrix (Windows is optional).
184+
if: >-
185+
always() &&
186+
!cancelled() &&
187+
needs.sign-and-notarize-macos.result == 'success' &&
188+
needs.build-binaries.result != 'cancelled'
189+
runs-on: ubuntu-latest
190+
steps:
191+
- uses: actions/download-artifact@v4
192+
with:
193+
pattern: client-binaries*
194+
path: release-assets
195+
merge-multiple: true
196+
- name: Upload to target GitHub release
197+
env:
198+
GH_TOKEN: ${{ secrets.TERRAPHIM_AI_RELEASE_TOKEN || secrets.GITHUB_TOKEN }}
199+
run: |
200+
TAG="${{ inputs.release_tag }}"
201+
REPO="terraphim/${{ inputs.target_repo }}"
202+
find release-assets -type f | sort
203+
gh release upload "$TAG" release-assets/* --repo "$REPO" --clobber

‎Cargo.toml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ members = [
1414
]
1515

1616
[workspace.package]
17-
version = "1.20.4"
17+
version = "1.21.0"
1818
edition = "2024"
1919
authors = ["Terraphim Team <team@terraphim.ai>"]
2020
documentation = "https://terraphim.ai"
@@ -23,6 +23,10 @@ repository = "https://git.terraphim.cloud/terraphim/terraphim-clients"
2323
license = "Apache-2.0"
2424
readme = "README.md"
2525

26+
[patch.crates-io]
27+
terraphim_service = { version = "1.20.5", registry = "terraphim" }
28+
rustls-webpki = { git = "https://github.com/rustls/webpki.git", tag = "v/0.103.12" }
29+
2630
[workspace.dependencies]
2731
tokio = { version = "1.0", features = ["full"] }
2832
reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false }
@@ -39,9 +43,6 @@ tempfile = "3.27"
3943
rustls = { version = "0.23", default-features = false }
4044
rustls-webpki = "0.103.12"
4145

42-
[patch.crates-io]
43-
rustls-webpki = { git = "https://github.com/rustls/webpki.git", tag = "v/0.103.12" }
44-
4546
[profile.release]
4647
panic = "unwind"
4748
lto = false

‎crates/terraphim_agent/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ terraphim_persistence = { version = "1.0.0" }
7474
terraphim_config = { version = "1.0.0" }
7575
terraphim_command_runtime = { path = "../terraphim_command_runtime", version = "0.1.0" }
7676
terraphim_automata = { version = "1.19.2" }
77-
terraphim_service = { version = "1.0.0", default-features = false }
77+
terraphim_service = { version = "1.20.4", default-features = false, registry = "terraphim" }
7878
terraphim_middleware = { version = "1.0.0" }
7979
terraphim_rolegraph = { version = "1.0.0" }
8080
terraphim_hooks = { path = "../terraphim_hooks", version = "1.0.0" }

‎crates/terraphim_cli/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ path = "src/main.rs"
1818

1919
[dependencies]
2020
# Core terraphim crates
21-
terraphim_service = { version = "1.0.0" }
21+
terraphim_service = { version = "1.20.4", registry = "terraphim" }
2222
terraphim_config = { version = "1.0.0" }
2323
terraphim_command_runtime = { path = "../terraphim_command_runtime", version = "0.1.0" }
2424
terraphim_types = { version = "1.0.0" }

‎crates/terraphim_grep/Cargo.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ log.workspace = true
2828
terraphim_types = { version = "1.15.0" }
2929
terraphim_rolegraph = { version = "1.15.0" }
3030
terraphim_automata = { version = "1.19.2" }
31-
terraphim_service = { version = "1.16.15", optional = true }
31+
terraphim_service = { version = "1.20.5", optional = true, registry = "terraphim" }
3232
terraphim_config = { version = "1.15.0" }
3333

3434
fff-search = { version = "0.8.4", optional = true }
@@ -44,8 +44,12 @@ clap = { version = "4", features = ["derive"] }
4444
# `code-search` ships in the default set so a plain `cargo build`/`cargo install`
4545
# produces a grep that actually searches file contents. Without it, `search_code`
4646
# compiles to a no-op stub that returns no matches (see hybrid_searcher.rs).
47+
# It is also required for the no-KG failover path.
4748
default = ["llm", "code-search"]
4849
llm = ["dep:terraphim_service"]
50+
# Enable fast file-finder code search. This is enabled by default so that
51+
# terraphim-grep can fall back to plain enhanced grep when no knowledge graph
52+
# thesaurus is configured.
4953
code-search = ["dep:fff-search"]
5054
# Enable OpenRouter provider support (required for live OpenRouter tests against free models)
5155
openrouter = ["llm", "terraphim_service/openrouter"]

‎crates/terraphim_grep/src/lib.rs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,4 +390,47 @@ mod tests {
390390
);
391391
assert!(result.answer.is_none(), "no LLM -> no synthesised answer");
392392
}
393+
394+
/// When no thesaurus is available, the searcher must still run the `fff-search` code path
395+
/// and return results with empty concepts. This is the "enhanced grep" failover mode.
396+
#[cfg(feature = "code-search")]
397+
#[tokio::test]
398+
async fn search_without_thesaurus_uses_fff_mode() {
399+
let tmp = tempfile::TempDir::new().expect("tempdir");
400+
for i in 0..3 {
401+
let path = tmp.path().join(format!("file_{i}.rs"));
402+
std::fs::write(&path, format!("fn target_{i}() {{ /* target */ }}\n")).unwrap();
403+
}
404+
405+
// Empty thesaurus => no KG configuration.
406+
let thesaurus = Thesaurus::new("test-role".to_string());
407+
assert!(thesaurus.is_empty());
408+
409+
let hybrid = HybridSearcher::new("test-role".to_string(), thesaurus)
410+
.expect("build hybrid searcher")
411+
.with_search_path(tmp.path().to_path_buf());
412+
let grep = TerraphimGrep::new(Arc::new(hybrid), Arc::new(SufficiencyJudge::default()));
413+
414+
let result = grep
415+
.search(
416+
"target",
417+
GrepOptions {
418+
haystack: Haystack::Code,
419+
max_results: 50,
420+
..GrepOptions::default()
421+
},
422+
)
423+
.await
424+
.expect("search should succeed without thesaurus");
425+
426+
assert!(
427+
!result.chunks.is_empty(),
428+
"expected fff-search to return chunks without KG"
429+
);
430+
assert!(
431+
result.concepts.is_empty(),
432+
"expected no KG concepts without thesaurus"
433+
);
434+
assert_eq!(result.stats.kg_hits, 0);
435+
}
393436
}

0 commit comments

Comments
 (0)