Skip to content

Commit ac88426

Browse files
pmarreckpmarreck
committed
search: extract gatherCandidates + filterCandidates from fn search
From fleet code review 2026-06-01 (`disorganized`, WARN — fn search was 312 lines). Two helpers carved out: - `gatherCandidates(allocator, db, embedder, query, options, &results, &vector_ranks, &lexical_ranks)`: dispatches on `options.mode` (lexical / vector / hybrid). Handles the FTS5 fetch for lexical, the embedding + vector fetch, and the hybrid bm25 merge (O(1) via the id→index map added in `31e1199e`). 70 lines extracted. - `filterCandidates(allocator, options, &results)`: in-place comments_only + lang/ext/kind filtering. Mutates `results` to a filtered subset and frees the dropped entries. 36 lines extracted. fn search now 214 lines (was 312). The intricate per-result scoring + intent inference + duplicate penalty + sort + dropoff + truncation logic remains inline — covered by many end-to-end search tests, and high-risk to rearrange without an even larger surface refactor (the scoring step reads ~12 fields from `options` and 4 intermediate locals). Zero behavior change; all 52 test binaries pass.
1 parent 25b7d1e commit ac88426

1 file changed

Lines changed: 138 additions & 101 deletions

File tree

‎src/search.zig‎

Lines changed: 138 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,142 @@ pub const SearchResult = struct {
9191
total_relevant: usize,
9292
};
9393

94+
/// Apply post-gather filtering to the candidate set:
95+
/// - comments_only: keep only symbols whose doc_comment is non-null
96+
/// - allowed_langs: keep only matching languages
97+
/// - allowed_exts: keep only matching file extensions
98+
/// - allowed_kinds: keep only matching symbol kinds (fn/struct/etc.)
99+
///
100+
/// In-place: mutates `results` by deinit'ing dropped entries and replacing
101+
/// the list with the filtered subset. Extracted from `pub fn search` (2026-06-02).
102+
fn filterCandidates(
103+
allocator: std.mem.Allocator,
104+
options: Options,
105+
results: *std.ArrayListUnmanaged(Result),
106+
) !void {
107+
if (options.comments_only) {
108+
var filtered = @as(std.ArrayListUnmanaged(Result), .empty);
109+
errdefer {
110+
for (filtered.items) |*res| res.deinit(allocator);
111+
filtered.deinit(allocator);
112+
}
113+
for (results.items) |res| {
114+
if (res.symbol.doc_comment != null) {
115+
try filtered.append(allocator, res);
116+
} else {
117+
var tmp = res;
118+
tmp.deinit(allocator);
119+
}
120+
}
121+
results.deinit(allocator);
122+
results.* = filtered;
123+
}
124+
125+
if (options.allowed_langs.len > 0 or options.allowed_exts.len > 0 or options.allowed_symbol_kinds.len > 0) {
126+
var filtered = @as(std.ArrayListUnmanaged(Result), .empty);
127+
errdefer {
128+
for (filtered.items) |*res| res.deinit(allocator);
129+
filtered.deinit(allocator);
130+
}
131+
for (results.items) |res| {
132+
if (matchesFilters(res.symbol, options)) {
133+
try filtered.append(allocator, res);
134+
} else {
135+
var tmp = res;
136+
tmp.deinit(allocator);
137+
}
138+
}
139+
results.deinit(allocator);
140+
results.* = filtered;
141+
}
142+
}
143+
144+
/// Gather the initial candidate set for `search`. Dispatches on
145+
/// `options.mode`:
146+
/// .lexical → only FTS5 candidates
147+
/// .vector → only vector candidates (populates `vector_ranks` for RRF)
148+
/// .hybrid → vector candidates first; then merge FTS candidates,
149+
/// transferring FTS bm25 onto pre-existing vector rows in O(1)
150+
/// via an id→index map (see hybrid-merge regression test).
151+
///
152+
/// Extracted from `pub fn search`'s 70-line candidate switch (2026-06-02).
153+
fn gatherCandidates(
154+
allocator: std.mem.Allocator,
155+
db: storage.Db,
156+
embedder: embedding.Embedder,
157+
query: []const u8,
158+
options: Options,
159+
results: *std.ArrayListUnmanaged(Result),
160+
vector_ranks: *std.AutoHashMap(i64, usize),
161+
lexical_ranks: *std.AutoHashMap(i64, usize),
162+
) !void {
163+
if (options.mode == .lexical) {
164+
const lexical = try lexicalCandidates(
165+
allocator,
166+
db,
167+
query,
168+
options.top_n * options.candidate_multiplier,
169+
options.comments_only,
170+
options.fts_mode,
171+
);
172+
for (lexical) |res| try results.append(allocator, res);
173+
allocator.free(lexical);
174+
return;
175+
}
176+
177+
const inputs = [_][]const u8{ query };
178+
const embeddings = try embedder.embed(embedder.ctx, allocator, &inputs);
179+
defer embedder.free(embedder.ctx, allocator, embeddings);
180+
if (embeddings.len != 1) return error.InvalidEmbeddingCount;
181+
182+
const limit = options.top_n * options.candidate_multiplier;
183+
const vector_results = try vectorCandidates(allocator, db, embeddings[0], limit, options.comments_only);
184+
185+
// Build vector rank map (candidates are ordered by distance, best first).
186+
for (vector_results, 0..) |res, i| {
187+
try vector_ranks.put(res.id, i + 1); // 1-based rank
188+
}
189+
190+
for (vector_results) |res| try results.append(allocator, res);
191+
allocator.free(vector_results);
192+
193+
if (options.mode != .hybrid) return;
194+
195+
// Map result.id -> index into results.items, so the FTS-bm25 merge below
196+
// can find the existing row in O(1) instead of a linear scan over
197+
// `results.items` per lexical hit. Matters when callers crank --top-n /
198+
// --candidate-multiplier high enough that the lexical and vector
199+
// candidate sets both hit the thousands.
200+
var id_to_index = std.AutoHashMap(i64, usize).init(allocator);
201+
defer id_to_index.deinit();
202+
for (results.items, 0..) |res, idx| {
203+
try id_to_index.put(res.id, idx);
204+
}
205+
206+
const lexical = try lexicalCandidates(allocator, db, query, limit, options.comments_only, options.fts_mode);
207+
defer allocator.free(lexical);
208+
209+
// Build lexical rank map (candidates are ordered by relevance, best first).
210+
for (lexical, 0..) |res, i| {
211+
try lexical_ranks.put(res.id, i + 1); // 1-based rank
212+
}
213+
214+
for (lexical) |res| {
215+
if (id_to_index.get(res.id)) |existing_idx| {
216+
// Symbol already came back from vector candidates with bm25=0.
217+
// Transfer the FTS bm25 onto the existing row so the scorer can
218+
// use it; otherwise we throw away the FTS signal and the merged
219+
// result scores like a vector-only hit.
220+
results.items[existing_idx].bm25 = res.bm25;
221+
var tmp = res;
222+
tmp.deinit(allocator);
223+
} else {
224+
try id_to_index.put(res.id, results.items.len);
225+
try results.append(allocator, res);
226+
}
227+
}
228+
}
229+
94230
pub fn search(
95231
allocator: std.mem.Allocator,
96232
db: storage.Db,
@@ -132,108 +268,9 @@ pub fn search(
132268
var lexical_ranks = std.AutoHashMap(i64, usize).init(allocator);
133269
defer lexical_ranks.deinit();
134270

135-
if (options.mode == .lexical) {
136-
const lexical = try lexicalCandidates(
137-
allocator,
138-
db,
139-
query,
140-
options.top_n * options.candidate_multiplier,
141-
options.comments_only,
142-
options.fts_mode,
143-
);
144-
for (lexical) |res| try results.append(allocator, res);
145-
allocator.free(lexical);
146-
} else {
147-
const inputs = [_][]const u8{ query };
148-
const embeddings = try embedder.embed(embedder.ctx, allocator, &inputs);
149-
defer embedder.free(embedder.ctx, allocator, embeddings);
150-
if (embeddings.len != 1) return error.InvalidEmbeddingCount;
151-
152-
const limit = options.top_n * options.candidate_multiplier;
153-
const vector_results = try vectorCandidates(allocator, db, embeddings[0], limit, options.comments_only);
154-
155-
// Build vector rank map (candidates are ordered by distance, best first).
156-
for (vector_results, 0..) |res, i| {
157-
try vector_ranks.put(res.id, i + 1); // 1-based rank
158-
}
159-
160-
for (vector_results) |res| try results.append(allocator, res);
161-
allocator.free(vector_results);
162-
163-
if (options.mode == .hybrid) {
164-
// Map result.id -> index into results.items, so the FTS-bm25
165-
// merge below can find the existing row in O(1) instead of a
166-
// linear scan over `results.items` per lexical hit. Matters when
167-
// callers crank --top-n / --candidate-multiplier high enough
168-
// that the lexical and vector candidate sets both hit the
169-
// thousands.
170-
var id_to_index = std.AutoHashMap(i64, usize).init(allocator);
171-
defer id_to_index.deinit();
172-
for (results.items, 0..) |res, idx| {
173-
try id_to_index.put(res.id, idx);
174-
}
175-
176-
const lexical = try lexicalCandidates(allocator, db, query, limit, options.comments_only, options.fts_mode);
177-
defer allocator.free(lexical);
178-
179-
// Build lexical rank map (candidates are ordered by relevance, best first).
180-
for (lexical, 0..) |res, i| {
181-
try lexical_ranks.put(res.id, i + 1); // 1-based rank
182-
}
183-
184-
for (lexical) |res| {
185-
if (id_to_index.get(res.id)) |existing_idx| {
186-
// Symbol already came back from vector candidates with
187-
// bm25=0. Transfer the FTS bm25 onto the existing row so
188-
// the scorer can use it; otherwise we throw away the FTS
189-
// signal and the merged result scores like a vector-only
190-
// hit.
191-
results.items[existing_idx].bm25 = res.bm25;
192-
var tmp = res;
193-
tmp.deinit(allocator);
194-
} else {
195-
try id_to_index.put(res.id, results.items.len);
196-
try results.append(allocator, res);
197-
}
198-
}
199-
}
200-
}
201-
202-
if (options.comments_only) {
203-
var filtered = @as(std.ArrayListUnmanaged(Result), .empty);
204-
errdefer {
205-
for (filtered.items) |*res| res.deinit(allocator);
206-
filtered.deinit(allocator);
207-
}
208-
for (results.items) |res| {
209-
if (res.symbol.doc_comment != null) {
210-
try filtered.append(allocator, res);
211-
} else {
212-
var tmp = res;
213-
tmp.deinit(allocator);
214-
}
215-
}
216-
results.deinit(allocator);
217-
results = filtered;
218-
}
271+
try gatherCandidates(allocator, db, embedder, query, options, &results, &vector_ranks, &lexical_ranks);
219272

220-
if (options.allowed_langs.len > 0 or options.allowed_exts.len > 0 or options.allowed_symbol_kinds.len > 0) {
221-
var filtered = @as(std.ArrayListUnmanaged(Result), .empty);
222-
errdefer {
223-
for (filtered.items) |*res| res.deinit(allocator);
224-
filtered.deinit(allocator);
225-
}
226-
for (results.items) |res| {
227-
if (matchesFilters(res.symbol, options)) {
228-
try filtered.append(allocator, res);
229-
} else {
230-
var tmp = res;
231-
tmp.deinit(allocator);
232-
}
233-
}
234-
results.deinit(allocator);
235-
results = filtered;
236-
}
273+
try filterCandidates(allocator, options, &results);
237274

238275
const query_trimmed = std.mem.trim(u8, query, " \t\r\n");
239276

0 commit comments

Comments
 (0)