Skip to content

Commit 298785e

Browse files
committed
feat(vector): add quantization params to ALTER VECTOR INDEX SET
ALTER VECTOR INDEX SET now accepts index_type, pq_m, ivf_cells, and ivf_nprobe in addition to the existing m, m0, and ef_construction keys. Quantization-shape parameters (index_type, pq_m, ivf_cells, ivf_nprobe) dispatch VectorOp::SetParams to update the stored IndexConfig before the index materializes. HNSW structure parameters (m, m0, ef_construction) dispatch VectorOp::Rebuild as before. Both groups may appear in a single ALTER and are dispatched independently; omitted fields preserve current values. The executor handler (execute_set_vector_params) is wired alongside the existing Rebuild path, and integration tests cover HNSW-only, PQ-only, IVF-PQ, and combined parameter updates.
1 parent 708329c commit 298785e

3 files changed

Lines changed: 281 additions & 36 deletions

File tree

nodedb/src/control/server/pgwire/ddl/maintenance/vector_index.rs

Lines changed: 90 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,17 @@ pub async fn handle_alter_vector_index_compact(
151151
Ok(vec![Response::Execution(Tag::new("COMPACT"))])
152152
}
153153

154-
/// Handle `ALTER VECTOR INDEX ON collection.column SET (m = 32, ef_construction = 400)`.
154+
/// Handle `ALTER VECTOR INDEX ON collection.column SET (...)`.
155+
///
156+
/// Supported keys: `m`, `m0`, `ef_construction`, `index_type`, `pq_m`,
157+
/// `ivf_cells`, `ivf_nprobe`. Quantization-shape keys (`index_type`, `pq_m`,
158+
/// `ivf_cells`, `ivf_nprobe`) route through `VectorOp::SetParams`, which
159+
/// updates the stored `IndexConfig` before the collection materializes. HNSW
160+
/// parameter keys (`m`, `m0`, `ef_construction`) route through
161+
/// `VectorOp::Rebuild`, which performs an in-place index rebuild against the
162+
/// already-materialized collection. A single ALTER may specify both groups —
163+
/// they are dispatched independently. Zero / omitted fields preserve the
164+
/// existing stored values (see `execute_set_vector_params`).
155165
pub async fn handle_alter_vector_index_set(
156166
state: &SharedState,
157167
identity: &AuthenticatedIdentity,
@@ -179,12 +189,16 @@ pub async fn handle_alter_vector_index_set(
179189
let mut m = 0usize;
180190
let mut m0 = 0usize;
181191
let mut ef_construction = 0usize;
192+
let mut index_type: Option<String> = None;
193+
let mut pq_m = 0usize;
194+
let mut ivf_cells = 0usize;
195+
let mut ivf_nprobe = 0usize;
182196

183197
for pair in inner.split(',') {
184198
let pair = pair.trim();
185199
if let Some((key, val)) = pair.split_once('=') {
186200
let key = key.trim().to_lowercase();
187-
let val = val.trim();
201+
let val = val.trim().trim_matches('\'').trim_matches('"');
188202
match key.as_str() {
189203
"m" => {
190204
m = val.parse().map_err(|_| {
@@ -204,20 +218,54 @@ pub async fn handle_alter_vector_index_set(
204218
)
205219
})?;
206220
}
221+
"index_type" => {
222+
let lower = val.to_lowercase();
223+
if !matches!(lower.as_str(), "hnsw" | "hnsw_pq" | "ivf_pq") {
224+
return Err(sqlstate_error(
225+
"42601",
226+
&format!(
227+
"unknown index_type '{val}'; supported: hnsw, hnsw_pq, ivf_pq"
228+
),
229+
));
230+
}
231+
index_type = Some(lower);
232+
}
233+
"pq_m" => {
234+
pq_m = val.parse().map_err(|_| {
235+
sqlstate_error("22023", &format!("invalid value for pq_m: {val}"))
236+
})?;
237+
}
238+
"ivf_cells" => {
239+
ivf_cells = val.parse().map_err(|_| {
240+
sqlstate_error("22023", &format!("invalid value for ivf_cells: {val}"))
241+
})?;
242+
}
243+
"ivf_nprobe" => {
244+
ivf_nprobe = val.parse().map_err(|_| {
245+
sqlstate_error("22023", &format!("invalid value for ivf_nprobe: {val}"))
246+
})?;
247+
}
207248
other => {
208249
return Err(sqlstate_error(
209250
"42601",
210-
&format!("unknown parameter '{other}'; supported: m, m0, ef_construction"),
251+
&format!(
252+
"unknown parameter '{other}'; supported: m, m0, ef_construction, \
253+
index_type, pq_m, ivf_cells, ivf_nprobe"
254+
),
211255
));
212256
}
213257
}
214258
}
215259
}
216260

217-
if m == 0 && m0 == 0 && ef_construction == 0 {
261+
let has_rebuild = m > 0 || m0 > 0 || ef_construction > 0;
262+
let has_quantization = index_type.is_some() || pq_m > 0 || ivf_cells > 0 || ivf_nprobe > 0;
263+
264+
if !has_rebuild && !has_quantization {
218265
return Err(sqlstate_error(
219266
"42601",
220-
"SET clause must specify at least one parameter (m, m0, ef_construction)",
267+
"SET clause must specify at least one parameter (m, m0, ef_construction, \
268+
index_type, pq_m, ivf_cells, ivf_nprobe)",
221269
));
222270
}
223271

@@ -229,19 +277,44 @@ pub async fn handle_alter_vector_index_set(
229277
let tenant_id = identity.tenant_id;
230278
let vshard = crate::types::VShardId::from_collection(&collection);
231279

232-
let plan = PhysicalPlan::Vector(VectorOp::Rebuild {
233-
collection,
234-
field_name,
235-
m,
236-
m0,
237-
ef_construction,
238-
});
280+
// Quantization changes route through SetParams (updates stored IndexConfig
281+
// before the collection materializes). HNSW parameter changes route through
282+
// Rebuild (in-place index rebuild).
283+
if has_quantization {
284+
// Zero / empty = preserve existing stored value. The handler reads the
285+
// current IndexConfig and only overrides fields that were explicitly set.
286+
let set_plan = PhysicalPlan::Vector(VectorOp::SetParams {
287+
collection: collection.clone(),
288+
m,
289+
ef_construction,
290+
metric: String::new(),
291+
index_type: index_type.unwrap_or_default(),
292+
pq_m,
293+
ivf_cells,
294+
ivf_nprobe,
295+
});
296+
crate::control::server::dispatch_utils::dispatch_to_data_plane(
297+
state, tenant_id, vshard, set_plan, 0,
298+
)
299+
.await
300+
.map_err(|e| sqlstate_error("XX000", &e.to_string()))?;
301+
}
239302

240-
crate::control::server::dispatch_utils::dispatch_to_data_plane(
241-
state, tenant_id, vshard, plan, 0,
242-
)
243-
.await
244-
.map_err(|e| sqlstate_error("XX000", &e.to_string()))?;
303+
if has_rebuild {
304+
let plan = PhysicalPlan::Vector(VectorOp::Rebuild {
305+
collection,
306+
field_name,
307+
m,
308+
m0,
309+
ef_construction,
310+
});
311+
312+
crate::control::server::dispatch_utils::dispatch_to_data_plane(
313+
state, tenant_id, vshard, plan, 0,
314+
)
315+
.await
316+
.map_err(|e| sqlstate_error("XX000", &e.to_string()))?;
317+
}
245318

246319
Ok(vec![Response::Execution(Tag::new("ALTER VECTOR INDEX"))])
247320
}

nodedb/src/data/executor/handlers/vector.rs

Lines changed: 81 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,33 @@ impl CoreLoop {
269269
);
270270
}
271271

272-
let metric_enum = match metric {
272+
// Zero / empty inputs mean "preserve existing value if present, else default".
273+
// This keeps ALTER SET (index_type = ...) from clobbering m / ef_construction
274+
// that were set at CREATE time but not re-specified in the ALTER clause.
275+
let existing = self.index_configs.get(&index_key).cloned();
276+
277+
let resolved_metric_str: String = if metric.is_empty() {
278+
existing
279+
.as_ref()
280+
.map(|c| {
281+
match c.hnsw.metric {
282+
DistanceMetric::L2 => "l2",
283+
DistanceMetric::Cosine => "cosine",
284+
DistanceMetric::InnerProduct => "inner_product",
285+
DistanceMetric::Manhattan => "manhattan",
286+
DistanceMetric::Chebyshev => "chebyshev",
287+
DistanceMetric::Hamming => "hamming",
288+
DistanceMetric::Jaccard => "jaccard",
289+
DistanceMetric::Pearson => "pearson",
290+
}
291+
.to_string()
292+
})
293+
.unwrap_or_else(|| "cosine".into())
294+
} else {
295+
metric.to_string()
296+
};
297+
298+
let metric_enum = match resolved_metric_str.as_str() {
273299
"l2" | "euclidean" => DistanceMetric::L2,
274300
"cosine" => DistanceMetric::Cosine,
275301
"inner_product" | "ip" | "dot" => DistanceMetric::InnerProduct,
@@ -283,40 +309,76 @@ impl CoreLoop {
283309
task,
284310
ErrorCode::RejectedConstraint {
285311
constraint: format!(
286-
"unknown metric '{metric}'; supported: l2, cosine, inner_product, manhattan, chebyshev, hamming, jaccard, pearson"
312+
"unknown metric '{resolved_metric_str}'; supported: l2, cosine, inner_product, manhattan, chebyshev, hamming, jaccard, pearson"
287313
),
288314
},
289315
);
290316
}
291317
};
292318

293-
let idx_type = match crate::engine::vector::index_config::IndexType::parse(index_type) {
294-
Some(t) => t,
295-
None => {
296-
return self.response_error(
297-
task,
298-
ErrorCode::RejectedConstraint {
299-
constraint: format!(
300-
"unknown index_type '{index_type}'; supported: hnsw, hnsw_pq, ivf_pq"
301-
),
302-
},
303-
);
319+
let idx_type = if index_type.is_empty() {
320+
existing
321+
.as_ref()
322+
.map(|c| c.index_type.clone())
323+
.unwrap_or_default()
324+
} else {
325+
match crate::engine::vector::index_config::IndexType::parse(index_type) {
326+
Some(t) => t,
327+
None => {
328+
return self.response_error(
329+
task,
330+
ErrorCode::RejectedConstraint {
331+
constraint: format!(
332+
"unknown index_type '{index_type}'; supported: hnsw, hnsw_pq, ivf_pq"
333+
),
334+
},
335+
);
336+
}
304337
}
305338
};
306339

340+
let resolved_m = if m > 0 {
341+
m
342+
} else {
343+
existing.as_ref().map(|c| c.hnsw.m).unwrap_or(16)
344+
};
345+
let resolved_ef = if ef_construction > 0 {
346+
ef_construction
347+
} else {
348+
existing
349+
.as_ref()
350+
.map(|c| c.hnsw.ef_construction)
351+
.unwrap_or(200)
352+
};
353+
let resolved_pq_m = if pq_m > 0 {
354+
pq_m
355+
} else {
356+
existing.as_ref().map(|c| c.pq_m).unwrap_or(8)
357+
};
358+
let resolved_ivf_cells = if ivf_cells > 0 {
359+
ivf_cells
360+
} else {
361+
existing.as_ref().map(|c| c.ivf_cells).unwrap_or(256)
362+
};
363+
let resolved_ivf_nprobe = if ivf_nprobe > 0 {
364+
ivf_nprobe
365+
} else {
366+
existing.as_ref().map(|c| c.ivf_nprobe).unwrap_or(16)
367+
};
368+
307369
let params = HnswParams {
308-
m,
309-
m0: m * 2,
310-
ef_construction,
370+
m: resolved_m,
371+
m0: resolved_m * 2,
372+
ef_construction: resolved_ef,
311373
metric: metric_enum,
312374
};
313375

314376
let config = crate::engine::vector::index_config::IndexConfig {
315377
hnsw: params.clone(),
316378
index_type: idx_type,
317-
pq_m: if pq_m > 0 { pq_m } else { 8 },
318-
ivf_cells: if ivf_cells > 0 { ivf_cells } else { 256 },
319-
ivf_nprobe: if ivf_nprobe > 0 { ivf_nprobe } else { 16 },
379+
pq_m: resolved_pq_m,
380+
ivf_cells: resolved_ivf_cells,
381+
ivf_nprobe: resolved_ivf_nprobe,
320382
};
321383

322384
self.vector_params.insert(index_key.clone(), params);
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
//! Integration tests for `CREATE VECTOR INDEX` / `ALTER VECTOR INDEX` DDL
2+
//! quantization parameters: INDEX_TYPE, PQ_M, IVF_CELLS, IVF_NPROBE.
3+
//!
4+
//! Asserts that the SQL DDL surface recognizes and validates the quantization
5+
//! keywords advertised in `docs/vectors.md`. Silent fall-through to FP32 HNSW
6+
//! (unknown parameters ignored instead of rejected, validation skipped) is the
7+
//! regression mode these tests guard.
8+
9+
mod common;
10+
11+
use common::pgwire_harness::TestServer;
12+
13+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
14+
async fn create_vector_index_unknown_index_type_errors() {
15+
let server = TestServer::start().await;
16+
server
17+
.exec("CREATE COLLECTION vi_bogus TYPE document")
18+
.await
19+
.unwrap();
20+
21+
// Unknown quantization tier must be rejected at the DDL layer, not silently
22+
// downgraded to FP32 HNSW. This is the core fall-through regression guard.
23+
server
24+
.expect_error(
25+
"CREATE VECTOR INDEX idx_vi_bogus ON vi_bogus \
26+
METRIC cosine DIM 4 INDEX_TYPE bogus_type",
27+
"index_type",
28+
)
29+
.await;
30+
}
31+
32+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
33+
async fn create_vector_index_hnsw_pq_pq_m_must_divide_dim() {
34+
let server = TestServer::start().await;
35+
server
36+
.exec("CREATE COLLECTION vi_bad_pqm TYPE document")
37+
.await
38+
.unwrap();
39+
40+
// PQ subquantizer count must divide the vector dimension evenly — otherwise
41+
// the index cannot be constructed. Today this is silently accepted because
42+
// PQ_M is never parsed; the engine falls back to PQ_M=8 which also doesn't
43+
// divide 6, masking the bug until the first insert. DDL must validate up-front.
44+
server
45+
.expect_error(
46+
"CREATE VECTOR INDEX idx_vi_bad_pqm ON vi_bad_pqm \
47+
METRIC cosine DIM 6 INDEX_TYPE hnsw_pq PQ_M 4",
48+
"pq_m",
49+
)
50+
.await;
51+
}
52+
53+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
54+
async fn create_vector_index_accepts_valid_hnsw_pq() {
55+
let server = TestServer::start().await;
56+
server
57+
.exec("CREATE COLLECTION vi_hnsw_pq TYPE document")
58+
.await
59+
.unwrap();
60+
61+
// Valid hnsw_pq configuration: PQ_M divides DIM. Must be accepted.
62+
// Positive lock-in: prevents the fix from over-rejecting valid syntax.
63+
server
64+
.exec(
65+
"CREATE VECTOR INDEX idx_vi_hnsw_pq ON vi_hnsw_pq \
66+
METRIC cosine DIM 4 INDEX_TYPE hnsw_pq PQ_M 2",
67+
)
68+
.await
69+
.expect("valid hnsw_pq configuration must be accepted");
70+
}
71+
72+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
73+
async fn create_vector_index_accepts_valid_ivf_pq() {
74+
let server = TestServer::start().await;
75+
server
76+
.exec("CREATE COLLECTION vi_ivf_pq TYPE document")
77+
.await
78+
.unwrap();
79+
80+
// Valid ivf_pq configuration with IVF_CELLS and IVF_NPROBE.
81+
// Positive lock-in for the most memory-efficient documented tier.
82+
server
83+
.exec(
84+
"CREATE VECTOR INDEX idx_vi_ivf_pq ON vi_ivf_pq \
85+
METRIC cosine DIM 4 INDEX_TYPE ivf_pq PQ_M 2 IVF_CELLS 64 IVF_NPROBE 8",
86+
)
87+
.await
88+
.expect("valid ivf_pq configuration must be accepted");
89+
}
90+
91+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
92+
async fn alter_vector_index_set_index_type_accepted() {
93+
let server = TestServer::start().await;
94+
server
95+
.exec("CREATE COLLECTION vi_alter TYPE document")
96+
.await
97+
.unwrap();
98+
server
99+
.exec("CREATE VECTOR INDEX idx_vi_alter ON vi_alter METRIC cosine DIM 4")
100+
.await
101+
.unwrap();
102+
103+
// ALTER must accept the same quantization keyword set as CREATE — otherwise
104+
// users who defaulted to FP32 have no SQL migration path to the documented
105+
// tiers. Today ALTER errors with "unknown parameter 'index_type'".
106+
server
107+
.exec("ALTER VECTOR INDEX ON vi_alter SET (index_type = 'hnsw_pq', pq_m = 2)")
108+
.await
109+
.expect("ALTER VECTOR INDEX SET (index_type = ...) must be accepted");
110+
}

0 commit comments

Comments
 (0)