Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile.nidx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# For a version that uses pre-built binaries (outside Docker) see `Dockerfile.nidx_prebuild`
#

FROM rust:1.94.0-trixie AS builder
FROM rust:1.95.0-trixie AS builder
RUN apt update && apt install -y protobuf-compiler
COPY nucliadb_protos/*proto /app/nucliadb_protos/
COPY nidx /app/nidx
Expand Down
4 changes: 2 additions & 2 deletions nidx/nidx_paragraph/src/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ use nidx_protos::{OrderBy, ParagraphItem, ParagraphSearchResponse, StreamRequest
use nidx_types::prefilter::PrefilterResult;
use tantivy::collector::{Collector, Count, FacetCollector, TopDocs};
use tantivy::query::{AllQuery, Query};
use tantivy::{DateTime, Order, schema::*};
use tantivy::{DocAddress, Index, IndexReader};
use tantivy::schema::{Facet, Field, Value};
use tantivy::{DateTime, DocAddress, Index, IndexReader, Order, TantivyDocument};
use tracing::*;

use super::schema::ParagraphSchema;
Expand Down
2 changes: 1 addition & 1 deletion nidx/nidx_vector/src/data_store/v2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ impl DataStoreV2 {
None
};

for (idx, elem) in (0..).zip(entries.into_iter()) {
for (idx, elem) in (0..).zip(entries) {
let (first_vector, _) = vectors.write(idx, elem.vectors.iter().map(|v| config.vector_type.encode(v)))?;
if let Some(quantized) = &mut quantized {
for v in &elem.vectors {
Expand Down
6 changes: 1 addition & 5 deletions nidx/nidx_vector/src/hnsw/search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,11 +207,7 @@ impl<'a, DR: DataRetriever> HnswSearcher<'a, DR> {

let mut preloaded = 0;

loop {
let Some(Cnx(candidate, candidate_similarity)) = candidates.pop() else {
break;
};

while let Some(Cnx(candidate, candidate_similarity)) = candidates.pop() {
if candidate_similarity < self.retriever.min_score() {
break;
}
Expand Down
41 changes: 19 additions & 22 deletions nidx/nidx_vector/src/indexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,29 +65,26 @@ impl<'a> ResourceWrapper<'a> {

pub fn fields(&self) -> impl Iterator<Item = (&String, impl Iterator<Item = ParagraphVectors<'_>>)> {
self.resource.paragraphs.iter().map(|(field_id, paragraphs_wrapper)| {
let sentences_iterator = paragraphs_wrapper
.paragraphs
.iter()
.filter_map(|(_paragraph_id, paragraph)| {
let sentences = if let Some(vectorset) = &self.vectorset {
// indexing a vectorset, we should return only paragraphs from this vectorset.
// If vectorset is not found, we'll skip this paragraph
if let Some(vectorset_sentences) = paragraph.vectorsets_sentences.get(vectorset) {
Some(&vectorset_sentences.sentences)
} else if self.fallback_to_default_vectorset {
Some(&paragraph.sentences)
} else {
None
}
} else {
// Default vectors index (no vectorset)
let sentences_iterator = paragraphs_wrapper.paragraphs.values().filter_map(|paragraph| {
let sentences = if let Some(vectorset) = &self.vectorset {
// indexing a vectorset, we should return only paragraphs from this vectorset.
// If vectorset is not found, we'll skip this paragraph
if let Some(vectorset_sentences) = paragraph.vectorsets_sentences.get(vectorset) {
Some(&vectorset_sentences.sentences)
} else if self.fallback_to_default_vectorset {
Some(&paragraph.sentences)
};
sentences.map(|s| ParagraphVectors {
vectors: s,
labels: &paragraph.labels,
})
});
} else {
None
}
} else {
// Default vectors index (no vectorset)
Some(&paragraph.sentences)
};
sentences.map(|s| ParagraphVectors {
vectors: s,
labels: &paragraph.labels,
})
});
(field_id, sentences_iterator)
})
}
Expand Down
2 changes: 1 addition & 1 deletion nidx/nidx_vector/src/segment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -823,7 +823,7 @@ mod test {
let path_merged = tempdir()?;
let merged_dp = merge(path_merged.path(), work, &config)?;

for (i, (elem, mut labels)) in elems1.into_iter().chain(elems2.into_iter()).enumerate() {
for (i, (elem, mut labels)) in elems1.into_iter().chain(elems2).enumerate() {
let vector = merged_dp.data_store.get_vector(VectorAddr(i as u32));
assert_eq!(config.vector_type.encode(&elem.vectors[0]), vector.vector());

Expand Down
2 changes: 1 addition & 1 deletion nidx/src/searcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ async fn refresher_task(mut rx: Receiver<(IndexId, bool)>, index_cache: Arc<Inde
}

let mut unique_indexes = HashMap::new();
for (index_id, force) in try_later.drain(std::ops::RangeFull).chain(recv_buf.into_iter()) {
for (index_id, force) in try_later.drain(std::ops::RangeFull).chain(recv_buf) {
unique_indexes
.entry(index_id)
.and_modify(|f| *f = *f || force)
Expand Down
4 changes: 2 additions & 2 deletions nidx/src/searcher/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ pub async fn run_sync(

// Remove deleted indexes
let deleted = Index::marked_to_delete(&meta.pool).await?;
for (shard_id, index_id) in deleted.into_iter().map(|i| (i.shard_id, i.id)).chain(indexes_to_delete.into_iter()) {
for (shard_id, index_id) in deleted.into_iter().map(|i| (i.shard_id, i.id)).chain(indexes_to_delete) {
if shutdown.is_cancelled() {
break;
}
Expand All @@ -140,7 +140,7 @@ pub async fn run_sync(

let sync_semaphore = Arc::new(Semaphore::new(settings.parallel_index_syncs));
let mut tasks = JoinSet::new();
for index in indexes.into_iter().chain(retry_indexes.into_iter()).chain(indexes_to_sync.into_iter()) {
for index in indexes.into_iter().chain(retry_indexes).chain(indexes_to_sync) {
let index_id = index.id;
let meta2 = meta.clone();
let index_metadata2 = Arc::clone(&index_metadata);
Expand Down
2 changes: 1 addition & 1 deletion rust-toolchain.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[toolchain]
# if this changes, also change the version in the Dockerfiles
channel = "1.94.0"
channel = "1.95.0"
Loading