Skip to content

Commit 19d0d48

Browse files
committed
refactor(ftindex): make FullTextArchiveReader generic over SeekRead
Allow both whole-file (SliceReader) and streaming/range-read strategies for full-text archive access. Adds from_seek_read constructor; keeps from_input_file as a convenience wrapper for the whole-file case. Addresses coordination with global-index full-text path.
1 parent 260de52 commit 19d0d48

1 file changed

Lines changed: 28 additions & 12 deletions

File tree

crates/paimon/src/ftindex/reader.rs

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
//! Reader over the `paimon-ftindex-core` v1 archive format.
1919
20-
use paimon_ftindex_core::io::SliceReader;
20+
use paimon_ftindex_core::io::{SeekRead, SliceReader};
2121
use paimon_ftindex_core::{FullTextIndexReader, FullTextSearchResult};
2222
use roaring::RoaringTreemap;
2323

@@ -42,19 +42,22 @@ impl From<FullTextSearchResult> for FullTextHits {
4242

4343
/// Reader over a single full-text index archive, backed by the shared
4444
/// `paimon-ftindex-core` engine (v1 archive format).
45-
pub struct FullTextArchiveReader {
46-
inner: FullTextIndexReader<SliceReader>,
45+
///
46+
/// Generic over any `SeekRead` implementation, allowing both whole-file
47+
/// (via `SliceReader`) and streaming/range-read strategies.
48+
pub struct FullTextArchiveReader<R: SeekRead + 'static> {
49+
inner: FullTextIndexReader<R>,
4750
}
4851

49-
impl FullTextArchiveReader {
50-
/// Read the whole archive from `input` into memory and open the engine
51-
/// reader over it. Archives are single index files, so a whole-read keeps
52-
/// peak memory at one archive.
53-
pub async fn from_input_file(input: &InputFile) -> crate::Result<Self> {
54-
let bytes = input.read().await?;
55-
let reader =
56-
FullTextIndexReader::open(SliceReader::new(bytes.to_vec())).map_err(map_ft_err)?;
57-
Ok(Self { inner: reader })
52+
impl<R: SeekRead + 'static> FullTextArchiveReader<R> {
53+
/// Open a full-text archive reader over any `SeekRead` implementation.
54+
///
55+
/// Use this for streaming/range-read strategies (e.g., remote FileIO with
56+
/// bounded concurrency). For convenience when reading whole files into
57+
/// memory, see [`from_input_file`](Self::from_input_file).
58+
pub fn from_seek_read(reader: R) -> crate::Result<Self> {
59+
let inner = FullTextIndexReader::open(reader).map_err(map_ft_err)?;
60+
Ok(Self { inner })
5861
}
5962

6063
/// Search with a JSON DSL query (see the engine's query spec), returning up
@@ -88,6 +91,19 @@ impl FullTextArchiveReader {
8891
}
8992
}
9093

94+
impl FullTextArchiveReader<SliceReader> {
95+
/// Read the whole archive from `input` into memory and open the engine
96+
/// reader over it. Archives are single index files, so a whole-read keeps
97+
/// peak memory at one archive.
98+
///
99+
/// This is a convenience wrapper over [`from_seek_read`](Self::from_seek_read)
100+
/// for the common whole-file case.
101+
pub async fn from_input_file(input: &InputFile) -> crate::Result<Self> {
102+
let bytes = input.read().await?;
103+
Self::from_seek_read(SliceReader::new(bytes.to_vec()))
104+
}
105+
}
106+
91107
fn map_ft_err(e: paimon_ftindex_core::FtIndexError) -> crate::Error {
92108
crate::Error::UnexpectedError {
93109
message: format!("full-text index engine error: {e}"),

0 commit comments

Comments
 (0)