From a9edfff73524966a82041c6fc1997f661c2f60c6 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:00:40 +0800 Subject: [PATCH 01/30] feat(storage): add column group metadata to blocks --- .../it/storages/fuse/bloom_index_meta_size.rs | 1 + .../storages/common/cache/src/manager.rs | 1 + .../common/table_meta/src/meta/current/mod.rs | 1 + .../common/table_meta/src/meta/v2/mod.rs | 1 + .../common/table_meta/src/meta/v2/segment.rs | 66 +++++++++++++++++++ .../src/meta/v3/frozen/block_meta.rs | 1 + .../fuse/src/io/write/block_writer.rs | 1 + .../fuse/src/io/write/stream/block_builder.rs | 1 + .../storages/fuse/src/operations/changes.rs | 1 + 9 files changed, 74 insertions(+) diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index eff08a15cba53..4bfc02314476c 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -330,6 +330,7 @@ fn build_test_segment_info( file_size: 0, col_stats: col_stats.clone(), col_metas, + column_groups: vec![], cluster_stats: None, location: block_location, bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)), diff --git a/src/query/storages/common/cache/src/manager.rs b/src/query/storages/common/cache/src/manager.rs index 39914f449644c..b9244c16bc48d 100644 --- a/src/query/storages/common/cache/src/manager.rs +++ b/src/query/storages/common/cache/src/manager.rs @@ -1297,6 +1297,7 @@ mod tests { file_size: 0, col_stats: Default::default(), col_metas: Default::default(), + column_groups: vec![], cluster_stats: None, location: ("".to_string(), 0), bloom_filter_index_location: None, diff --git a/src/query/storages/common/table_meta/src/meta/current/mod.rs b/src/query/storages/common/table_meta/src/meta/current/mod.rs index 4e0719702e9e9..d8b45be536174 100644 --- a/src/query/storages/common/table_meta/src/meta/current/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/current/mod.rs @@ -16,6 +16,7 @@ pub use v0::ColumnMeta as SingleColumnMeta; pub use v2::AdditionalStatsMeta; pub use v2::BlockMeta; pub use v2::ClusterStatistics; +pub use v2::ColumnGroupFileMeta; pub use v2::ColumnMeta; pub use v2::ColumnStatistics; pub use v2::DraftVirtualBlockMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/mod.rs b/src/query/storages/common/table_meta/src/meta/v2/mod.rs index 8db7b00c17bf3..0ba8b8ad9e03b 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/mod.rs @@ -19,6 +19,7 @@ pub mod statistics; mod table_snapshot_statistics; pub use segment::BlockMeta; +pub use segment::ColumnGroupFileMeta; pub use segment::ColumnMeta; pub use segment::DraftVirtualBlockMeta; pub use segment::DraftVirtualColumnMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 85c0acd262c4b..d396a93e090ea 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -170,6 +170,20 @@ pub struct DraftVirtualBlockMeta { pub virtual_location: Location, } +/// Metadata of one physical column-group file in a logical block. +/// +/// A file may still contain column chunks that are no longer active. Readers must only use the +/// chunks listed in [`Self::active_column_ids`]. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] +pub struct ColumnGroupFileMeta { + pub active_column_ids: Vec, + pub location: Location, + pub format_version: FormatVersion, + pub file_size: u64, + pub uncompressed_size: u64, + pub leaf_column_metas: HashMap, +} + /// Meta information of a block /// Part of and kept inside the [SegmentInfo] #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -180,6 +194,12 @@ pub struct BlockMeta { #[serde(deserialize_with = "crate::meta::v2::statistics::deserialize_col_stats")] pub col_stats: HashMap, pub col_metas: HashMap, + /// Physical files that contain the active columns of this logical block. + /// + /// An empty vector is the legacy single-file representation described by `location`, + /// `file_size`, `block_size`, and `col_metas`. + #[serde(default)] + pub column_groups: Vec, pub cluster_stats: Option, /// location of data block pub location: Location, @@ -233,6 +253,7 @@ impl BlockMeta { file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats, location, bloom_filter_index_location, @@ -372,6 +393,7 @@ impl BlockMeta { file_size: s.file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats: None, location: (s.location.path.clone(), 0), bloom_filter_index_location: None, @@ -404,6 +426,7 @@ impl BlockMeta { file_size: s.file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats: None, location: s.location.clone(), bloom_filter_index_location: s.bloom_filter_index_location.clone(), @@ -434,3 +457,46 @@ impl From<(v0::SegmentInfo, &[TableField])> for SegmentInfo { SegmentInfo::from_v0(v, fields) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deserialize_legacy_block_meta_without_column_groups() { + let block_meta = BlockMeta::new( + 10, + 300, + 30, + HashMap::new(), + HashMap::new(), + None, + ("old.parquet".to_string(), 2), + None, + 0, + None, + None, + None, + None, + None, + None, + None, + None, + Compression::Zstd, + None, + ); + let location = block_meta.location.clone(); + let mut value = serde_json::to_value(block_meta).unwrap(); + assert!( + value + .as_object_mut() + .unwrap() + .remove("column_groups") + .is_some() + ); + + let decoded: BlockMeta = serde_json::from_value(value).unwrap(); + assert!(decoded.column_groups.is_empty()); + assert_eq!(decoded.location, location); + } +} diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index f73a26d3a8b5f..a3c1b2e2cd9ff 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -57,6 +57,7 @@ impl From for crate::meta::BlockMeta { .into_iter() .map(|(k, v)| (k, v.into())) .collect(), + column_groups: vec![], cluster_stats: value.cluster_stats.map(|v| v.into()), location: value.location, bloom_filter_index_location: value.bloom_filter_index_location, diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index b77492abebef7..c6bd8048a0137 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -267,6 +267,7 @@ impl BlockBuilder { file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), diff --git a/src/query/storages/fuse/src/io/write/stream/block_builder.rs b/src/query/storages/fuse/src/io/write/stream/block_builder.rs index dc98dad6c17b2..5d2122cc2bd32 100644 --- a/src/query/storages/fuse/src/io/write/stream/block_builder.rs +++ b/src/query/storages/fuse/src/io/write/stream/block_builder.rs @@ -404,6 +404,7 @@ impl StreamBlockBuilder { file_size: file_size as u64, col_stats, col_metas, + column_groups: vec![], cluster_stats, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index 582474c2e3944..568e6817799a8 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -1152,6 +1152,7 @@ mod tests { file_size: block_size, col_stats, col_metas: HashMap::new(), + column_groups: vec![], cluster_stats: None, location: (test_block_path(uuid), 0), bloom_filter_index_location: None, From 3b51cc599564eb958f7427af16d1abbb78520007 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Thu, 23 Jul 2026 00:12:54 +0800 Subject: [PATCH 02/30] feat(storage): read blocks from column groups --- .../it/storages/fuse/operations/prewhere.rs | 6 +- src/query/storages/fuse/src/fuse_part.rs | 13 +- .../agg_index/agg_index_reader_parquet.rs | 9 +- .../io/read/block/block_reader_deserialize.rs | 13 +- .../io/read/block/block_reader_merge_io.rs | 63 ++++---- .../read/block/block_reader_merge_io_async.rs | 33 ++++ .../fuse/src/io/read/block/parquet/mod.rs | 60 ++++++- src/query/storages/fuse/src/lib.rs | 1 + .../mutation/processors/mutation_source.rs | 10 +- .../operations/read/block_format/parquet.rs | 23 --- .../operations/read/parquet_rows_fetcher.rs | 49 +++--- .../src/operations/read/read_block_context.rs | 29 ++-- .../fuse/src/operations/read_partitions.rs | 151 +++++++++++++++--- .../fuse/src/pruning/expr_runtime_pruner.rs | 4 +- .../column_oriented_block_prune.rs | 7 +- 15 files changed, 318 insertions(+), 153 deletions(-) diff --git a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs index 0c089ee730095..0d747d19732bf 100644 --- a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs +++ b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs @@ -44,6 +44,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_storages_fuse::FuseBlockPartInfo; +use databend_common_storages_fuse::FuseColumnGroupPartInfo; use databend_common_storages_fuse::io::BlockReader; use databend_common_storages_fuse::io::DataItem; use databend_common_storages_fuse::io::WriteSettings; @@ -332,7 +333,10 @@ async fn prepare_prewhere_data() -> Result { bloom_filter_index_size: 0, create_on: None, nums_rows: num_rows, - columns_meta: column_metas.clone(), + column_groups: vec![FuseColumnGroupPartInfo { + location: "test_block".to_string(), + columns_meta: column_metas.clone(), + }], columns_stat: None, compression, sort_min_max: None, diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index e18e306cb6ef0..205f49d3645c8 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -35,6 +35,13 @@ use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; use databend_storages_common_table_meta::meta::Location; +/// Projected column chunks to read from one physical column-group file. +#[derive(Clone, serde::Serialize, serde::Deserialize, PartialEq, Debug)] +pub struct FuseColumnGroupPartInfo { + pub location: String, + pub columns_meta: HashMap, +} + /// Fuse table partition information. #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] pub struct FuseBlockPartInfo { @@ -45,7 +52,7 @@ pub struct FuseBlockPartInfo { pub create_on: Option>, pub nums_rows: usize, - pub columns_meta: HashMap, + pub column_groups: Vec, pub columns_stat: Option>, pub compression: Compression, @@ -83,7 +90,7 @@ impl FuseBlockPartInfo { bloom_filter_index_location: Option, bloom_filter_index_size: u64, rows_count: u64, - columns_meta: HashMap, + column_groups: Vec, columns_stat: Option>, compression: Compression, sort_min_max: Option<(Scalar, Scalar)>, @@ -95,7 +102,7 @@ impl FuseBlockPartInfo { bloom_filter_index_location, bloom_filter_index_size, create_on, - columns_meta, + column_groups, nums_rows: rows_count as usize, compression, sort_min_max, diff --git a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs index a2dbe5c192f7d..7caa74166fd1c 100644 --- a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs +++ b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs @@ -28,14 +28,7 @@ impl AggIndexReader { ) -> Result { let columns_chunks = data.columns_chunks()?; let part = FuseBlockPartInfo::from_part(&part)?; - let block = self.reader.deserialize_parquet_chunks( - part.nums_rows, - &part.columns_meta, - columns_chunks, - &part.compression, - &part.location, - None, - )?; + let block = self.reader.deserialize_part(part, columns_chunks, None)?; self.apply_agg_info(block) } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index 9dfe85a7b817d..611ae6a69b6c3 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -40,15 +40,10 @@ impl BlockReader { storage_format: &FuseStorageFormat, ) -> Result { let part = FuseBlockPartInfo::from_part(&part)?; - - self.deserialize_chunks( - &part.location, - part.nums_rows, - &part.compression, - &part.columns_meta, - chunks, - storage_format, - ) + match storage_format { + FuseStorageFormat::Parquet => self.deserialize_part(part, chunks, None), + FuseStorageFormat::Unsupported => Err(unsupported_storage_format_error()), + } } pub fn deserialize_chunks( diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs index cdb61afcc436a..10c42e30e733f 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs @@ -15,7 +15,6 @@ use std::collections::HashMap; use std::sync::Arc; -use bytes::Bytes; use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_storages_common_cache::ColumnData; @@ -34,7 +33,7 @@ pub enum DataItem<'a> { } pub struct BlockReadResult { - merge_io_result: MergeIOReadResult, + merge_io_results: Vec, pub(crate) cached_column_data: CachedColumnData, pub(crate) cached_column_array: CachedColumnArray, } @@ -46,22 +45,46 @@ impl BlockReadResult { cached_column_array: CachedColumnArray, ) -> BlockReadResult { BlockReadResult { - merge_io_result, + merge_io_results: vec![merge_io_result], + cached_column_data, + cached_column_array, + } + } + + pub(crate) fn merge(results: Vec) -> BlockReadResult { + let mut merge_io_results = Vec::with_capacity(results.len()); + let mut cached_column_data = vec![]; + let mut cached_column_array = vec![]; + + for result in results { + merge_io_results.extend(result.merge_io_results); + cached_column_data.extend(result.cached_column_data); + cached_column_array.extend(result.cached_column_array); + } + + BlockReadResult { + merge_io_results, cached_column_data, cached_column_array, } } pub fn columns_chunks(&self) -> Result>> { - let mut res = HashMap::with_capacity(self.merge_io_result.columns_chunk_offsets.len()); + let capacity = self + .merge_io_results + .iter() + .map(|result| result.columns_chunk_offsets.len()) + .sum(); + let mut res = HashMap::with_capacity(capacity); // merge column data fetched from object storage - for (column_id, (chunk_idx, range)) in &self.merge_io_result.columns_chunk_offsets { - let chunk = self - .merge_io_result - .owner_memory - .get_chunk(*chunk_idx, &self.merge_io_result.block_path)?; - res.insert(*column_id, DataItem::RawData(chunk.slice(range.clone()))); + for merge_io_result in &self.merge_io_results { + for (column_id, (chunk_idx, range)) in &merge_io_result.columns_chunk_offsets { + let chunk = merge_io_result + .owner_memory + .get_chunk(*chunk_idx, &merge_io_result.block_path)?; + res.insert(*column_id, DataItem::RawData(chunk.slice(range.clone()))); + } } // merge column data from cache @@ -76,24 +99,4 @@ impl BlockReadResult { Ok(res) } - - pub fn column_buffers(&self) -> Result> { - let mut res = HashMap::with_capacity(self.merge_io_result.columns_chunk_offsets.len()); - - // merge column data fetched from object storage - for (column_id, (chunk_idx, range)) in &self.merge_io_result.columns_chunk_offsets { - let chunk = self - .merge_io_result - .owner_memory - .get_chunk(*chunk_idx, &self.merge_io_result.block_path)?; - res.insert(*column_id, chunk.slice(range.clone()).to_bytes()); - } - - // merge column data from cache - for (column_id, data) in &self.cached_column_data { - res.insert(*column_id, data.bytes()); - } - - Ok(res) - } } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs index 1d0be8fa3cb0c..592ae9a8b0e46 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs @@ -29,10 +29,23 @@ use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::ColumnMeta; use crate::BlockReadResult; +use crate::FuseColumnGroupPartInfo; use crate::io::BlockReadContext; use crate::io::BlockReader; impl BlockReader { + #[async_backtrace::framed] + pub(crate) async fn read_column_groups_data_by_merge_io( + &self, + settings: &ReadSettings, + column_groups: &[FuseColumnGroupPartInfo], + ignore_column_ids: &Option>, + ) -> Result { + self.read_context() + .read_column_groups_data_by_merge_io(settings, column_groups, ignore_column_ids) + .await + } + #[async_backtrace::framed] pub async fn read_columns_data_by_merge_io( &self, @@ -48,6 +61,26 @@ impl BlockReader { } impl BlockReadContext { + #[async_backtrace::framed] + pub(crate) async fn read_column_groups_data_by_merge_io( + &self, + settings: &ReadSettings, + column_groups: &[FuseColumnGroupPartInfo], + ignore_column_ids: &Option>, + ) -> Result { + let reads = column_groups.iter().map(|group| { + self.read_columns_data_by_merge_io( + settings, + &group.location, + &group.columns_meta, + ignore_column_ids, + ) + }); + Ok(BlockReadResult::merge( + futures::future::try_join_all(reads).await?, + )) + } + #[async_backtrace::framed] pub async fn read_columns_data_by_merge_io( &self, diff --git a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs index aaeb22d5291b3..3bcbcc95f3316 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs @@ -42,6 +42,7 @@ pub use deserialize::column_chunks_to_record_batch; pub use row_selection::RowSelection; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::io::BlockReader; use crate::io::read::block::block_reader_merge_io::DataItem; @@ -52,12 +53,35 @@ impl BlockReader { column_chunks: HashMap, selection: Option<&RowSelection>, ) -> databend_common_exception::Result { - self.deserialize_parquet_chunks( + self.deserialize_column_groups( part.nums_rows, - &part.columns_meta, + &part.column_groups, column_chunks, &part.compression, - &part.location, + selection, + ) + } + + pub(crate) fn deserialize_column_groups( + &self, + num_rows: usize, + column_groups: &[FuseColumnGroupPartInfo], + column_chunks: HashMap, + compression: &Compression, + selection: Option<&RowSelection>, + ) -> databend_common_exception::Result { + self.deserialize_parquet_chunks_with_cache_key( + num_rows, + column_chunks, + compression, + |column_id| { + column_groups.iter().find_map(|group| { + group.columns_meta.get(&column_id).map(|column_meta| { + let (offset, len) = column_meta.offset_length(); + TableDataCacheKey::new(&group.location, column_id, offset, len) + }) + }) + }, selection, ) } @@ -71,6 +95,31 @@ impl BlockReader { block_path: &str, selection: Option<&RowSelection>, ) -> databend_common_exception::Result { + self.deserialize_parquet_chunks_with_cache_key( + num_rows, + column_chunks, + compression, + |column_id| { + column_metas.get(&column_id).map(|column_meta| { + let (offset, len) = column_meta.offset_length(); + TableDataCacheKey::new(block_path, column_id, offset, len) + }) + }, + selection, + ) + } + + fn deserialize_parquet_chunks_with_cache_key( + &self, + num_rows: usize, + column_chunks: HashMap, + compression: &Compression, + cache_key_for_column: F, + selection: Option<&RowSelection>, + ) -> databend_common_exception::Result + where + F: Fn(ColumnId) -> Option, + { let result_rows = selection.map(|s| s.selected_rows).unwrap_or(num_rows); // If projection is empty, return a DataBlock with the appropriate row count but no columns if self.projected_schema.fields.is_empty() { @@ -128,10 +177,7 @@ impl BlockReader { let arrow_array = column_by_name(&record_batch, &name_paths[i]); if !column_node.is_nested { if let Some(cache) = &array_cache { - let meta = column_metas.get(&field.column_id).unwrap(); - let (offset, len) = meta.offset_length(); - let key = - TableDataCacheKey::new(block_path, field.column_id, offset, len); + let key = cache_key_for_column(field.column_id).unwrap(); let array_memory_size = arrow_array.get_array_memory_size(); cache.insert(key.into(), (arrow_array.clone(), array_memory_size)); } diff --git a/src/query/storages/fuse/src/lib.rs b/src/query/storages/fuse/src/lib.rs index 9bebd92396acf..75641f43bd6ac 100644 --- a/src/query/storages/fuse/src/lib.rs +++ b/src/query/storages/fuse/src/lib.rs @@ -72,6 +72,7 @@ use databend_common_catalog::table::TableStatistics; pub use databend_common_catalog::table_context::TableContext; pub use fuse_column::FuseTableColumnStatisticsProvider; pub use fuse_part::FuseBlockPartInfo; +pub use fuse_part::FuseColumnGroupPartInfo; pub use fuse_part::FuseLazyPartInfo; pub use fuse_table::FuseTable; pub use fuse_table::RetentionPolicy; diff --git a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs index 951224fb743b9..4038ac251d8a2 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs @@ -385,10 +385,9 @@ impl Processor for MutationSource { } else { let read_res = self .block_reader - .read_columns_data_by_merge_io( + .read_column_groups_data_by_merge_io( &settings, - &fuse_part.location, - &fuse_part.columns_meta, + &fuse_part.column_groups, &None, ) .await?; @@ -407,10 +406,9 @@ impl Processor for MutationSource { let settings = ReadSettings::from_ctx(&self.ctx)?; let read_res = remain_reader - .read_columns_data_by_merge_io( + .read_column_groups_data_by_merge_io( &settings, - &fuse_part.location, - &fuse_part.columns_meta, + &fuse_part.column_groups, &None, ) .await?; diff --git a/src/query/storages/fuse/src/operations/read/block_format/parquet.rs b/src/query/storages/fuse/src/operations/read/block_format/parquet.rs index 3f78bf4882858..9e6990851b577 100644 --- a/src/query/storages/fuse/src/operations/read/block_format/parquet.rs +++ b/src/query/storages/fuse/src/operations/read/block_format/parquet.rs @@ -12,19 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; -use std::collections::HashSet; - -use databend_common_exception::Result; -use databend_common_expression::ColumnId; use databend_common_storage::read_metadata_async; -use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::ColumnMeta; use opendal::Operator; use super::ReadBlockMeta; -use crate::io::BlockReadContext; -use crate::io::BlockReadResult; use crate::io::build_columns_meta; pub struct FuseParquetBlockFormat; @@ -34,20 +25,6 @@ impl FuseParquetBlockFormat { Self } - /// Reads raw column data from the given block location. - pub async fn read_data_by_merge_io( - &self, - read_ctx: &BlockReadContext, - settings: &ReadSettings, - location: &str, - columns_meta: &HashMap, - ignore_column_ids: &Option>, - ) -> Result { - read_ctx - .read_columns_data_by_merge_io(settings, location, columns_meta, ignore_column_ids) - .await - } - /// Reads the metadata needed to fetch an arbitrary block location. pub async fn read_block_meta( &self, diff --git a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs index 375acb536a130..7ca09a46796c8 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs @@ -27,7 +27,6 @@ use databend_common_catalog::plan::split_row_id; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; use databend_common_storage::ColumnNodes; @@ -37,7 +36,6 @@ use databend_storages_common_cache::InMemoryLruCache; use databend_storages_common_cache::LoadParams; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockMeta; -use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::Compression; use databend_storages_common_table_meta::meta::TableSnapshot; use futures_util::stream::FuturesUnordered; @@ -50,6 +48,7 @@ use super::fuse_rows_fetcher::RowsFetchMetadata; use super::fuse_rows_fetcher::RowsFetcher; use crate::BlockReadResult; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::FuseTable; use crate::io::BlockReader; use crate::io::CompactSegmentInfoReader; @@ -93,10 +92,9 @@ pub struct RowsFetchMetadataImpl { // block_bytes after projection pub block_bytes: usize, - pub location: String, pub nums_rows: usize, pub compression: Compression, - pub columns_meta: HashMap, + pub column_groups: Vec, } impl RowsFetchMetadata for RowsFetchMetadataImpl { @@ -328,12 +326,7 @@ impl ParquetRowsFetcher { .await .expect("row-fetch io semaphore never closed"); let chunk = reader - .read_columns_data_by_merge_io( - &settings, - &metadata.location, - &metadata.columns_meta, - &None, - ) + .read_column_groups_data_by_merge_io(&settings, &metadata.column_groups, &None) .await?; Ok(( @@ -363,20 +356,24 @@ impl ParquetRowsFetcher { let compression_ratio = block_meta.block_size as f64 / block_meta.file_size as f64; let mut block_bytes = 0; let mut average_bytes = 0; - for (column_id, column_meta) in &fuse_part.columns_meta { - if let Some(columns_stat) = &fuse_part.columns_stat { - if let Some(column_stat) = columns_stat.get(column_id) { - average_bytes += column_stat.in_memory_size as usize / fuse_part.nums_rows; - block_bytes += column_stat.in_memory_size as usize; - - continue; + for group in &fuse_part.column_groups { + for (column_id, column_meta) in &group.columns_meta { + if let Some(columns_stat) = &fuse_part.columns_stat { + if let Some(column_stat) = columns_stat.get(column_id) { + average_bytes += + column_stat.in_memory_size as usize / fuse_part.nums_rows; + block_bytes += column_stat.in_memory_size as usize; + + continue; + } } - } - let compressed_size = column_meta.read_bytes(&None); - let estimate_memory_size = (compressed_size as f64 * compression_ratio) as usize; - block_bytes += estimate_memory_size; - average_bytes += estimate_memory_size / fuse_part.nums_rows; + let compressed_size = column_meta.read_bytes(&None); + let estimate_memory_size = + (compressed_size as f64 * compression_ratio) as usize; + block_bytes += estimate_memory_size; + average_bytes += estimate_memory_size / fuse_part.nums_rows; + } } metadata.push(RowsFetchMetadataImpl { @@ -384,8 +381,7 @@ impl ParquetRowsFetcher { block_bytes, nums_rows: fuse_part.nums_rows, compression: fuse_part.compression, - location: fuse_part.location.clone(), - columns_meta: fuse_part.columns_meta.clone(), + column_groups: fuse_part.column_groups.clone(), }); } @@ -398,12 +394,11 @@ impl ParquetRowsFetcher { chunk: BlockReadResult, ) -> Result { let columns_chunks = chunk.columns_chunks()?; - reader.deserialize_parquet_chunks( + reader.deserialize_column_groups( metadata.nums_rows, - &metadata.columns_meta, + &metadata.column_groups, columns_chunks, &metadata.compression, - &metadata.location, None, ) } diff --git a/src/query/storages/fuse/src/operations/read/read_block_context.rs b/src/query/storages/fuse/src/operations/read/read_block_context.rs index a0121ee638a9e..a37a1ca1974e9 100644 --- a/src/query/storages/fuse/src/operations/read/read_block_context.rs +++ b/src/query/storages/fuse/src/operations/read/read_block_context.rs @@ -23,6 +23,7 @@ use log::debug; use super::block_format::FuseParquetBlockFormat; use super::parquet_data_source::ParquetDataSource; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; use crate::io::AggIndexReader; use crate::io::BlockReadContext; @@ -77,12 +78,10 @@ impl ReadBlockContext { .and_then(|source| source.ignore_column_ids.clone()); let data = self - .block_format - .read_data_by_merge_io( - &self.block_read_ctx, + .block_read_ctx + .read_column_groups_data_by_merge_io( &self.read_settings, - &fuse_part.location, - &fuse_part.columns_meta, + &fuse_part.column_groups, &ignore_column_ids, ) .await?; @@ -112,15 +111,13 @@ impl ReadBlockContext { return Ok(None); }; - let data = match self - .block_format - .read_data_by_merge_io( - &index_block_read_ctx, - &self.read_settings, - &location, - &block_meta.columns_meta, - &None, - ) + let num_rows = block_meta.num_rows; + let column_groups = vec![FuseColumnGroupPartInfo { + location: location.clone(), + columns_meta: block_meta.columns_meta, + }]; + let data = match index_block_read_ctx + .read_column_groups_data_by_merge_io(&self.read_settings, &column_groups, &None) .await { Ok(data) => data, @@ -134,8 +131,8 @@ impl ReadBlockContext { location, None, 0, - block_meta.num_rows, - block_meta.columns_meta, + num_rows, + column_groups, None, index_reader.compression().into(), None, diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 37fecc118202c..90b8cd074bc57 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -87,6 +87,7 @@ use crate::FuseLazyPartInfo; use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; +use crate::fuse_part::FuseColumnGroupPartInfo; use crate::io::BloomIndexRebuilder; use crate::pruning::BlockPruner; use crate::pruning::FusePruner; @@ -1369,29 +1370,68 @@ impl FuseTable { (statistics, partitions) } + fn projected_column_groups( + meta: &BlockMeta, + projected_column_ids: &HashSet, + ) -> Vec { + if meta.column_groups.is_empty() { + let columns_meta = projected_column_ids + .iter() + .filter_map(|column_id| { + meta.col_metas + .get(column_id) + .map(|column_meta| (*column_id, column_meta.clone())) + }) + .collect(); + return vec![FuseColumnGroupPartInfo { + location: meta.location.0.clone(), + columns_meta, + }]; + } + + let mut column_groups = Vec::with_capacity(meta.column_groups.len()); + for group in &meta.column_groups { + let mut group_columns_meta = HashMap::new(); + for column_id in &group.active_column_ids { + if !projected_column_ids.contains(column_id) { + continue; + } + if let Some(column_meta) = group.leaf_column_metas.get(column_id) { + group_columns_meta.insert(*column_id, column_meta.clone()); + } + } + if !group_columns_meta.is_empty() { + column_groups.push(FuseColumnGroupPartInfo { + location: group.location.0.clone(), + columns_meta: group_columns_meta, + }); + } + } + column_groups + } + pub fn all_columns_part( schema: Option<&TableSchemaRef>, block_meta_index: &Option, top_k: &Option<(TopK, Scalar)>, meta: &BlockMeta, ) -> PartInfoPtr { - let mut columns_meta = HashMap::with_capacity(meta.col_metas.len()); let mut columns_stats = HashMap::with_capacity(meta.col_stats.len()); let mut spatial_stats = HashMap::new(); - for column_id in meta.col_metas.keys() { - // ignore all deleted field - if let Some(schema) = schema { - if schema.is_column_deleted(*column_id) { - continue; - } - } - - // ignore column this block dose not exist - if let Some(meta) = meta.col_metas.get(column_id) { - columns_meta.insert(*column_id, meta.clone()); - } + let mut projected_column_ids = if meta.column_groups.is_empty() { + meta.col_metas.keys().copied().collect::>() + } else { + meta.column_groups + .iter() + .flat_map(|group| group.active_column_ids.iter().copied()) + .collect() + }; + if let Some(schema) = schema { + projected_column_ids.retain(|column_id| !schema.is_column_deleted(*column_id)); + } + for column_id in &projected_column_ids { if let Some(stat) = meta.col_stats.get(column_id) { columns_stats.insert(*column_id, stat.clone()); } @@ -1403,6 +1443,7 @@ impl FuseTable { spatial_stats.insert(*column_id, stats.clone()); } } + let column_groups = Self::projected_column_groups(meta, &projected_column_ids); let rows_count = meta.row_count; let location = meta.location.0.clone(); @@ -1420,7 +1461,7 @@ impl FuseTable { meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, rows_count, - columns_meta, + column_groups, Some(columns_stats), meta.compression(), sort_min_max, @@ -1436,17 +1477,14 @@ impl FuseTable { top_k: Option<(TopK, Scalar)>, projection: &Projection, ) -> PartInfoPtr { - let mut columns_meta = HashMap::with_capacity(projection.len()); let mut columns_stat = HashMap::with_capacity(projection.len()); let mut spatial_stats = HashMap::new(); + let mut projected_column_ids = HashSet::with_capacity(projection.len()); let columns = projection.project_column_nodes(column_nodes).unwrap(); for column in &columns { for column_id in &column.leaf_column_ids { - // ignore column this block dose not exist - if let Some(column_meta) = meta.col_metas.get(column_id) { - columns_meta.insert(*column_id, column_meta.clone()); - } + projected_column_ids.insert(*column_id); if let Some(column_stat) = meta.col_stats.get(column_id) { columns_stat.insert(*column_id, column_stat.clone()); } @@ -1459,6 +1497,7 @@ impl FuseTable { } } } + let column_groups = Self::projected_column_groups(meta, &projected_column_ids); let rows_count = meta.row_count; let location = meta.location.0.clone(); @@ -1478,7 +1517,7 @@ impl FuseTable { meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, rows_count, - columns_meta, + column_groups, Some(columns_stat), meta.compression(), sort_min_max, @@ -1490,6 +1529,11 @@ impl FuseTable { #[cfg(test)] mod tests { + use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; + use databend_storages_common_table_meta::meta::ColumnMeta; + use databend_storages_common_table_meta::meta::Compression; + use databend_storages_common_table_meta::meta::SingleColumnMeta; + use super::*; fn segment_location(segment_idx: usize, location: &str, snapshot_loc: &str) -> SegmentLocation { @@ -1517,4 +1561,71 @@ mod tests { changed_segment[1].location = ("segment-c".to_string(), 1); assert!(!same_segment_locations(&original, &changed_segment)); } + + #[test] + fn test_projected_column_groups() { + let column_1_meta = ColumnMeta::Parquet(SingleColumnMeta::new(0, 10, 3)); + let stale_column_2_meta = ColumnMeta::Parquet(SingleColumnMeta::new(10, 10, 3)); + let active_column_2_meta = ColumnMeta::Parquet(SingleColumnMeta::new(0, 12, 3)); + let mut block_meta = BlockMeta::new( + 3, + 30, + 22, + HashMap::new(), + HashMap::from([ + (1, column_1_meta.clone()), + (2, active_column_2_meta.clone()), + ]), + None, + ("group-2.parquet".to_string(), 2), + None, + 0, + None, + None, + None, + None, + None, + None, + None, + None, + Compression::Zstd, + None, + ); + + let legacy_groups = FuseTable::projected_column_groups(&block_meta, &HashSet::from([2])); + assert_eq!(legacy_groups.len(), 1); + assert_eq!(legacy_groups[0].location, "group-2.parquet"); + assert_eq!( + legacy_groups[0].columns_meta, + HashMap::from([(2, active_column_2_meta.clone())]) + ); + + block_meta.column_groups = vec![ + ColumnGroupFileMeta { + active_column_ids: vec![1], + location: ("group-1.parquet".to_string(), 2), + format_version: 2, + file_size: 10, + uncompressed_size: 10, + leaf_column_metas: HashMap::from([(1, column_1_meta), (2, stale_column_2_meta)]), + }, + ColumnGroupFileMeta { + active_column_ids: vec![2], + location: block_meta.location.clone(), + format_version: 2, + file_size: 12, + uncompressed_size: 12, + leaf_column_metas: HashMap::from([(2, active_column_2_meta.clone())]), + }, + ]; + + let column_groups = FuseTable::projected_column_groups(&block_meta, &HashSet::from([2])); + + assert_eq!(column_groups.len(), 1); + assert_eq!(column_groups[0].location, "group-2.parquet"); + assert_eq!( + column_groups[0].columns_meta, + HashMap::from([(2, active_column_2_meta)]) + ); + } } diff --git a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs index 85db98015890f..055d92866f997 100644 --- a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs +++ b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs @@ -538,7 +538,7 @@ mod tests { bloom_filter_index_location, bloom_filter_index_size, 4, - HashMap::new(), + vec![], Some(column_stats), Compression::Lz4Raw, None, @@ -728,7 +728,7 @@ mod tests { bloom_filter_index_location, bloom_filter_index_size, 1000, - HashMap::new(), + vec![], Some(column_stats), Compression::Lz4Raw, None, diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index 29227a2fdf313..0fd7dda5332cc 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -42,6 +42,7 @@ use tokio::sync::OwnedSemaphorePermit; use super::PrunedColumnOrientedSegmentMeta; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::pruning::BlockPruner; use crate::pruning_pipeline::RuntimeFilterPruneContext; @@ -277,12 +278,16 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { } } + let column_groups = vec![FuseColumnGroupPartInfo { + location: location_path.clone(), + columns_meta, + }]; let part_info = FuseBlockPartInfo::create( location_path, bloom_filter_index_location, bloom_filter_index_size, row_count, - columns_meta, + column_groups, Some(columns_stat), compression, None, // TODO(Sky): sort_min_max From 2e9975cd4bc2bfe7cd601914c43d986cd1f8fe4f Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:52:19 +0800 Subject: [PATCH 03/30] feat(storage): support partial updates --- .../common/table_option_validation.rs | 3 + .../src/interpreters/interpreter_mutation.rs | 1 + .../interpreters/interpreter_table_create.rs | 2 + .../interpreter_table_set_options.rs | 12 + .../physical_column_mutation.rs | 104 +++++--- .../src/physical_plans/physical_mutation.rs | 179 ++++++++++++- .../physical_mutation_source.rs | 50 +++- .../physical_plans/physical_plan_builder.rs | 3 +- .../it/storages/fuse/bloom_index_meta_size.rs | 1 + .../storages/fuse/operations/mutation/mod.rs | 1 + .../fuse/operations/mutation/update.rs | 160 +++++++++++ .../it/storages/fuse/operations/prewhere.rs | 1 + src/query/settings/src/settings_default.rs | 7 + .../settings/src/settings_getter_setter.rs | 4 + .../src/planner/execution/stream_column.rs | 55 +++- .../storages/common/cache/src/manager.rs | 1 + .../common/table_meta/src/meta/current/mod.rs | 1 + .../common/table_meta/src/meta/v2/mod.rs | 1 + .../common/table_meta/src/meta/v2/segment.rs | 32 ++- .../src/meta/v3/frozen/block_meta.rs | 1 + src/query/storages/fuse/src/constants.rs | 1 + src/query/storages/fuse/src/fuse_part.rs | 5 + .../io/read/block/block_reader_deserialize.rs | 62 ++++- .../fuse/src/io/write/block_writer.rs | 248 ++++++++++++++++++ .../fuse/src/io/write/stream/block_builder.rs | 1 + .../src/io/write/virtual_column_builder.rs | 6 + .../storages/fuse/src/operations/changes.rs | 1 + .../processors/transform_serialize_block.rs | 47 +++- src/query/storages/fuse/src/operations/gc.rs | 24 +- .../operations/mutation/meta/mutation_meta.rs | 5 + .../operations/mutation/meta/mutation_part.rs | 5 +- .../mutation/processors/compact_source.rs | 43 ++- .../mutation/processors/mutation_source.rs | 13 +- .../fuse/src/operations/mutation_source.rs | 20 +- .../src/operations/read/read_block_context.rs | 1 + .../fuse/src/operations/read_partitions.rs | 2 + .../mutator/replace_into_operation_agg.rs | 29 +- .../fuse/src/operations/table_index.rs | 2 + .../storages/fuse/src/operations/util.rs | 26 +- .../storages/fuse/src/pruning/block_pruner.rs | 1 + .../storages/fuse/src/pruning/bloom_pruner.rs | 178 +++++++++++-- .../fuse/src/pruning/expr_runtime_pruner.rs | 2 + .../column_oriented_block_prune.rs | 2 + .../fuse/src/table_functions/fuse_block.rs | 51 ++++ 44 files changed, 1254 insertions(+), 140 deletions(-) create mode 100644 src/query/service/tests/it/storages/fuse/operations/mutation/update.rs diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index cdafb74907eac..b8d4db9e1df3c 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -37,6 +37,7 @@ use databend_common_storages_fuse::FUSE_OPT_KEY_DATA_RETENTION_PERIOD_IN_HOURS; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARQUET_DICTIONARY; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN; use databend_common_storages_fuse::FUSE_OPT_KEY_FILE_SIZE; use databend_common_storages_fuse::FUSE_OPT_KEY_RECLUSTER_DEPTH; @@ -91,7 +92,9 @@ pub static CREATE_FUSE_OPTIONS: LazyLock> = LazyLock::new( r.insert(FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP); r.insert(FUSE_OPT_KEY_ENABLE_AUTO_VACUUM); r.insert(FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE); + r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN); + r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD); r.insert(OPT_KEY_BLOOM_INDEX_COLUMNS); diff --git a/src/query/service/src/interpreters/interpreter_mutation.rs b/src/query/service/src/interpreters/interpreter_mutation.rs index 1ac94e0c36abf..03e3430a7f3c4 100644 --- a/src/query/service/src/interpreters/interpreter_mutation.rs +++ b/src/query/service/src/interpreters/interpreter_mutation.rs @@ -246,6 +246,7 @@ pub async fn build_mutation_info( partitions, statistics, table_meta_timestamps, + partial_update: false, }) } diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index bfe7c753e8f3e..43f127815e719 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -47,6 +47,7 @@ use databend_common_storages_fuse::FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER; use databend_common_storages_fuse::FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FuseSegmentFormat; use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::io::MetaReaders; @@ -499,6 +500,7 @@ impl CreateTableInterpreter { is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_ENABLE_AUTO_VACUUM)?; // check enable auto analyze. is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE)?; + is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE)?; is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER)?; is_valid_option_of_type::( &table_meta.options, diff --git a/src/query/service/src/interpreters/interpreter_table_set_options.rs b/src/query/service/src/interpreters/interpreter_table_set_options.rs index 5441a63e12fc7..75537850a61f2 100644 --- a/src/query/service/src/interpreters/interpreter_table_set_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_set_options.rs @@ -30,6 +30,7 @@ use databend_common_storages_fuse::FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER; use databend_common_storages_fuse::FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FuseSegmentFormat; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::SegmentsIO; @@ -159,6 +160,10 @@ impl Interpreter for SetOptionsInterpreter { // Same as settings of FUSE_OPT_KEY_ENABLE_AUTO_VACUUM, expect value type is unsigned integer is_valid_option_of_type::(&self.plan.set_options, FUSE_OPT_KEY_ENABLE_AUTO_VACUUM)?; is_valid_option_of_type::(&self.plan.set_options, FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER)?; + is_valid_option_of_type::( + &self.plan.set_options, + FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, + )?; is_valid_option_of_type::( &self.plan.set_options, FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD, @@ -284,6 +289,13 @@ async fn set_segment_format( .await?; for segment in segments { let segment = segment?; + if segment.blocks.iter().any(|block| { + !block.column_groups.is_empty() || !block.bloom_index_files.is_empty() + }) { + return Err(ErrorCode::TableOptionInvalid( + "cannot convert segments containing partial-update files to the column-oriented format; rewrite the blocks to the single-file layout first", + )); + } for block in segment.blocks { segment_builder.add_block(block.as_ref().clone())?; } diff --git a/src/query/service/src/physical_plans/physical_column_mutation.rs b/src/query/service/src/physical_plans/physical_column_mutation.rs index b4c90345ce8cd..9e0abfa496185 100644 --- a/src/query/service/src/physical_plans/physical_column_mutation.rs +++ b/src/query/service/src/physical_plans/physical_column_mutation.rs @@ -29,6 +29,7 @@ use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::TransformSerializeBlock; +use databend_common_storages_fuse::statistics::ClusterStatsGenerator; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ColumnMutationFormatter; @@ -51,6 +52,7 @@ pub struct ColumnMutation { pub has_filter_column: bool, pub table_meta_timestamps: TableMetaTimestamps, pub udf_col_num: usize, + pub partial_update_fields: Option>, } #[typetag::serde] @@ -98,6 +100,7 @@ impl IPhysicalPlan for ColumnMutation { has_filter_column: self.has_filter_column, table_meta_timestamps: self.table_meta_timestamps, udf_col_num: self.udf_col_num, + partial_update_fields: self.partial_update_fields.clone(), }) } @@ -114,11 +117,10 @@ impl IPhysicalPlan for ColumnMutation { let mut exprs = Vec::with_capacity(mutation_expr.len()); for (id, remote_expr) in mutation_expr { let expr = remote_expr.as_expr(&BUILTIN_FUNCTIONS); - let schema_index = field_id_to_schema_index.get(id).unwrap(); - schema_offset_to_new_offset.insert(*schema_index, next_column_offset); - field_id_to_schema_index - .entry(*id) - .and_modify(|e| *e = next_column_offset); + if let Some(schema_index) = field_id_to_schema_index.get(id) { + schema_offset_to_new_offset.insert(*schema_index, next_column_offset); + } + field_id_to_schema_index.insert(*id, next_column_offset); next_column_offset += 1; exprs.push(expr); } @@ -139,21 +141,15 @@ impl IPhysicalPlan for ColumnMutation { remote_expr .as_expr(&BUILTIN_FUNCTIONS) .project_column_ref(|index| { - schema_offset_to_new_offset + Ok(schema_offset_to_new_offset .get(index) - .ok_or_else(|| { - ErrorCode::BadArguments(format!( - "Unable to get field named \"{}\"", - index - )) - }) .copied() + .unwrap_or(*index)) })?; - let schema_index = field_id_to_schema_index.get(id).unwrap(); - schema_offset_to_new_offset.insert(*schema_index, next_column_offset); - field_id_to_schema_index - .entry(*id) - .and_modify(|e| *e = next_column_offset); + if let Some(schema_index) = field_id_to_schema_index.get(id) { + schema_offset_to_new_offset.insert(*schema_index, next_column_offset); + } + field_id_to_schema_index.insert(*id, next_column_offset); next_column_offset += 1; exprs.push(expr); } @@ -166,12 +162,38 @@ impl IPhysicalPlan for ColumnMutation { // Keep only table fields in their schema order. Mutation input may // carry derived UDF argument/result columns that must not be // serialized back into the table. - let mut projection = field_id_to_schema_index.iter().collect::>(); - projection.sort_by_key(|(field_id, _)| *field_id); - let projection = projection - .into_iter() - .map(|(_, schema_index)| *schema_index) - .collect::>(); + let projection = if let Some(updated_fields) = &self.partial_update_fields { + let table = builder + .ctx + .build_table_by_table_info(&self.table_info, None)?; + let table = FuseTable::try_from_table(table.as_ref())?; + let schema_with_stream = table.schema_with_stream(); + let source_schema = schema_with_stream.remove_virtual_computed_fields(); + updated_fields + .iter() + .map(|source_index| { + let column_id = source_schema.field(*source_index).column_id(); + let field_id = schema_with_stream + .fields() + .iter() + .position(|field| field.column_id() == column_id) + .ok_or_else(|| ErrorCode::Internal("updated field is not in schema"))?; + field_id_to_schema_index + .get(&field_id) + .copied() + .ok_or_else(|| { + ErrorCode::Internal("updated field is not in mutation output") + }) + }) + .collect::>>()? + } else { + let mut projection = field_id_to_schema_index.iter().collect::>(); + projection.sort_by_key(|(field_id, _)| *field_id); + projection + .into_iter() + .map(|(_, schema_index)| *schema_index) + .collect::>() + }; block_operators.push(BlockOperator::Project { projection }); builder.main_pipeline.add_transformer(|| { @@ -188,8 +210,12 @@ impl IPhysicalPlan for ColumnMutation { .build_table_by_table_info(&self.table_info, None)?; let table = FuseTable::try_from_table(table.as_ref())?; + let write_column_group = self.partial_update_fields.is_some(); + let block_thresholds = table.get_block_thresholds(); - let cluster_stats_gen = if matches!(self.mutation_kind, MutationKind::Delete) { + let cluster_stats_gen = if write_column_group { + ClusterStatsGenerator::default() + } else if matches!(self.mutation_kind, MutationKind::Delete) { let input_schema = DataSchema::from(table.schema_with_stream()).into(); table.get_cluster_stats_gen(builder.ctx.clone(), 0, block_thresholds, input_schema)? } else { @@ -202,15 +228,27 @@ impl IPhysicalPlan for ColumnMutation { }; builder.main_pipeline.add_transform(|input, output| { - let proc = TransformSerializeBlock::try_create( - builder.ctx.clone(), - input, - output, - table, - cluster_stats_gen.clone(), - self.mutation_kind, - self.table_meta_timestamps, - )?; + let proc = if write_column_group { + TransformSerializeBlock::try_create_for_update( + builder.ctx.clone(), + input, + output, + table, + cluster_stats_gen.clone(), + self.partial_update_fields.clone().unwrap(), + self.table_meta_timestamps, + )? + } else { + TransformSerializeBlock::try_create( + builder.ctx.clone(), + input, + output, + table, + cluster_stats_gen.clone(), + self.mutation_kind, + self.table_meta_timestamps, + )? + }; proc.into_processor() }) } diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index e37402097604b..97e11579d2c4b 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -57,11 +57,13 @@ use databend_common_sql::binder::wrap_cast; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::parse_cluster_keys; use databend_common_sql::parse_computed_field_index_expr; use databend_common_sql::plans::BoundColumnRef; use databend_common_sql::plans::ConstantExpr; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::TruncateMode; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_storages_common_table_meta::meta::Location; @@ -91,6 +93,135 @@ use crate::sessions::TableContext; // The predicate column symbol should not conflict with update expr column bindings. pub const PREDICATE_COLUMN_INDEX: Symbol = Symbol::DUMMY_COLUMN; +struct PartialUpdateInfo { + required_columns: ColumnSet, + updated_field_indices: Vec, +} + +#[allow(clippy::too_many_arguments)] +fn build_partial_update_info( + ctx: Arc, + table: &FuseTable, + bind_context: &BindContext, + update_list: &HashMap, + direct_filter: &[ScalarExpr], + database: Option<&str>, + table_name: &str, +) -> Result> { + if !ctx.get_settings().get_enable_partial_update()? + || !table.get_option(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, false) + || table.is_column_oriented() + || !table.get_table_info().meta.indexes.is_empty() + || update_list.is_empty() + { + return Ok(None); + } + + let schema_with_stream = table.schema_with_stream(); + let table_schema = table.schema(); + let computed_schema: DataSchemaRef = Arc::new(table_schema.as_ref().into()); + let mut updated_column_ids = update_list + .keys() + .map(|index| schema_with_stream.field(*index).column_id()) + .collect::>(); + let mut computed_dependencies = BTreeSet::new(); + + for field in table_schema.fields() { + let Some(ComputedExpr::Stored(stored_expr)) = field.computed_expr() else { + continue; + }; + let expr = + parse_computed_field_index_expr(ctx.clone(), computed_schema.clone(), stored_expr)?; + let dependencies = expr + .column_refs() + .into_iter() + .map(|(index, _)| index) + .collect::>(); + if dependencies + .iter() + .any(|index| update_list.contains_key(index)) + { + updated_column_ids.insert(field.column_id()); + computed_dependencies.extend(dependencies); + } + } + + for stream_column in table.stream_columns() { + updated_column_ids.insert(stream_column.column_id()); + } + + if let Some(cluster_keys) = table.resolve_cluster_keys() { + let cluster_exprs = parse_cluster_keys(ctx.clone(), Arc::new(table.clone()), cluster_keys)?; + let updates_cluster_key = cluster_exprs.iter().any(|expr| { + expr.column_refs().iter().any(|(index, _)| { + updated_column_ids.contains(&schema_with_stream.field(*index).column_id()) + }) + }); + if updates_cluster_key { + return Ok(None); + } + } + + let source_schema = schema_with_stream.remove_virtual_computed_fields(); + let updated_field_indices = source_schema + .fields() + .iter() + .enumerate() + .filter_map(|(index, field)| { + updated_column_ids + .contains(&field.column_id()) + .then_some(index) + }) + .collect::>(); + if updated_field_indices.is_empty() + || updated_field_indices.len() >= source_schema.fields().len() + { + return Ok(None); + } + + let find_symbol = |field_index: FieldIndex| { + let field = schema_with_stream.field(field_index); + bind_context + .columns + .iter() + .find(|binding| { + BindContext::match_column_binding(database, Some(table_name), field.name(), binding) + }) + .map(|binding| binding.index) + }; + + let mut required_columns = update_list + .values() + .flat_map(ScalarExpr::used_columns) + .chain(direct_filter.iter().flat_map(ScalarExpr::used_columns)) + .collect::(); + for field_index in update_list + .keys() + .copied() + .chain(computed_dependencies.into_iter()) + { + let Some(symbol) = find_symbol(field_index) else { + return Ok(None); + }; + required_columns.insert(symbol); + } + for (field_index, field) in schema_with_stream.fields().iter().enumerate() { + if updated_column_ids.contains(&field.column_id()) && field.computed_expr().is_none() { + let Some(symbol) = find_symbol(field_index) else { + return Ok(None); + }; + // STORED computed columns are regenerated from their dependencies and do not need + // their previous values. Stream columns do need their previous values. + required_columns.insert(symbol); + } + } + + Ok(Some(PartialUpdateInfo { + required_columns, + updated_field_indices, + })) +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Mutation { pub meta: PhysicalPlanMeta, @@ -269,6 +400,41 @@ impl PhysicalPlanBuilder { .. } = mutation; + let table = self + .ctx + .get_table(catalog_name, database_name, table_name) + .await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let partial_table_name = table_name_alias + .as_ref() + .map(|name| name.to_lowercase()) + .unwrap_or_else(|| table_name.clone()); + let partial_database = table_name_alias.is_none().then_some(database_name.as_str()); + let partial_update = if *strategy == MutationStrategy::Direct { + matched_evaluators + .first() + .and_then(|evaluator| evaluator.update.as_ref()) + .map(|update_list| { + build_partial_update_info( + self.ctx.clone(), + fuse_table, + bind_context, + update_list, + direct_filter, + partial_database, + &partial_table_name, + ) + }) + .transpose()? + .flatten() + } else { + None + }; + if let Some(info) = &partial_update { + required = info.required_columns.clone(); + } + self.mutation_build_info.as_mut().unwrap().partial_update = partial_update.is_some(); + let mut maybe_udfs = BTreeSet::new(); for matched_evaluator in matched_evaluators { if let Some(condition) = &matched_evaluator.condition { @@ -308,10 +474,6 @@ impl PhysicalPlanBuilder { return Ok(plan); } - let table = self - .ctx - .get_table(catalog_name, database_name, table_name) - .await?; let table_info = table.get_table_info(); let table_name = table_name.clone(); @@ -399,6 +561,7 @@ impl PhysicalPlanBuilder { has_filter_column: predicate_column_index.is_some(), table_meta_timestamps: mutation_build_info.table_meta_timestamps, udf_col_num, + partial_update_fields: partial_update.map(|info| info.updated_field_indices), }); if *distributed { @@ -1069,10 +1232,10 @@ fn build_field_id_to_schema_index( column_binding, ) { let column_index = column_binding.index; - let schema_index = mutation_input_schema - .index_of(&column_index.to_string()) - .unwrap(); - field_id_to_schema_index.insert(field_id, schema_index); + if let Ok(schema_index) = mutation_input_schema.index_of(&column_index.to_string()) + { + field_id_to_schema_index.insert(field_id, schema_index); + } break; } } diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index eb1c525c516e7..5e637d0bcc23d 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -73,6 +73,7 @@ pub struct MutationSource { pub output_schema: DataSchemaRef, pub input_type: MutationType, pub read_partition_columns: ColumnSet, + pub partial_update: bool, pub truncate_table: bool, pub partitions: Partitions, @@ -117,6 +118,7 @@ impl IPhysicalPlan for MutationSource { output_schema: self.output_schema.clone(), input_type: self.input_type.clone(), read_partition_columns: self.read_partition_columns.clone(), + partial_update: self.partial_update, truncate_table: self.truncate_table, partitions: self.partitions.clone(), statistics: self.statistics.clone(), @@ -216,16 +218,28 @@ impl IPhysicalPlan for MutationSource { col_indices, &mut builder.main_pipeline, mutation_action, + self.partial_update, )?; if table.change_tracking_enabled() { - let stream_ctx = StreamContext::try_create( - builder.ctx.get_function_context()?, - table.schema_with_stream(), - table.get_table_info().ident.seq, - is_delete, - update_mutation_with_filter, - )?; + let stream_ctx = if self.partial_update { + StreamContext::try_create_projected( + builder.ctx.get_function_context()?, + table.schema_with_stream(), + &read_partition_columns, + table.get_table_info().ident.seq, + is_delete, + update_mutation_with_filter, + )? + } else { + StreamContext::try_create( + builder.ctx.get_function_context()?, + table.schema_with_stream(), + table.get_table_info().ident.seq, + is_delete, + update_mutation_with_filter, + )? + }; builder .main_pipeline .add_transformer(|| TransformAddStreamColumns::new(stream_ctx.clone())); @@ -239,6 +253,7 @@ impl PhysicalPlanBuilder { pub async fn build_mutation_source( &mut self, mutation_source: &databend_common_sql::plans::MutationSource, + required: ColumnSet, ) -> Result { let all_predicates: Vec = mutation_source .secure_predicates @@ -264,9 +279,21 @@ impl PhysicalPlanBuilder { }; let mutation_info = self.mutation_build_info.as_ref().unwrap(); + let partial_update = self + .mutation_build_info + .as_ref() + .is_some_and(|info| info.partial_update); let metadata = self.metadata.read(); let mut fields = Vec::with_capacity(mutation_source.columns.len()); for column_index in mutation_source.columns.iter() { + if partial_update + && !required.contains(column_index) + && !mutation_source + .read_partition_columns + .contains(column_index) + { + continue; + } let column = metadata.column(*column_index); // Ignore virtual computed columns. if let Ok(column_id) = mutation_source.schema.index_of(&column.name()) { @@ -275,6 +302,12 @@ impl PhysicalPlanBuilder { } fields.sort_by_key(|(_, _, id)| *id); + let read_partition_columns = if partial_update { + fields.iter().map(|(_, index, _)| *index).collect() + } else { + mutation_source.read_partition_columns.clone() + }; + let mut fields = fields .into_iter() .map(|(name, index, _)| { @@ -302,7 +335,8 @@ impl PhysicalPlanBuilder { display_filters, has_hidden_secure_filters: !mutation_source.secure_predicates.is_empty(), input_type: mutation_source.mutation_type.clone(), - read_partition_columns: mutation_source.read_partition_columns.clone(), + read_partition_columns, + partial_update, truncate_table, meta: PhysicalPlanMeta::new("MutationSource"), partitions: mutation_info.partitions.clone(), diff --git a/src/query/service/src/physical_plans/physical_plan_builder.rs b/src/query/service/src/physical_plans/physical_plan_builder.rs index 8616c62826182..5e189e1598d50 100644 --- a/src/query/service/src/physical_plans/physical_plan_builder.rs +++ b/src/query/service/src/physical_plans/physical_plan_builder.rs @@ -169,7 +169,7 @@ impl PhysicalPlanBuilder { self.build_mutation(s_expr, mutation, required).await } RelOperator::MutationSource(mutation_source) => { - self.build_mutation_source(mutation_source).await + self.build_mutation_source(mutation_source, required).await } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, RelOperator::MaterializedCTE(materialized_cte) => { @@ -471,4 +471,5 @@ pub struct MutationBuildInfo { pub partitions: Partitions, pub statistics: PartStatistics, pub table_meta_timestamps: TableMetaTimestamps, + pub partial_update: bool, } diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index 4bfc02314476c..943550667f962 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -334,6 +334,7 @@ fn build_test_segment_info( cluster_stats: None, location: block_location, bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)), + bloom_index_files: vec![], bloom_filter_index_size: 0, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs index cdeb474f52580..9160af3163a83 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs @@ -16,6 +16,7 @@ mod block_compact_mutator; mod deletion; mod recluster_mutator; mod segments_compact_mutator; +mod update; pub use segments_compact_mutator::CompactSegmentTestFixture; pub use segments_compact_mutator::compact_segment; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs new file mode 100644 index 0000000000000..e7bbea0952370 --- /dev/null +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -0,0 +1,160 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; +use databend_query::test_kits::*; +use databend_storages_common_cache::LoadParams; +use databend_storages_common_table_meta::meta::BlockMeta; + +async fn latest_block_meta(fixture: &TestFixture) -> anyhow::Result> { + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let snapshot = fuse_table.read_table_snapshot().await?.unwrap(); + let segment_reader = + MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()); + let (segment_location, segment_version) = &snapshot.segments[0]; + let segment = segment_reader + .read(&LoadParams { + location: segment_location.clone(), + len_hint: None, + ver: *segment_version, + put_cache: false, + }) + .await?; + let blocks = segment.block_metas()?; + assert_eq!(blocks.len(), 1); + Ok(blocks[0].clone()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, value int) engine=fuse \ + bloom_index_columns='id,value' enable_partial_update=true change_tracking=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10), (2, 20)" + )) + .await?; + + let table = fixture.latest_default_table().await?; + let schema = table.schema(); + let id_column_id = schema.field(0).column_id(); + let value_column_id = schema.field(1).column_id(); + let origin = latest_block_meta(&fixture).await?; + + fixture + .execute_command("set enable_partial_update = 1") + .await?; + + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let updated = latest_block_meta(&fixture).await?; + + assert_eq!(updated.column_groups.len(), 2); + let unchanged_group = updated + .column_groups + .iter() + .find(|group| group.location == origin.location) + .unwrap(); + assert_eq!(unchanged_group.active_column_ids, vec![id_column_id]); + let changed_group = updated + .column_groups + .iter() + .find(|group| group.location == updated.location) + .unwrap(); + assert!(changed_group.active_column_ids.contains(&value_column_id)); + assert_eq!(changed_group.active_column_ids.len(), 4); + assert_eq!( + updated.col_stats.get(&id_column_id), + origin.col_stats.get(&id_column_id) + ); + assert!(updated.bloom_filter_index_location.is_none()); + assert_eq!(updated.bloom_index_files.len(), 2); + assert_eq!( + updated.bloom_filter_index_size, + updated + .bloom_index_files + .iter() + .map(|file| file.file_size) + .sum::() + ); + + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 2" + )) + .await?; + let updated_again = latest_block_meta(&fixture).await?; + assert_eq!(updated_again.column_groups.len(), 2); + assert!( + updated_again + .column_groups + .iter() + .any(|group| group.location == origin.location) + ); + assert!( + updated_again + .column_groups + .iter() + .all(|group| group.location != updated.location) + ); + assert_eq!(updated_again.bloom_index_files.len(), 2); + assert!(updated_again.bloom_filter_index_location.is_none()); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11) or (id = 2 and value = 21)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + fixture + .execute_command("set enable_partial_update = 0") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + let fully_rewritten = latest_block_meta(&fixture).await?; + assert!(fully_rewritten.column_groups.is_empty()); + assert!(fully_rewritten.bloom_index_files.is_empty()); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 12) or (id = 2 and value = 21)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} diff --git a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs index 0d747d19732bf..f5ee8d7659980 100644 --- a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs +++ b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs @@ -331,6 +331,7 @@ async fn prepare_prewhere_data() -> Result { location: "test_block".to_string(), bloom_filter_index_location: None, bloom_filter_index_size: 0, + bloom_index_files: vec![], create_on: None, nums_rows: num_rows, column_groups: vec![FuseColumnGroupPartInfo { diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index ff305876f01f6..57e2c494bd31c 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -1605,6 +1605,13 @@ impl DefaultSettings { scope: SettingScope::Both, range: Some(SettingRange::Numeric(0..=1)), }), + ("enable_partial_update", DefaultSettingValue { + value: UserSettingValue::UInt64(0), + desc: "Enables eligible direct UPDATE statements to write column groups.", + mode: SettingMode::Both, + scope: SettingScope::Both, + range: Some(SettingRange::Numeric(0..=1)), + }), ("enable_auto_vacuum", DefaultSettingValue { value: UserSettingValue::UInt64(0), desc: "Whether to automatically trigger VACUUM operations on tables (using vacuum2)", diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index 6e698fd984f63..396589259ede3 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -1120,6 +1120,10 @@ impl Settings { Ok(self.try_get_u64("error_on_nondeterministic_update")? == 1) } + pub fn get_enable_partial_update(&self) -> Result { + Ok(self.try_get_u64("enable_partial_update")? == 1) + } + pub fn get_max_query_memory_usage(&self) -> Result { self.try_get_u64("max_query_memory_usage") } diff --git a/src/query/sql/src/planner/execution/stream_column.rs b/src/query/sql/src/planner/execution/stream_column.rs index e1b1f3f3c3484..d457b655b500b 100644 --- a/src/query/sql/src/planner/execution/stream_column.rs +++ b/src/query/sql/src/planner/execution/stream_column.rs @@ -52,9 +52,46 @@ impl StreamContext { table_version: u64, is_delete: bool, update_mutation_with_filter: bool, + ) -> Result { + Self::try_create_inner( + func_ctx, + schema, + None, + table_version, + is_delete, + update_mutation_with_filter, + ) + } + + pub fn try_create_projected( + func_ctx: FunctionContext, + schema: Arc, + projection: &[usize], + table_version: u64, + is_delete: bool, + update_mutation_with_filter: bool, + ) -> Result { + Self::try_create_inner( + func_ctx, + schema, + Some(projection), + table_version, + is_delete, + update_mutation_with_filter, + ) + } + + fn try_create_inner( + func_ctx: FunctionContext, + schema: Arc, + projection: Option<&[usize]>, + table_version: u64, + is_delete: bool, + update_mutation_with_filter: bool, ) -> Result { let input_schema = schema.remove_virtual_computed_fields(); - let num_fields = input_schema.fields().len() + update_mutation_with_filter as usize; + let projected_len = projection.map_or(input_schema.fields().len(), <[usize]>::len); + let num_fields = projected_len + update_mutation_with_filter as usize; let stream_columns = [ StreamColumn::new(ORIGIN_VERSION_COL_NAME, StreamColumnType::OriginVersion), @@ -69,12 +106,20 @@ impl StreamContext { let mut exprs = Vec::with_capacity(stream_columns.len()); for stream_column in stream_columns.iter() { let schema_index = input_schema.index_of(stream_column.column_name()).unwrap(); + let input_index = projection + .map(|projection| { + projection + .iter() + .position(|index| *index == schema_index) + .expect("partial update must read stream columns") + }) + .unwrap_or(schema_index); let origin_stream_column_scalar_expr = ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column: ColumnBindingBuilder::new( stream_column.column_name().to_string(), - Symbol::from_field_index(schema_index), + Symbol::from_field_index(input_index), Box::new(stream_column.data_type()), Visibility::Visible, ) @@ -83,14 +128,14 @@ impl StreamContext { let current_stream_column_scalar_expr = match stream_column.column_type() { StreamColumnType::OriginVersion => { - new_schema_index.insert(schema_index, num_fields + 2); + new_schema_index.insert(input_index, num_fields + 2); ScalarExpr::ConstantExpr(ConstantExpr { span: None, value: table_version.into(), }) } StreamColumnType::OriginBlockId => { - new_schema_index.insert(schema_index, num_fields + 3); + new_schema_index.insert(input_index, num_fields + 3); ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column: ColumnBindingBuilder::new( @@ -103,7 +148,7 @@ impl StreamContext { }) } StreamColumnType::OriginRowNum => { - new_schema_index.insert(schema_index, num_fields + 4); + new_schema_index.insert(input_index, num_fields + 4); ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column: ColumnBindingBuilder::new( diff --git a/src/query/storages/common/cache/src/manager.rs b/src/query/storages/common/cache/src/manager.rs index b9244c16bc48d..c98c19a7c4016 100644 --- a/src/query/storages/common/cache/src/manager.rs +++ b/src/query/storages/common/cache/src/manager.rs @@ -1301,6 +1301,7 @@ mod tests { cluster_stats: None, location: ("".to_string(), 0), bloom_filter_index_location: None, + bloom_index_files: vec![], bloom_filter_index_size: 0, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/storages/common/table_meta/src/meta/current/mod.rs b/src/query/storages/common/table_meta/src/meta/current/mod.rs index d8b45be536174..f0b1e3dc5c161 100644 --- a/src/query/storages/common/table_meta/src/meta/current/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/current/mod.rs @@ -15,6 +15,7 @@ pub use v0::ColumnMeta as SingleColumnMeta; pub use v2::AdditionalStatsMeta; pub use v2::BlockMeta; +pub use v2::BloomIndexFileMeta; pub use v2::ClusterStatistics; pub use v2::ColumnGroupFileMeta; pub use v2::ColumnMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/mod.rs b/src/query/storages/common/table_meta/src/meta/v2/mod.rs index 0ba8b8ad9e03b..d3c4c66c56c64 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/mod.rs @@ -19,6 +19,7 @@ pub mod statistics; mod table_snapshot_statistics; pub use segment::BlockMeta; +pub use segment::BloomIndexFileMeta; pub use segment::ColumnGroupFileMeta; pub use segment::ColumnMeta; pub use segment::DraftVirtualBlockMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index d396a93e090ea..8dc3d90118a83 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -184,6 +184,18 @@ pub struct ColumnGroupFileMeta { pub leaf_column_metas: HashMap, } +/// Metadata of one physical Bloom index file referenced by a logical block. +/// +/// A file may still contain filters that are no longer active. Readers must only use the filters +/// whose column ids are listed in [`Self::active_column_ids`]. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] +pub struct BloomIndexFileMeta { + pub active_column_ids: Vec, + pub location: Location, + pub format_version: FormatVersion, + pub file_size: u64, +} + /// Meta information of a block /// Part of and kept inside the [SegmentInfo] #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -206,6 +218,13 @@ pub struct BlockMeta { /// location of bloom filter index pub bloom_filter_index_location: Option, + /// Physical files that contain the active Bloom filters of this logical block. + /// + /// An empty vector is the legacy single-file representation described by + /// `bloom_filter_index_location` and `bloom_filter_index_size`. + #[serde(default)] + pub bloom_index_files: Vec, + #[serde(default)] pub bloom_filter_index_size: u64, pub inverted_index_size: Option, @@ -257,6 +276,7 @@ impl BlockMeta { cluster_stats, location, bloom_filter_index_location, + bloom_index_files: vec![], bloom_filter_index_size, inverted_index_size, ngram_filter_index_size, @@ -397,6 +417,7 @@ impl BlockMeta { cluster_stats: None, location: (s.location.path.clone(), 0), bloom_filter_index_location: None, + bloom_index_files: vec![], bloom_filter_index_size: 0, compression: Compression::Lz4, inverted_index_size: None, @@ -430,6 +451,7 @@ impl BlockMeta { cluster_stats: None, location: s.location.clone(), bloom_filter_index_location: s.bloom_filter_index_location.clone(), + bloom_index_files: vec![], bloom_filter_index_size: s.bloom_filter_index_size, compression: s.compression, inverted_index_size: None, @@ -463,7 +485,7 @@ mod tests { use super::*; #[test] - fn test_deserialize_legacy_block_meta_without_column_groups() { + fn test_deserialize_legacy_block_meta_without_file_lists() { let block_meta = BlockMeta::new( 10, 300, @@ -494,9 +516,17 @@ mod tests { .remove("column_groups") .is_some() ); + assert!( + value + .as_object_mut() + .unwrap() + .remove("bloom_index_files") + .is_some() + ); let decoded: BlockMeta = serde_json::from_value(value).unwrap(); assert!(decoded.column_groups.is_empty()); + assert!(decoded.bloom_index_files.is_empty()); assert_eq!(decoded.location, location); } } diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index a3c1b2e2cd9ff..04678a2790d2f 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -61,6 +61,7 @@ impl From for crate::meta::BlockMeta { cluster_stats: value.cluster_stats.map(|v| v.into()), location: value.location, bloom_filter_index_location: value.bloom_filter_index_location, + bloom_index_files: vec![], bloom_filter_index_size: value.bloom_filter_index_size, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/storages/fuse/src/constants.rs b/src/query/storages/fuse/src/constants.rs index a73e0667807d0..d5da27468466b 100644 --- a/src/query/storages/fuse/src/constants.rs +++ b/src/query/storages/fuse/src/constants.rs @@ -24,6 +24,7 @@ pub const FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP: &str = "data_retention_num_snapshots_to_keep"; pub const FUSE_OPT_KEY_ENABLE_AUTO_VACUUM: &str = "enable_auto_vacuum"; pub const FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE: &str = "enable_auto_analyze"; +pub const FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; pub const FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN: &str = "enable_virtual_column"; pub const FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD: &str = "auto_compaction_imperfect_blocks_threshold"; diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 205f49d3645c8..b995d30fb8e9b 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -30,6 +30,7 @@ use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_storages_common_pruner::BlockMetaIndex; +use databend_storages_common_table_meta::meta::BloomIndexFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; @@ -49,6 +50,8 @@ pub struct FuseBlockPartInfo { pub bloom_filter_index_location: Option, pub bloom_filter_index_size: u64, + #[serde(default)] + pub bloom_index_files: Vec, pub create_on: Option>, pub nums_rows: usize, @@ -89,6 +92,7 @@ impl FuseBlockPartInfo { location: String, bloom_filter_index_location: Option, bloom_filter_index_size: u64, + bloom_index_files: Vec, rows_count: u64, column_groups: Vec, columns_stat: Option>, @@ -101,6 +105,7 @@ impl FuseBlockPartInfo { location, bloom_filter_index_location, bloom_filter_index_size, + bloom_index_files, create_on, column_groups, nums_rows: rows_count as usize, diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index 611ae6a69b6c3..a5893b7d51bdc 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::future::Future; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::Result; @@ -27,11 +28,52 @@ use databend_storages_common_table_meta::meta::column_oriented_segment::BlockRea use super::BlockReader; use crate::BlockReadResult; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; use crate::io::read::block::block_reader_merge_io::DataItem; use crate::unsupported_storage_format_error; impl BlockReader { + pub(crate) fn projected_column_groups(&self, meta: &BlockMeta) -> Vec { + let projected_column_ids = self + .project_column_nodes + .iter() + .flat_map(|node| node.leaf_column_ids.iter().copied()) + .collect::>(); + if meta.column_groups.is_empty() { + return vec![FuseColumnGroupPartInfo { + location: meta.location.0.clone(), + columns_meta: meta + .col_metas + .iter() + .filter(|(column_id, _)| projected_column_ids.contains(column_id)) + .map(|(column_id, column_meta)| (*column_id, column_meta.clone())) + .collect(), + }]; + } + + meta.column_groups + .iter() + .filter_map(|group| { + let columns_meta = group + .active_column_ids + .iter() + .filter(|column_id| projected_column_ids.contains(column_id)) + .filter_map(|column_id| { + group + .leaf_column_metas + .get(column_id) + .map(|column_meta| (*column_id, column_meta.clone())) + }) + .collect::>(); + (!columns_meta.is_empty()).then(|| FuseColumnGroupPartInfo { + location: group.location.0.clone(), + columns_meta, + }) + }) + .collect() + } + /// Deserialize column chunks data from parquet format to DataBlock. pub fn deserialize_chunks_with_part_info( &self, @@ -77,11 +119,21 @@ impl BlockReader { storage_format: &FuseStorageFormat, ) -> Result { // Get the merged IO read result. - let merge_io_read_result = self - .read_columns_data_by_merge_io(settings, &meta.location.0, &meta.col_metas, &None) - .await?; - - self.deserialize_chunks_with_meta(&meta.into(), storage_format, merge_io_read_result) + let column_groups = self.projected_column_groups(meta); + let read: std::pin::Pin> + Send + '_>> = + Box::pin(self.read_column_groups_data_by_merge_io(settings, &column_groups, &None)); + let merge_io_read_result = read.await?; + let column_chunks = merge_io_read_result.columns_chunks()?; + match storage_format { + FuseStorageFormat::Parquet => self.deserialize_column_groups( + meta.row_count as usize, + &column_groups, + column_chunks, + &meta.compression, + None, + ), + FuseStorageFormat::Unsupported => Err(unsupported_storage_format_error()), + } } pub fn deserialize_chunks_with_meta( diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index c6bd8048a0137..d8c1c46c5dcda 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -14,12 +14,14 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; use chrono::Utc; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfo; use databend_common_expression::ColumnId; @@ -49,7 +51,9 @@ use databend_storages_common_blocks::blocks_to_parquet_with_stats; use databend_storages_common_index::NgramArgs; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::BloomIndexFileMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; +use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; use databend_storages_common_table_meta::meta::StatisticsOfColumns; @@ -271,6 +275,7 @@ impl BlockBuilder { cluster_stats, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), + bloom_index_files: vec![], bloom_filter_index_size: bloom_index_state .as_ref() .map(|v| v.size) @@ -312,6 +317,249 @@ impl BlockBuilder { }; Ok(serialized) } + + /// Serialize only the fields changed by an UPDATE and merge their physical + /// metadata back into the original logical block. + pub fn build_column_group( + &self, + data_block: DataBlock, + origin: &BlockMeta, + updated_field_indices: &[FieldIndex], + ) -> Result { + if data_block.num_rows() as u64 != origin.row_count { + return Err(ErrorCode::Internal( + "column-group update changed the block row count", + )); + } + + let mut updated_field_indices = updated_field_indices.to_vec(); + updated_field_indices.sort_unstable(); + updated_field_indices.dedup(); + if updated_field_indices.is_empty() { + return Err(ErrorCode::Internal( + "column-group update has no updated fields", + )); + } + if updated_field_indices + .iter() + .any(|index| *index >= self.source_schema.fields().len()) + { + return Err(ErrorCode::Internal( + "column-group update field is outside the table schema", + )); + } + + let updated_fields = updated_field_indices + .iter() + .copied() + .collect::>(); + let updated_schema = Arc::new(self.source_schema.project(&updated_field_indices)); + if data_block.num_columns() != updated_schema.fields().len() { + return Err(ErrorCode::Internal( + "column-group update block does not match updated fields", + )); + } + let updated_block = data_block; + let updated_column_ids = updated_schema.to_leaf_column_ids(); + let updated_column_id_set = updated_column_ids.iter().copied().collect::>(); + + let (data_location, block_id) = self + .meta_locations + .gen_block_location(self.table_meta_timestamps); + + let updated_bloom_columns_map = updated_field_indices + .iter() + .enumerate() + .filter_map(|(updated_index, source_index)| { + self.bloom_columns_map + .get(source_index) + .cloned() + .map(|field| (updated_index, field)) + }) + .collect::>(); + let updated_bloom_column_ids = updated_bloom_columns_map + .values() + .map(|field| field.column_id()) + .collect::>(); + let rebuild_bloom_index = !updated_bloom_columns_map.is_empty(); + let bloom_index_state = if rebuild_bloom_index { + let location = self.meta_locations.block_bloom_index_location(&block_id); + BloomIndexState::from_data_block( + self.ctx.clone(), + &updated_block, + location, + self.write_settings.bloom_index_type, + updated_bloom_columns_map, + &[], + )? + } else { + None + }; + + let mut column_distinct_count = bloom_index_state + .as_ref() + .map(|index| index.column_distinct_count.clone()) + .unwrap_or_default(); + let updated_ndv_columns_map = updated_field_indices + .iter() + .enumerate() + .filter_map(|(updated_index, source_index)| { + self.ndv_columns_map + .get(source_index) + .cloned() + .map(|field| (updated_index, field)) + }) + .collect::>(); + let column_hlls = build_column_hlls(&updated_block, &updated_ndv_columns_map)?; + if let Some(hlls) = &column_hlls { + for (column_id, hll) in hlls { + if let Entry::Vacant(entry) = column_distinct_count.entry(*column_id) { + entry.insert(hll.count()); + } + } + } + + let rebuild_virtual_columns = self + .virtual_column_builder + .as_ref() + .is_some_and(|builder| builder.is_affected(&updated_fields)); + let virtual_column_state = None; + + let updated_col_stats = gen_columns_statistics( + &updated_block, + Some(column_distinct_count), + &updated_schema, + &self.write_settings.col_stats_truncate_lens, + )?; + let uncompressed_size = + updated_block.estimate_block_size(updated_block.num_columns()) as u64; + let (updated_col_metas, buffer) = serialize_block_with_column_stats( + &self.write_settings, + &updated_schema, + Some(&updated_col_stats), + updated_block, + )?; + let file_size = buffer.len() as u64; + + let current_column_ids = self.source_schema.to_leaf_column_id_set(); + let mut column_groups = if origin.column_groups.is_empty() { + vec![ColumnGroupFileMeta { + active_column_ids: self + .source_schema + .to_leaf_column_ids() + .into_iter() + .filter(|column_id| origin.col_metas.contains_key(column_id)) + .collect(), + location: origin.location.clone(), + format_version: origin.location.1, + file_size: origin.file_size, + uncompressed_size: origin.block_size, + leaf_column_metas: origin.col_metas.clone(), + }] + } else { + origin.column_groups.clone() + }; + for group in &mut column_groups { + group.active_column_ids.retain(|column_id| { + current_column_ids.contains(column_id) && !updated_column_id_set.contains(column_id) + }); + } + column_groups.retain(|group| !group.active_column_ids.is_empty()); + column_groups.push(ColumnGroupFileMeta { + active_column_ids: updated_column_ids, + location: data_location.clone(), + format_version: data_location.1, + file_size, + uncompressed_size, + leaf_column_metas: updated_col_metas.clone(), + }); + + let mut block_meta = origin.clone(); + block_meta.location = data_location.clone(); + block_meta.file_size = column_groups.iter().map(|group| group.file_size).sum(); + block_meta.block_size = column_groups + .iter() + .map(|group| group.uncompressed_size) + .sum(); + block_meta.column_groups = column_groups; + block_meta + .col_metas + .retain(|column_id, _| current_column_ids.contains(column_id)); + block_meta.col_metas.extend(updated_col_metas); + block_meta + .col_stats + .retain(|column_id, _| current_column_ids.contains(column_id)); + block_meta.col_stats.extend(updated_col_stats); + block_meta.create_on = Some(Utc::now()); + + if rebuild_bloom_index { + let mut bloom_index_files = if origin.bloom_index_files.is_empty() { + origin + .bloom_filter_index_location + .as_ref() + .map(|location| BloomIndexFileMeta { + active_column_ids: self + .bloom_columns_map + .values() + .map(|field| field.column_id()) + .filter(|column_id| !updated_bloom_column_ids.contains(column_id)) + .collect(), + location: location.clone(), + format_version: location.1, + file_size: origin.bloom_filter_index_size, + }) + .into_iter() + .collect() + } else { + origin.bloom_index_files.clone() + }; + for file in &mut bloom_index_files { + file.active_column_ids + .retain(|column_id| !updated_bloom_column_ids.contains(column_id)); + } + bloom_index_files.retain(|file| !file.active_column_ids.is_empty()); + if let Some(state) = &bloom_index_state { + let mut active_column_ids = + updated_bloom_column_ids.iter().copied().collect::>(); + active_column_ids.sort_unstable(); + bloom_index_files.push(BloomIndexFileMeta { + active_column_ids, + location: state.location.clone(), + format_version: state.location.1, + file_size: state.size, + }); + } + block_meta.bloom_filter_index_location = None; + block_meta.bloom_filter_index_size = + bloom_index_files.iter().map(|file| file.file_size).sum(); + block_meta.bloom_index_files = bloom_index_files; + block_meta.ngram_filter_index_size = None; + } + if rebuild_virtual_columns { + block_meta.virtual_block_meta = None; + } + + let column_hlls = column_hlls + .map(|hlls| { + if self.serialize_hll { + encode_column_hll(&hlls).map(BlockHLLState::Serialized) + } else { + Ok(BlockHLLState::Deserialized(hlls)) + } + }) + .transpose()?; + + Ok(BlockSerialization { + block_raw_data: buffer, + block_meta, + bloom_index_state, + inverted_index_states: vec![], + virtual_column_state, + vector_index_state: None, + spatial_index_state: None, + column_hlls, + }) + } } pub struct BlockWriter; diff --git a/src/query/storages/fuse/src/io/write/stream/block_builder.rs b/src/query/storages/fuse/src/io/write/stream/block_builder.rs index 5d2122cc2bd32..1d196e083ac85 100644 --- a/src/query/storages/fuse/src/io/write/stream/block_builder.rs +++ b/src/query/storages/fuse/src/io/write/stream/block_builder.rs @@ -408,6 +408,7 @@ impl StreamBlockBuilder { cluster_stats, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), + bloom_index_files: vec![], bloom_filter_index_size: bloom_index_state .as_ref() .map(|v| v.size) diff --git a/src/query/storages/fuse/src/io/write/virtual_column_builder.rs b/src/query/storages/fuse/src/io/write/virtual_column_builder.rs index 2eeddde0c5550..2cfb46f61cf14 100644 --- a/src/query/storages/fuse/src/io/write/virtual_column_builder.rs +++ b/src/query/storages/fuse/src/io/write/virtual_column_builder.rs @@ -142,6 +142,12 @@ impl VirtualColumnBuilder { }) } + pub(crate) fn is_affected(&self, updated_fields: &HashSet) -> bool { + self.variant_offsets + .iter() + .any(|offset| updated_fields.contains(offset)) + } + pub fn add_block(&mut self, block: &DataBlock) -> Result<()> { let num_rows = block.num_rows(); diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index 568e6817799a8..cd11bc3aaaec3 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -1156,6 +1156,7 @@ mod tests { cluster_stats: None, location: (test_block_path(uuid), 0), bloom_filter_index_location: None, + bloom_index_files: vec![], bloom_filter_index_size: 0, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index ccfb9c70aaf63..c15731180bb6d 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -23,6 +23,7 @@ use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; +use databend_common_expression::FieldIndex; use databend_common_expression::TableSchema; use databend_common_metrics::storage::metrics_inc_recluster_write_block_nums; use databend_common_pipeline::core::Event; @@ -35,6 +36,7 @@ use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_storage::MutationStatus; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::RangeIndex; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use opendal::Operator; @@ -60,6 +62,7 @@ enum State { block: DataBlock, stats_type: ClusterStatsGenType, index: Option, + origin_block_meta: Option>, }, Serialized { serialized: BlockSerialization, @@ -78,6 +81,7 @@ pub struct TransformSerializeBlock { table_id: Option, // Only used in multi table insert kind: MutationKind, pending_insert_rows: u64, + updated_field_indices: Option>, } impl TransformSerializeBlock { @@ -98,6 +102,7 @@ impl TransformSerializeBlock { cluster_stats_gen, kind, false, + None, table_meta_timestamps, ) } @@ -119,6 +124,30 @@ impl TransformSerializeBlock { cluster_stats_gen, kind, true, + None, + table_meta_timestamps, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn try_create_for_update( + ctx: Arc, + input: Arc, + output: Arc, + table: &FuseTable, + cluster_stats_gen: ClusterStatsGenerator, + updated_field_indices: Vec, + table_meta_timestamps: TableMetaTimestamps, + ) -> Result { + Self::do_create( + ctx, + input, + output, + table, + cluster_stats_gen, + MutationKind::Update, + false, + Some(updated_field_indices), table_meta_timestamps, ) } @@ -131,6 +160,7 @@ impl TransformSerializeBlock { cluster_stats_gen: ClusterStatsGenerator, kind: MutationKind, with_tid: bool, + updated_field_indices: Option>, table_meta_timestamps: TableMetaTimestamps, ) -> Result { let schema = table.schema(); @@ -217,6 +247,7 @@ impl TransformSerializeBlock { table_id: if with_tid { Some(table.get_id()) } else { None }, kind, pending_insert_rows: 0, + updated_field_indices, }) } @@ -313,6 +344,7 @@ impl Processor for TransformSerializeBlock { block: input_data, stats_type: serialize_block.stats_type, index: Some(serialize_block.index), + origin_block_meta: serialize_block.origin_block_meta, }; Ok(Event::Sync) } @@ -344,6 +376,7 @@ impl Processor for TransformSerializeBlock { block: input_data, stats_type: ClusterStatsGenType::Generally, index: None, + origin_block_meta: None, }; Ok(Event::Sync) } @@ -355,11 +388,20 @@ impl Processor for TransformSerializeBlock { block, stats_type, index, + origin_block_meta, } => { // Check if the datablock is valid, this is needed to ensure data is correct block.check_valid()?; - let serialized = + let serialized = if let (Some(updated_field_indices), Some(origin_block_meta)) = + (&self.updated_field_indices, origin_block_meta) + { + self.block_builder.build_column_group( + block, + &origin_block_meta, + updated_field_indices, + )? + } else { self.block_builder .build(block, |block, generator| match &stats_type { ClusterStatsGenType::Generally => generator.gen_stats_for_append(block), @@ -368,7 +410,8 @@ impl Processor for TransformSerializeBlock { .gen_with_origin_stats(&block, origin_stats.clone())?; Ok((cluster_stats, block)) } - })?; + })? + }; self.state = State::Serialized { serialized, index }; } diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs index 3629d9d297367..81dd3ea6f4d65 100644 --- a/src/query/storages/fuse/src/operations/gc.rs +++ b/src/query/storages/fuse/src/operations/gc.rs @@ -803,9 +803,27 @@ impl TryFrom> for LocationTuple { let mut hll_location = HashSet::new(); let block_metas = value.block_metas()?; for block_meta in block_metas.into_iter() { - block_location.insert(block_meta.location.0.clone()); - if let Some(bloom_loc) = &block_meta.bloom_filter_index_location { - bloom_location.insert(bloom_loc.0.clone()); + if block_meta.column_groups.is_empty() { + block_location.insert(block_meta.location.0.clone()); + } else { + block_location.extend( + block_meta + .column_groups + .iter() + .map(|group| group.location.0.clone()), + ); + } + if block_meta.bloom_index_files.is_empty() { + if let Some(bloom_loc) = &block_meta.bloom_filter_index_location { + bloom_location.insert(bloom_loc.0.clone()); + } + } else { + bloom_location.extend( + block_meta + .bloom_index_files + .iter() + .map(|file| file.location.0.clone()), + ); } } if let Some(loc) = value.as_ref().summary.additional_stats_loc() { diff --git a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs index 4f276045351d4..73b9e30bf8a2e 100644 --- a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs +++ b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs @@ -21,6 +21,7 @@ use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use crate::BlockReadResult; +use crate::FuseColumnGroupPartInfo; use crate::operations::common::BlockMetaIndex; use crate::operations::mutation::CompactExtraInfo; use crate::operations::mutation::DeletedSegmentInfo; @@ -54,6 +55,7 @@ pub struct SerializeBlock { pub index: BlockMetaIndex, pub stats_type: ClusterStatsGenType, pub insert_rows: u64, + pub origin_block_meta: Option>, } impl SerializeBlock { @@ -61,11 +63,13 @@ impl SerializeBlock { index: BlockMetaIndex, stats_type: ClusterStatsGenType, insert_rows: u64, + origin_block_meta: Option>, ) -> Self { SerializeBlock { index, stats_type, insert_rows, + origin_block_meta, } } } @@ -74,6 +78,7 @@ pub enum CompactSourceMeta { Concat { read_res: Vec, metas: Vec>, + column_groups: Vec>, index: BlockMetaIndex, }, Extras(CompactExtraInfo), diff --git a/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs b/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs index 5a8d2564c9052..7cb314339fea3 100644 --- a/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs +++ b/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs @@ -16,13 +16,14 @@ use std::any::Any; use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; +use std::sync::Arc; use databend_common_catalog::plan::PartInfo; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_storages_common_pruner::BlockMetaIndex; -use databend_storages_common_table_meta::meta::ClusterStatistics; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::Statistics; use crate::operations::mutation::SegmentIndex; @@ -82,7 +83,7 @@ impl DeletedSegmentInfo { #[derive(serde::Serialize, serde::Deserialize, PartialEq)] pub struct MutationPartInfo { pub index: BlockMetaIndex, - pub cluster_stats: Option, + pub block_meta: Arc, pub inner_part: PartInfoPtr, pub whole_block_mutation: bool, } diff --git a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs index 7da9ed30eea32..6a3cb628d6203 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs @@ -19,6 +19,7 @@ use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_metrics::storage::*; @@ -89,14 +90,15 @@ impl PrefetchAsyncSource for CompactSource { metrics_inc_compact_block_read_bytes(block.block_size); } - block_reader - .read_columns_data_by_merge_io( + let column_groups = block_reader.projected_column_groups(&block); + let read_res = block_reader + .read_column_groups_data_by_merge_io( &settings, - &block.location.0, - &block.col_metas, + &column_groups, &None, ) - .await + .await?; + Ok::<_, ErrorCode>((read_res, column_groups)) }) .await .unwrap() @@ -105,7 +107,10 @@ impl PrefetchAsyncSource for CompactSource { let start = Instant::now(); - let read_res = futures::future::try_join_all(task_futures).await?; + let (read_res, column_groups) = futures::future::try_join_all(task_futures) + .await? + .into_iter() + .unzip(); // Perf. { metrics_inc_compact_block_read_milliseconds(start.elapsed().as_millis() as u64); @@ -113,6 +118,7 @@ impl PrefetchAsyncSource for CompactSource { Box::new(CompactSourceMeta::Concat { read_res, metas: task.blocks.clone(), + column_groups, index: task.index.clone(), }) } @@ -157,17 +163,29 @@ impl BlockMetaTransform for CompactTransform { CompactSourceMeta::Concat { read_res, metas, + column_groups, index, } => { let blocks = read_res .into_iter() .zip(metas.into_iter()) - .map(|(data, meta)| { - let mut block = self.block_reader.deserialize_chunks_with_meta( - &meta.as_ref().into(), - &self.storage_format, - data, - )?; + .zip(column_groups.into_iter()) + .map(|((data, meta), column_groups)| { + let chunks = data.columns_chunks()?; + let mut block = match self.storage_format { + FuseStorageFormat::Parquet => { + self.block_reader.deserialize_column_groups( + meta.row_count as usize, + &column_groups, + chunks, + &meta.compression, + None, + )? + } + FuseStorageFormat::Unsupported => { + return Err(crate::unsupported_storage_format_error()); + } + }; self.scan_progress.incr(&ProgressValues { rows: block.num_rows(), @@ -188,6 +206,7 @@ impl BlockMetaTransform for CompactTransform { index, ClusterStatsGenType::Generally, 0, + None, ))); let new_block = block.add_meta(Some(meta))?; Ok(vec![new_block]) diff --git a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs index 4038ac251d8a2..37f9a59e02c16 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs @@ -38,6 +38,7 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_sql::evaluator::BlockOperator; use databend_common_storage::MutationStatus; use databend_storages_common_io::ReadSettings; +use databend_storages_common_table_meta::meta::BlockMeta; use crate::BlockReadResult; use crate::FuseStorageFormat; @@ -90,6 +91,7 @@ pub struct MutationSource { index: BlockMetaIndex, stats_type: ClusterStatsGenType, update_rows: u64, + origin_block_meta: Option>, } impl MutationSource { @@ -119,6 +121,7 @@ impl MutationSource { index: BlockMetaIndex::default(), stats_type: ClusterStatsGenType::Generally, update_rows: 0, + origin_block_meta: None, }))) } } @@ -223,6 +226,7 @@ impl Processor for MutationSource { self.index.clone(), self.stats_type.clone(), 0, + None, ), )); self.state = State::Output( @@ -325,6 +329,7 @@ impl Processor for MutationSource { self.index.clone(), self.stats_type.clone(), update_rows, + self.origin_block_meta.take(), ))); let meta: BlockMetaInfoPtr = if self.update_stream_columns { Box::new(gen_mutation_stream_meta(Some(inner_meta), &path)?) @@ -359,8 +364,11 @@ impl Processor for MutationSource { block_idx: part.index.block_idx, }; if matches!(self.action, MutationAction::Deletion) { - self.stats_type = - ClusterStatsGenType::WithOrigin(part.cluster_stats.clone()); + self.stats_type = ClusterStatsGenType::WithOrigin( + part.block_meta.cluster_stats.clone(), + ); + } else { + self.origin_block_meta = Some(part.block_meta.clone()); } let inner_part = part.inner_part.clone(); @@ -376,6 +384,7 @@ impl Processor for MutationSource { self.index.clone(), self.stats_type.clone(), 0, + None, ), )); self.state = State::Output( diff --git a/src/query/storages/fuse/src/operations/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation_source.rs index 378e98345dcd3..15d2d833627df 100644 --- a/src/query/storages/fuse/src/operations/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation_source.rs @@ -56,14 +56,17 @@ impl FuseTable { col_indices: Vec, pipeline: &mut Pipeline, mutation_action: MutationAction, + partial_update: bool, ) -> Result<()> { let all_column_indices = self.all_column_indices(); - let col_indices = + let mut col_indices = if matches!(mutation_action, MutationAction::Deletion) || !col_indices.is_empty() { col_indices } else { all_column_indices.clone() }; + col_indices.sort_unstable(); + col_indices.dedup(); let projection = Projection::Columns(col_indices.clone()); let update_stream_columns = self.change_tracking_enabled(); let block_reader = self.create_block_reader(ctx.clone(), projection, false)?; @@ -76,10 +79,14 @@ impl FuseTable { })); let num_column_indices = self.schema_with_stream().fields().len(); - let remain_column_indices: Vec = all_column_indices - .into_iter() - .filter(|index| !col_indices.contains(index)) - .collect(); + let remain_column_indices: Vec = if partial_update { + vec![] + } else { + all_column_indices + .into_iter() + .filter(|index| !col_indices.contains(index)) + .collect() + }; let mut source_col_indices = col_indices; if matches!(mutation_action, MutationAction::Deletion) && update_stream_columns || matches!(mutation_action, MutationAction::Update) && filter_expr.is_some() @@ -260,13 +267,12 @@ impl FuseTable { .into_iter() .zip(inner_parts.partitions.into_iter()) .map(|((index, block_meta), inner_part)| { - let cluster_stats = block_meta.cluster_stats.clone(); let key = (index.segment_idx, index.block_idx); let whole_block_mutation = whole_block_deletions.contains(&key); let part_info_ptr: PartInfoPtr = Arc::new(Box::new(Mutation::MutationPartInfo(MutationPartInfo { index, - cluster_stats, + block_meta, inner_part, whole_block_mutation, }))); diff --git a/src/query/storages/fuse/src/operations/read/read_block_context.rs b/src/query/storages/fuse/src/operations/read/read_block_context.rs index a37a1ca1974e9..b6e912a30253b 100644 --- a/src/query/storages/fuse/src/operations/read/read_block_context.rs +++ b/src/query/storages/fuse/src/operations/read/read_block_context.rs @@ -131,6 +131,7 @@ impl ReadBlockContext { location, None, 0, + vec![], num_rows, column_groups, None, diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 90b8cd074bc57..cccd65f93f733 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -1460,6 +1460,7 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, + meta.bloom_index_files.clone(), rows_count, column_groups, Some(columns_stats), @@ -1516,6 +1517,7 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, + meta.bloom_index_files.clone(), rows_count, column_groups, Some(columns_stat), diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 4d22b1aad512f..020f62d4a097a 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -55,6 +55,7 @@ use log::warn; use opendal::Operator; use tokio::sync::Semaphore; +use crate::FuseStorageFormat; use crate::FuseTable; use crate::io::BlockBuilder; use crate::io::BlockReader; @@ -633,13 +634,9 @@ impl AggregationContext { } async fn read_block(&self, reader: &BlockReader, block_meta: &BlockMeta) -> Result { + let column_groups = reader.projected_column_groups(block_meta); let merged_io_read_result = reader - .read_columns_data_by_merge_io( - &self.read_settings, - &block_meta.location.0, - &block_meta.col_metas, - &None, - ) + .read_column_groups_data_by_merge_io(&self.read_settings, &column_groups, &None) .await?; // deserialize block data @@ -650,14 +647,18 @@ impl AggregationContext { GlobalIORuntime::instance() .spawn(async move { let column_chunks = merged_io_read_result.columns_chunks()?; - reader.deserialize_chunks( - block_meta_ptr.location.0.as_str(), - block_meta_ptr.row_count as usize, - &block_meta_ptr.compression, - &block_meta_ptr.col_metas, - column_chunks, - &storage_format, - ) + match storage_format { + FuseStorageFormat::Parquet => reader.deserialize_column_groups( + block_meta_ptr.row_count as usize, + &column_groups, + column_chunks, + &block_meta_ptr.compression, + None, + ), + FuseStorageFormat::Unsupported => { + Err(crate::unsupported_storage_format_error()) + } + } }) .await .map_err(|e| { diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index 38fe38a02df69..fe7db597c3bff 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -794,6 +794,8 @@ impl AsyncTransform for NgramIndexTransform { }; let state = BloomIndexState::from_bloom_index(&bloom_index, index_location)?; + new_block_meta.bloom_filter_index_location = Some(state.location.clone()); + new_block_meta.bloom_index_files.clear(); new_block_meta.bloom_filter_index_size = state.size(); new_block_meta.ngram_filter_index_size = state.ngram_size(); BlockWriter::write_down_bloom_index_state(&self.operator, Some(state)).await?; diff --git a/src/query/storages/fuse/src/operations/util.rs b/src/query/storages/fuse/src/operations/util.rs index c2a65329b937d..8f893ad5eca4c 100644 --- a/src/query/storages/fuse/src/operations/util.rs +++ b/src/query/storages/fuse/src/operations/util.rs @@ -119,13 +119,9 @@ pub async fn read_block( block_meta: &BlockMeta, read_settings: &ReadSettings, ) -> Result { + let column_groups = reader.projected_column_groups(block_meta); let merged_io_read_result = reader - .read_columns_data_by_merge_io( - read_settings, - &block_meta.location.0, - &block_meta.col_metas, - &None, - ) + .read_column_groups_data_by_merge_io(read_settings, &column_groups, &None) .await?; // deserialize block data @@ -136,14 +132,16 @@ pub async fn read_block( GlobalIORuntime::instance() .spawn(async move { let column_chunks = merged_io_read_result.columns_chunks()?; - reader.deserialize_chunks( - block_meta_ptr.location.0.as_str(), - block_meta_ptr.row_count as usize, - &block_meta_ptr.compression, - &block_meta_ptr.col_metas, - column_chunks, - &storage_format, - ) + match storage_format { + FuseStorageFormat::Parquet => reader.deserialize_column_groups( + block_meta_ptr.row_count as usize, + &column_groups, + column_chunks, + &block_meta_ptr.compression, + None, + ), + FuseStorageFormat::Unsupported => Err(crate::unsupported_storage_format_error()), + } }) .await .map_err(|e| { diff --git a/src/query/storages/fuse/src/pruning/block_pruner.rs b/src/query/storages/fuse/src/pruning/block_pruner.rs index fac061b199b43..848fecc1728a0 100644 --- a/src/query/storages/fuse/src/pruning/block_pruner.rs +++ b/src/query/storages/fuse/src/pruning/block_pruner.rs @@ -346,6 +346,7 @@ impl BlockPruner { bloom_pruner.should_keep( &block_meta.bloom_filter_index_location, block_meta.bloom_filter_index_size, + &block_meta.bloom_index_files, &block_meta.col_stats, column_ids, &block_meta.as_ref().into(), diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 5d7fd27389c27..b2bbfbe2f199c 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -23,6 +23,7 @@ use databend_common_expression::ColumnId; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; @@ -33,8 +34,10 @@ use databend_storages_common_index::FilterEvalResult; use databend_storages_common_index::NgramArgs; use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; +use databend_storages_common_table_meta::meta::BloomIndexFileMeta; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; +use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use log::info; use log::warn; @@ -54,6 +57,7 @@ pub trait BloomPruner { &self, index_location: &Option, index_length: u64, + index_files: &[BloomIndexFileMeta], column_stats: &StatisticsOfColumns, column_ids: Vec, block_meta: &BlockReadInfo, @@ -68,10 +72,6 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( expr: &Expr, part: &FuseBlockPartInfo, ) -> Result { - let Some(index_location) = part.bloom_filter_index_location.as_ref() else { - return Ok(false); - }; - if part.bloom_filter_index_size == 0 { return Ok(false); } @@ -95,26 +95,48 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( } } - let index_columns = result - .bloom_fields - .iter() - .map(|field| BloomIndex::build_filter_bloom_name(index_location.1, field)) - .collect::>>()?; - let filter = index_location - .read_block_filter( - dal.clone(), + let bloom_index = if part.bloom_index_files.is_empty() { + let Some(index_location) = part.bloom_filter_index_location.as_ref() else { + return Ok(false); + }; + let index_columns = result + .bloom_fields + .iter() + .map(|field| BloomIndex::build_filter_bloom_name(index_location.1, field)) + .collect::>>()?; + let filter = index_location + .read_block_filter( + dal.clone(), + settings, + &index_columns, + part.bloom_filter_index_size, + ) + .await?; + BloomIndex::from_filter_block( + func_ctx.clone(), + filter.filter_schema, + filter.filters, + index_location.1, + )? + } else { + let Some(filter) = read_multi_file_block_filter( + dal, settings, - &index_columns, - part.bloom_filter_index_size, + &result.bloom_fields, + &[], + &part.bloom_index_files, ) - .await?; - - let bloom_index = BloomIndex::from_filter_block( - func_ctx.clone(), - filter.filter_schema, - filter.filters, - index_location.1, - )?; + .await? + else { + return Ok(false); + }; + BloomIndex::from_filter_block( + func_ctx.clone(), + filter.filter_schema, + filter.filters, + BlockFilter::VERSION, + )? + }; let like_scalar_map = HashMap::new(); let empty_stats = StatisticsOfColumns::new(); @@ -129,6 +151,72 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( )? == FilterEvalResult::MustFalse) } +async fn read_multi_file_block_filter( + dal: &Operator, + settings: &ReadSettings, + index_fields: &[TableField], + ngram_args: &[NgramArgs], + index_files: &[BloomIndexFileMeta], +) -> Result> { + let mut merged_fields = Vec::new(); + let mut merged_filters = Vec::new(); + + for file in index_files { + // V2 filters use legacy scalar encoding and cannot be mixed with the digest encoding of + // current files. Keeping the block is the safe compatibility behavior. + if file.format_version == 2 { + return Ok(None); + } + + let mut rename = HashMap::new(); + let mut columns = Vec::new(); + for field in index_fields { + if !file.active_column_ids.contains(&field.column_id()) { + continue; + } + if let Some(arg) = ngram_args.iter().find(|arg| arg.field() == field) { + let name = BloomIndex::build_filter_ngram_name( + field.column_id(), + arg.gram_size(), + arg.bloom_size(), + ); + rename.insert(name.clone(), name.clone()); + columns.push(name); + } else { + let physical = BloomIndex::build_filter_bloom_name(file.format_version, field)?; + let normalized = BloomIndex::build_filter_bloom_name(BlockFilter::VERSION, field)?; + rename.insert(physical.clone(), normalized); + columns.push(physical); + } + } + if columns.is_empty() { + continue; + } + + let filter = file + .location + .read_block_filter(dal.clone(), settings, &columns, file.file_size) + .await?; + for (field, value) in filter + .filter_schema + .fields() + .iter() + .zip(filter.filters.into_iter()) + { + let Some(name) = rename.get(field.name()) else { + continue; + }; + merged_fields.push(TableField::new(name, TableDataType::Binary)); + merged_filters.push(value); + } + } + + Ok(Some(BlockFilter { + filter_schema: Arc::new(TableSchema::new(merged_fields)), + filters: merged_filters, + })) +} + pub struct BloomPrunerCreator { func_ctx: FunctionContext, @@ -329,6 +417,38 @@ impl BloomPrunerCreator { } } + async fn apply_files( + &self, + index_files: &[BloomIndexFileMeta], + column_stats: &StatisticsOfColumns, + ) -> Result { + let Some(filter) = read_multi_file_block_filter( + &self.dal, + &self.settings, + &self.index_fields, + &self.ngram_args, + index_files, + ) + .await? + else { + return Ok(true); + }; + Ok(BloomIndex::from_filter_block( + self.func_ctx.clone(), + filter.filter_schema, + filter.filters, + BlockFilter::VERSION, + )? + .apply( + self.filter_expression.clone(), + &self.eq_scalar_map, + &self.like_scalar_map, + &self.ngram_args, + column_stats, + self.data_schema.clone(), + )? != FilterEvalResult::MustFalse) + } + async fn try_rebuild_missing_bloom_index( &self, bloom_index_location: &Location, @@ -387,11 +507,23 @@ impl BloomPruner for BloomPrunerCreator { &self, index_location: &Option, index_length: u64, + index_files: &[BloomIndexFileMeta], column_stats: &StatisticsOfColumns, column_ids: Vec, block_meta: &BlockReadInfo, ) -> bool { - if let Some(loc) = index_location { + if !index_files.is_empty() { + match self.apply_files(index_files, column_stats).await { + Ok(value) => value, + Err(e) => { + warn!( + "failed to apply multi-file bloom pruner, returning true. {}", + e + ); + true + } + } + } else if let Some(loc) = index_location { // load filter, and try pruning according to filter expression match self .apply(loc, index_length, column_stats, column_ids, block_meta) diff --git a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs index 055d92866f997..f4b65db5a0f48 100644 --- a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs +++ b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs @@ -537,6 +537,7 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, + vec![], 4, vec![], Some(column_stats), @@ -727,6 +728,7 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, + vec![], 1000, vec![], Some(column_stats), diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index 0fd7dda5332cc..945a792030159 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -223,6 +223,7 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { .should_keep( &bloom_filter_index_location, bloom_filter_index_size, + &[], &columns_stat, column_ids.clone(), &block_read_info, @@ -286,6 +287,7 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { location_path, bloom_filter_index_location, bloom_filter_index_size, + vec![], row_count, column_groups, Some(columns_stat), diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index f75d4094fda72..104e86307b559 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use databend_common_catalog::table::Table; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockEntry; use databend_common_expression::Column; @@ -28,6 +29,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::VariantType; use databend_common_expression::types::string::StringColumnBuilder; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; @@ -79,6 +81,8 @@ impl TableMetaFunc for FuseBlock { "virtual_column_size", TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))), ), + TableField::new("column_groups", TableDataType::Variant), + TableField::new("bloom_index_files", TableDataType::Variant), ]) } @@ -104,6 +108,8 @@ impl TableMetaFunc for FuseBlock { let mut vector_index_size = Vec::with_capacity(len); let mut spatial_index_size = Vec::with_capacity(len); let mut virtual_column_size = Vec::with_capacity(len); + let mut column_groups = Vec::with_capacity(len); + let mut bloom_index_files = Vec::with_capacity(len); let segments_io = SegmentsIO::create(ctx.clone(), tbl.operator.clone(), tbl.schema()); @@ -140,6 +146,49 @@ impl TableMetaFunc for FuseBlock { .as_ref() .map(|m| m.virtual_column_size), ); + column_groups.push( + jsonb::parse_value( + serde_json::to_string( + &block + .column_groups + .iter() + .map(|group| { + serde_json::json!({ + "active_column_ids": group.active_column_ids, + "location": group.location.0, + "format_version": group.format_version, + "file_size": group.file_size, + "uncompressed_size": group.uncompressed_size, + }) + }) + .collect::>(), + )? + .as_bytes(), + ) + .map_err(|error| ErrorCode::Internal(error.to_string()))? + .to_vec(), + ); + bloom_index_files.push( + jsonb::parse_value( + serde_json::to_string( + &block + .bloom_index_files + .iter() + .map(|file| { + serde_json::json!({ + "active_column_ids": file.active_column_ids, + "location": file.location.0, + "format_version": file.format_version, + "file_size": file.file_size, + }) + }) + .collect::>(), + )? + .as_bytes(), + ) + .map_err(|error| ErrorCode::Internal(error.to_string()))? + .to_vec(), + ); num_rows += 1; if num_rows >= limit { @@ -164,6 +213,8 @@ impl TableMetaFunc for FuseBlock { UInt64Type::from_opt_data(vector_index_size).into(), UInt64Type::from_opt_data(spatial_index_size).into(), UInt64Type::from_opt_data(virtual_column_size).into(), + VariantType::from_data(column_groups).into(), + VariantType::from_data(bloom_index_files).into(), ], num_rows, )) From 9ade1e04fc775b205623e008210239ec41fd1002 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:53:26 +0800 Subject: [PATCH 04/30] fix(storage): harden partial update metadata handling --- .../common/table_option_validation.rs | 1 - .../physical_column_mutation.rs | 4 +- .../fuse/operations/mutation/update.rs | 128 +++++++++++++++++- src/query/storages/fuse/src/fuse_part.rs | 48 +++++++ .../io/read/block/block_reader_deserialize.rs | 34 +---- .../fuse/src/io/write/block_writer.rs | 96 ++++++------- .../src/io/write/virtual_column_builder.rs | 6 - .../transform_mutation_aggregator.rs | 44 +++++- .../fuse/src/operations/read_partitions.rs | 49 +------ .../mutator/replace_into_operation_agg.rs | 47 +------ .../fuse/src/table_functions/fuse_encoding.rs | 51 ++++--- .../fuse/src/table_functions/fuse_page.rs | 85 ++++++------ .../09_0054_partial_update.test | 70 ++++++++++ 13 files changed, 431 insertions(+), 232 deletions(-) create mode 100644 tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index b8d4db9e1df3c..eae2eff1728ea 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -94,7 +94,6 @@ pub static CREATE_FUSE_OPTIONS: LazyLock> = LazyLock::new( r.insert(FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE); r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN); - r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD); r.insert(OPT_KEY_BLOOM_INDEX_COLUMNS); diff --git a/src/query/service/src/physical_plans/physical_column_mutation.rs b/src/query/service/src/physical_plans/physical_column_mutation.rs index 9e0abfa496185..5a0a2756a0026 100644 --- a/src/query/service/src/physical_plans/physical_column_mutation.rs +++ b/src/query/service/src/physical_plans/physical_column_mutation.rs @@ -20,6 +20,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; +use databend_common_expression::FieldIndex; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; @@ -52,7 +53,8 @@ pub struct ColumnMutation { pub has_filter_column: bool, pub table_meta_timestamps: TableMetaTimestamps, pub udf_col_num: usize, - pub partial_update_fields: Option>, + /// Field indices in the physical table schema after virtual computed fields are removed. + pub partial_update_fields: Option>, } #[typetag::serde] diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index e7bbea0952370..c72fdc6848b00 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -18,7 +18,9 @@ use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; +use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::decode_column_hll; async fn latest_block_meta(fixture: &TestFixture) -> anyhow::Result> { let table = fixture.latest_default_table().await?; @@ -40,6 +42,33 @@ async fn latest_block_meta(fixture: &TestFixture) -> anyhow::Result anyhow::Result { + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let snapshot = fuse_table.read_table_snapshot().await?.unwrap(); + let segment_reader = + MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()); + let (segment_location, segment_version) = &snapshot.segments[0]; + let segment = segment_reader + .read(&LoadParams { + location: segment_location.clone(), + len_hint: None, + ver: *segment_version, + put_cache: false, + }) + .await?; + let (stats_location, stats_version) = segment.summary.additional_stats_loc().unwrap(); + let stats = MetaReaders::segment_stats_reader(fuse_table.get_operator()) + .read(&LoadParams { + location: stats_location, + len_hint: None, + ver: stats_version, + put_cache: false, + }) + .await?; + Ok(decode_column_hll(&stats.block_hlls[0])?.unwrap()) +} + #[tokio::test(flavor = "multi_thread")] async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; @@ -50,7 +79,8 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { fixture .execute_command(&format!( "create table {db}.{table_name} (id int, value int) engine=fuse \ - bloom_index_columns='id,value' enable_partial_update=true change_tracking=true" + bloom_index_columns='id,value' approx_distinct_columns='id,value' \ + enable_partial_update=true change_tracking=true" )) .await?; fixture @@ -64,6 +94,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { let id_column_id = schema.field(0).column_id(); let value_column_id = schema.field(1).column_id(); let origin = latest_block_meta(&fixture).await?; + let origin_hll = latest_block_hll(&fixture).await?; fixture .execute_command("set enable_partial_update = 1") @@ -105,6 +136,36 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { .map(|file| file.file_size) .sum::() ); + let updated_hll = latest_block_hll(&fixture).await?; + assert_eq!( + updated_hll.get(&id_column_id), + origin_hll.get(&id_column_id) + ); + assert_ne!( + updated_hll.get(&value_column_id), + origin_hll.get(&value_column_id) + ); + + let rows = fixture + .execute_query(&format!( + "select count(distinct column_name) from fuse_page('{db}', '{table_name}') \ + where column_name in ('id', 'value')" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + let rows = fixture + .execute_query(&format!( + "select count(distinct column_name) from fuse_encoding('{db}', '{table_name}') \ + where column_name in ('id', 'value')" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + fixture + .execute_command(&format!( + "alter table {db}.{table_name} set options(bloom_index_columns='id')" + )) + .await?; fixture .execute_command(&format!( @@ -125,8 +186,15 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { .iter() .all(|group| group.location != updated.location) ); - assert_eq!(updated_again.bloom_index_files.len(), 2); assert!(updated_again.bloom_filter_index_location.is_none()); + assert_eq!(updated_again.bloom_index_files.len(), 1); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where value = 21" + )) + .await?; + assert_eq!(query_count(rows).await?, 1); let rows = fixture .execute_query(&format!( @@ -158,3 +226,59 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, payload variant) engine=fuse \ + enable_partial_update=true enable_virtual_column=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values \ + (1, parse_json('{{\"x\": 1}}')), (2, parse_json('{{\"x\": 2}}'))" + )) + .await?; + assert!( + latest_block_meta(&fixture) + .await? + .virtual_block_meta + .is_some() + ); + + fixture + .execute_command(&format!( + "alter table {db}.{table_name} set options(enable_virtual_column=false)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set payload = parse_json('{{\"x\": 3}}') where id = 1" + )) + .await?; + + assert!( + latest_block_meta(&fixture) + .await? + .virtual_block_meta + .is_none() + ); + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where payload:x::int in (2, 3)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index b995d30fb8e9b..32685469db02b 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -13,7 +13,9 @@ // limitations under the License. use std::any::Any; +use std::borrow::Cow; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; @@ -30,7 +32,9 @@ use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_storages_common_pruner::BlockMetaIndex; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BloomIndexFileMeta; +use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; @@ -43,6 +47,50 @@ pub struct FuseColumnGroupPartInfo { pub columns_meta: HashMap, } +/// Normalize a legacy single-file block and a column-group block to the same physical-file view. +pub(crate) fn normalized_column_group_files(meta: &BlockMeta) -> Cow<'_, [ColumnGroupFileMeta]> { + if !meta.column_groups.is_empty() { + return Cow::Borrowed(&meta.column_groups); + } + + let mut active_column_ids = meta.col_metas.keys().copied().collect::>(); + active_column_ids.sort_unstable(); + Cow::Owned(vec![ColumnGroupFileMeta { + active_column_ids, + location: meta.location.clone(), + format_version: meta.location.1, + file_size: meta.file_size, + uncompressed_size: meta.block_size, + leaf_column_metas: meta.col_metas.clone(), + }]) +} + +pub(crate) fn project_column_groups( + meta: &BlockMeta, + projected_column_ids: &HashSet, +) -> Vec { + normalized_column_group_files(meta) + .iter() + .filter_map(|group| { + let columns_meta = group + .active_column_ids + .iter() + .filter(|column_id| projected_column_ids.contains(column_id)) + .filter_map(|column_id| { + group + .leaf_column_metas + .get(column_id) + .map(|column_meta| (*column_id, column_meta.clone())) + }) + .collect::>(); + (!columns_meta.is_empty()).then(|| FuseColumnGroupPartInfo { + location: group.location.0.clone(), + columns_meta, + }) + }) + .collect() +} + /// Fuse table partition information. #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] pub struct FuseBlockPartInfo { diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index a5893b7d51bdc..86027702243c2 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -30,6 +30,7 @@ use crate::BlockReadResult; use crate::FuseBlockPartInfo; use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; +use crate::fuse_part::project_column_groups; use crate::io::read::block::block_reader_merge_io::DataItem; use crate::unsupported_storage_format_error; @@ -40,38 +41,7 @@ impl BlockReader { .iter() .flat_map(|node| node.leaf_column_ids.iter().copied()) .collect::>(); - if meta.column_groups.is_empty() { - return vec![FuseColumnGroupPartInfo { - location: meta.location.0.clone(), - columns_meta: meta - .col_metas - .iter() - .filter(|(column_id, _)| projected_column_ids.contains(column_id)) - .map(|(column_id, column_meta)| (*column_id, column_meta.clone())) - .collect(), - }]; - } - - meta.column_groups - .iter() - .filter_map(|group| { - let columns_meta = group - .active_column_ids - .iter() - .filter(|column_id| projected_column_ids.contains(column_id)) - .filter_map(|column_id| { - group - .leaf_column_metas - .get(column_id) - .map(|column_meta| (*column_id, column_meta.clone())) - }) - .collect::>(); - (!columns_meta.is_empty()).then(|| FuseColumnGroupPartInfo { - location: group.location.0.clone(), - columns_meta, - }) - }) - .collect() + project_column_groups(meta, &projected_column_ids) } /// Deserialize column chunks data from parquet format to DataBlock. diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index d8c1c46c5dcda..656ec54959bfd 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; @@ -27,6 +26,7 @@ use databend_common_expression::BlockMetaInfo; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; +use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::local_block_meta_serde; @@ -48,7 +48,9 @@ use databend_common_metrics::storage::metrics_inc_block_write_milliseconds; use databend_common_metrics::storage::metrics_inc_block_write_nums; use databend_storages_common_blocks::SerializedParquet; use databend_storages_common_blocks::blocks_to_parquet_with_stats; +use databend_storages_common_index::BloomIndex; use databend_storages_common_index::NgramArgs; +use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BloomIndexFileMeta; @@ -165,6 +167,31 @@ pub struct BlockBuilder { } impl BlockBuilder { + fn add_hll_distinct_counts( + column_distinct_count: &mut HashMap, + column_hlls: &Option, + ) { + if let Some(hlls) = column_hlls { + for (column_id, hll) in hlls { + column_distinct_count + .entry(*column_id) + .or_insert_with(|| hll.count()); + } + } + } + + fn finalize_column_hlls(&self, column_hlls: Option) -> Result> { + column_hlls + .map(|hlls| { + if self.serialize_hll { + encode_column_hll(&hlls).map(BlockHLLState::Serialized) + } else { + Ok(BlockHLLState::Deserialized(hlls)) + } + }) + .transpose() + } + pub fn build(&self, data_block: DataBlock, f: F) -> Result where F: Fn(DataBlock, &ClusterStatsGenerator) -> Result<(Option, DataBlock)> { @@ -188,13 +215,7 @@ impl BlockBuilder { .unwrap_or_default(); let column_hlls = build_column_hlls(&data_block, &self.ndv_columns_map)?; - if let Some(hlls) = &column_hlls { - for (key, val) in hlls { - if let Entry::Vacant(entry) = column_distinct_count.entry(*key) { - entry.insert(val.count()); - } - } - } + Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); let mut inverted_index_states = Vec::with_capacity(self.inverted_index_builders.len()); for inverted_index_builder in &self.inverted_index_builders { @@ -296,15 +317,7 @@ impl BlockBuilder { create_on: Some(Utc::now()), }; - let column_hlls = column_hlls - .map(|hlls| { - if self.serialize_hll { - encode_column_hll(&hlls).map(BlockHLLState::Serialized) - } else { - Ok(BlockHLLState::Deserialized(hlls)) - } - }) - .transpose()?; + let column_hlls = self.finalize_column_hlls(column_hlls)?; let serialized = BlockSerialization { block_raw_data: buffer, block_meta, @@ -349,10 +362,6 @@ impl BlockBuilder { )); } - let updated_fields = updated_field_indices - .iter() - .copied() - .collect::>(); let updated_schema = Arc::new(self.source_schema.project(&updated_field_indices)); if data_block.num_columns() != updated_schema.fields().len() { return Err(ErrorCode::Internal( @@ -381,6 +390,12 @@ impl BlockBuilder { .values() .map(|field| field.column_id()) .collect::>(); + let invalidated_bloom_column_ids = updated_field_indices + .iter() + .map(|index| self.source_schema.field(*index)) + .filter(|field| BloomIndex::supported_type(field.data_type())) + .map(TableField::column_id) + .collect::>(); let rebuild_bloom_index = !updated_bloom_columns_map.is_empty(); let bloom_index_state = if rebuild_bloom_index { let location = self.meta_locations.block_bloom_index_location(&block_id); @@ -411,18 +426,15 @@ impl BlockBuilder { }) .collect::>(); let column_hlls = build_column_hlls(&updated_block, &updated_ndv_columns_map)?; - if let Some(hlls) = &column_hlls { - for (column_id, hll) in hlls { - if let Entry::Vacant(entry) = column_distinct_count.entry(*column_id) { - entry.insert(hll.count()); - } - } - } - - let rebuild_virtual_columns = self - .virtual_column_builder - .as_ref() - .is_some_and(|builder| builder.is_affected(&updated_fields)); + Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); + + let invalidate_virtual_columns = updated_field_indices.iter().any(|index| { + self.source_schema + .field(*index) + .data_type() + .remove_nullable() + == TableDataType::Variant + }); let virtual_column_state = None; let updated_col_stats = gen_columns_statistics( @@ -492,7 +504,7 @@ impl BlockBuilder { block_meta.col_stats.extend(updated_col_stats); block_meta.create_on = Some(Utc::now()); - if rebuild_bloom_index { + if !invalidated_bloom_column_ids.is_empty() { let mut bloom_index_files = if origin.bloom_index_files.is_empty() { origin .bloom_filter_index_location @@ -502,7 +514,7 @@ impl BlockBuilder { .bloom_columns_map .values() .map(|field| field.column_id()) - .filter(|column_id| !updated_bloom_column_ids.contains(column_id)) + .filter(|column_id| !invalidated_bloom_column_ids.contains(column_id)) .collect(), location: location.clone(), format_version: location.1, @@ -515,7 +527,7 @@ impl BlockBuilder { }; for file in &mut bloom_index_files { file.active_column_ids - .retain(|column_id| !updated_bloom_column_ids.contains(column_id)); + .retain(|column_id| !invalidated_bloom_column_ids.contains(column_id)); } bloom_index_files.retain(|file| !file.active_column_ids.is_empty()); if let Some(state) = &bloom_index_state { @@ -535,19 +547,11 @@ impl BlockBuilder { block_meta.bloom_index_files = bloom_index_files; block_meta.ngram_filter_index_size = None; } - if rebuild_virtual_columns { + if invalidate_virtual_columns { block_meta.virtual_block_meta = None; } - let column_hlls = column_hlls - .map(|hlls| { - if self.serialize_hll { - encode_column_hll(&hlls).map(BlockHLLState::Serialized) - } else { - Ok(BlockHLLState::Deserialized(hlls)) - } - }) - .transpose()?; + let column_hlls = self.finalize_column_hlls(column_hlls)?; Ok(BlockSerialization { block_raw_data: buffer, diff --git a/src/query/storages/fuse/src/io/write/virtual_column_builder.rs b/src/query/storages/fuse/src/io/write/virtual_column_builder.rs index 2cfb46f61cf14..2eeddde0c5550 100644 --- a/src/query/storages/fuse/src/io/write/virtual_column_builder.rs +++ b/src/query/storages/fuse/src/io/write/virtual_column_builder.rs @@ -142,12 +142,6 @@ impl VirtualColumnBuilder { }) } - pub(crate) fn is_affected(&self, updated_fields: &HashSet) -> bool { - self.variant_offsets - .iter() - .any(|offset| updated_fields.contains(offset)) - } - pub fn add_block(&mut self, block: &DataBlock) -> Result<()> { let num_rows = block.num_rows(); diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs index ed38b7de229df..a6cd97fe33279 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs @@ -25,6 +25,7 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::BlockThresholds; +use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::TableSchemaRef; @@ -47,6 +48,8 @@ use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::VirtualBlockMeta; +use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::table::ClusterType; use itertools::Itertools; @@ -539,8 +542,19 @@ impl TableMutationAggregator { }) .collect::>(); - for (idx, new_meta) in segment_mutation.replaced_blocks { - block_editor.insert(idx, new_meta); + for (idx, (new_meta, new_hll)) in segment_mutation.replaced_blocks { + let new_hll = if let Some(updated_column_ids) = new_meta + .column_groups + .last() + .map(|group| group.active_column_ids.as_slice()) + { + let previous_hll = + block_editor.get(&idx).and_then(|(_, hll)| hll.as_ref()); + replace_partial_block_hll(previous_hll, new_hll, updated_column_ids)? + } else { + new_hll + }; + block_editor.insert(idx, (new_meta, new_hll)); } for idx in segment_mutation.deleted_blocks { block_editor.remove(&idx); @@ -953,3 +967,29 @@ fn generate_segment_stats(hlls: Vec>) -> Result, + replacement: Option, + updated_column_ids: &[ColumnId], +) -> Result> { + let mut merged = previous + .map(decode_column_hll) + .transpose()? + .flatten() + .unwrap_or_default(); + for column_id in updated_column_ids { + merged.remove(column_id); + } + if let Some(replacement) = replacement { + if let Some(replacement) = decode_column_hll(&replacement)? { + merged.extend(replacement); + } + } + + if merged.is_empty() { + Ok(None) + } else { + Ok(Some(encode_column_hll(&merged)?)) + } +} diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index cccd65f93f733..7e4f5a7cd4927 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -88,6 +88,8 @@ use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; use crate::fuse_part::FuseColumnGroupPartInfo; +use crate::fuse_part::normalized_column_group_files; +use crate::fuse_part::project_column_groups; use crate::io::BloomIndexRebuilder; use crate::pruning::BlockPruner; use crate::pruning::FusePruner; @@ -1374,40 +1376,7 @@ impl FuseTable { meta: &BlockMeta, projected_column_ids: &HashSet, ) -> Vec { - if meta.column_groups.is_empty() { - let columns_meta = projected_column_ids - .iter() - .filter_map(|column_id| { - meta.col_metas - .get(column_id) - .map(|column_meta| (*column_id, column_meta.clone())) - }) - .collect(); - return vec![FuseColumnGroupPartInfo { - location: meta.location.0.clone(), - columns_meta, - }]; - } - - let mut column_groups = Vec::with_capacity(meta.column_groups.len()); - for group in &meta.column_groups { - let mut group_columns_meta = HashMap::new(); - for column_id in &group.active_column_ids { - if !projected_column_ids.contains(column_id) { - continue; - } - if let Some(column_meta) = group.leaf_column_metas.get(column_id) { - group_columns_meta.insert(*column_id, column_meta.clone()); - } - } - if !group_columns_meta.is_empty() { - column_groups.push(FuseColumnGroupPartInfo { - location: group.location.0.clone(), - columns_meta: group_columns_meta, - }); - } - } - column_groups + project_column_groups(meta, projected_column_ids) } pub fn all_columns_part( @@ -1419,14 +1388,10 @@ impl FuseTable { let mut columns_stats = HashMap::with_capacity(meta.col_stats.len()); let mut spatial_stats = HashMap::new(); - let mut projected_column_ids = if meta.column_groups.is_empty() { - meta.col_metas.keys().copied().collect::>() - } else { - meta.column_groups - .iter() - .flat_map(|group| group.active_column_ids.iter().copied()) - .collect() - }; + let mut projected_column_ids = normalized_column_group_files(meta) + .iter() + .flat_map(|group| group.active_column_ids.iter().copied()) + .collect::>(); if let Some(schema) = schema { projected_column_ids.retain(|column_id| !schema.is_column_deleted(*column_id)); } diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 020f62d4a097a..9d76b8b5e9ccd 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -27,7 +27,6 @@ use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; -use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; use databend_common_expression::FromData; use databend_common_expression::Scalar; @@ -55,7 +54,6 @@ use log::warn; use opendal::Operator; use tokio::sync::Semaphore; -use crate::FuseStorageFormat; use crate::FuseTable; use crate::io::BlockBuilder; use crate::io::BlockReader; @@ -512,8 +510,13 @@ impl AggregationContext { metrics_inc_replace_row_number_totally_loaded(block_meta.row_count); // read the remaining columns - let remain_columns_data = - self.read_block(remain_columns_reader, block_meta).await?; + let remain_columns_data = read_block( + self.write_settings.storage_format, + remain_columns_reader, + block_meta, + &self.read_settings, + ) + .await?; // remove the deleted rows let remain_columns_data_after_deletion = @@ -633,42 +636,6 @@ impl AggregationContext { } } - async fn read_block(&self, reader: &BlockReader, block_meta: &BlockMeta) -> Result { - let column_groups = reader.projected_column_groups(block_meta); - let merged_io_read_result = reader - .read_column_groups_data_by_merge_io(&self.read_settings, &column_groups, &None) - .await?; - - // deserialize block data - // cpu intensive task, send them to dedicated thread pool - let storage_format = self.write_settings.storage_format; - let block_meta_ptr = block_meta.clone(); - let reader = reader.clone(); - GlobalIORuntime::instance() - .spawn(async move { - let column_chunks = merged_io_read_result.columns_chunks()?; - match storage_format { - FuseStorageFormat::Parquet => reader.deserialize_column_groups( - block_meta_ptr.row_count as usize, - &column_groups, - column_chunks, - &block_meta_ptr.compression, - None, - ), - FuseStorageFormat::Unsupported => { - Err(crate::unsupported_storage_format_error()) - } - } - }) - .await - .map_err(|e| { - ErrorCode::Internal( - "unexpected, failed to join aggregation context read block tasks for replace into.", - ) - .add_message_back(e.to_string()) - })? - } - // return true if the block is pruned, otherwise false async fn apply_bloom_pruning( &self, diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index b44d0257d2746..29e99ee0060ea 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -48,6 +48,7 @@ use databend_common_expression::types::UInt64Type; use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use futures::stream; @@ -60,6 +61,7 @@ use parquet::basic::Type as ParquetPhysicalType; use crate::FuseStorageFormat; use crate::FuseTable; +use crate::fuse_part::normalized_column_group_files; use crate::io::SegmentsIO; use crate::io::read::meta::read_thrift_file_metadata; use crate::sessions::TableContext; @@ -306,7 +308,7 @@ impl<'a> FuseEncodingImpl<'a> { storage_format: FuseStorageFormat, table_name: Arc, fields: Arc>, - column_id_to_index: Arc>, + column_metas: Arc>, column_name_filter: Arc>, ) -> Result> { let file_meta = read_thrift_file_metadata(operator, &location, Some(file_size)).await?; @@ -329,11 +331,14 @@ impl<'a> FuseEncodingImpl<'a> { continue; } let column_id = field.column_id; - let Some(column_idx) = column_id_to_index.get(&column_id) else { + let Some(column_meta) = column_metas.get(&column_id) else { continue; }; - let Some(column_chunk) = columns.get(*column_idx) else { - // Missing column caused by schema evolutions + let column_range = column_meta.offset_length(); + let Some(column_chunk) = columns + .iter() + .find(|column| column.byte_range() == column_range) + else { continue; }; let compressed_size = u64::try_from(column_chunk.compressed_size()).map_err(|_| { @@ -381,14 +386,6 @@ impl<'a> FuseEncodingImpl<'a> { ) -> Result<()> { let schema = table.schema(); let fields = Arc::new(schema.fields().clone()); - let column_id_to_index: Arc> = Arc::new( - schema - .to_leaf_column_ids() - .into_iter() - .enumerate() - .map(|(idx, id)| (id, idx)) - .collect(), - ); let column_name_filter = Arc::new(self.column_name_filter.clone()); let table_name = Arc::new(table.name().to_string()); let storage_format = table.storage_format; @@ -401,28 +398,42 @@ impl<'a> FuseEncodingImpl<'a> { let segments = segments_io .read_segments::(chunk, false) .await?; - let mut block_locations = Vec::new(); + let mut block_groups = Vec::new(); for segment in segments { let segment = segment?; for block in segment.blocks.iter() { - block_locations.push((block.location.0.clone(), block.file_size)); + for group in normalized_column_group_files(block).iter() { + let column_metas = group + .active_column_ids + .iter() + .filter_map(|column_id| { + group + .leaf_column_metas + .get(column_id) + .map(|meta| (*column_id, meta.clone())) + }) + .collect(); + block_groups.push(( + group.location.0.clone(), + group.file_size, + Arc::new(column_metas), + )); + } } } - if block_locations.is_empty() { + if block_groups.is_empty() { continue; } let table_operator = table.operator.clone(); let fields_arc = fields.clone(); - let column_id_to_index_arc = column_id_to_index.clone(); let table_name_arc = table_name.clone(); let column_name_filter_arc = column_name_filter.clone(); - let mut block_stream = stream::iter(block_locations.into_iter().map( - move |(location, file_size)| { + let mut block_stream = stream::iter(block_groups.into_iter().map( + move |(location, file_size, column_metas)| { let operator = table_operator.clone(); let fields = fields_arc.clone(); - let column_id_to_index = column_id_to_index_arc.clone(); let table_name = table_name_arc.clone(); let column_name_filter = column_name_filter_arc.clone(); async move { @@ -433,7 +444,7 @@ impl<'a> FuseEncodingImpl<'a> { storage_format, table_name, fields, - column_id_to_index, + column_metas, column_name_filter, ) .await diff --git a/src/query/storages/fuse/src/table_functions/fuse_page.rs b/src/query/storages/fuse/src/table_functions/fuse_page.rs index f98117460e3dd..726f326967015 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_page.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_page.rs @@ -43,6 +43,7 @@ use thrift::protocol::TCompactInputProtocol; use crate::FuseStorageFormat; use crate::FuseTable; +use crate::fuse_part::normalized_column_group_files; use crate::io::SegmentsIO; use crate::sessions::TableContext; use crate::table_functions::TableMetaFuncTemplate; @@ -144,48 +145,52 @@ impl TableMetaFunc for FusePage { let segment = segment?; for block in segment.blocks.iter() { let block = block.as_ref(); - for field in schema.fields().iter() { - if field.is_nested() { - continue; - } - let column_id = field.column_id; - let Some(column_meta) = block.col_metas.get(&column_id) else { - continue; - }; - let Some(parquet_meta) = column_meta.as_parquet() else { - continue; - }; + for group in normalized_column_group_files(block).iter() { + for field in schema.fields().iter() { + if field.is_nested() + || !group.active_column_ids.contains(&field.column_id) + { + continue; + } + let Some(column_meta) = group.leaf_column_metas.get(&field.column_id) + else { + continue; + }; + let Some(parquet_meta) = column_meta.as_parquet() else { + continue; + }; - let column_bytes = read_parquet_column_chunk( - &tbl.operator, - &block.location.0, - parquet_meta.offset, - parquet_meta.len, - ) - .await?; - for (page_ordinal_in_column, page) in parse_page_headers( - field.name(), - parquet_meta.offset, - column_bytes.as_slice(), - )? - .into_iter() - .enumerate() - { - block_location.put_and_commit(&block.location.0); - column_name.put_and_commit(page.column_name); - page_type.put_and_commit(page.page_type); - encoding.put_and_commit(page.encoding); - page_ordinal.push(page_ordinal_in_column as u64); - num_values.push(page.num_values); - num_rows.push(page.num_rows); - compressed_size.push(page.compressed_size); - uncompressed_size.push(page.uncompressed_size); - header_size.push(page.header_size); - page_offset.push(page.page_offset); + let column_bytes = read_parquet_column_chunk( + &tbl.operator, + &group.location.0, + parquet_meta.offset, + parquet_meta.len, + ) + .await?; + for (page_ordinal_in_column, page) in parse_page_headers( + field.name(), + parquet_meta.offset, + column_bytes.as_slice(), + )? + .into_iter() + .enumerate() + { + block_location.put_and_commit(&group.location.0); + column_name.put_and_commit(page.column_name); + page_type.put_and_commit(page.page_type); + encoding.put_and_commit(page.encoding); + page_ordinal.push(page_ordinal_in_column as u64); + num_values.push(page.num_values); + num_rows.push(page.num_rows); + compressed_size.push(page.compressed_size); + uncompressed_size.push(page.uncompressed_size); + header_size.push(page.header_size); + page_offset.push(page.page_offset); - rows += 1; - if rows >= limit { - break 'FOR; + rows += 1; + if rows >= limit { + break 'FOR; + } } } } diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test new file mode 100644 index 0000000000000..6651608a753ca --- /dev/null +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -0,0 +1,70 @@ +statement ok +drop table if exists t_partial_update + +statement ok +set enable_partial_update = 0 + +statement ok +create table t_partial_update(id int, value int) + bloom_index_columns = 'id,value' + enable_partial_update = true + +statement ok +insert into t_partial_update values (1, 10), (2, 20) + +# The table option alone is not enough to enable partial updates. +statement ok +update t_partial_update set value = value + 1 where id = 1 + +query B +select column_groups = parse_json('[]') from fuse_block('default', 't_partial_update') +---- +1 + +statement ok +set enable_partial_update = 1 + +statement ok +update t_partial_update set value = value + 1 where id = 2 + +query II +select * from t_partial_update order by id +---- +1 11 +2 21 + +query BB +select column_groups != parse_json('[]'), bloom_index_files != parse_json('[]') +from fuse_block('default', 't_partial_update') +---- +1 1 + +query I +select count(distinct column_name) +from fuse_page('default', 't_partial_update') +where column_name in ('id', 'value') +---- +2 + +query I +select count(distinct column_name) +from fuse_encoding('default', 't_partial_update') +where column_name in ('id', 'value') +---- +2 + +# Disabling the setting forces the next rewrite back to the single-file layout. +statement ok +set enable_partial_update = 0 + +statement ok +update t_partial_update set value = value + 1 where id = 1 + +query BB +select column_groups = parse_json('[]'), bloom_index_files = parse_json('[]') +from fuse_block('default', 't_partial_update') +---- +1 1 + +statement ok +drop table t_partial_update From 6f1cda075ef66eb978b7262d1b15bd46210f3cd5 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:49:00 +0800 Subject: [PATCH 05/30] fix(storage): protect partial update physical files --- .../fuse/operations/vacuum_table_v2.rs | 32 +++++- .../fuse/operations/computed_columns.rs | 48 ++++++++ .../it/storages/fuse/operations/vacuum2.rs | 108 ++++++++++++++++++ .../physical_mutation_source.rs | 55 ++++++--- .../fuse/operations/mutation/update.rs | 7 ++ .../column_oriented_segment/block_meta.rs | 33 ++++++ src/query/storages/fuse/src/fuse_part.rs | 17 +++ .../fuse/src/operations/read_partitions.rs | 12 +- .../fuse/src/table_functions/fuse_column.rs | 32 +++--- .../09_0054_partial_update.test | 7 ++ 10 files changed, 317 insertions(+), 34 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index 9c26393cea85f..19fc1c9fd79e2 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -134,8 +134,33 @@ pub async fn do_vacuum2( .read_segments::>(&protected_segments, false) .await?; let mut gc_root_blocks = HashSet::new(); + let mut gc_root_indexes = HashSet::new(); for segment in segments { - gc_root_blocks.extend(segment?.block_metas()?.iter().map(|b| b.location.0.clone())); + for block in segment?.block_metas()? { + if block.column_groups.is_empty() { + gc_root_blocks.insert(block.location.0.clone()); + } else { + gc_root_blocks.extend( + block + .column_groups + .iter() + .map(|group| group.location.0.clone()), + ); + } + + if block.bloom_index_files.is_empty() { + if let Some(location) = &block.bloom_filter_index_location { + gc_root_indexes.insert(location.0.clone()); + } + } else { + gc_root_indexes.extend( + block + .bloom_index_files + .iter() + .map(|file| file.location.0.clone()), + ); + } + } } ctx.set_status_info(&format!( "Read segments for table {}, elapsed: {:?}, total protected blocks: {}", @@ -203,6 +228,7 @@ pub async fn do_vacuum2( &ctx, table_info.desc.as_str(), &blocks_to_gc, + &gc_root_indexes, &table_agg_index_ids, inverted_indexes, &mut files_to_gc, @@ -262,6 +288,7 @@ async fn purge_block_chunks( ctx: &Arc, table_desc: &str, blocks_to_gc: &[String], + protected_index_paths: &HashSet, table_agg_index_ids: &[u64], inverted_indexes: &BTreeMap, files_to_gc: &mut Vec, @@ -280,8 +307,9 @@ async fn purge_block_chunks( return Err(err.with_context("failed to vacuum block chunk")); } - let indexes_to_gc = + let mut indexes_to_gc = collect_block_index_locations(block_chunk, table_agg_index_ids, inverted_indexes); + indexes_to_gc.retain(|path| !protected_index_paths.contains(path)); ctx.set_status_info(&format!( "Collected indexes_to_gc for table {}, elapsed: {:?}, block chunk: {}/{}, blocks in chunk: {}, indexes_to_gc: {:?}", table_desc, diff --git a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs index 1733ecfc3acde..e206655da5a6d 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs @@ -153,3 +153,51 @@ async fn test_computed_column() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_with_virtual_column_and_change_tracking() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} \ + (id int, value int, virtual_value bigint as (value + 1) virtual) engine=fuse \ + enable_partial_update=true change_tracking=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name}(id, value) values (1, 10), (2, 20)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11 and virtual_value = 12) \ + or (id = 2 and value = 20 and virtual_value = 21)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + let rows = fixture + .execute_query(&format!( + "select count(*) from fuse_block('{db}', '{table_name}') \ + where column_groups != parse_json('[]')" + )) + .await?; + assert_eq!(query_count(rows).await?, 1); + + Ok(()) +} diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs index 971a7feaf9f88..4af37e054ff54 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs @@ -13,14 +13,39 @@ // limitations under the License. use std::path::Path; +use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; use databend_enterprise_query::test_kits::context::EESetup; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContextTableAccess; use databend_query::test_kits::TestFixture; +use databend_storages_common_cache::LoadParams; use databend_storages_common_io::dedup_file_locations; +use databend_storages_common_table_meta::meta::BlockMeta; + +async fn latest_default_block_meta(fixture: &TestFixture) -> anyhow::Result> { + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let snapshot = fuse_table.read_table_snapshot().await?.unwrap(); + let segment_reader = + MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()); + let (segment_location, segment_version) = &snapshot.segments[0]; + let segment = segment_reader + .read(&LoadParams { + location: segment_location.clone(), + len_hint: None, + ver: *segment_version, + put_cache: false, + }) + .await?; + let blocks = segment.block_metas()?; + assert_eq!(blocks.len(), 1); + Ok(blocks[0].clone()) +} // TODO investigate this // NOTE: SHOULD specify flavor = "multi_thread", otherwise query execution might be hanged @@ -118,6 +143,89 @@ async fn test_vacuum2_all() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum2_preserves_active_partial_update_files() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, b int) engine=fuse \ + bloom_index_columns='id,a,b' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, 20), (2, 30, 40)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + let obsolete_group = latest_default_block_meta(&fixture) + .await? + .location + .0 + .clone(); + + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set b = b + 1 where id = 1" + )) + .await?; + + let current = latest_default_block_meta(&fixture).await?; + assert!( + current + .column_groups + .iter() + .all(|group| group.location.0 != obsolete_group) + ); + + fixture + .execute_command(&format!("call system$fuse_vacuum2('{db}', '{table_name}')")) + .await?; + + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + for group in ¤t.column_groups { + operator.stat(&group.location.0).await?; + } + for file in ¤t.bloom_index_files { + operator.stat(&file.location.0).await?; + } + assert!(operator.stat(&obsolete_group).await.is_err()); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and a = 12 and b = 21) \ + or (id = 2 and a = 30 and b = 40)" + )) + .await?; + assert_eq!(databend_query::test_kits::query_count(rows).await?, 2); + + Ok(()) +} + /// Verifies that dedup_file_locations correctly removes duplicates and reports samples. #[test] fn test_dedup_file_locations() { diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index 5e637d0bcc23d..a5d6a8dcbd947 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -28,6 +28,7 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::FieldIndex; use databend_common_expression::FunctionContext; use databend_common_expression::type_check::check_function; use databend_common_expression::types::DataType; @@ -73,6 +74,10 @@ pub struct MutationSource { pub output_schema: DataSchemaRef, pub input_type: MutationType, pub read_partition_columns: ColumnSet, + /// Physical schema projection used by partial updates. Metadata Symbols remain in + /// `read_partition_columns` for expression/output naming and must not be used as field indices. + #[serde(default)] + pub partial_update_projection: Option>, pub partial_update: bool, pub truncate_table: bool, @@ -118,6 +123,7 @@ impl IPhysicalPlan for MutationSource { output_schema: self.output_schema.clone(), input_type: self.input_type.clone(), read_partition_columns: self.read_partition_columns.clone(), + partial_update_projection: self.partial_update_projection.clone(), partial_update: self.partial_update, truncate_table: self.truncate_table, partitions: self.partitions.clone(), @@ -154,11 +160,31 @@ impl IPhysicalPlan for MutationSource { ); } - let read_partition_columns: Vec = self - .read_partition_columns - .iter() - .map(|idx| idx.as_field_index()) - .collect(); + let physical_schema = table.schema_with_stream().remove_virtual_computed_fields(); + let read_partition_field_indices: Vec = + if let Some(projection) = &self.partial_update_projection { + let storage_schema = table.schema_with_stream(); + projection + .iter() + .map(|field_index| { + let column_id = physical_schema.field(*field_index).column_id(); + storage_schema + .fields() + .iter() + .position(|field| field.column_id() == column_id) + .expect("physical field must exist in storage schema") + }) + .collect() + } else { + self.read_partition_columns + .iter() + .map(|idx| idx.as_field_index()) + .collect() + }; + let stream_projection = self + .partial_update_projection + .as_deref() + .unwrap_or(&read_partition_field_indices); let is_lazy = self.partitions.partitions_type() == PartInfoType::LazyLevel && is_delete; if is_lazy { @@ -166,7 +192,7 @@ impl IPhysicalPlan for MutationSource { let table_clone = table.clone(); let ctx_clone = builder.ctx.clone(); let filters_clone = self.filters.clone(); - let projection = Projection::Columns(read_partition_columns.clone()); + let projection = Projection::Columns(read_partition_field_indices.clone()); let mut segment_locations = Vec::with_capacity(self.partitions.partitions.len()); for part in &self.partitions.partitions { // Safe to downcast because we know the partition is lazy @@ -205,17 +231,12 @@ impl IPhysicalPlan for MutationSource { } else { MutationAction::Update }; - let col_indices = self - .read_partition_columns - .iter() - .map(|idx| idx.as_field_index()) - .collect(); let update_mutation_with_filter = self.input_type == MutationType::Update && filter.is_some(); table.add_mutation_source( builder.ctx.clone(), filter, - col_indices, + read_partition_field_indices.clone(), &mut builder.main_pipeline, mutation_action, self.partial_update, @@ -226,7 +247,7 @@ impl IPhysicalPlan for MutationSource { StreamContext::try_create_projected( builder.ctx.get_function_context()?, table.schema_with_stream(), - &read_partition_columns, + stream_projection, table.get_table_info().ident.seq, is_delete, update_mutation_with_filter, @@ -302,6 +323,13 @@ impl PhysicalPlanBuilder { } fields.sort_by_key(|(_, _, id)| *id); + let partial_update_projection = partial_update.then(|| { + fields + .iter() + .map(|(_, _, field_index)| *field_index) + .collect::>() + }); + let read_partition_columns = if partial_update { fields.iter().map(|(_, index, _)| *index).collect() } else { @@ -336,6 +364,7 @@ impl PhysicalPlanBuilder { has_hidden_secure_filters: !mutation_source.secure_predicates.is_empty(), input_type: mutation_source.mutation_type.clone(), read_partition_columns, + partial_update_projection, partial_update, truncate_table, meta: PhysicalPlanMeta::new("MutationSource"), diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index c72fdc6848b00..257de74d22a7e 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -153,6 +153,13 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { )) .await?; assert_eq!(query_count(rows).await?, 2); + let rows = fixture + .execute_query(&format!( + "select count(distinct block_location) from fuse_column('{db}', '{table_name}') \ + where column_name in ('id', 'value')" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); let rows = fixture .execute_query(&format!( "select count(distinct column_name) from fuse_encoding('{db}', '{table_name}') \ diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs index b8e3d94d4f76c..dcf7b5cc85818 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs @@ -29,6 +29,12 @@ pub trait AbstractBlockMeta: Send + Sync + 'static + Sized { fn row_count(&self) -> u64; fn location_path(&self) -> String; fn col_metas(&self, col_ids: &HashSet) -> HashMap; + fn col_metas_by_location( + &self, + col_ids: &HashSet, + ) -> Vec<(String, HashMap)> { + vec![(self.location_path(), self.col_metas(col_ids))] + } fn virtual_block_meta(&self) -> Option; } @@ -54,6 +60,33 @@ impl AbstractBlockMeta for BlockMeta { col_metas } + fn col_metas_by_location( + &self, + col_ids: &HashSet, + ) -> Vec<(String, HashMap)> { + if self.column_groups.is_empty() { + return vec![(self.location_path(), self.col_metas(col_ids))]; + } + + self.column_groups + .iter() + .filter_map(|group| { + let col_metas = group + .active_column_ids + .iter() + .filter(|column_id| col_ids.contains(column_id)) + .filter_map(|column_id| { + group + .leaf_column_metas + .get(column_id) + .map(|meta| (*column_id, meta.clone())) + }) + .collect::>(); + (!col_metas.is_empty()).then(|| (group.location.0.clone(), col_metas)) + }) + .collect() + } + fn location_path(&self) -> String { self.location.0.to_string() } diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 32685469db02b..1b9003ab36a5f 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -69,6 +69,23 @@ pub(crate) fn project_column_groups( meta: &BlockMeta, projected_column_ids: &HashSet, ) -> Vec { + if meta.column_groups.is_empty() { + let columns_meta = meta + .col_metas + .iter() + .filter(|(column_id, _)| projected_column_ids.contains(column_id)) + .map(|(column_id, column_meta)| (*column_id, column_meta.clone())) + .collect::>(); + return if columns_meta.is_empty() { + vec![] + } else { + vec![FuseColumnGroupPartInfo { + location: meta.location.0.clone(), + columns_meta, + }] + }; + } + normalized_column_group_files(meta) .iter() .filter_map(|group| { diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 7e4f5a7cd4927..8ba6a5979de4b 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -1388,10 +1388,14 @@ impl FuseTable { let mut columns_stats = HashMap::with_capacity(meta.col_stats.len()); let mut spatial_stats = HashMap::new(); - let mut projected_column_ids = normalized_column_group_files(meta) - .iter() - .flat_map(|group| group.active_column_ids.iter().copied()) - .collect::>(); + let mut projected_column_ids = if meta.column_groups.is_empty() { + meta.col_metas.keys().copied().collect::>() + } else { + normalized_column_group_files(meta) + .iter() + .flat_map(|group| group.active_column_ids.iter().copied()) + .collect::>() + }; if let Some(schema) = schema { projected_column_ids.retain(|column_id| !schema.is_column_deleted(*column_id)); } diff --git a/src/query/storages/fuse/src/table_functions/fuse_column.rs b/src/query/storages/fuse/src/table_functions/fuse_column.rs index 7f098b858404d..a9078ab097038 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_column.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_column.rs @@ -130,27 +130,29 @@ impl FuseColumn { for segment in segments { let segment = segment?; for block in segment.block_metas()? { - for (id, column) in block.col_metas(&col_ids).iter() { - if let Some(f) = leaf_fields.iter().find(|f| f.column_id == *id) { - block_location.put_and_commit(block.location_path()); - block_size.push(block.block_size()); - file_size.push(block.file_size()); - row_count.push(column.total_rows() as u64); + for (location, col_metas) in block.col_metas_by_location(&col_ids) { + for (id, column) in &col_metas { + if let Some(f) = leaf_fields.iter().find(|f| f.column_id == *id) { + block_location.put_and_commit(&location); + block_size.push(block.block_size()); + file_size.push(block.file_size()); + row_count.push(column.total_rows() as u64); - column_name.put_and_commit(&f.name); + column_name.put_and_commit(&f.name); - column_type.put_and_commit(f.data_type.to_string()); + column_type.put_and_commit(f.data_type.to_string()); - column_id.push(*id); + column_id.push(*id); - let (offset, length) = column.offset_length(); - block_offset.push(offset); - bytes_compressed.push(length); + let (offset, length) = column.offset_length(); + block_offset.push(offset); + bytes_compressed.push(length); - num_rows += 1; + num_rows += 1; - if num_rows >= limit { - break 'FOR; + if num_rows >= limit { + break 'FOR; + } } } } diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test index 6651608a753ca..29fa3af587e4c 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -53,6 +53,13 @@ where column_name in ('id', 'value') ---- 2 +query I +select count(distinct block_location) +from fuse_column('default', 't_partial_update') +where column_name in ('id', 'value') +---- +2 + # Disabling the setting forces the next rewrite back to the single-file layout. statement ok set enable_partial_update = 0 From 84aee80dd66ac53276e02b8418436edd5eb60f20 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:53:01 +0800 Subject: [PATCH 06/30] fix(storage): preserve partial update file invariants --- .../fuse/operations/vacuum_table_v2.rs | 33 ++-- .../it/storages/fuse/operations/vacuum2.rs | 25 +-- .../physical_mutation_source.rs | 8 +- src/query/service/src/test_kits/fuse.rs | 37 +++++ .../fuse/operations/mutation/update.rs | 147 +++++++++++++----- .../column_oriented_segment/block_meta.rs | 23 +-- .../common/table_meta/src/meta/v2/segment.rs | 104 +++++++++++++ .../table_meta/src/table/table_compression.rs | 16 ++ src/query/storages/fuse/src/fuse_part.rs | 60 +------ .../fuse/src/io/write/block_writer.rs | 36 +++-- .../processors/transform_serialize_block.rs | 104 ++++++++++++- src/query/storages/fuse/src/operations/gc.rs | 32 ++-- .../fuse/src/operations/read_partitions.rs | 19 +-- .../fuse/src/table_functions/fuse_encoding.rs | 3 +- .../fuse/src/table_functions/fuse_page.rs | 3 +- .../09_0054_partial_update.test | 30 ++++ 16 files changed, 448 insertions(+), 232 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index 19fc1c9fd79e2..fe284cc2f0ab8 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -137,29 +137,16 @@ pub async fn do_vacuum2( let mut gc_root_indexes = HashSet::new(); for segment in segments { for block in segment?.block_metas()? { - if block.column_groups.is_empty() { - gc_root_blocks.insert(block.location.0.clone()); - } else { - gc_root_blocks.extend( - block - .column_groups - .iter() - .map(|group| group.location.0.clone()), - ); - } - - if block.bloom_index_files.is_empty() { - if let Some(location) = &block.bloom_filter_index_location { - gc_root_indexes.insert(location.0.clone()); - } - } else { - gc_root_indexes.extend( - block - .bloom_index_files - .iter() - .map(|file| file.location.0.clone()), - ); - } + gc_root_blocks.extend( + block + .data_file_locations() + .map(|location| location.0.clone()), + ); + gc_root_indexes.extend( + block + .bloom_index_file_locations() + .map(|location| location.0.clone()), + ); } } ctx.set_status_info(&format!( diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs index 4af37e054ff54..40a516d723615 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs @@ -13,39 +13,16 @@ // limitations under the License. use std::path::Path; -use std::sync::Arc; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::io::MetaReaders; use databend_enterprise_query::test_kits::context::EESetup; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContextTableAccess; use databend_query::test_kits::TestFixture; -use databend_storages_common_cache::LoadParams; +use databend_query::test_kits::latest_default_block_meta; use databend_storages_common_io::dedup_file_locations; -use databend_storages_common_table_meta::meta::BlockMeta; - -async fn latest_default_block_meta(fixture: &TestFixture) -> anyhow::Result> { - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot = fuse_table.read_table_snapshot().await?.unwrap(); - let segment_reader = - MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()); - let (segment_location, segment_version) = &snapshot.segments[0]; - let segment = segment_reader - .read(&LoadParams { - location: segment_location.clone(), - len_hint: None, - ver: *segment_version, - put_cache: false, - }) - .await?; - let blocks = segment.block_metas()?; - assert_eq!(blocks.len(), 1); - Ok(blocks[0].clone()) -} // TODO investigate this // NOTE: SHOULD specify flavor = "multi_thread", otherwise query execution might be hanged diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index a5d6a8dcbd947..8dd9beca6522c 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -78,7 +78,6 @@ pub struct MutationSource { /// `read_partition_columns` for expression/output naming and must not be used as field indices. #[serde(default)] pub partial_update_projection: Option>, - pub partial_update: bool, pub truncate_table: bool, pub partitions: Partitions, @@ -124,7 +123,6 @@ impl IPhysicalPlan for MutationSource { input_type: self.input_type.clone(), read_partition_columns: self.read_partition_columns.clone(), partial_update_projection: self.partial_update_projection.clone(), - partial_update: self.partial_update, truncate_table: self.truncate_table, partitions: self.partitions.clone(), statistics: self.statistics.clone(), @@ -161,6 +159,7 @@ impl IPhysicalPlan for MutationSource { } let physical_schema = table.schema_with_stream().remove_virtual_computed_fields(); + let partial_update = self.partial_update_projection.is_some(); let read_partition_field_indices: Vec = if let Some(projection) = &self.partial_update_projection { let storage_schema = table.schema_with_stream(); @@ -239,11 +238,11 @@ impl IPhysicalPlan for MutationSource { read_partition_field_indices.clone(), &mut builder.main_pipeline, mutation_action, - self.partial_update, + partial_update, )?; if table.change_tracking_enabled() { - let stream_ctx = if self.partial_update { + let stream_ctx = if partial_update { StreamContext::try_create_projected( builder.ctx.get_function_context()?, table.schema_with_stream(), @@ -365,7 +364,6 @@ impl PhysicalPlanBuilder { input_type: mutation_source.mutation_type.clone(), read_partition_columns, partial_update_projection, - partial_update, truncate_table, meta: PhysicalPlanMeta::new("MutationSource"), partitions: mutation_info.partitions.clone(), diff --git a/src/query/service/src/test_kits/fuse.rs b/src/query/service/src/test_kits/fuse.rs index c176f705bac7e..44735d60b6305 100644 --- a/src/query/service/src/test_kits/fuse.rs +++ b/src/query/service/src/test_kits/fuse.rs @@ -19,6 +19,7 @@ use std::vec; use chrono::DateTime; use chrono::Duration; use chrono::Utc; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; @@ -32,14 +33,17 @@ use databend_common_storages_factory::Table; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_common_storages_fuse::statistics::merge_statistics; use databend_common_storages_fuse::statistics::reducers::reduce_block_metas; +use databend_storages_common_cache::LoadParams; use databend_storages_common_cache::SegmentStatistics; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; @@ -62,6 +66,39 @@ use crate::interpreters::MutationInterpreter; use crate::sessions::QueryContext; /// This file contains some helper functions for testing fuse table. +pub async fn latest_default_segment(fixture: &TestFixture) -> Result> { + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let snapshot = fuse_table + .read_table_snapshot() + .await? + .ok_or_else(|| ErrorCode::Internal("default test table has no snapshot"))?; + let (segment_location, segment_version) = snapshot + .segments + .first() + .ok_or_else(|| ErrorCode::Internal("default test table has no segment"))?; + MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()) + .read(&LoadParams { + location: segment_location.clone(), + len_hint: None, + ver: *segment_version, + put_cache: false, + }) + .await +} + +pub async fn latest_default_block_meta(fixture: &TestFixture) -> Result> { + let segment = latest_default_segment(fixture).await?; + let blocks = segment.block_metas()?; + if blocks.len() != 1 { + return Err(ErrorCode::Internal(format!( + "expected one block in default test table, got {}", + blocks.len() + ))); + } + Ok(blocks[0].clone()) +} + pub async fn generate_snapshot_with_segments( fuse_table: &FuseTable, segment_locations: Vec, diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 257de74d22a7e..4fa943654d5cc 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -12,51 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::BlockHLL; -use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::decode_column_hll; -async fn latest_block_meta(fixture: &TestFixture) -> anyhow::Result> { - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot = fuse_table.read_table_snapshot().await?.unwrap(); - let segment_reader = - MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()); - let (segment_location, segment_version) = &snapshot.segments[0]; - let segment = segment_reader - .read(&LoadParams { - location: segment_location.clone(), - len_hint: None, - ver: *segment_version, - put_cache: false, - }) - .await?; - let blocks = segment.block_metas()?; - assert_eq!(blocks.len(), 1); - Ok(blocks[0].clone()) -} - async fn latest_block_hll(fixture: &TestFixture) -> anyhow::Result { let table = fixture.latest_default_table().await?; let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot = fuse_table.read_table_snapshot().await?.unwrap(); - let segment_reader = - MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()); - let (segment_location, segment_version) = &snapshot.segments[0]; - let segment = segment_reader - .read(&LoadParams { - location: segment_location.clone(), - len_hint: None, - ver: *segment_version, - put_cache: false, - }) - .await?; + let segment = latest_default_segment(fixture).await?; let (stats_location, stats_version) = segment.summary.additional_stats_loc().unwrap(); let stats = MetaReaders::segment_stats_reader(fuse_table.get_operator()) .read(&LoadParams { @@ -93,7 +59,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { let schema = table.schema(); let id_column_id = schema.field(0).column_id(); let value_column_id = schema.field(1).column_id(); - let origin = latest_block_meta(&fixture).await?; + let origin = latest_default_block_meta(&fixture).await?; let origin_hll = latest_block_hll(&fixture).await?; fixture @@ -106,7 +72,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { )) .await?; - let updated = latest_block_meta(&fixture).await?; + let updated = latest_default_block_meta(&fixture).await?; assert_eq!(updated.column_groups.len(), 2); let unchanged_group = updated @@ -179,7 +145,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { "update {db}.{table_name} set value = value + 1 where id = 2" )) .await?; - let updated_again = latest_block_meta(&fixture).await?; + let updated_again = latest_default_block_meta(&fixture).await?; assert_eq!(updated_again.column_groups.len(), 2); assert!( updated_again @@ -219,7 +185,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { "update {db}.{table_name} set value = value + 1 where id = 1" )) .await?; - let fully_rewritten = latest_block_meta(&fixture).await?; + let fully_rewritten = latest_default_block_meta(&fixture).await?; assert!(fully_rewritten.column_groups.is_empty()); assert!(fully_rewritten.bloom_index_files.is_empty()); @@ -234,6 +200,105 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_preserves_block_compression() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, b int) engine=fuse \ + compression='zstd' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, 20), (2, 30, 40)" + )) + .await?; + let origin_compression = latest_default_block_meta(&fixture).await?.compression; + + fixture + .execute_command(&format!( + "alter table {db}.{table_name} set options(compression='snappy')" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set b = b + 1 where id = 2" + )) + .await?; + + let updated = latest_default_block_meta(&fixture).await?; + assert_eq!(updated.compression, origin_compression); + assert_eq!(updated.column_groups.len(), 3); + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and a = 11 and b = 20) \ + or (id = 2 and a = 30 and b = 41)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_reads_legacy_bloom_schema() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, value int, other int) engine=fuse \ + bloom_index_columns='id' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, 20), (2, 30, 40)" + )) + .await?; + let table = fixture.latest_default_table().await?; + let id_column_id = table.schema().field_with_name("id")?.column_id(); + + fixture + .execute_command(&format!( + "alter table {db}.{table_name} set options(bloom_index_columns='value')" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set other = other + 1 where id = 1" + )) + .await?; + + let updated = latest_default_block_meta(&fixture).await?; + assert!(updated.bloom_filter_index_location.is_none()); + assert_eq!(updated.bloom_index_files.len(), 1); + assert_eq!(updated.bloom_index_files[0].active_column_ids, vec![ + id_column_id + ]); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; @@ -254,7 +319,7 @@ async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyh )) .await?; assert!( - latest_block_meta(&fixture) + latest_default_block_meta(&fixture) .await? .virtual_block_meta .is_some() @@ -275,7 +340,7 @@ async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyh .await?; assert!( - latest_block_meta(&fixture) + latest_default_block_meta(&fixture) .await? .virtual_block_meta .is_none() diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs index dcf7b5cc85818..fba09071c244a 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs @@ -64,26 +64,9 @@ impl AbstractBlockMeta for BlockMeta { &self, col_ids: &HashSet, ) -> Vec<(String, HashMap)> { - if self.column_groups.is_empty() { - return vec![(self.location_path(), self.col_metas(col_ids))]; - } - - self.column_groups - .iter() - .filter_map(|group| { - let col_metas = group - .active_column_ids - .iter() - .filter(|column_id| col_ids.contains(column_id)) - .filter_map(|column_id| { - group - .leaf_column_metas - .get(column_id) - .map(|meta| (*column_id, meta.clone())) - }) - .collect::>(); - (!col_metas.is_empty()).then(|| (group.location.0.clone(), col_metas)) - }) + BlockMeta::project_column_groups(self, col_ids) + .into_iter() + .map(|group| (group.location.0, group.leaf_column_metas)) .collect() } diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 8dc3d90118a83..2adeb518f8694 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; use std::collections::HashMap; +use std::collections::HashSet; use std::ops::Range; use std::sync::Arc; @@ -296,6 +298,108 @@ impl BlockMeta { self.compression } + /// Active physical data files referenced by this logical block. + pub fn data_file_locations(&self) -> impl Iterator { + self.column_groups + .is_empty() + .then_some(&self.location) + .into_iter() + .chain(self.column_groups.iter().map(|group| &group.location)) + } + + /// Active physical Bloom files referenced by this logical block. + pub fn bloom_index_file_locations(&self) -> impl Iterator { + self.bloom_index_files + .is_empty() + .then_some(self.bloom_filter_index_location.as_ref()) + .flatten() + .into_iter() + .chain(self.bloom_index_files.iter().map(|file| &file.location)) + } + + /// Normalize legacy and split layouts to the active physical data-file view. + pub fn physical_column_groups(&self) -> Cow<'_, [ColumnGroupFileMeta]> { + if !self.column_groups.is_empty() { + return Cow::Borrowed(&self.column_groups); + } + + let mut active_column_ids = self.col_metas.keys().copied().collect::>(); + active_column_ids.sort_unstable(); + Cow::Owned(vec![ColumnGroupFileMeta { + active_column_ids, + location: self.location.clone(), + format_version: self.location.1, + file_size: self.file_size, + uncompressed_size: self.block_size, + leaf_column_metas: self.col_metas.clone(), + }]) + } + + /// Project active leaf metadata while preserving each owning physical file. + pub fn project_column_groups( + &self, + projected_column_ids: &HashSet, + ) -> Vec { + let project_group = + |active_column_ids: &[ColumnId], + location: &Location, + format_version: FormatVersion, + file_size: u64, + uncompressed_size: u64, + leaf_column_metas: &HashMap| { + let active_column_ids = active_column_ids + .iter() + .filter(|column_id| projected_column_ids.contains(column_id)) + .filter(|column_id| leaf_column_metas.contains_key(column_id)) + .copied() + .collect::>(); + if active_column_ids.is_empty() { + return None; + } + let leaf_column_metas = active_column_ids + .iter() + .map(|column_id| (*column_id, leaf_column_metas[column_id].clone())) + .collect(); + Some(ColumnGroupFileMeta { + active_column_ids, + location: location.clone(), + format_version, + file_size, + uncompressed_size, + leaf_column_metas, + }) + }; + + if self.column_groups.is_empty() { + let mut active_column_ids = self.col_metas.keys().copied().collect::>(); + active_column_ids.sort_unstable(); + return project_group( + &active_column_ids, + &self.location, + self.location.1, + self.file_size, + self.block_size, + &self.col_metas, + ) + .into_iter() + .collect(); + } + + self.column_groups + .iter() + .filter_map(|group| { + project_group( + &group.active_column_ids, + &group.location, + group.format_version, + group.file_size, + group.uncompressed_size, + &group.leaf_column_metas, + ) + }) + .collect() + } + /// Get the page size of the block. /// /// For the parquet format, the page size is its row count. diff --git a/src/query/storages/common/table_meta/src/table/table_compression.rs b/src/query/storages/common/table_meta/src/table/table_compression.rs index f6c1f9eb31b4f..a736437fe961e 100644 --- a/src/query/storages/common/table_meta/src/table/table_compression.rs +++ b/src/query/storages/common/table_meta/src/table/table_compression.rs @@ -24,6 +24,8 @@ pub enum TableCompression { None, LZ4, Snappy, + /// Kept for rewriting legacy blocks whose metadata records Gzip compression. + Gzip, #[default] Zstd, } @@ -55,11 +57,24 @@ impl From for meta::Compression { // Map to meta Lz4Raw. TableCompression::LZ4 => meta::Compression::Lz4Raw, TableCompression::Snappy => meta::Compression::Snappy, + TableCompression::Gzip => meta::Compression::Gzip, TableCompression::Zstd => meta::Compression::Zstd, } } } +impl From for TableCompression { + fn from(value: meta::Compression) -> Self { + match value { + meta::Compression::Lz4 | meta::Compression::Lz4Raw => TableCompression::LZ4, + meta::Compression::Snappy => TableCompression::Snappy, + meta::Compression::Zstd => TableCompression::Zstd, + meta::Compression::Gzip => TableCompression::Gzip, + meta::Compression::None => TableCompression::None, + } + } +} + /// Convert to parquet Compression. impl From for ParquetCompression { fn from(value: TableCompression) -> Self { @@ -67,6 +82,7 @@ impl From for ParquetCompression { TableCompression::None => ParquetCompression::UNCOMPRESSED, TableCompression::LZ4 => ParquetCompression::LZ4_RAW, TableCompression::Snappy => ParquetCompression::SNAPPY, + TableCompression::Gzip => ParquetCompression::GZIP(GzipLevel::default()), TableCompression::Zstd => ParquetCompression::ZSTD(ZstdLevel::default()), } } diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 1b9003ab36a5f..52666f345ab3f 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::any::Any; -use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; use std::collections::hash_map::DefaultHasher; @@ -34,7 +33,6 @@ use databend_common_expression::Scalar; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BloomIndexFileMeta; -use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; @@ -47,63 +45,15 @@ pub struct FuseColumnGroupPartInfo { pub columns_meta: HashMap, } -/// Normalize a legacy single-file block and a column-group block to the same physical-file view. -pub(crate) fn normalized_column_group_files(meta: &BlockMeta) -> Cow<'_, [ColumnGroupFileMeta]> { - if !meta.column_groups.is_empty() { - return Cow::Borrowed(&meta.column_groups); - } - - let mut active_column_ids = meta.col_metas.keys().copied().collect::>(); - active_column_ids.sort_unstable(); - Cow::Owned(vec![ColumnGroupFileMeta { - active_column_ids, - location: meta.location.clone(), - format_version: meta.location.1, - file_size: meta.file_size, - uncompressed_size: meta.block_size, - leaf_column_metas: meta.col_metas.clone(), - }]) -} - pub(crate) fn project_column_groups( meta: &BlockMeta, projected_column_ids: &HashSet, ) -> Vec { - if meta.column_groups.is_empty() { - let columns_meta = meta - .col_metas - .iter() - .filter(|(column_id, _)| projected_column_ids.contains(column_id)) - .map(|(column_id, column_meta)| (*column_id, column_meta.clone())) - .collect::>(); - return if columns_meta.is_empty() { - vec![] - } else { - vec![FuseColumnGroupPartInfo { - location: meta.location.0.clone(), - columns_meta, - }] - }; - } - - normalized_column_group_files(meta) - .iter() - .filter_map(|group| { - let columns_meta = group - .active_column_ids - .iter() - .filter(|column_id| projected_column_ids.contains(column_id)) - .filter_map(|column_id| { - group - .leaf_column_metas - .get(column_id) - .map(|column_meta| (*column_id, column_meta.clone())) - }) - .collect::>(); - (!columns_meta.is_empty()).then(|| FuseColumnGroupPartInfo { - location: group.location.0.clone(), - columns_meta, - }) + meta.project_column_groups(projected_column_ids) + .into_iter() + .map(|group| FuseColumnGroupPartInfo { + location: group.location.0, + columns_meta: group.leaf_column_metas, }) .collect() } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 656ec54959bfd..4224ac8c7219f 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -338,6 +338,7 @@ impl BlockBuilder { data_block: DataBlock, origin: &BlockMeta, updated_field_indices: &[FieldIndex], + legacy_bloom_column_ids: Option<&[ColumnId]>, ) -> Result { if data_block.num_rows() as u64 != origin.row_count { return Err(ErrorCode::Internal( @@ -445,8 +446,10 @@ impl BlockBuilder { )?; let uncompressed_size = updated_block.estimate_block_size(updated_block.num_columns()) as u64; + let mut write_settings = self.write_settings.clone(); + write_settings.table_compression = origin.compression.into(); let (updated_col_metas, buffer) = serialize_block_with_column_stats( - &self.write_settings, + &write_settings, &updated_schema, Some(&updated_col_stats), updated_block, @@ -506,22 +509,21 @@ impl BlockBuilder { if !invalidated_bloom_column_ids.is_empty() { let mut bloom_index_files = if origin.bloom_index_files.is_empty() { - origin - .bloom_filter_index_location - .as_ref() - .map(|location| BloomIndexFileMeta { - active_column_ids: self - .bloom_columns_map - .values() - .map(|field| field.column_id()) - .filter(|column_id| !invalidated_bloom_column_ids.contains(column_id)) - .collect(), - location: location.clone(), - format_version: location.1, - file_size: origin.bloom_filter_index_size, - }) - .into_iter() - .collect() + let mut files = Vec::new(); + if let Some(location) = &origin.bloom_filter_index_location { + let active_column_ids = legacy_bloom_column_ids.ok_or_else(|| { + ErrorCode::Internal("legacy Bloom file columns were not loaded") + })?; + if !active_column_ids.is_empty() { + files.push(BloomIndexFileMeta { + active_column_ids: active_column_ids.to_vec(), + location: location.clone(), + format_version: location.1, + file_size: origin.bloom_filter_index_size, + }); + } + } + files } else { origin.bloom_index_files.clone() }; diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index c15731180bb6d..9640446ae83b3 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::any::Any; +use std::collections::HashSet; use std::sync::Arc; use databend_common_base::base::ProgressValues; @@ -21,6 +22,7 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoDowncast; +use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; @@ -48,6 +50,7 @@ use crate::io::SpatialIndexBuilder; use crate::io::VectorIndexBuilder; use crate::io::VirtualColumnBuilder; use crate::io::create_inverted_index_builders; +use crate::io::read::bloom::block_filter_reader::load_index_meta; use crate::operations::common::BlockMetaIndex; use crate::operations::common::MutationLogEntry; use crate::operations::common::MutationLogs; @@ -58,11 +61,18 @@ use crate::statistics::ClusterStatsGenerator; #[allow(clippy::large_enum_variant)] enum State { Consume, + NeedLegacyBloomMeta { + block: DataBlock, + stats_type: ClusterStatsGenType, + index: Option, + origin_block_meta: Arc, + }, NeedSerialize { block: DataBlock, stats_type: ClusterStatsGenType, index: Option, origin_block_meta: Option>, + legacy_bloom_column_ids: Option>, }, Serialized { serialized: BlockSerialization, @@ -266,6 +276,48 @@ impl TransformSerializeBlock { self.block_builder.clone() } + fn needs_legacy_bloom_meta(&self, origin: &BlockMeta) -> bool { + origin.bloom_index_files.is_empty() + && origin.bloom_filter_index_location.is_some() + && self.updated_field_indices.as_ref().is_some_and(|indices| { + indices.iter().any(|index| { + BloomIndex::supported_type( + self.block_builder.source_schema.field(*index).data_type(), + ) + }) + }) + } + + async fn load_legacy_bloom_column_ids(&self, origin: &BlockMeta) -> Result> { + let location = origin + .bloom_filter_index_location + .as_ref() + .ok_or_else(|| ErrorCode::Internal("legacy Bloom location is missing"))?; + let meta = load_index_meta( + self.dal.clone(), + &location.0, + origin.bloom_filter_index_size, + None, + ) + .await?; + let stored_filter_names = meta + .columns + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(); + let mut active_column_ids = Vec::new(); + for field in self.block_builder.source_schema.fields() { + if !BloomIndex::supported_type(field.data_type()) { + continue; + } + let filter_name = BloomIndex::build_filter_bloom_name(location.1, field)?; + if stored_filter_names.contains(filter_name.as_str()) { + active_column_ids.push(field.column_id()); + } + } + Ok(active_column_ids) + } + fn mutation_logs(entry: MutationLogEntry) -> DataBlock { let meta = MutationLogs { entries: vec![entry], @@ -285,6 +337,9 @@ impl Processor for TransformSerializeBlock { } fn event(&mut self) -> Result { + if matches!(self.state, State::NeedLegacyBloomMeta { .. }) { + return Ok(Event::Async); + } if matches!(self.state, State::NeedSerialize { .. }) { return Ok(Event::Sync); } @@ -340,13 +395,28 @@ impl Processor for TransformSerializeBlock { } else { // replace the old block self.pending_insert_rows = serialize_block.insert_rows; - self.state = State::NeedSerialize { - block: input_data, - stats_type: serialize_block.stats_type, - index: Some(serialize_block.index), - origin_block_meta: serialize_block.origin_block_meta, - }; - Ok(Event::Sync) + let origin_block_meta = serialize_block.origin_block_meta; + if origin_block_meta + .as_deref() + .is_some_and(|origin| self.needs_legacy_bloom_meta(origin)) + { + self.state = State::NeedLegacyBloomMeta { + block: input_data, + stats_type: serialize_block.stats_type, + index: Some(serialize_block.index), + origin_block_meta: origin_block_meta.unwrap(), + }; + Ok(Event::Async) + } else { + self.state = State::NeedSerialize { + block: input_data, + stats_type: serialize_block.stats_type, + index: Some(serialize_block.index), + origin_block_meta, + legacy_bloom_column_ids: None, + }; + Ok(Event::Sync) + } } } SerializeDataMeta::CompactExtras(compact_extras) => { @@ -377,6 +447,7 @@ impl Processor for TransformSerializeBlock { stats_type: ClusterStatsGenType::Generally, index: None, origin_block_meta: None, + legacy_bloom_column_ids: None, }; Ok(Event::Sync) } @@ -389,6 +460,7 @@ impl Processor for TransformSerializeBlock { stats_type, index, origin_block_meta, + legacy_bloom_column_ids, } => { // Check if the datablock is valid, this is needed to ensure data is correct block.check_valid()?; @@ -400,6 +472,7 @@ impl Processor for TransformSerializeBlock { block, &origin_block_meta, updated_field_indices, + legacy_bloom_column_ids.as_deref(), )? } else { self.block_builder @@ -423,6 +496,23 @@ impl Processor for TransformSerializeBlock { #[async_backtrace::framed] async fn async_process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { + State::NeedLegacyBloomMeta { + block, + stats_type, + index, + origin_block_meta, + } => { + let legacy_bloom_column_ids = self + .load_legacy_bloom_column_ids(&origin_block_meta) + .await?; + self.state = State::NeedSerialize { + block, + stats_type, + index, + origin_block_meta: Some(origin_block_meta), + legacy_bloom_column_ids: Some(legacy_bloom_column_ids), + }; + } State::Serialized { serialized, index } => { let insert_rows = std::mem::take(&mut self.pending_insert_rows); let extended_block_meta = BlockWriter::write_down(&self.dal, serialized).await?; diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs index 81dd3ea6f4d65..33ae1d0d0738f 100644 --- a/src/query/storages/fuse/src/operations/gc.rs +++ b/src/query/storages/fuse/src/operations/gc.rs @@ -803,28 +803,16 @@ impl TryFrom> for LocationTuple { let mut hll_location = HashSet::new(); let block_metas = value.block_metas()?; for block_meta in block_metas.into_iter() { - if block_meta.column_groups.is_empty() { - block_location.insert(block_meta.location.0.clone()); - } else { - block_location.extend( - block_meta - .column_groups - .iter() - .map(|group| group.location.0.clone()), - ); - } - if block_meta.bloom_index_files.is_empty() { - if let Some(bloom_loc) = &block_meta.bloom_filter_index_location { - bloom_location.insert(bloom_loc.0.clone()); - } - } else { - bloom_location.extend( - block_meta - .bloom_index_files - .iter() - .map(|file| file.location.0.clone()), - ); - } + block_location.extend( + block_meta + .data_file_locations() + .map(|location| location.0.clone()), + ); + bloom_location.extend( + block_meta + .bloom_index_file_locations() + .map(|location| location.0.clone()), + ); } if let Some(loc) = value.as_ref().summary.additional_stats_loc() { hll_location.insert(loc.0); diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 8ba6a5979de4b..1e6ee9d0b85e1 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -87,8 +87,6 @@ use crate::FuseLazyPartInfo; use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; -use crate::fuse_part::FuseColumnGroupPartInfo; -use crate::fuse_part::normalized_column_group_files; use crate::fuse_part::project_column_groups; use crate::io::BloomIndexRebuilder; use crate::pruning::BlockPruner; @@ -1372,13 +1370,6 @@ impl FuseTable { (statistics, partitions) } - fn projected_column_groups( - meta: &BlockMeta, - projected_column_ids: &HashSet, - ) -> Vec { - project_column_groups(meta, projected_column_ids) - } - pub fn all_columns_part( schema: Option<&TableSchemaRef>, block_meta_index: &Option, @@ -1391,7 +1382,7 @@ impl FuseTable { let mut projected_column_ids = if meta.column_groups.is_empty() { meta.col_metas.keys().copied().collect::>() } else { - normalized_column_group_files(meta) + meta.physical_column_groups() .iter() .flat_map(|group| group.active_column_ids.iter().copied()) .collect::>() @@ -1412,7 +1403,7 @@ impl FuseTable { spatial_stats.insert(*column_id, stats.clone()); } } - let column_groups = Self::projected_column_groups(meta, &projected_column_ids); + let column_groups = project_column_groups(meta, &projected_column_ids); let rows_count = meta.row_count; let location = meta.location.0.clone(); @@ -1467,7 +1458,7 @@ impl FuseTable { } } } - let column_groups = Self::projected_column_groups(meta, &projected_column_ids); + let column_groups = project_column_groups(meta, &projected_column_ids); let rows_count = meta.row_count; let location = meta.location.0.clone(); @@ -1563,7 +1554,7 @@ mod tests { None, ); - let legacy_groups = FuseTable::projected_column_groups(&block_meta, &HashSet::from([2])); + let legacy_groups = project_column_groups(&block_meta, &HashSet::from([2])); assert_eq!(legacy_groups.len(), 1); assert_eq!(legacy_groups[0].location, "group-2.parquet"); assert_eq!( @@ -1590,7 +1581,7 @@ mod tests { }, ]; - let column_groups = FuseTable::projected_column_groups(&block_meta, &HashSet::from([2])); + let column_groups = project_column_groups(&block_meta, &HashSet::from([2])); assert_eq!(column_groups.len(), 1); assert_eq!(column_groups[0].location, "group-2.parquet"); diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index 29e99ee0060ea..f93676207acc3 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -61,7 +61,6 @@ use parquet::basic::Type as ParquetPhysicalType; use crate::FuseStorageFormat; use crate::FuseTable; -use crate::fuse_part::normalized_column_group_files; use crate::io::SegmentsIO; use crate::io::read::meta::read_thrift_file_metadata; use crate::sessions::TableContext; @@ -402,7 +401,7 @@ impl<'a> FuseEncodingImpl<'a> { for segment in segments { let segment = segment?; for block in segment.blocks.iter() { - for group in normalized_column_group_files(block).iter() { + for group in block.physical_column_groups().iter() { let column_metas = group .active_column_ids .iter() diff --git a/src/query/storages/fuse/src/table_functions/fuse_page.rs b/src/query/storages/fuse/src/table_functions/fuse_page.rs index 726f326967015..a1f721c9eb514 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_page.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_page.rs @@ -43,7 +43,6 @@ use thrift::protocol::TCompactInputProtocol; use crate::FuseStorageFormat; use crate::FuseTable; -use crate::fuse_part::normalized_column_group_files; use crate::io::SegmentsIO; use crate::sessions::TableContext; use crate::table_functions::TableMetaFuncTemplate; @@ -145,7 +144,7 @@ impl TableMetaFunc for FusePage { let segment = segment?; for block in segment.blocks.iter() { let block = block.as_ref(); - for group in normalized_column_group_files(block).iter() { + for group in block.physical_column_groups().iter() { for field in schema.fields().iter() { if field.is_nested() || !group.active_column_ids.contains(&field.column_id) diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test index 29fa3af587e4c..c338a0f736b88 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -75,3 +75,33 @@ from fuse_block('default', 't_partial_update') statement ok drop table t_partial_update + +# A partial group keeps the logical block codec even if the table option changes. +statement ok +create table t_partial_update_compression(id int, a int, b int) + compression = 'zstd' + enable_partial_update = true + +statement ok +insert into t_partial_update_compression values (1, 10, 20), (2, 30, 40) + +statement ok +alter table t_partial_update_compression set options(compression = 'snappy') + +statement ok +set enable_partial_update = 1 + +statement ok +update t_partial_update_compression set a = a + 1 where id = 1 + +statement ok +update t_partial_update_compression set b = b + 1 where id = 2 + +query III +select * from t_partial_update_compression order by id +---- +1 11 20 +2 30 41 + +statement ok +drop table t_partial_update_compression From 6beff4acc86faf3c7505948945ddf8e57013e7f4 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:51:12 +0800 Subject: [PATCH 07/30] fix(storage): close partial update integration gaps --- .../fuse/operations/computed_columns.rs | 45 +++++ src/query/service/src/physical_plans/mod.rs | 15 ++ .../physical_column_mutation.rs | 25 ++- .../src/physical_plans/physical_mutation.rs | 52 +++++- .../physical_mutation_source.rs | 38 ++-- .../storages/fuse/operations/replace_into.rs | 69 +++++++ .../common/table_meta/src/meta/v2/segment.rs | 55 +++--- .../table_meta/src/table/table_compression.rs | 9 +- .../fuse/src/io/write/block_writer.rs | 18 +- .../storages/fuse/src/operations/append.rs | 2 +- .../mutator/replace_into_operation_agg.rs | 168 ++++++++++-------- .../storages/fuse/src/pruning/bloom_pruner.rs | 2 +- src/query/storages/fuse/src/pruning/mod.rs | 1 + ...001_partial_update_cluster_dependency.test | 54 ++++++ 14 files changed, 394 insertions(+), 159 deletions(-) create mode 100644 tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test diff --git a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs index e206655da5a6d..eedd1f990a6a6 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs @@ -201,3 +201,48 @@ async fn test_partial_update_with_virtual_column_and_change_tracking() -> anyhow Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_rejects_indirect_cluster_key_update() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} \ + (id int, a int, k bigint as (a + 1) virtual) engine=fuse \ + enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name}(id, a) values (1, 10), (2, 20)" + )) + .await?; + fixture + .execute_command(&format!("alter table {db}.{table_name} cluster by(k)")) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + + let updated = latest_default_block_meta(&fixture).await?; + assert!(updated.column_groups.is_empty()); + assert!(updated.cluster_stats.is_some()); + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and a = 11 and k = 12) or (id = 2 and a = 20 and k = 21)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} diff --git a/src/query/service/src/physical_plans/mod.rs b/src/query/service/src/physical_plans/mod.rs index dd6571eb52bc0..c96201e1627fd 100644 --- a/src/query/service/src/physical_plans/mod.rs +++ b/src/query/service/src/physical_plans/mod.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_expression::FieldIndex; +use databend_common_expression::TableSchema; + mod physical_add_stream_column; mod physical_aggregate_expand; mod physical_aggregate_final; @@ -113,6 +116,18 @@ pub use physical_window_partition::*; pub use runtime_filter::PhysicalRuntimeFilter; pub use runtime_filter::PhysicalRuntimeFilters; +pub(crate) fn physical_field_to_storage_index( + physical_schema: &TableSchema, + storage_schema: &TableSchema, + physical_index: FieldIndex, +) -> Option { + let column_id = physical_schema.field(physical_index).column_id(); + storage_schema + .fields() + .iter() + .position(|field| field.column_id() == column_id) +} + pub mod explain; mod format; mod physical_cte_consumer; diff --git a/src/query/service/src/physical_plans/physical_column_mutation.rs b/src/query/service/src/physical_plans/physical_column_mutation.rs index 5a0a2756a0026..21c8175a927b0 100644 --- a/src/query/service/src/physical_plans/physical_column_mutation.rs +++ b/src/query/service/src/physical_plans/physical_column_mutation.rs @@ -35,6 +35,7 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ColumnMutationFormatter; use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::physical_field_to_storage_index; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; @@ -174,12 +175,12 @@ impl IPhysicalPlan for ColumnMutation { updated_fields .iter() .map(|source_index| { - let column_id = source_schema.field(*source_index).column_id(); - let field_id = schema_with_stream - .fields() - .iter() - .position(|field| field.column_id() == column_id) - .ok_or_else(|| ErrorCode::Internal("updated field is not in schema"))?; + let field_id = physical_field_to_storage_index( + &source_schema, + &schema_with_stream, + *source_index, + ) + .ok_or_else(|| ErrorCode::Internal("updated field is not in schema"))?; field_id_to_schema_index .get(&field_id) .copied() @@ -215,17 +216,23 @@ impl IPhysicalPlan for ColumnMutation { let write_column_group = self.partial_update_fields.is_some(); let block_thresholds = table.get_block_thresholds(); + let physical_table_schema = table.schema_with_stream().remove_virtual_computed_fields(); + let physical_input_schema = DataSchema::from(&physical_table_schema).into(); let cluster_stats_gen = if write_column_group { ClusterStatsGenerator::default() } else if matches!(self.mutation_kind, MutationKind::Delete) { - let input_schema = DataSchema::from(table.schema_with_stream()).into(); - table.get_cluster_stats_gen(builder.ctx.clone(), 0, block_thresholds, input_schema)? + table.get_cluster_stats_gen( + builder.ctx.clone(), + 0, + block_thresholds, + physical_input_schema, + )? } else { table.cluster_gen_for_append( builder.ctx.clone(), &mut builder.main_pipeline, block_thresholds, - None, + Some(physical_input_schema), )? }; diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 97e11579d2c4b..f41d868c3b560 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -21,6 +21,7 @@ use databend_common_catalog::plan::NUM_ROW_ID_PREFIX_BITS; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; @@ -98,6 +99,39 @@ struct PartialUpdateInfo { updated_field_indices: Vec, } +fn expand_computed_column_dependencies( + ctx: Arc, + table_schema: &TableSchema, + computed_schema: &DataSchemaRef, + dependency_ids: &mut BTreeSet, +) -> Result<()> { + loop { + let previous_len = dependency_ids.len(); + for field in table_schema.fields() { + if !dependency_ids.contains(&field.column_id()) { + continue; + } + let computed_expr = match field.computed_expr() { + Some(ComputedExpr::Stored(expr)) | Some(ComputedExpr::Virtual(expr)) => expr, + None => continue, + }; + let expr = parse_computed_field_index_expr( + ctx.clone(), + computed_schema.clone(), + computed_expr, + )?; + dependency_ids.extend( + expr.column_refs() + .into_iter() + .map(|(index, _)| table_schema.field(index).column_id()), + ); + } + if dependency_ids.len() == previous_len { + return Ok(()); + } + } +} + #[allow(clippy::too_many_arguments)] fn build_partial_update_info( ctx: Arc, @@ -152,12 +186,18 @@ fn build_partial_update_info( if let Some(cluster_keys) = table.resolve_cluster_keys() { let cluster_exprs = parse_cluster_keys(ctx.clone(), Arc::new(table.clone()), cluster_keys)?; - let updates_cluster_key = cluster_exprs.iter().any(|expr| { - expr.column_refs().iter().any(|(index, _)| { - updated_column_ids.contains(&schema_with_stream.field(*index).column_id()) - }) - }); - if updates_cluster_key { + let mut cluster_dependency_ids = cluster_exprs + .iter() + .flat_map(|expr| expr.column_refs()) + .map(|(index, _)| schema_with_stream.field(index).column_id()) + .collect::>(); + expand_computed_column_dependencies( + ctx.clone(), + table_schema.as_ref(), + &computed_schema, + &mut cluster_dependency_ids, + )?; + if !updated_column_ids.is_disjoint(&cluster_dependency_ids) { return Ok(None); } } diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index 8dd9beca6522c..f306f312eafff 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -55,6 +55,7 @@ use databend_common_storages_fuse::operations::VirtualSchemaMode; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::MutationSourceFormatter; use crate::physical_plans::format::PhysicalFormat; +use crate::physical_plans::physical_field_to_storage_index; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; @@ -160,26 +161,23 @@ impl IPhysicalPlan for MutationSource { let physical_schema = table.schema_with_stream().remove_virtual_computed_fields(); let partial_update = self.partial_update_projection.is_some(); - let read_partition_field_indices: Vec = - if let Some(projection) = &self.partial_update_projection { - let storage_schema = table.schema_with_stream(); - projection - .iter() - .map(|field_index| { - let column_id = physical_schema.field(*field_index).column_id(); - storage_schema - .fields() - .iter() - .position(|field| field.column_id() == column_id) - .expect("physical field must exist in storage schema") - }) - .collect() - } else { - self.read_partition_columns - .iter() - .map(|idx| idx.as_field_index()) - .collect() - }; + let read_partition_field_indices: Vec = if let Some(projection) = + &self.partial_update_projection + { + let storage_schema = table.schema_with_stream(); + projection + .iter() + .map(|field_index| { + physical_field_to_storage_index(&physical_schema, &storage_schema, *field_index) + .expect("physical field must exist in storage schema") + }) + .collect() + } else { + self.read_partition_columns + .iter() + .map(|idx| idx.as_field_index()) + .collect() + }; let stream_projection = self .partial_update_projection .as_deref() diff --git a/src/query/service/tests/it/storages/fuse/operations/replace_into.rs b/src/query/service/tests/it/storages/fuse/operations/replace_into.rs index 4288c6c3fc006..af17c3abb44ed 100644 --- a/src/query/service/tests/it/storages/fuse/operations/replace_into.rs +++ b/src/query/service/tests/it/storages/fuse/operations/replace_into.rs @@ -13,6 +13,8 @@ // limitations under the License. use databend_common_storages_fuse::FuseTable; +use databend_query::test_kits::TestFixture; +use databend_query::test_kits::latest_default_block_meta; use itertools::Itertools; #[test] @@ -58,3 +60,70 @@ fn test_partition() -> anyhow::Result<()> { } Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn test_replace_bloom_prunes_partial_update_block() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, b int) engine=fuse \ + cluster by(id) bloom_index_columns='id,a,b' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, 100), (100, 30, 200)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + let block_meta = latest_default_block_meta(&fixture).await?; + assert_eq!(block_meta.bloom_index_files.len(), 2); + + // Make the active data files unreadable while leaving both Bloom files intact. A key that + // might conflict must still try to read the block, proving that the missing data is visible. + // The absent key below can succeed only if REPLACE reads and combines the split Bloom files. + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + for group in block_meta.physical_column_groups().iter() { + operator.delete(&group.location.0).await?; + } + assert!( + fixture + .execute_command(&format!( + "replace into {db}.{table_name} on(id, a) values (1, 11, 999)" + )) + .await + .is_err() + ); + + fixture + .execute_command(&format!( + "replace into {db}.{table_name} on(id, a) values (50, 20, 999)" + )) + .await?; + let latest_table = fixture.latest_default_table().await?; + let latest_fuse = FuseTable::try_from_table(latest_table.as_ref())?; + assert_eq!( + latest_fuse + .read_table_snapshot() + .await? + .unwrap() + .summary + .row_count, + 3 + ); + + Ok(()) +} diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 2adeb518f8694..d984692c296a0 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -317,22 +317,40 @@ impl BlockMeta { .chain(self.bloom_index_files.iter().map(|file| &file.location)) } - /// Normalize legacy and split layouts to the active physical data-file view. - pub fn physical_column_groups(&self) -> Cow<'_, [ColumnGroupFileMeta]> { - if !self.column_groups.is_empty() { - return Cow::Borrowed(&self.column_groups); - } - - let mut active_column_ids = self.col_metas.keys().copied().collect::>(); + fn legacy_column_group( + &self, + projected_column_ids: Option<&HashSet>, + ) -> ColumnGroupFileMeta { + let mut active_column_ids = self + .col_metas + .keys() + .filter(|column_id| { + projected_column_ids.is_none_or(|projected| projected.contains(column_id)) + }) + .copied() + .collect::>(); active_column_ids.sort_unstable(); - Cow::Owned(vec![ColumnGroupFileMeta { + let leaf_column_metas = active_column_ids + .iter() + .map(|column_id| (*column_id, self.col_metas[column_id].clone())) + .collect(); + ColumnGroupFileMeta { active_column_ids, location: self.location.clone(), format_version: self.location.1, file_size: self.file_size, uncompressed_size: self.block_size, - leaf_column_metas: self.col_metas.clone(), - }]) + leaf_column_metas, + } + } + + /// Normalize legacy and split layouts to the active physical data-file view. + pub fn physical_column_groups(&self) -> Cow<'_, [ColumnGroupFileMeta]> { + if !self.column_groups.is_empty() { + return Cow::Borrowed(&self.column_groups); + } + + Cow::Owned(vec![self.legacy_column_group(None)]) } /// Project active leaf metadata while preserving each owning physical file. @@ -371,18 +389,11 @@ impl BlockMeta { }; if self.column_groups.is_empty() { - let mut active_column_ids = self.col_metas.keys().copied().collect::>(); - active_column_ids.sort_unstable(); - return project_group( - &active_column_ids, - &self.location, - self.location.1, - self.file_size, - self.block_size, - &self.col_metas, - ) - .into_iter() - .collect(); + let group = self.legacy_column_group(Some(projected_column_ids)); + return (!group.active_column_ids.is_empty()) + .then_some(group) + .into_iter() + .collect(); } self.column_groups diff --git a/src/query/storages/common/table_meta/src/table/table_compression.rs b/src/query/storages/common/table_meta/src/table/table_compression.rs index a736437fe961e..1485fd1ae20ee 100644 --- a/src/query/storages/common/table_meta/src/table/table_compression.rs +++ b/src/query/storages/common/table_meta/src/table/table_compression.rs @@ -90,13 +90,6 @@ impl From for ParquetCompression { impl From for ParquetCompression { fn from(value: meta::Compression) -> Self { - match value { - meta::Compression::Lz4Raw => ParquetCompression::LZ4_RAW, - meta::Compression::Snappy => ParquetCompression::SNAPPY, - meta::Compression::Zstd => ParquetCompression::ZSTD(ZstdLevel::default()), - meta::Compression::None => ParquetCompression::UNCOMPRESSED, - meta::Compression::Lz4 => ParquetCompression::LZ4_RAW, - meta::Compression::Gzip => ParquetCompression::GZIP(GzipLevel::default()), - } + TableCompression::from(value).into() } } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 4224ac8c7219f..73156e3c02605 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -457,23 +457,7 @@ impl BlockBuilder { let file_size = buffer.len() as u64; let current_column_ids = self.source_schema.to_leaf_column_id_set(); - let mut column_groups = if origin.column_groups.is_empty() { - vec![ColumnGroupFileMeta { - active_column_ids: self - .source_schema - .to_leaf_column_ids() - .into_iter() - .filter(|column_id| origin.col_metas.contains_key(column_id)) - .collect(), - location: origin.location.clone(), - format_version: origin.location.1, - file_size: origin.file_size, - uncompressed_size: origin.block_size, - leaf_column_metas: origin.col_metas.clone(), - }] - } else { - origin.column_groups.clone() - }; + let mut column_groups = origin.physical_column_groups().into_owned(); for group in &mut column_groups { group.active_column_ids.retain(|column_id| { current_column_ids.contains(column_id) && !updated_column_id_set.contains(column_id) diff --git a/src/query/storages/fuse/src/operations/append.rs b/src/query/storages/fuse/src/operations/append.rs index da7984aa16eec..da1e74183c185 100644 --- a/src/query/storages/fuse/src/operations/append.rs +++ b/src/query/storages/fuse/src/operations/append.rs @@ -181,12 +181,12 @@ impl FuseTable { ) -> Result { let input_schema = modified_schema.unwrap_or(DataSchema::from(self.schema_with_stream()).into()); + let num_input_columns = input_schema.num_fields(); let cluster_stats_gen = self.get_cluster_stats_gen(ctx.clone(), 0, block_thresholds, input_schema)?; let operators = cluster_stats_gen.operators.clone(); if !operators.is_empty() { - let num_input_columns = self.schema().fields().len(); let func_ctx2 = cluster_stats_gen.func_ctx.clone(); pipeline.add_transformer(move || { diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 9d76b8b5e9ccd..91d6078715d5d 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -41,6 +41,7 @@ use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_cache::LoadParams; use databend_storages_common_index::BloomIndex; +use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_io::ReadSettings; @@ -49,6 +50,7 @@ use databend_storages_common_table_meta::meta::BlockSlotDescription; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; +use databend_storages_common_table_meta::meta::Versioned; use log::info; use log::warn; use opendal::Operator; @@ -74,6 +76,7 @@ use crate::operations::replace_into::meta::ReplaceIntoOperation; use crate::operations::replace_into::meta::UniqueKeyDigest; use crate::operations::replace_into::mutator::DeletionAccumulator; use crate::operations::replace_into::mutator::row_hash_of_columns; +use crate::pruning::read_multi_file_block_filter; struct AggregationContext { segment_locations: AHashMap, @@ -646,93 +649,108 @@ impl AggregationContext { if bloom_on_conflict_field_index.is_empty() { return false; } - if let Some(loc) = &block_meta.bloom_filter_index_location { - match self - .load_bloom_filter( - loc, - block_meta.bloom_filter_index_size, - bloom_on_conflict_field_index, - ) - .await - { - Ok(filters) => { - // the caller ensures that the input_hashes is not empty - let row_count = input_hashes[0].len(); - - // let assume that the target block is prunable - let mut block_pruned = true; - for row in 0..row_count { - // for each row, by default, assume that columns of this row do have conflict with the target block. - let mut row_not_prunable = true; - for (col_idx, col_hash) in input_hashes.iter().enumerate() { - // For each column of current row, check if the corresponding bloom - // filter contains the digest of the column. - // - // Any one of the columns NOT contains by the corresponding bloom filter, - // indicates that the row is prunable(thus, we do not stop on the first column that - // the bloom filter contains). - - // - if bloom filter presents, check if the column is contained - // - if bloom filter absents, do nothing(since by default, we assume that the row is not-prunable) - if let Some(col_filter) = &filters[col_idx] { - let hash = col_hash[row]; - if hash == 0 || !col_filter.contains_digest(hash) { - // - hash == 0 indicates that the column value is null, which equals nothing. - // - NOT `contains_digest`, indicates that this column of row does not match - row_not_prunable = false; - // if one column not match, we do not need to check other columns - break; - } + match self + .load_bloom_filter(block_meta, bloom_on_conflict_field_index) + .await + { + Ok(Some(filters)) => { + // the caller ensures that the input_hashes is not empty + let row_count = input_hashes[0].len(); + + // let assume that the target block is prunable + let mut block_pruned = true; + for row in 0..row_count { + // for each row, by default, assume that columns of this row do have conflict with the target block. + let mut row_not_prunable = true; + for (col_idx, col_hash) in input_hashes.iter().enumerate() { + // For each column of current row, check if the corresponding bloom + // filter contains the digest of the column. + // + // Any one of the columns NOT contains by the corresponding bloom filter, + // indicates that the row is prunable(thus, we do not stop on the first column that + // the bloom filter contains). + + // - if bloom filter presents, check if the column is contained + // - if bloom filter absents, do nothing(since by default, we assume that the row is not-prunable) + if let Some(col_filter) = &filters[col_idx] { + let hash = col_hash[row]; + if hash == 0 || !col_filter.contains_digest(hash) { + // - hash == 0 indicates that the column value is null, which equals nothing. + // - NOT `contains_digest`, indicates that this column of row does not match + row_not_prunable = false; + // if one column not match, we do not need to check other columns + break; } } - if row_not_prunable { - // any row not prunable indicates that the target block is not prunable - block_pruned = false; - break; - } } - block_pruned - } - Err(e) => { - // broken index should not stop us: - warn!("failed to build bloom index column name: {}", e); - // failed to load bloom filter, do not prune - false + if row_not_prunable { + // any row not prunable indicates that the target block is not prunable + block_pruned = false; + break; + } } + block_pruned + } + Ok(None) => false, + Err(e) => { + // broken index should not stop us: + warn!("failed to build bloom index column name: {}", e); + // failed to load bloom filter, do not prune + false } - } else { - // no bloom filter, no pruning - false } } async fn load_bloom_filter( &self, - location: &Location, - index_len: u64, + block_meta: &BlockMeta, bloom_on_conflict_field_index: &[FieldIndex], - ) -> Result>>> { + ) -> Result>>>> { // different block may have different version of bloom filter index - let mut col_names = Vec::with_capacity(bloom_on_conflict_field_index.len()); - - for idx in bloom_on_conflict_field_index { - let bloom_column_name = BloomIndex::build_filter_bloom_name( - location.1, - &self.on_conflict_fields[*idx].table_field, - )?; - col_names.push(bloom_column_name); - } - - // using load_bloom_filter_by_columns is attractive, - // but it do not care about the version of the bloom filter index - let block_filter = location - .read_block_filter( - self.data_accessor.clone(), + let (block_filter, col_names) = if block_meta.bloom_index_files.is_empty() { + let Some(location) = &block_meta.bloom_filter_index_location else { + return Ok(None); + }; + let col_names = bloom_on_conflict_field_index + .iter() + .map(|idx| { + BloomIndex::build_filter_bloom_name( + location.1, + &self.on_conflict_fields[*idx].table_field, + ) + }) + .collect::>>()?; + let block_filter = location + .read_block_filter( + self.data_accessor.clone(), + &self.read_settings, + &col_names, + block_meta.bloom_filter_index_size, + ) + .await?; + (block_filter, col_names) + } else { + let fields = bloom_on_conflict_field_index + .iter() + .map(|idx| self.on_conflict_fields[*idx].table_field.clone()) + .collect::>(); + let Some(block_filter) = read_multi_file_block_filter( + &self.data_accessor, &self.read_settings, - &col_names, - index_len, + &fields, + &[], + &block_meta.bloom_index_files, ) - .await?; + .await? + else { + return Ok(None); + }; + let col_names = fields + .iter() + .map(|field| BloomIndex::build_filter_bloom_name(BlockFilter::VERSION, field)) + .collect::>>()?; + (block_filter, col_names) + }; // reorder the filter according to the order of bloom_on_conflict_field let mut filters = Vec::with_capacity(bloom_on_conflict_field_index.len()); @@ -744,14 +762,14 @@ impl AggregationContext { Err(_) => { info!( "bloom filter column {} not found for block {}", - filter_col_name, location.0 + filter_col_name, block_meta.location.0 ); filters.push(None); } } } - Ok(filters) + Ok(Some(filters)) } } diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index b2bbfbe2f199c..7194fd37f9874 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -151,7 +151,7 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( )? == FilterEvalResult::MustFalse) } -async fn read_multi_file_block_filter( +pub(crate) async fn read_multi_file_block_filter( dal: &Operator, settings: &ReadSettings, index_fields: &[TableField], diff --git a/src/query/storages/fuse/src/pruning/mod.rs b/src/query/storages/fuse/src/pruning/mod.rs index 89864254d5d43..72fc6f0a7f0ff 100644 --- a/src/query/storages/fuse/src/pruning/mod.rs +++ b/src/query/storages/fuse/src/pruning/mod.rs @@ -29,6 +29,7 @@ mod virtual_column_pruner; pub use block_pruner::BlockPruner; pub use bloom_pruner::BloomPruner; pub use bloom_pruner::BloomPrunerCreator; +pub(crate) use bloom_pruner::read_multi_file_block_filter; pub use expr_bloom_filter::ExprBloomFilter; pub use expr_runtime_pruner::ExprRuntimePruner; pub use expr_runtime_pruner::RuntimeFilterExpr; diff --git a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test new file mode 100644 index 0000000000000..a72803b9be06b --- /dev/null +++ b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test @@ -0,0 +1,54 @@ +## Copyright 2023 Databend Cloud +## +## Licensed under the Elastic License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## https://www.elastic.co/licensing/elastic-license +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +statement ok +create or replace database partial_update_cluster_dependency + +statement ok +use partial_update_cluster_dependency + +# A cluster key that references an updated column through a VIRTUAL computed column must use the +# full-update path and refresh its cluster statistics. +statement ok +create table t( + id int, + a int, + k bigint as (a + 1) virtual +) enable_partial_update = true + +statement ok +insert into t(id, a) values (1, 10), (2, 20) + +statement ok +alter table t cluster by(k) + +statement ok +set enable_partial_update = 1 + +statement ok +update t set a = a + 1 where id = 1 + +query B +select column_groups = parse_json('[]') from fuse_block('partial_update_cluster_dependency', 't') +---- +1 + +query III +select id, a, k from t order by id +---- +1 11 12 +2 20 21 + +statement ok +drop database partial_update_cluster_dependency From 62fda6f11f182679f39ccad74b3898215ce7e060 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:53:08 +0800 Subject: [PATCH 08/30] fix(storage): harden partial update index compatibility --- .../fuse/operations/computed_columns.rs | 54 +++- .../src/physical_plans/physical_mutation.rs | 85 +---- .../it/indexes/ngram_index/index_refresh.rs | 117 ++++++- .../fuse/operations/mutation/update.rs | 130 ++++++++ .../block_read_info.rs | 15 + .../common/table_meta/src/meta/current/mod.rs | 1 + .../common/table_meta/src/meta/v2/mod.rs | 1 + .../common/table_meta/src/meta/v2/segment.rs | 34 +- src/query/storages/fuse/src/fuse_part.rs | 17 + .../fuse/src/io/write/block_writer.rs | 22 +- .../fuse/src/io/write/bloom_index_writer.rs | 154 +++++++-- .../processors/transform_serialize_block.rs | 37 ++- .../mutator/replace_into_operation_agg.rs | 90 +++--- .../fuse/src/operations/table_index.rs | 236 ++++++-------- .../storages/fuse/src/pruning/bloom_pruner.rs | 302 ++++++++++++++---- .../column_oriented_block_prune.rs | 1 + .../fuse/src/table_functions/fuse_block.rs | 80 +++-- ...001_partial_update_cluster_dependency.test | 4 +- 18 files changed, 961 insertions(+), 419 deletions(-) diff --git a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs index eedd1f990a6a6..bd11215955733 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs @@ -155,7 +155,7 @@ async fn test_computed_column() -> anyhow::Result<()> { } #[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_with_virtual_column_and_change_tracking() -> anyhow::Result<()> { +async fn test_partial_update_rejects_virtual_column_with_change_tracking() -> anyhow::Result<()> { let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; let db = fixture.default_db_name(); let table_name = fixture.default_table_name(); @@ -194,7 +194,7 @@ async fn test_partial_update_with_virtual_column_and_change_tracking() -> anyhow let rows = fixture .execute_query(&format!( "select count(*) from fuse_block('{db}', '{table_name}') \ - where column_groups != parse_json('[]')" + where column_groups = parse_json('[]')" )) .await?; assert_eq!(query_count(rows).await?, 1); @@ -203,7 +203,55 @@ async fn test_partial_update_with_virtual_column_and_change_tracking() -> anyhow } #[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_rejects_indirect_cluster_key_update() -> anyhow::Result<()> { +async fn test_partial_update_rejects_stored_computed_column_table() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} \ + (id int, value int, stored_value bigint as (value + 1) stored) engine=fuse \ + enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name}(id, value) values (1, 10), (2, 20)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11 and stored_value = 12) \ + or (id = 2 and value = 20 and stored_value = 21)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + let rows = fixture + .execute_query(&format!( + "select count(*) from fuse_block('{db}', '{table_name}') \ + where column_groups = parse_json('[]')" + )) + .await?; + assert_eq!(query_count(rows).await?, 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_rejects_computed_column_table() -> anyhow::Result<()> { let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; let db = fixture.default_db_name(); let table_name = fixture.default_table_name(); diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index f41d868c3b560..9e5f0969356c9 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -21,7 +21,6 @@ use databend_common_catalog::plan::NUM_ROW_ID_PREFIX_BITS; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; use databend_common_expression::ConstantFolder; use databend_common_expression::DataField; @@ -99,39 +98,6 @@ struct PartialUpdateInfo { updated_field_indices: Vec, } -fn expand_computed_column_dependencies( - ctx: Arc, - table_schema: &TableSchema, - computed_schema: &DataSchemaRef, - dependency_ids: &mut BTreeSet, -) -> Result<()> { - loop { - let previous_len = dependency_ids.len(); - for field in table_schema.fields() { - if !dependency_ids.contains(&field.column_id()) { - continue; - } - let computed_expr = match field.computed_expr() { - Some(ComputedExpr::Stored(expr)) | Some(ComputedExpr::Virtual(expr)) => expr, - None => continue, - }; - let expr = parse_computed_field_index_expr( - ctx.clone(), - computed_schema.clone(), - computed_expr, - )?; - dependency_ids.extend( - expr.column_refs() - .into_iter() - .map(|(index, _)| table_schema.field(index).column_id()), - ); - } - if dependency_ids.len() == previous_len { - return Ok(()); - } - } -} - #[allow(clippy::too_many_arguments)] fn build_partial_update_info( ctx: Arc, @@ -147,38 +113,20 @@ fn build_partial_update_info( || table.is_column_oriented() || !table.get_table_info().meta.indexes.is_empty() || update_list.is_empty() + || table + .schema() + .fields() + .iter() + .any(|field| field.computed_expr().is_some()) { return Ok(None); } let schema_with_stream = table.schema_with_stream(); - let table_schema = table.schema(); - let computed_schema: DataSchemaRef = Arc::new(table_schema.as_ref().into()); let mut updated_column_ids = update_list .keys() .map(|index| schema_with_stream.field(*index).column_id()) .collect::>(); - let mut computed_dependencies = BTreeSet::new(); - - for field in table_schema.fields() { - let Some(ComputedExpr::Stored(stored_expr)) = field.computed_expr() else { - continue; - }; - let expr = - parse_computed_field_index_expr(ctx.clone(), computed_schema.clone(), stored_expr)?; - let dependencies = expr - .column_refs() - .into_iter() - .map(|(index, _)| index) - .collect::>(); - if dependencies - .iter() - .any(|index| update_list.contains_key(index)) - { - updated_column_ids.insert(field.column_id()); - computed_dependencies.extend(dependencies); - } - } for stream_column in table.stream_columns() { updated_column_ids.insert(stream_column.column_id()); @@ -186,18 +134,13 @@ fn build_partial_update_info( if let Some(cluster_keys) = table.resolve_cluster_keys() { let cluster_exprs = parse_cluster_keys(ctx.clone(), Arc::new(table.clone()), cluster_keys)?; - let mut cluster_dependency_ids = cluster_exprs + let updates_cluster_key = cluster_exprs .iter() .flat_map(|expr| expr.column_refs()) - .map(|(index, _)| schema_with_stream.field(index).column_id()) - .collect::>(); - expand_computed_column_dependencies( - ctx.clone(), - table_schema.as_ref(), - &computed_schema, - &mut cluster_dependency_ids, - )?; - if !updated_column_ids.is_disjoint(&cluster_dependency_ids) { + .any(|(index, _)| { + updated_column_ids.contains(&schema_with_stream.field(index).column_id()) + }); + if updates_cluster_key { return Ok(None); } } @@ -235,11 +178,7 @@ fn build_partial_update_info( .flat_map(ScalarExpr::used_columns) .chain(direct_filter.iter().flat_map(ScalarExpr::used_columns)) .collect::(); - for field_index in update_list - .keys() - .copied() - .chain(computed_dependencies.into_iter()) - { + for field_index in update_list.keys().copied() { let Some(symbol) = find_symbol(field_index) else { return Ok(None); }; @@ -250,8 +189,6 @@ fn build_partial_update_info( let Some(symbol) = find_symbol(field_index) else { return Ok(None); }; - // STORED computed columns are regenerated from their dependencies and do not need - // their previous values. Stream columns do need their previous values. required_columns.insert(symbol); } } diff --git a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs index 4f208d640c523..17812bb1035ca 100644 --- a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; use std::sync::Arc; use databend_common_exception::Result; @@ -20,7 +21,9 @@ use databend_common_storages_fuse::io::read::bloom::block_filter_reader::load_bl use databend_query::sessions::TableContext; use databend_query::storages::index::filters::BlockFilter; use databend_query::test_kits::TestFixture; +use databend_query::test_kits::latest_default_block_meta; use databend_storages_common_io::ReadSettings; +use databend_storages_common_table_meta::meta::Versioned; use futures_util::StreamExt; #[tokio::test(flavor = "multi_thread")] @@ -44,7 +47,7 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { .execute_command("CREATE NGRAM INDEX idx2 ON default.t3(d);") .await?; - let block_filter_0 = get_block_filter(&fixture, &[ + let block_filter_0 = get_block_filter(&fixture, "default", "t3", &[ "Bloom(0)".to_string(), "Bloom(1)".to_string(), "Bloom(2)".to_string(), @@ -57,7 +60,7 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { fixture .execute_command("REFRESH NGRAM INDEX idx2 ON default.t3;") .await?; - let block_filter_1 = get_block_filter(&fixture, &[ + let block_filter_1 = get_block_filter(&fixture, "default", "t3", &[ "Bloom(0)".to_string(), "Bloom(1)".to_string(), "Bloom(2)".to_string(), @@ -88,7 +91,7 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { fixture .execute_command("REFRESH NGRAM INDEX idx2 ON default.t3;") .await?; - let block_filter_2 = get_block_filter(&fixture, &[ + let block_filter_2 = get_block_filter(&fixture, "default", "t3", &[ "Bloom(0)".to_string(), "Bloom(1)".to_string(), "Bloom(2)".to_string(), @@ -107,11 +110,111 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { Ok(()) } -async fn get_block_filter(fixture: &TestFixture, columns: &[String]) -> Result { +#[tokio::test(flavor = "multi_thread")] +async fn test_refresh_ngram_preserves_split_bloom_files() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, d string, e string) engine=fuse \ + bloom_index_columns='id,a' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values \ + (1, 10, 'aaaaaaaaaa', 'xxxxxxxxxx'), \ + (2, 20, 'bbbbbbbbbb', 'yyyyyyyyyy')" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + + let split_meta = latest_default_block_meta(&fixture).await?; + assert_eq!(split_meta.bloom_index_files.len(), 2); + let table = fixture.latest_default_table().await?; + let id_column_id = table.schema().field(0).column_id(); + let a_column_id = table.schema().field(1).column_id(); + let d_column_id = table.schema().field(2).column_id(); + let e_column_id = table.schema().field(3).column_id(); + let active_column_ids = split_meta + .bloom_index_files + .iter() + .flat_map(|file| file.active_column_ids.iter().copied()) + .collect::>(); + assert_eq!( + active_column_ids, + HashSet::from([id_column_id, a_column_id]) + ); + + let operator = DataOperator::instance().operator(); + let split_files = split_meta + .bloom_index_files + .iter() + .map(|file| (file.location.0.clone(), file.file_size)) + .collect::>(); + fixture + .execute_command(&format!( + "create ngram index idx_e on {db}.{table_name}(e) gram_size=4" + )) + .await?; + fixture + .execute_command(&format!("create ngram index idx_d on {db}.{table_name}(d)")) + .await?; + fixture + .execute_command(&format!("refresh ngram index idx_d on {db}.{table_name}")) + .await?; + + let refreshed_meta = latest_default_block_meta(&fixture).await?; + assert!(refreshed_meta.bloom_index_files.is_empty()); + let refreshed_location = refreshed_meta + .bloom_filter_index_location + .as_ref() + .expect("refreshed Bloom index location"); + assert_eq!(refreshed_location.1, BlockFilter::VERSION); + assert!( + split_files + .iter() + .all(|(path, _)| path != &refreshed_location.0) + ); + for (path, size) in &split_files { + assert_eq!(operator.stat(path).await?.content_length(), *size); + } + + let columns = [ + format!("Bloom({id_column_id})"), + format!("Bloom({a_column_id})"), + format!("Ngram({d_column_id})_3_1048576"), + format!("Ngram({e_column_id})_4_1048576"), + ]; + let block_filter = get_block_filter(&fixture, &db, &table_name, &columns).await?; + assert_eq!(block_filter.filters.len(), columns.len()); + for column in columns { + assert!(block_filter.filter_schema.index_of(&column).is_ok()); + } + + Ok(()) +} + +async fn get_block_filter( + fixture: &TestFixture, + database: &str, + table: &str, + columns: &[String], +) -> Result { let block = fixture - .execute_query( - "select bloom_filter_location, bloom_filter_size from fuse_block('default', 't3');", - ) + .execute_query(&format!( + "select bloom_filter_location, bloom_filter_size \ + from fuse_block('{database}', '{table}')" + )) .await? .next() .await diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 4fa943654d5cc..087279bf5155f 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -299,6 +299,136 @@ async fn test_partial_update_reads_legacy_bloom_schema() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_auto_fix_missing_split_bloom_index() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, value int) engine=fuse \ + bloom_index_columns='id,value' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10), (3, 30)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + let value_column_id = table.schema().field_with_name("value")?.column_id(); + let block_meta = latest_default_block_meta(&fixture).await?; + assert_eq!(block_meta.bloom_index_files.len(), 2); + let missing_file = block_meta + .bloom_index_files + .iter() + .find(|file| file.active_column_ids.contains(&value_column_id)) + .unwrap(); + let existing_file = block_meta + .bloom_index_files + .iter() + .find(|file| file.location != missing_file.location) + .unwrap(); + let existing_data = operator.read(&existing_file.location.0).await?.to_bytes(); + operator.delete(&missing_file.location.0).await?; + assert!(!operator.exists(&missing_file.location.0).await?); + + fixture + .execute_command("set enable_auto_fix_missing_bloom_index = 1") + .await?; + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where value = 20" + )) + .await?; + assert_eq!(query_count(rows).await?, 0); + assert!(operator.exists(&missing_file.location.0).await?); + assert_eq!( + operator.read(&existing_file.location.0).await?.to_bytes(), + existing_data + ); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where id = 1 and value = 11" + )) + .await?; + assert_eq!(query_count(rows).await?, 1); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_auto_fix_missing_legacy_bloom_index_from_split_data() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, flag boolean) engine=fuse \ + bloom_index_columns='id' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, true), (3, false)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set flag = false where flag" + )) + .await?; + + let block_meta = latest_default_block_meta(&fixture).await?; + assert_eq!(block_meta.column_groups.len(), 2); + assert!(block_meta.bloom_index_files.is_empty()); + let bloom_location = block_meta.bloom_filter_index_location.as_ref().unwrap(); + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + operator.delete(&bloom_location.0).await?; + assert!(!operator.exists(&bloom_location.0).await?); + + fixture + .execute_command("set enable_auto_fix_missing_bloom_index = 1") + .await?; + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where id = 2" + )) + .await?; + assert_eq!(query_count(rows).await?, 0); + assert!(operator.exists(&bloom_location.0).await?); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where id = 1 and not flag" + )) + .await?; + assert_eq!(query_count(rows).await?, 1); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs index a5422703484d4..45e7265b2eba7 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs @@ -13,10 +13,12 @@ // limitations under the License. use std::collections::HashMap; +use std::collections::HashSet; use databend_common_expression::ColumnId; use crate::meta::BlockMeta; +use crate::meta::ColumnGroupFileMeta; use crate::meta::ColumnMeta; use crate::meta::Compression; @@ -25,16 +27,29 @@ pub struct BlockReadInfo { pub location: String, pub row_count: u64, pub col_metas: HashMap, + /// Active physical files that make up this logical block. + /// + /// An empty vector is the legacy single-file representation described by `location` and + /// `col_metas`. + #[serde(default)] + pub column_groups: Vec, pub compression: Compression, pub block_size: u64, } impl From<&BlockMeta> for BlockReadInfo { fn from(meta: &BlockMeta) -> Self { + let column_groups = if meta.column_groups.is_empty() { + vec![] + } else { + let projected_column_ids = meta.col_metas.keys().copied().collect::>(); + meta.project_column_groups(&projected_column_ids) + }; BlockReadInfo { location: meta.location.0.clone(), row_count: meta.row_count, col_metas: meta.col_metas.clone(), + column_groups, compression: meta.compression, block_size: meta.block_size, } diff --git a/src/query/storages/common/table_meta/src/meta/current/mod.rs b/src/query/storages/common/table_meta/src/meta/current/mod.rs index f0b1e3dc5c161..7ea5f05c4e522 100644 --- a/src/query/storages/common/table_meta/src/meta/current/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/current/mod.rs @@ -16,6 +16,7 @@ pub use v0::ColumnMeta as SingleColumnMeta; pub use v2::AdditionalStatsMeta; pub use v2::BlockMeta; pub use v2::BloomIndexFileMeta; +pub use v2::BloomIndexLayout; pub use v2::ClusterStatistics; pub use v2::ColumnGroupFileMeta; pub use v2::ColumnMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/mod.rs b/src/query/storages/common/table_meta/src/meta/v2/mod.rs index d3c4c66c56c64..bfafc7f1ab1de 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/mod.rs @@ -20,6 +20,7 @@ mod table_snapshot_statistics; pub use segment::BlockMeta; pub use segment::BloomIndexFileMeta; +pub use segment::BloomIndexLayout; pub use segment::ColumnGroupFileMeta; pub use segment::ColumnMeta; pub use segment::DraftVirtualBlockMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index d984692c296a0..1da5904c36597 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -198,6 +198,18 @@ pub struct BloomIndexFileMeta { pub file_size: u64, } +/// Borrowed view of the physical Bloom-index layout used by a logical block. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BloomIndexLayout<'a> { + Legacy { + location: &'a Location, + file_size: u64, + }, + Split { + files: &'a [BloomIndexFileMeta], + }, +} + /// Meta information of a block /// Part of and kept inside the [SegmentInfo] #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -215,7 +227,11 @@ pub struct BlockMeta { #[serde(default)] pub column_groups: Vec, pub cluster_stats: Option, - /// location of data block + /// Compatibility anchor for this logical block's data. + /// + /// In the legacy layout this is the only data-file location. In a split layout it identifies + /// the newest column-group file and does not cover the other active files; use + /// [`Self::physical_column_groups`] or [`Self::data_file_locations`] for physical reads. pub location: Location, /// location of bloom filter index pub bloom_filter_index_location: Option, @@ -298,6 +314,22 @@ impl BlockMeta { self.compression } + /// Normalize optional legacy and split Bloom metadata into one physical-layout view. + pub fn bloom_index_layout(&self) -> Option> { + if self.bloom_index_files.is_empty() { + self.bloom_filter_index_location + .as_ref() + .map(|location| BloomIndexLayout::Legacy { + location, + file_size: self.bloom_filter_index_size, + }) + } else { + Some(BloomIndexLayout::Split { + files: &self.bloom_index_files, + }) + } + } + /// Active physical data files referenced by this logical block. pub fn data_file_locations(&self) -> impl Iterator { self.column_groups diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 52666f345ab3f..74e4d7d09daa7 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -33,6 +33,7 @@ use databend_common_expression::Scalar; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BloomIndexFileMeta; +use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; @@ -102,6 +103,22 @@ impl PartInfo for FuseBlockPartInfo { } impl FuseBlockPartInfo { + /// Normalize optional legacy and split Bloom metadata into one physical-layout view. + pub fn bloom_index_layout(&self) -> Option> { + if self.bloom_index_files.is_empty() { + self.bloom_filter_index_location + .as_ref() + .map(|location| BloomIndexLayout::Legacy { + location, + file_size: self.bloom_filter_index_size, + }) + } else { + Some(BloomIndexLayout::Split { + files: &self.bloom_index_files, + }) + } + } + #[allow(clippy::too_many_arguments)] pub fn create( location: String, diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 73156e3c02605..c33f818e3233e 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -54,6 +54,7 @@ use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BloomIndexFileMeta; +use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; @@ -492,24 +493,27 @@ impl BlockBuilder { block_meta.create_on = Some(Utc::now()); if !invalidated_bloom_column_ids.is_empty() { - let mut bloom_index_files = if origin.bloom_index_files.is_empty() { - let mut files = Vec::new(); - if let Some(location) = &origin.bloom_filter_index_location { + let mut bloom_index_files = match origin.bloom_index_layout() { + None => Vec::new(), + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { let active_column_ids = legacy_bloom_column_ids.ok_or_else(|| { ErrorCode::Internal("legacy Bloom file columns were not loaded") })?; if !active_column_ids.is_empty() { - files.push(BloomIndexFileMeta { + vec![BloomIndexFileMeta { active_column_ids: active_column_ids.to_vec(), location: location.clone(), format_version: location.1, - file_size: origin.bloom_filter_index_size, - }); + file_size, + }] + } else { + Vec::new() } } - files - } else { - origin.bloom_index_files.clone() + Some(BloomIndexLayout::Split { files }) => files.to_vec(), }; for file in &mut bloom_index_files { file.active_column_ids diff --git a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs index 6a1447a9d0089..865749d0fa206 100644 --- a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs @@ -14,10 +14,12 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; use databend_common_catalog::plan::Projection; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; @@ -31,6 +33,7 @@ use databend_storages_common_index::BloomIndexType; use databend_storages_common_index::NgramArgs; use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; +use databend_storages_common_table_meta::meta::BloomIndexFileMeta; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; @@ -38,6 +41,7 @@ use databend_storages_common_table_meta::table::TableCompression; use opendal::Buffer; use opendal::Operator; +use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; use crate::io::BlockReader; @@ -139,11 +143,19 @@ pub struct BloomIndexRebuilder { } impl BloomIndexRebuilder { - pub async fn bloom_index_state_from_block_meta( - &self, - bloom_index_location: &Location, - block_read_info: &BlockReadInfo, - ) -> Result> { + pub(crate) fn validate_rebuild_version(bloom_index_location: &Location) -> Result<()> { + if bloom_index_location.1 != BlockFilter::VERSION { + return Err(ErrorCode::DeprecatedIndexFormat(format!( + "cannot rebuild Bloom index {:?} in legacy format {} with current format {}", + bloom_index_location, + bloom_index_location.1, + BlockFilter::VERSION + ))); + } + Ok(()) + } + + async fn read_data_block(&self, block_read_info: &BlockReadInfo) -> Result { let ctx = self.table_ctx.clone(); let projection = @@ -158,29 +170,77 @@ impl BloomIndexRebuilder { )?; let settings = ReadSettings::from_ctx(&self.table_ctx)?; + let column_groups = if block_read_info.column_groups.is_empty() { + vec![FuseColumnGroupPartInfo { + location: block_read_info.location.clone(), + columns_meta: block_read_info.col_metas.clone(), + }] + } else { + block_read_info + .column_groups + .iter() + .map(|group| FuseColumnGroupPartInfo { + location: group.location.0.clone(), + columns_meta: group + .active_column_ids + .iter() + .filter_map(|column_id| { + group + .leaf_column_metas + .get(column_id) + .map(|column_meta| (*column_id, column_meta.clone())) + }) + .collect(), + }) + .collect() + }; let merge_io_read_result = block_reader - .read_columns_data_by_merge_io( - &settings, - &block_read_info.location, - &block_read_info.col_metas, - &None, - ) + .read_column_groups_data_by_merge_io(&settings, &column_groups, &None) .await?; - let data_block = block_reader.deserialize_chunks_with_meta( - block_read_info, - &self.storage_format, - merge_io_read_result, - )?; + let column_chunks = merge_io_read_result.columns_chunks()?; + match self.storage_format { + FuseStorageFormat::Parquet => block_reader.deserialize_column_groups( + block_read_info.row_count as usize, + &column_groups, + column_chunks, + &block_read_info.compression, + None, + ), + FuseStorageFormat::Unsupported => Err(crate::unsupported_storage_format_error()), + } + } - assert_eq!(bloom_index_location.1, BlockFilter::VERSION); + fn bloom_index_state_from_data_block( + &self, + bloom_index_location: &Location, + data_block: &DataBlock, + active_column_ids: Option<&HashSet>, + ) -> Result> { + Self::validate_rebuild_version(bloom_index_location)?; + let bloom_columns_map = self + .bloom_columns_map + .iter() + .filter(|(_, field)| { + active_column_ids.is_none_or(|ids| ids.contains(&field.column_id())) + }) + .map(|(index, field)| (*index, field.clone())) + .collect(); + let ngram_args = self + .ngram_args + .iter() + .filter(|arg| { + active_column_ids.is_none_or(|ids| ids.contains(&arg.field().column_id())) + }) + .cloned() + .collect::>(); let mut builder = BloomIndexBuilder::create( self.table_ctx.get_function_context()?, self.bloom_index_type, - self.bloom_columns_map.clone(), - &self.ngram_args, + bloom_columns_map, + &ngram_args, )?; - builder.add_block(&data_block)?; + builder.add_block(data_block)?; let maybe_bloom_index = builder.finalize()?; match maybe_bloom_index { @@ -191,4 +251,58 @@ impl BloomIndexRebuilder { ))), } } + + pub async fn bloom_index_state_from_block_meta( + &self, + bloom_index_location: &Location, + block_read_info: &BlockReadInfo, + ) -> Result> { + let data_block = self.read_data_block(block_read_info).await?; + self.bloom_index_state_from_data_block(bloom_index_location, &data_block, None) + } + + pub(crate) async fn bloom_index_states_from_block_meta( + &self, + bloom_index_files: &[BloomIndexFileMeta], + block_read_info: &BlockReadInfo, + ) -> Result>> { + let data_block = self.read_data_block(block_read_info).await?; + let mut states = Vec::with_capacity(bloom_index_files.len()); + for file in bloom_index_files { + let active_column_ids = file + .active_column_ids + .iter() + .copied() + .collect::>(); + let Some((state, _)) = self.bloom_index_state_from_data_block( + &file.location, + &data_block, + Some(&active_column_ids), + )? + else { + return Ok(None); + }; + states.push(state); + } + Ok(Some(states)) + } +} + +#[cfg(test)] +mod tests { + use databend_common_exception::ErrorCode; + use databend_storages_common_index::filters::BlockFilter; + use databend_storages_common_table_meta::meta::Versioned; + + use super::BloomIndexRebuilder; + + #[test] + fn test_validate_rebuild_version_rejects_legacy_format() { + let legacy_location = ("legacy-bloom".to_string(), BlockFilter::VERSION - 1); + let err = BloomIndexRebuilder::validate_rebuild_version(&legacy_location).unwrap_err(); + assert_eq!(err.code(), ErrorCode::DEPRECATED_INDEX_FORMAT); + + let current_location = ("current-bloom".to_string(), BlockFilter::VERSION); + BloomIndexRebuilder::validate_rebuild_version(¤t_location).unwrap(); + } } diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index 9640446ae83b3..b8dc37b8cd3b1 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -39,6 +39,7 @@ use databend_common_storage::MutationStatus; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use opendal::Operator; @@ -277,29 +278,27 @@ impl TransformSerializeBlock { } fn needs_legacy_bloom_meta(&self, origin: &BlockMeta) -> bool { - origin.bloom_index_files.is_empty() - && origin.bloom_filter_index_location.is_some() - && self.updated_field_indices.as_ref().is_some_and(|indices| { - indices.iter().any(|index| { - BloomIndex::supported_type( - self.block_builder.source_schema.field(*index).data_type(), - ) - }) + matches!( + origin.bloom_index_layout(), + Some(BloomIndexLayout::Legacy { .. }) + ) && self.updated_field_indices.as_ref().is_some_and(|indices| { + indices.iter().any(|index| { + BloomIndex::supported_type( + self.block_builder.source_schema.field(*index).data_type(), + ) }) + }) } async fn load_legacy_bloom_column_ids(&self, origin: &BlockMeta) -> Result> { - let location = origin - .bloom_filter_index_location - .as_ref() - .ok_or_else(|| ErrorCode::Internal("legacy Bloom location is missing"))?; - let meta = load_index_meta( - self.dal.clone(), - &location.0, - origin.bloom_filter_index_size, - None, - ) - .await?; + let Some(BloomIndexLayout::Legacy { + location, + file_size, + }) = origin.bloom_index_layout() + else { + return Err(ErrorCode::Internal("legacy Bloom location is missing")); + }; + let meta = load_index_meta(self.dal.clone(), &location.0, file_size, None).await?; let stored_filter_names = meta .columns .iter() diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 91d6078715d5d..11b52c6419e79 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -47,6 +47,7 @@ use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BlockSlotDescription; +use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; @@ -707,49 +708,58 @@ impl AggregationContext { bloom_on_conflict_field_index: &[FieldIndex], ) -> Result>>>> { // different block may have different version of bloom filter index - let (block_filter, col_names) = if block_meta.bloom_index_files.is_empty() { - let Some(location) = &block_meta.bloom_filter_index_location else { - return Ok(None); - }; - let col_names = bloom_on_conflict_field_index - .iter() - .map(|idx| { - BloomIndex::build_filter_bloom_name( - location.1, - &self.on_conflict_fields[*idx].table_field, + let (block_filter, col_names) = match block_meta.bloom_index_layout() { + None => return Ok(None), + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { + let col_names = bloom_on_conflict_field_index + .iter() + .map(|idx| { + BloomIndex::build_filter_bloom_name( + location.1, + &self.on_conflict_fields[*idx].table_field, + ) + }) + .collect::>>()?; + let block_filter = location + .read_block_filter( + self.data_accessor.clone(), + &self.read_settings, + &col_names, + file_size, ) - }) - .collect::>>()?; - let block_filter = location - .read_block_filter( - self.data_accessor.clone(), + .await?; + (block_filter, col_names) + } + Some(BloomIndexLayout::Split { files }) => { + let fields = bloom_on_conflict_field_index + .iter() + .map(|idx| self.on_conflict_fields[*idx].table_field.clone()) + .collect::>(); + let Some(filter) = read_multi_file_block_filter( + &self.data_accessor, &self.read_settings, - &col_names, - block_meta.bloom_filter_index_size, + &fields, + &[], + files, ) - .await?; - (block_filter, col_names) - } else { - let fields = bloom_on_conflict_field_index - .iter() - .map(|idx| self.on_conflict_fields[*idx].table_field.clone()) - .collect::>(); - let Some(block_filter) = read_multi_file_block_filter( - &self.data_accessor, - &self.read_settings, - &fields, - &[], - &block_meta.bloom_index_files, - ) - .await? - else { - return Ok(None); - }; - let col_names = fields - .iter() - .map(|field| BloomIndex::build_filter_bloom_name(BlockFilter::VERSION, field)) - .collect::>>()?; - (block_filter, col_names) + .await? + else { + return Ok(None); + }; + // REPLACE carries current digest hashes instead of scalar values, so it cannot + // safely evaluate legacy V2 filters. Keep the block in that compatibility case. + if filter.format_version != BlockFilter::VERSION { + return Ok(None); + } + let col_names = fields + .iter() + .map(|field| BloomIndex::build_filter_bloom_name(filter.format_version, field)) + .collect::>>()?; + (filter.block_filter, col_names) + } }; // reorder the filter according to the order of bloom_on_conflict_field diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index fe7db597c3bff..61434b2dafec1 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -28,9 +28,8 @@ use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; -use databend_common_expression::TableDataType; +use databend_common_expression::FieldIndex; use databend_common_expression::TableField; -use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::local_block_meta_serde; use databend_common_meta_app::schema::TableIndex; @@ -42,10 +41,8 @@ use databend_common_pipeline::sources::AsyncSourcer; use databend_common_pipeline_transforms::AsyncTransform; use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::executor::physical_plans::MutationKind; -use databend_storages_common_cache::CacheAccessor; -use databend_storages_common_cache::CacheManager; -use databend_storages_common_cache::FilterImpl; use databend_storages_common_cache::LoadParams; +use databend_storages_common_index::BloomIndexType; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; @@ -55,17 +52,15 @@ use databend_storages_common_table_meta::meta::RawBlockHLL; use databend_storages_common_table_meta::meta::SegmentStatistics; use databend_storages_common_table_meta::meta::SingleColumnMeta; use databend_storages_common_table_meta::meta::Statistics; -use databend_storages_common_table_meta::meta::Versioned; use log::info; use opendal::Operator; +use uuid::Uuid; use crate::FuseStorageFormat; use crate::FuseTable; use crate::index::BloomIndex; use crate::index::BloomIndexBuilder; use crate::index::NgramArgs; -use crate::index::filters::BlockFilter; -use crate::index::filters::Filter; use crate::io::BlockReader; use crate::io::BlockWriter; use crate::io::BloomIndexState; @@ -73,7 +68,6 @@ use crate::io::MetaReaders; use crate::io::SpatialIndexBuilder; use crate::io::TableMetaLocationGenerator; use crate::io::VectorIndexBuilder; -use crate::io::read::bloom::block_filter_reader::load_bloom_filter_by_columns; use crate::io::read::bloom::block_filter_reader::load_index_meta; use crate::io::read::load_spatial_index_meta; use crate::io::read::load_vector_index_meta; @@ -118,13 +112,26 @@ pub async fn do_refresh_table_index( info!("Start refresh {} index {}", index_type, index_name); let table_schema = fuse_table.schema(); + let table_meta = &fuse_table.get_table_info().meta; + let index_arg = build_refresh_index_arg( + fuse_table, + &index_name, + &index_type, + table_meta, + &index_schema, + &table_schema, + )?; - // Collect field indices used by index. - let mut field_indices = Vec::with_capacity(index_schema.fields.len()); - for field in &index_schema.fields { - let field_index = table_schema.index_of(field.name())?; - field_indices.push(field_index); - } + let field_indices = match &index_arg { + // A refresh publishes one new current-format Bloom file. Rebuild every configured Bloom + // and Ngram filter from the logical row so old V2/V3 encodings are never relabeled V4. + RefreshIndexArg::Ngram(arg) => arg.source_field_indices.clone(), + RefreshIndexArg::Vector(_) | RefreshIndexArg::Spatial(_) => index_schema + .fields + .iter() + .map(|field| table_schema.index_of(field.name())) + .collect::>>()?, + }; // Read data here to keep the order of blocks in segment. let projection = Projection::Columns(field_indices); @@ -132,15 +139,14 @@ pub async fn do_refresh_table_index( let block_reader = fuse_table.create_block_reader(ctx.clone(), projection, false)?; let meta_locations = fuse_table.meta_location_generator().clone(); - let segment_reader = MetaReaders::segment_info_reader(fuse_table.get_operator(), table_schema); + let segment_reader = + MetaReaders::segment_info_reader(fuse_table.get_operator(), table_schema.clone()); if snapshot.segments.is_empty() { return Ok(0); } let operator = fuse_table.get_operator_ref(); - let table_meta = &fuse_table.get_table_info().meta; - let index_arg = build_refresh_index_arg(&index_name, &index_type, table_meta, &index_schema)?; let target_segments = segment_locs.map(|locs| locs.into_iter().collect::>()); // Read the segment infos and collect the block metas that need to generate the index. @@ -219,10 +225,10 @@ pub async fn do_refresh_table_index( NgramIndexTransform::new( ctx.clone(), operator.clone(), - settings, - ngram_index_arg.index_ngram_args.clone(), - ngram_index_arg.ngram_index_names.clone(), - ngram_index_arg.existing_names_prefix.clone(), + ngram_index_arg.bloom_index_type, + ngram_index_arg.bloom_columns_map.clone(), + ngram_index_arg.ngram_args.clone(), + meta_locations.clone(), ) }); } @@ -295,21 +301,18 @@ pub async fn do_refresh_table_index( // build the index arguments used for refresh fn build_refresh_index_arg( + fuse_table: &FuseTable, index_name: &String, index_type: &TableIndexType, table_meta: &TableMeta, index_schema: &TableSchemaRef, + table_schema: &TableSchemaRef, ) -> Result { match index_type { TableIndexType::Ngram => { let index_ngram_args = FuseTable::create_ngram_index_args(&table_meta.indexes, index_schema, false)?; - let existing_names_prefix = index_ngram_args - .iter() - .map(|arg| format!("Ngram({})", arg.column_id())) - .collect::>(); - let ngram_index_names = index_ngram_args .iter() .map(|arg| { @@ -321,10 +324,26 @@ fn build_refresh_index_arg( }) .collect::>(); + let source_schema: TableSchemaRef = + table_schema.remove_virtual_computed_fields().into(); + let source_field_indices = source_schema + .fields() + .iter() + .map(|field| table_schema.index_of(field.name())) + .collect::>>()?; + let ngram_arg = RefreshNgramIndexArg { - index_ngram_args, ngram_index_names, - existing_names_prefix, + source_field_indices, + bloom_index_type: fuse_table.bloom_index_type(), + bloom_columns_map: fuse_table + .bloom_index_cols() + .bloom_index_fields(source_schema.clone(), BloomIndex::supported_type)?, + ngram_args: FuseTable::create_ngram_index_args( + &table_meta.indexes, + &source_schema, + false, + )?, }; Ok(RefreshIndexArg::Ngram(ngram_arg)) } @@ -419,37 +438,30 @@ async fn check_ngram_index_generated( stats: Option>, ngram_index_arg: &RefreshNgramIndexArg, ) -> Result> { - let index_location = TableMetaLocationGenerator::gen_bloom_index_location_from_block_location( - &block_meta.location.0, - ); - // only generate bloom index if it is not exist. - let index_columns = if let Ok(content_length) = operator - .stat(&index_location) - .await - .map(|meta| meta.content_length()) - { - let bloom_index_meta = - load_index_meta(operator.clone(), &index_location, content_length, None).await?; - - let mut all_generated = true; - for ngram_index_name in &ngram_index_arg.ngram_index_names { - if !bloom_index_meta - .columns - .iter() - .any(|(column_name, _)| column_name == ngram_index_name) + // A partial update may split active Bloom filters across several immutable files. Such a + // block must be consolidated when adding Ngram filters, so do not mistake the Bloom file + // derived from the newest data group for the complete logical index. + if block_meta.bloom_index_files.is_empty() { + if let Some((index_path, _)) = &block_meta.bloom_filter_index_location { + if let Ok(content_length) = operator + .stat(index_path) + .await + .map(|meta| meta.content_length()) { - all_generated = false; - break; + let bloom_index_meta = + load_index_meta(operator.clone(), index_path, content_length, None).await?; + + if ngram_index_arg.ngram_index_names.iter().all(|name| { + bloom_index_meta + .columns + .iter() + .any(|(column_name, _)| column_name == name) + }) { + return Ok(None); + } } } - if all_generated { - return Ok(None); - } - - Some(bloom_index_meta.columns.clone()) - } else { - None - }; + } let ngram_index_meta = RefreshIndexMeta { index: BlockMetaIndex { segment_idx, @@ -460,7 +472,7 @@ async fn check_ngram_index_generated( .as_ref() .and_then(|v| v.block_hlls.get(block_idx)) .cloned(), - index_columns, + index_columns: None, index_meta: None, }; Ok(Some(ngram_index_meta)) @@ -683,28 +695,28 @@ impl AsyncSource for IndexSource { pub struct NgramIndexTransform { ctx: Arc, operator: Operator, - settings: ReadSettings, - index_ngram_args: Vec, - ngram_index_names: Vec, - existing_names_prefix: Vec, + bloom_index_type: BloomIndexType, + bloom_columns_map: BTreeMap, + ngram_args: Vec, + meta_locations: TableMetaLocationGenerator, } impl NgramIndexTransform { pub fn new( ctx: Arc, operator: Operator, - settings: ReadSettings, - index_ngram_args: Vec, - ngram_index_names: Vec, - existing_names_prefix: Vec, + bloom_index_type: BloomIndexType, + bloom_columns_map: BTreeMap, + ngram_args: Vec, + meta_locations: TableMetaLocationGenerator, ) -> Self { Self { ctx, operator, - settings, - index_ngram_args, - ngram_index_names, - existing_names_prefix, + bloom_index_type, + bloom_columns_map, + ngram_args, + meta_locations, } } } @@ -719,79 +731,27 @@ impl AsyncTransform for NgramIndexTransform { index, block_meta, column_hlls, - index_columns, + index_columns: _, index_meta: _index_meta, } = data_block .get_meta() .and_then(RefreshIndexMeta::downcast_ref_from) .unwrap(); - let index_path = TableMetaLocationGenerator::gen_bloom_index_location_from_block_location( - &block_meta.location.0, - ); - let mut new_block_meta = Arc::unwrap_or_clone(block_meta.clone()); - let index_location = (index_path.clone(), BlockFilter::VERSION); let mut builder = BloomIndexBuilder::create( self.ctx.get_function_context()?, - databend_storages_common_index::BloomIndexType::default(), - BTreeMap::new(), - &self.index_ngram_args, + self.bloom_index_type, + self.bloom_columns_map.clone(), + &self.ngram_args, )?; builder.add_block(&data_block)?; - if let Some(new_ngram_index) = builder.finalize()? { - let mut old_ngram_names = Vec::new(); - let bloom_index = if let Some(old_index_columns) = index_columns { - let mut index_columns = Vec::with_capacity(old_index_columns.len()); - for column in old_index_columns { - let name = column.0.to_string(); - if self - .existing_names_prefix - .iter() - .any(|name_prefix| name.starts_with(name_prefix)) - { - old_ngram_names.push(name); - continue; - } - index_columns.push(name); - } - - let mut block_filter = load_bloom_filter_by_columns( - self.operator.clone(), - &self.settings, - &index_columns, - &index_path, - block_meta.bloom_filter_index_size, - ) - .await?; - let new_ngram_columns = new_ngram_index.serialize_to_data_block()?.take_columns(); - for new_ngram_column in new_ngram_columns { - let (new_filter, _) = FilterImpl::from_bytes( - unsafe { new_ngram_column.index_unchecked(0) } - .as_binary() - .unwrap(), - )?; - block_filter.filters.push(Arc::new(new_filter)); - } - - let mut new_filter_schema = TableSchema::clone(&block_filter.filter_schema); - for ngram_index_name in &self.ngram_index_names { - new_filter_schema - .add_columns(&[TableField::new(ngram_index_name, TableDataType::Binary)])?; - } - block_filter.filter_schema = Arc::new(new_filter_schema); - - BloomIndex::from_filter_block( - self.ctx.get_function_context()?, - block_filter.filter_schema, - block_filter.filters, - index_location.1, - )? - } else { - new_ngram_index - }; + if let Some(bloom_index) = builder.finalize()? { + let index_location = self + .meta_locations + .block_bloom_index_location(&Uuid::now_v7()); let state = BloomIndexState::from_bloom_index(&bloom_index, index_location)?; new_block_meta.bloom_filter_index_location = Some(state.location.clone()); @@ -799,16 +759,6 @@ impl AsyncTransform for NgramIndexTransform { new_block_meta.bloom_filter_index_size = state.size(); new_block_meta.ngram_filter_index_size = state.ngram_size(); BlockWriter::write_down_bloom_index_state(&self.operator, Some(state)).await?; - - // remove old bloom index meta and filter - if let Some(cache) = CacheManager::instance().get_bloom_index_meta_cache() { - cache.evict(&index_path); - } - if let Some(cache) = CacheManager::instance().get_bloom_index_filter_cache() { - for old_ngram_name in old_ngram_names { - cache.evict(&format!("{index_path}-{}", old_ngram_name)); - } - } } else { return Err(ErrorCode::RefreshIndexError( "Refresh Ngram index failed".to_string(), @@ -1060,9 +1010,11 @@ enum RefreshIndexArg { } struct RefreshNgramIndexArg { - index_ngram_args: Vec, ngram_index_names: Vec, - existing_names_prefix: Vec, + source_field_indices: Vec, + bloom_index_type: BloomIndexType, + bloom_columns_map: BTreeMap, + ngram_args: Vec, } struct RefreshVectorIndexArg { diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 7194fd37f9874..2cb55f5afbfa8 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -35,6 +35,7 @@ use databend_storages_common_index::NgramArgs; use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BloomIndexFileMeta; +use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::Versioned; @@ -95,47 +96,41 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( } } - let bloom_index = if part.bloom_index_files.is_empty() { - let Some(index_location) = part.bloom_filter_index_location.as_ref() else { - return Ok(false); - }; - let index_columns = result - .bloom_fields - .iter() - .map(|field| BloomIndex::build_filter_bloom_name(index_location.1, field)) - .collect::>>()?; - let filter = index_location - .read_block_filter( - dal.clone(), - settings, - &index_columns, - part.bloom_filter_index_size, - ) - .await?; - BloomIndex::from_filter_block( - func_ctx.clone(), - filter.filter_schema, - filter.filters, - index_location.1, - )? - } else { - let Some(filter) = read_multi_file_block_filter( - dal, - settings, - &result.bloom_fields, - &[], - &part.bloom_index_files, - ) - .await? - else { - return Ok(false); - }; - BloomIndex::from_filter_block( - func_ctx.clone(), - filter.filter_schema, - filter.filters, - BlockFilter::VERSION, - )? + let bloom_index = match part.bloom_index_layout() { + None => return Ok(false), + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { + let index_columns = result + .bloom_fields + .iter() + .map(|field| BloomIndex::build_filter_bloom_name(location.1, field)) + .collect::>>()?; + let filter = location + .read_block_filter(dal.clone(), settings, &index_columns, file_size) + .await?; + BloomIndex::from_filter_block( + func_ctx.clone(), + filter.filter_schema, + filter.filters, + location.1, + )? + } + Some(BloomIndexLayout::Split { files }) => { + let Some(filter) = + read_multi_file_block_filter(dal, settings, &result.bloom_fields, &[], files) + .await? + else { + return Ok(false); + }; + BloomIndex::from_filter_block( + func_ctx.clone(), + filter.block_filter.filter_schema, + filter.block_filter.filters, + filter.format_version, + )? + } }; let like_scalar_map = HashMap::new(); @@ -151,23 +146,53 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( )? == FilterEvalResult::MustFalse) } +pub(crate) struct MultiFileBlockFilter { + pub block_filter: BlockFilter, + pub format_version: u64, +} + +fn multi_file_filter_version( + index_fields: &[TableField], + index_files: &[BloomIndexFileMeta], +) -> Option { + let mut has_v2 = false; + let mut has_digest_encoding = false; + for file in index_files { + if !index_fields + .iter() + .any(|field| file.active_column_ids.contains(&field.column_id())) + { + continue; + } + if file.format_version == 2 { + has_v2 = true; + } else { + has_digest_encoding = true; + } + } + match (has_v2, has_digest_encoding) { + (true, true) => None, + (true, false) => Some(2), + _ => Some(BlockFilter::VERSION), + } +} + pub(crate) async fn read_multi_file_block_filter( dal: &Operator, settings: &ReadSettings, index_fields: &[TableField], ngram_args: &[NgramArgs], index_files: &[BloomIndexFileMeta], -) -> Result> { +) -> Result> { + let Some(format_version) = multi_file_filter_version(index_fields, index_files) else { + // V2 filters use legacy scalar encoding and cannot be evaluated together with the digest + // encoding of newer files. Keeping the block is the safe compatibility behavior. + return Ok(None); + }; let mut merged_fields = Vec::new(); let mut merged_filters = Vec::new(); for file in index_files { - // V2 filters use legacy scalar encoding and cannot be mixed with the digest encoding of - // current files. Keeping the block is the safe compatibility behavior. - if file.format_version == 2 { - return Ok(None); - } - let mut rename = HashMap::new(); let mut columns = Vec::new(); for field in index_fields { @@ -184,7 +209,7 @@ pub(crate) async fn read_multi_file_block_filter( columns.push(name); } else { let physical = BloomIndex::build_filter_bloom_name(file.format_version, field)?; - let normalized = BloomIndex::build_filter_bloom_name(BlockFilter::VERSION, field)?; + let normalized = BloomIndex::build_filter_bloom_name(format_version, field)?; rename.insert(physical.clone(), normalized); columns.push(physical); } @@ -211,9 +236,12 @@ pub(crate) async fn read_multi_file_block_filter( } } - Ok(Some(BlockFilter { - filter_schema: Arc::new(TableSchema::new(merged_fields)), - filters: merged_filters, + Ok(Some(MultiFileBlockFilter { + block_filter: BlockFilter { + filter_schema: Arc::new(TableSchema::new(merged_fields)), + filters: merged_filters, + }, + format_version, })) } @@ -421,23 +449,53 @@ impl BloomPrunerCreator { &self, index_files: &[BloomIndexFileMeta], column_stats: &StatisticsOfColumns, + block_meta: &BlockReadInfo, ) -> Result { - let Some(filter) = read_multi_file_block_filter( + let maybe_filter = read_multi_file_block_filter( &self.dal, &self.settings, &self.index_fields, &self.ngram_args, index_files, ) - .await? - else { + .await; + let maybe_filter = match (&maybe_filter, &self.bloom_index_builder) { + (Err(_), Some(bloom_index_builder)) => { + match self + .try_rebuild_missing_bloom_index_files( + index_files, + bloom_index_builder, + block_meta, + ) + .await + { + Ok(true) => { + read_multi_file_block_filter( + &self.dal, + &self.settings, + &self.index_fields, + &self.ngram_args, + index_files, + ) + .await + } + Ok(false) => maybe_filter, + Err(e) => { + info!("failed to re-build missing split Bloom indexes: {}", e); + maybe_filter + } + } + } + _ => maybe_filter, + }; + let Some(filter) = maybe_filter? else { return Ok(true); }; Ok(BloomIndex::from_filter_block( self.func_ctx.clone(), - filter.filter_schema, - filter.filters, - BlockFilter::VERSION, + filter.block_filter.filter_schema, + filter.block_filter.filters, + filter.format_version, )? .apply( self.filter_expression.clone(), @@ -449,6 +507,64 @@ impl BloomPrunerCreator { )? != FilterEvalResult::MustFalse) } + async fn try_rebuild_missing_bloom_index_files( + &self, + index_files: &[BloomIndexFileMeta], + bloom_index_builder: &BloomIndexRebuilder, + block_read_info: &BlockReadInfo, + ) -> Result { + let needed_column_ids = self + .index_fields + .iter() + .map(TableField::column_id) + .collect::>(); + let mut missing_files = Vec::new(); + for file in index_files { + if !file + .active_column_ids + .iter() + .any(|column_id| needed_column_ids.contains(column_id)) + { + continue; + } + if self.dal.exists(&file.location.0).await? { + continue; + } + if file.format_version != BlockFilter::VERSION { + info!( + "cannot rebuild missing split Bloom index {:?} with legacy format {}", + file.location, file.format_version + ); + return Ok(false); + } + missing_files.push(file.clone()); + } + if missing_files.is_empty() { + return Ok(false); + } + + let Some(states) = bloom_index_builder + .bloom_index_states_from_block_meta(&missing_files, block_read_info) + .await? + else { + return Ok(false); + }; + if states + .iter() + .zip(&missing_files) + .any(|(state, file)| state.size() != file.file_size) + { + info!("cannot rebuild missing split Bloom indexes because serialized sizes changed"); + return Ok(false); + } + for (state, file) in states.into_iter().zip(&missing_files) { + BlockWriter::write_down_bloom_index_state(&bloom_index_builder.table_dal, Some(state)) + .await?; + info!("re-created missing split Bloom index {:?}", file.location); + } + Ok(true) + } + async fn try_rebuild_missing_bloom_index( &self, bloom_index_location: &Location, @@ -456,6 +572,14 @@ impl BloomPrunerCreator { index_columns: &[String], block_read_info: &BlockReadInfo, ) -> Result> { + if let Err(e) = BloomIndexRebuilder::validate_rebuild_version(bloom_index_location) { + info!( + "cannot rebuild missing legacy Bloom index {:?}: {}", + bloom_index_location, e + ); + return Ok(None); + } + if self.dal.exists(bloom_index_location.0.as_str()).await? { info!("bloom index exists, ignore"); return Ok(None); @@ -513,7 +637,10 @@ impl BloomPruner for BloomPrunerCreator { block_meta: &BlockReadInfo, ) -> bool { if !index_files.is_empty() { - match self.apply_files(index_files, column_stats).await { + match self + .apply_files(index_files, column_stats, block_meta) + .await + { Ok(value) => value, Err(e) => { warn!( @@ -541,3 +668,60 @@ impl BloomPruner for BloomPrunerCreator { } } } + +#[cfg(test)] +mod tests { + use databend_common_expression::TableDataType; + use databend_common_expression::TableField; + use databend_common_expression::types::NumberDataType; + use databend_storages_common_index::filters::BlockFilter; + use databend_storages_common_table_meta::meta::BloomIndexFileMeta; + use databend_storages_common_table_meta::meta::Versioned; + + use super::multi_file_filter_version; + + fn field(column_id: u32) -> TableField { + let mut field = TableField::new( + &format!("c{column_id}"), + TableDataType::Number(NumberDataType::UInt64), + ); + field.column_id = column_id; + field + } + + fn file(active_column_ids: Vec, format_version: u64) -> BloomIndexFileMeta { + BloomIndexFileMeta { + active_column_ids, + location: (format!("bloom-{format_version}"), format_version), + format_version, + file_size: 1, + } + } + + #[test] + fn test_multi_file_filter_version() { + let fields = vec![field(1)]; + assert_eq!( + multi_file_filter_version(&fields, &[file(vec![1], 2)]), + Some(2) + ); + assert_eq!( + multi_file_filter_version(&fields, &[file(vec![1], 2), file(vec![2], 4)]), + Some(2), + "an irrelevant current file does not create a mixed-codec read" + ); + assert_eq!( + multi_file_filter_version(&fields, &[file(vec![1], 2), file(vec![1], 4)]), + None, + "V2 and digest encodings for the requested field cannot be merged" + ); + assert_eq!( + multi_file_filter_version(&fields, &[file(vec![1], 3), file(vec![1], 4)]), + Some(BlockFilter::VERSION) + ); + assert_eq!( + multi_file_filter_version(&fields, &[file(vec![2], 2)]), + Some(BlockFilter::VERSION) + ); + } +} diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index 945a792030159..dd6568160d8e2 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -215,6 +215,7 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { location: location_path.clone(), row_count, col_metas: columns_meta.clone(), + column_groups: vec![], compression, block_size, }; diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index 104e86307b559..857f8cadb4546 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -33,6 +33,7 @@ use databend_common_expression::types::VariantType; use databend_common_expression::types::string::StringColumnBuilder; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; +use serde::Serialize; use crate::FuseTable; use crate::io::SegmentsIO; @@ -43,6 +44,13 @@ use crate::table_functions::function_template::TableMetaFunc; pub struct FuseBlock; pub type FuseBlockFunc = TableMetaFuncTemplate; +fn serialize_variant(value: &impl Serialize) -> Result> { + let json = serde_json::to_vec(value)?; + Ok(jsonb::parse_value(&json) + .map_err(|error| ErrorCode::Internal(error.to_string()))? + .to_vec()) +} + #[async_trait::async_trait] impl TableMetaFunc for FuseBlock { fn schema() -> Arc { @@ -146,49 +154,35 @@ impl TableMetaFunc for FuseBlock { .as_ref() .map(|m| m.virtual_column_size), ); - column_groups.push( - jsonb::parse_value( - serde_json::to_string( - &block - .column_groups - .iter() - .map(|group| { - serde_json::json!({ - "active_column_ids": group.active_column_ids, - "location": group.location.0, - "format_version": group.format_version, - "file_size": group.file_size, - "uncompressed_size": group.uncompressed_size, - }) - }) - .collect::>(), - )? - .as_bytes(), - ) - .map_err(|error| ErrorCode::Internal(error.to_string()))? - .to_vec(), - ); - bloom_index_files.push( - jsonb::parse_value( - serde_json::to_string( - &block - .bloom_index_files - .iter() - .map(|file| { - serde_json::json!({ - "active_column_ids": file.active_column_ids, - "location": file.location.0, - "format_version": file.format_version, - "file_size": file.file_size, - }) - }) - .collect::>(), - )? - .as_bytes(), - ) - .map_err(|error| ErrorCode::Internal(error.to_string()))? - .to_vec(), - ); + column_groups.push(serialize_variant( + &block + .column_groups + .iter() + .map(|group| { + serde_json::json!({ + "active_column_ids": group.active_column_ids, + "location": group.location.0, + "format_version": group.format_version, + "file_size": group.file_size, + "uncompressed_size": group.uncompressed_size, + }) + }) + .collect::>(), + )?); + bloom_index_files.push(serialize_variant( + &block + .bloom_index_files + .iter() + .map(|file| { + serde_json::json!({ + "active_column_ids": file.active_column_ids, + "location": file.location.0, + "format_version": file.format_version, + "file_size": file.file_size, + }) + }) + .collect::>(), + )?); num_rows += 1; if num_rows >= limit { diff --git a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test index a72803b9be06b..cbac9fc64cad1 100644 --- a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test +++ b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test @@ -18,8 +18,8 @@ create or replace database partial_update_cluster_dependency statement ok use partial_update_cluster_dependency -# A cluster key that references an updated column through a VIRTUAL computed column must use the -# full-update path and refresh its cluster statistics. +# A table with a computed column must use the full-update path. The cluster key also verifies that +# the fallback pipeline uses the persisted physical schema and refreshes its cluster statistics. statement ok create table t( id int, From 1443f8c82eab67ba6c3ebea42170ca9b468c2438 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:10:36 +0800 Subject: [PATCH 09/30] fix(storage): close partial update bloom lifecycle gaps --- .../fuse/operations/vacuum_table_v2.rs | 122 +++++++++++++++- .../it/storages/fuse/operations/vacuum2.rs | 102 ++++++++++++++ .../fuse/operations/mutation/update.rs | 132 ++++++++++++++++++ .../common/table_meta/src/meta/v2/segment.rs | 106 ++++++++++++-- src/query/storages/fuse/src/fuse_part.rs | 17 +-- .../io/read/block/block_reader_deserialize.rs | 2 + .../fuse/src/io/write/block_writer.rs | 31 +++- .../fuse/src/io/write/bloom_index_writer.rs | 10 +- .../processors/transform_serialize_block.rs | 27 ++-- .../storages/fuse/src/pruning/block_pruner.rs | 4 +- .../storages/fuse/src/pruning/bloom_pruner.rs | 60 ++++---- .../column_oriented_block_prune.rs | 9 +- .../fuse/src/table_functions/fuse_encoding.rs | 10 +- 13 files changed, 539 insertions(+), 93 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index fe284cc2f0ab8..d8cf13984dc81 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -29,11 +29,14 @@ use databend_common_meta_app::schema::TableIndex; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::SegmentsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; +use databend_common_storages_fuse::operations::ASSUMPTION_MAX_TXN_DURATION; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_io::Files; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::Location; +use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; +use databend_storages_common_table_meta::meta::uuid_from_date_time; use log::info; const VACUUM2_BLOCK_DELETE_CHUNK_SIZE: usize = 1000; @@ -190,6 +193,24 @@ pub async fn do_vacuum2( slice_summary(&blocks_to_gc) )); + // Bloom files are immutable, but their lifetime is not tied to the data file whose UUID was + // originally used for the index path. Ngram refresh, for example, consolidates a split Bloom + // layout into a fresh file while retaining the active data groups. List the Bloom directory + // independently so those superseded files do not leak forever. + let start = std::time::Instant::now(); + let bloom_indexes_before_gc_root = + list_bloom_indexes_before_gc_root(fuse_table, gc_root_timestamp, gc_root_meta_ts).await?; + let bloom_indexes_to_gc = bloom_indexes_before_gc_root + .into_iter() + .filter(|path| !gc_root_indexes.contains(path)) + .collect::>(); + ctx.set_status_info(&format!( + "Filtered bloom_indexes_to_gc for table {}, elapsed: {:?}, bloom_indexes_to_gc: {:?}", + table_info.desc, + start.elapsed(), + slice_summary(&bloom_indexes_to_gc) + )); + let start = std::time::Instant::now(); let catalog = ctx.get_default_catalog()?; let table_agg_index_ids = catalog @@ -205,9 +226,18 @@ pub async fn do_vacuum2( blocks_to_gc.len() * (table_agg_index_ids.len() + inverted_indexes.len() + 2) + stats_to_gc.len() + segments_to_gc.len() - + snapshots_to_gc.len(), + + snapshots_to_gc.len() + + bloom_indexes_to_gc.len(), ); + // Bloom indexes must be removed before their data blocks. Keep the removed paths so the + // block-derived compatibility cleanup below does not issue duplicate deletes. + if !bloom_indexes_to_gc.is_empty() { + op.remove_file_in_batch(&bloom_indexes_to_gc).await?; + files_to_gc.extend(bloom_indexes_to_gc.iter().cloned()); + } + let removed_bloom_indexes = bloom_indexes_to_gc.into_iter().collect::>(); + // order is important // indexes should be removed before their blocks, because index locations to gc are generated from block locations. purge_block_chunks( @@ -216,6 +246,7 @@ pub async fn do_vacuum2( table_info.desc.as_str(), &blocks_to_gc, &gc_root_indexes, + &removed_bloom_indexes, &table_agg_index_ids, inverted_indexes, &mut files_to_gc, @@ -276,6 +307,7 @@ async fn purge_block_chunks( table_desc: &str, blocks_to_gc: &[String], protected_index_paths: &HashSet, + removed_bloom_index_paths: &HashSet, table_agg_index_ids: &[u64], inverted_indexes: &BTreeMap, files_to_gc: &mut Vec, @@ -296,7 +328,9 @@ async fn purge_block_chunks( let mut indexes_to_gc = collect_block_index_locations(block_chunk, table_agg_index_ids, inverted_indexes); - indexes_to_gc.retain(|path| !protected_index_paths.contains(path)); + indexes_to_gc.retain(|path| { + !protected_index_paths.contains(path) && !removed_bloom_index_paths.contains(path) + }); ctx.set_status_info(&format!( "Collected indexes_to_gc for table {}, elapsed: {:?}, block chunk: {}/{}, blocks in chunk: {}, indexes_to_gc: {:?}", table_desc, @@ -328,6 +362,56 @@ async fn purge_block_chunks( Ok(()) } +async fn list_bloom_indexes_before_gc_root( + fuse_table: &FuseTable, + gc_root_timestamp: DateTime, + gc_root_meta_ts: DateTime, +) -> Result> { + let gc_root_uuid = uuid_from_date_time(gc_root_timestamp).simple().to_string(); + let gc_root_timestamp_prefix = &gc_root_uuid[..12]; + fuse_table + .list_files( + fuse_table + .meta_location_generator() + .block_bloom_index_prefix() + .to_string(), + |path, modified| { + is_bloom_index_gc_candidate( + &path, + modified, + gc_root_timestamp_prefix, + gc_root_meta_ts, + ) + }, + ) + .await +} + +fn is_bloom_index_gc_candidate( + path: &str, + modified: DateTime, + gc_root_timestamp_prefix: &str, + gc_root_meta_ts: DateTime, +) -> bool { + if let Ok(uuid) = try_extract_uuid_str_from_path(path) { + let bytes = uuid.as_bytes(); + if bytes.len() == 32 + && bytes + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + && bytes[12] == b'7' + { + // UUID V7 stores its millisecond timestamp in the first 12 hexadecimal digits. Use a + // strict bound, matching Vacuum2's segment/block listing behavior at the GC root. + return &uuid[..12] < gc_root_timestamp_prefix; + } + } + + // Old and malformed object names have no trustworthy creation time in their key. Preserve + // the existing Vacuum2 compatibility window so an in-flight old writer cannot lose its file. + modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts +} + fn collect_block_index_locations( blocks_to_gc: &[String], table_agg_index_ids: &[u64], @@ -416,6 +500,7 @@ fn slice_summary(s: &[T]) -> String { #[cfg(test)] mod tests { + use chrono::Duration; use databend_common_meta_app::schema::TableIndexType; use super::*; @@ -455,4 +540,37 @@ mod tests { TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(&blocks[1]), ]); } + + #[test] + fn test_bloom_index_gc_candidate_timestamp_safety() { + let gc_root_meta_ts = Utc::now(); + let recent = gc_root_meta_ts - Duration::hours(1); + let old = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION - Duration::hours(1); + let gc_root_timestamp_prefix = "0191114d30fd"; + + assert!(is_bloom_index_gc_candidate( + "1/2/_i_b_v2/0191114d30fc70000000000000000000_v4.parquet", + recent, + gc_root_timestamp_prefix, + gc_root_meta_ts, + )); + assert!(!is_bloom_index_gc_candidate( + "1/2/_i_b_v2/0191114d30fd70000000000000000000_v4.parquet", + old, + gc_root_timestamp_prefix, + gc_root_meta_ts, + )); + assert!(!is_bloom_index_gc_candidate( + "1/2/_i_b_v2/0191114D30FC70000000000000000000_v4.parquet", + recent, + gc_root_timestamp_prefix, + gc_root_meta_ts, + )); + assert!(is_bloom_index_gc_candidate( + "1/2/_i_b_v2/0123456789ab4def8123456789abcdef_v4.parquet", + old, + gc_root_timestamp_prefix, + gc_root_meta_ts, + )); + } } diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs index 40a516d723615..cf9c4a314ffcf 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs @@ -203,6 +203,108 @@ async fn test_vacuum2_preserves_active_partial_update_files() -> anyhow::Result< Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum2_removes_superseded_split_bloom_files() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, content string) engine=fuse \ + bloom_index_columns='id,a' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values \ + (1, 10, 'aaaaaaaaaa'), (2, 20, 'bbbbbbbbbb')" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + + let split_meta = latest_default_block_meta(&fixture).await?; + assert_eq!(split_meta.bloom_index_files.len(), 2); + let superseded_bloom_files = split_meta + .bloom_index_files + .iter() + .map(|file| file.location.0.clone()) + .collect::>(); + + fixture + .execute_command(&format!( + "create ngram index idx_content on {db}.{table_name}(content)" + )) + .await?; + fixture + .execute_command(&format!( + "refresh ngram index idx_content on {db}.{table_name}" + )) + .await?; + + let current = latest_default_block_meta(&fixture).await?; + assert!(current.bloom_index_files.is_empty()); + let current_bloom_file = current + .bloom_filter_index_location + .as_ref() + .expect("consolidated Bloom file") + .0 + .clone(); + assert!( + superseded_bloom_files + .iter() + .all(|path| path != ¤t_bloom_file) + ); + let active_data_files = current + .data_file_locations() + .map(|location| location.0.clone()) + .collect::>(); + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + for path in &superseded_bloom_files { + operator.stat(path).await?; + } + + fixture + .execute_command(&format!("call system$fuse_vacuum2('{db}', '{table_name}')")) + .await?; + + for path in superseded_bloom_files { + assert!( + operator.stat(&path).await.is_err(), + "{path} was not removed" + ); + } + operator.stat(¤t_bloom_file).await?; + for path in active_data_files { + operator.stat(&path).await?; + } + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and a = 11 and content = 'aaaaaaaaaa') \ + or (id = 2 and a = 20 and content = 'bbbbbbbbbb')" + )) + .await?; + assert_eq!(databend_query::test_kits::query_count(rows).await?, 2); + + Ok(()) +} + /// Verifies that dedup_file_locations correctly removes duplicates and reports samples. #[test] fn test_dedup_file_locations() { diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 087279bf5155f..81468c7220613 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -299,6 +299,138 @@ async fn test_partial_update_reads_legacy_bloom_schema() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_drops_obsolete_split_bloom_file_after_drop_column() +-> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, value int, flag boolean) engine=fuse \ + bloom_index_columns='id,value' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, true), (2, 20, false)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let table = fixture.latest_default_table().await?; + let value_column_id = table.schema().field_with_name("value")?.column_id(); + let split_meta = latest_default_block_meta(&fixture).await?; + let obsolete_bloom_location = split_meta + .bloom_index_files + .iter() + .find(|file| file.active_column_ids.contains(&value_column_id)) + .unwrap() + .location + .clone(); + + fixture + .execute_command(&format!("alter table {db}.{table_name} drop column value")) + .await?; + // Boolean is not supported by the ordinary Bloom filter. This exercises metadata cleanup + // even when the current UPDATE neither invalidates nor writes a Bloom filter. + fixture + .execute_command(&format!( + "update {db}.{table_name} set flag = not flag where id = 1" + )) + .await?; + + let updated = latest_default_block_meta(&fixture).await?; + assert!( + updated + .bloom_index_files + .iter() + .all(|file| file.location != obsolete_bloom_location + && !file.active_column_ids.contains(&value_column_id)) + ); + assert_eq!( + updated.bloom_filter_index_size, + updated + .bloom_index_files + .iter() + .map(|file| file.file_size) + .sum::() + ); + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and not flag) or (id = 2 and not flag)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_ignores_missing_legacy_bloom_file() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, value int, flag boolean) engine=fuse \ + bloom_index_columns='id,value' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, true), (2, 20, false)" + )) + .await?; + + let table = fixture.latest_default_table().await?; + let value_column_id = table.schema().field_with_name("value")?.column_id(); + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + let origin = latest_default_block_meta(&fixture).await?; + let missing_location = origin.bloom_filter_index_location.as_ref().unwrap().clone(); + operator.delete(&missing_location.0).await?; + + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let updated = latest_default_block_meta(&fixture).await?; + assert!(updated.bloom_filter_index_location.is_none()); + assert_eq!(updated.bloom_index_files.len(), 1); + assert_eq!(updated.bloom_index_files[0].active_column_ids, vec![ + value_column_id + ]); + assert_ne!(updated.bloom_index_files[0].location, missing_location); + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11 and flag) \ + or (id = 2 and value = 20 and not flag)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn test_auto_fix_missing_split_bloom_index() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 1da5904c36597..0b47d7f350b7f 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -186,6 +186,17 @@ pub struct ColumnGroupFileMeta { pub leaf_column_metas: HashMap, } +impl ColumnGroupFileMeta { + /// Active leaf metadata in this physical file. + pub fn active_leaf_column_metas(&self) -> impl Iterator { + self.active_column_ids.iter().filter_map(|column_id| { + self.leaf_column_metas + .get(column_id) + .map(|column_meta| (*column_id, column_meta)) + }) + } +} + /// Metadata of one physical Bloom index file referenced by a logical block. /// /// A file may still contain filters that are no longer active. Readers must only use the filters @@ -210,6 +221,24 @@ pub enum BloomIndexLayout<'a> { }, } +impl<'a> BloomIndexLayout<'a> { + /// Normalize optional legacy and split Bloom metadata into one physical-layout view. + pub fn from_metadata( + legacy_location: Option<&'a Location>, + legacy_file_size: u64, + split_files: &'a [BloomIndexFileMeta], + ) -> Option { + if split_files.is_empty() { + legacy_location.map(|location| Self::Legacy { + location, + file_size: legacy_file_size, + }) + } else { + Some(Self::Split { files: split_files }) + } + } +} + /// Meta information of a block /// Part of and kept inside the [SegmentInfo] #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -316,18 +345,11 @@ impl BlockMeta { /// Normalize optional legacy and split Bloom metadata into one physical-layout view. pub fn bloom_index_layout(&self) -> Option> { - if self.bloom_index_files.is_empty() { - self.bloom_filter_index_location - .as_ref() - .map(|location| BloomIndexLayout::Legacy { - location, - file_size: self.bloom_filter_index_size, - }) - } else { - Some(BloomIndexLayout::Split { - files: &self.bloom_index_files, - }) - } + BloomIndexLayout::from_metadata( + self.bloom_filter_index_location.as_ref(), + self.bloom_filter_index_size, + &self.bloom_index_files, + ) } /// Active physical data files referenced by this logical block. @@ -631,6 +653,66 @@ impl From<(v0::SegmentInfo, &[TableField])> for SegmentInfo { mod tests { use super::*; + #[test] + fn test_bloom_index_layout_normalizes_legacy_and_split_metadata() { + let legacy_location = ("legacy-bloom".to_string(), 2); + assert_eq!( + BloomIndexLayout::from_metadata(Some(&legacy_location), 10, &[]), + Some(BloomIndexLayout::Legacy { + location: &legacy_location, + file_size: 10, + }) + ); + + let split_files = vec![BloomIndexFileMeta { + active_column_ids: vec![1], + location: ("split-bloom".to_string(), 4), + format_version: 4, + file_size: 20, + }]; + assert_eq!( + BloomIndexLayout::from_metadata(Some(&legacy_location), 10, &split_files), + Some(BloomIndexLayout::Split { + files: &split_files, + }) + ); + } + + #[test] + fn test_active_leaf_column_metas_excludes_inactive_and_missing_columns() { + let group = ColumnGroupFileMeta { + active_column_ids: vec![1, 3], + location: ("group.parquet".to_string(), 4), + format_version: 4, + file_size: 20, + uncompressed_size: 40, + leaf_column_metas: HashMap::from([ + ( + 1, + ColumnMeta::Parquet(v0::ColumnMeta { + offset: 10, + len: 11, + num_values: 12, + }), + ), + ( + 2, + ColumnMeta::Parquet(v0::ColumnMeta { + offset: 20, + len: 21, + num_values: 22, + }), + ), + ]), + }; + + let active_metas = group + .active_leaf_column_metas() + .map(|(column_id, column_meta)| (column_id, column_meta.offset_length())) + .collect::>(); + assert_eq!(active_metas, vec![(1, (10, 11))]); + } + #[test] fn test_deserialize_legacy_block_meta_without_file_lists() { let block_meta = BlockMeta::new( diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 74e4d7d09daa7..577a285758c0d 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -105,18 +105,11 @@ impl PartInfo for FuseBlockPartInfo { impl FuseBlockPartInfo { /// Normalize optional legacy and split Bloom metadata into one physical-layout view. pub fn bloom_index_layout(&self) -> Option> { - if self.bloom_index_files.is_empty() { - self.bloom_filter_index_location - .as_ref() - .map(|location| BloomIndexLayout::Legacy { - location, - file_size: self.bloom_filter_index_size, - }) - } else { - Some(BloomIndexLayout::Split { - files: &self.bloom_index_files, - }) - } + BloomIndexLayout::from_metadata( + self.bloom_filter_index_location.as_ref(), + self.bloom_filter_index_size, + &self.bloom_index_files, + ) } #[allow(clippy::too_many_arguments)] diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index 86027702243c2..e9e04a4f92730 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -90,6 +90,8 @@ impl BlockReader { ) -> Result { // Get the merged IO read result. let column_groups = self.projected_column_groups(meta); + // Type erasure breaks the recursive async future formed by virtual-column reads that + // return to `read_by_meta` through the merge-IO path. let read: std::pin::Pin> + Send + '_>> = Box::pin(self.read_column_groups_data_by_merge_io(settings, &column_groups, &None)); let merge_io_read_result = read.await?; diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index c33f818e3233e..42a95f88584c9 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -492,7 +492,30 @@ impl BlockBuilder { block_meta.col_stats.extend(updated_col_stats); block_meta.create_on = Some(Utc::now()); - if !invalidated_bloom_column_ids.is_empty() { + let current_bloom_column_ids = self + .source_schema + .fields() + .iter() + .filter(|field| BloomIndex::supported_type(field.data_type())) + .map(TableField::column_id) + .collect::>(); + let keep_legacy_bloom_layout = if matches!( + origin.bloom_index_layout(), + Some(BloomIndexLayout::Legacy { .. }) + ) { + let active_column_ids = legacy_bloom_column_ids + .ok_or_else(|| ErrorCode::Internal("legacy Bloom file columns were not loaded"))?; + !active_column_ids.is_empty() + && bloom_index_state.is_none() + && invalidated_bloom_column_ids.is_empty() + && active_column_ids + .iter() + .all(|column_id| current_bloom_column_ids.contains(column_id)) + } else { + false + }; + + if !keep_legacy_bloom_layout { let mut bloom_index_files = match origin.bloom_index_layout() { None => Vec::new(), Some(BloomIndexLayout::Legacy { @@ -516,8 +539,10 @@ impl BlockBuilder { Some(BloomIndexLayout::Split { files }) => files.to_vec(), }; for file in &mut bloom_index_files { - file.active_column_ids - .retain(|column_id| !invalidated_bloom_column_ids.contains(column_id)); + file.active_column_ids.retain(|column_id| { + current_bloom_column_ids.contains(column_id) + && !invalidated_bloom_column_ids.contains(column_id) + }); } bloom_index_files.retain(|file| !file.active_column_ids.is_empty()); if let Some(state) = &bloom_index_state { diff --git a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs index 865749d0fa206..c0e14b822e446 100644 --- a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs @@ -182,14 +182,8 @@ impl BloomIndexRebuilder { .map(|group| FuseColumnGroupPartInfo { location: group.location.0.clone(), columns_meta: group - .active_column_ids - .iter() - .filter_map(|column_id| { - group - .leaf_column_metas - .get(column_id) - .map(|column_meta| (*column_id, column_meta.clone())) - }) + .active_leaf_column_metas() + .map(|(column_id, column_meta)| (column_id, column_meta.clone())) .collect(), }) .collect() diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index b8dc37b8cd3b1..efae3c6420539 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -41,6 +41,7 @@ use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use log::warn; use opendal::Operator; use crate::FuseTable; @@ -278,16 +279,11 @@ impl TransformSerializeBlock { } fn needs_legacy_bloom_meta(&self, origin: &BlockMeta) -> bool { - matches!( - origin.bloom_index_layout(), - Some(BloomIndexLayout::Legacy { .. }) - ) && self.updated_field_indices.as_ref().is_some_and(|indices| { - indices.iter().any(|index| { - BloomIndex::supported_type( - self.block_builder.source_schema.field(*index).data_type(), - ) - }) - }) + self.updated_field_indices.is_some() + && matches!( + origin.bloom_index_layout(), + Some(BloomIndexLayout::Legacy { .. }) + ) } async fn load_legacy_bloom_column_ids(&self, origin: &BlockMeta) -> Result> { @@ -298,7 +294,16 @@ impl TransformSerializeBlock { else { return Err(ErrorCode::Internal("legacy Bloom location is missing")); }; - let meta = load_index_meta(self.dal.clone(), &location.0, file_size, None).await?; + let meta = match load_index_meta(self.dal.clone(), &location.0, file_size, None).await { + Ok(meta) => meta, + Err(error) => { + warn!( + "failed to load legacy Bloom metadata at {:?}; dropping its reference from the partial-update output: {}", + location, error + ); + return Ok(vec![]); + } + }; let stored_filter_names = meta .columns .iter() diff --git a/src/query/storages/fuse/src/pruning/block_pruner.rs b/src/query/storages/fuse/src/pruning/block_pruner.rs index 848fecc1728a0..21d7acb9d8c57 100644 --- a/src/query/storages/fuse/src/pruning/block_pruner.rs +++ b/src/query/storages/fuse/src/pruning/block_pruner.rs @@ -344,9 +344,7 @@ impl BlockPruner { .measure_async( PruningCostKind::BlocksBloom, bloom_pruner.should_keep( - &block_meta.bloom_filter_index_location, - block_meta.bloom_filter_index_size, - &block_meta.bloom_index_files, + block_meta.bloom_index_layout(), &block_meta.col_stats, column_ids, &block_meta.as_ref().into(), diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 2cb55f5afbfa8..09cfbe9075569 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -56,9 +56,7 @@ pub trait BloomPruner { // returns true, if target should NOT be pruned (false positive allowed) async fn should_keep( &self, - index_location: &Option, - index_length: u64, - index_files: &[BloomIndexFileMeta], + index_layout: Option>, column_stats: &StatisticsOfColumns, column_ids: Vec, block_meta: &BlockReadInfo, @@ -629,42 +627,42 @@ impl BloomPruner for BloomPrunerCreator { #[async_backtrace::framed] async fn should_keep( &self, - index_location: &Option, - index_length: u64, - index_files: &[BloomIndexFileMeta], + index_layout: Option>, column_stats: &StatisticsOfColumns, column_ids: Vec, block_meta: &BlockReadInfo, ) -> bool { - if !index_files.is_empty() { - match self - .apply_files(index_files, column_stats, block_meta) - .await - { - Ok(value) => value, - Err(e) => { - warn!( - "failed to apply multi-file bloom pruner, returning true. {}", - e - ); - true + match index_layout { + Some(BloomIndexLayout::Split { files }) => { + match self.apply_files(files, column_stats, block_meta).await { + Ok(value) => value, + Err(e) => { + warn!( + "failed to apply multi-file bloom pruner, returning true. {}", + e + ); + true + } } } - } else if let Some(loc) = index_location { - // load filter, and try pruning according to filter expression - match self - .apply(loc, index_length, column_stats, column_ids, block_meta) - .await - { - Ok(v) => v, - Err(e) => { - // swallow exceptions intentionally, corrupted index should not prevent execution - warn!("failed to apply bloom pruner, returning true. {}", e); - true + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { + // Load the filter and try pruning according to its expression. + match self + .apply(location, file_size, column_stats, column_ids, block_meta) + .await + { + Ok(value) => value, + Err(e) => { + // Swallow exceptions intentionally: a corrupt index must not fail a query. + warn!("failed to apply bloom pruner, returning true. {}", e); + true + } } } - } else { - true + None => true, } } } diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index dd6568160d8e2..9bdc2e2254796 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -32,6 +32,7 @@ use databend_common_pipeline::sinks::AsyncSink; use databend_common_pipeline::sinks::AsyncSinker; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_pruner::RangeIndexInput; +use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnMetaV0; use databend_storages_common_table_meta::meta::ColumnStatistics; @@ -222,9 +223,11 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { if !bloom_pruner .should_keep( - &bloom_filter_index_location, - bloom_filter_index_size, - &[], + BloomIndexLayout::from_metadata( + bloom_filter_index_location.as_ref(), + bloom_filter_index_size, + &[], + ), &columns_stat, column_ids.clone(), &block_read_info, diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index f93676207acc3..cc4c67a3f861e 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -403,14 +403,8 @@ impl<'a> FuseEncodingImpl<'a> { for block in segment.blocks.iter() { for group in block.physical_column_groups().iter() { let column_metas = group - .active_column_ids - .iter() - .filter_map(|column_id| { - group - .leaf_column_metas - .get(column_id) - .map(|meta| (*column_id, meta.clone())) - }) + .active_leaf_column_metas() + .map(|(column_id, column_meta)| (column_id, column_meta.clone())) .collect(); block_groups.push(( group.location.0.clone(), From 63cb0be299d890c42c2da43ae16a3ac049aaa77a Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:18:00 +0800 Subject: [PATCH 10/30] fix(storage): bound partial update read amplification --- .../fuse/operations/vacuum_table_v2.rs | 73 ++---- .../fuse/src/io/read/block/block_reader.rs | 4 + .../read/block/block_reader_merge_io_async.rs | 48 +++- .../fuse/src/io/write/block_writer.rs | 238 +++++++++++------- .../processors/transform_serialize_block.rs | 55 ++-- src/query/storages/fuse/src/operations/mod.rs | 2 + .../mutator/replace_into_operation_agg.rs | 38 ++- .../storages/fuse/src/operations/vacuum.rs | 173 ++++++++++++- .../fuse/src/table_functions/fuse_encoding.rs | 13 +- 9 files changed, 457 insertions(+), 187 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index d8cf13984dc81..7fc4342f7905e 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -29,14 +29,13 @@ use databend_common_meta_app::schema::TableIndex; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::SegmentsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::operations::ASSUMPTION_MAX_TXN_DURATION; +use databend_common_storages_fuse::operations::VacuumObjectKeyPolicy; +use databend_common_storages_fuse::operations::is_vacuum_object_gc_candidate; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_io::Files; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; -use databend_storages_common_table_meta::meta::uuid_from_date_time; use log::info; const VACUUM2_BLOCK_DELETE_CHUNK_SIZE: usize = 1000; @@ -367,8 +366,7 @@ async fn list_bloom_indexes_before_gc_root( gc_root_timestamp: DateTime, gc_root_meta_ts: DateTime, ) -> Result> { - let gc_root_uuid = uuid_from_date_time(gc_root_timestamp).simple().to_string(); - let gc_root_timestamp_prefix = &gc_root_uuid[..12]; + let key_policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; fuse_table .list_files( fuse_table @@ -376,42 +374,12 @@ async fn list_bloom_indexes_before_gc_root( .block_bloom_index_prefix() .to_string(), |path, modified| { - is_bloom_index_gc_candidate( - &path, - modified, - gc_root_timestamp_prefix, - gc_root_meta_ts, - ) + is_vacuum_object_gc_candidate(&path, modified, gc_root_meta_ts, key_policy) }, ) .await } -fn is_bloom_index_gc_candidate( - path: &str, - modified: DateTime, - gc_root_timestamp_prefix: &str, - gc_root_meta_ts: DateTime, -) -> bool { - if let Ok(uuid) = try_extract_uuid_str_from_path(path) { - let bytes = uuid.as_bytes(); - if bytes.len() == 32 - && bytes - .iter() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) - && bytes[12] == b'7' - { - // UUID V7 stores its millisecond timestamp in the first 12 hexadecimal digits. Use a - // strict bound, matching Vacuum2's segment/block listing behavior at the GC root. - return &uuid[..12] < gc_root_timestamp_prefix; - } - } - - // Old and malformed object names have no trustworthy creation time in their key. Preserve - // the existing Vacuum2 compatibility window so an in-flight old writer cannot lose its file. - modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts -} - fn collect_block_index_locations( blocks_to_gc: &[String], table_agg_index_ids: &[u64], @@ -502,6 +470,7 @@ fn slice_summary(s: &[T]) -> String { mod tests { use chrono::Duration; use databend_common_meta_app::schema::TableIndexType; + use databend_storages_common_table_meta::meta::uuid_from_date_time; use super::*; @@ -543,34 +512,40 @@ mod tests { #[test] fn test_bloom_index_gc_candidate_timestamp_safety() { + let gc_root_timestamp = Utc::now() - Duration::days(1); let gc_root_meta_ts = Utc::now(); let recent = gc_root_meta_ts - Duration::hours(1); - let old = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION - Duration::hours(1); - let gc_root_timestamp_prefix = "0191114d30fd"; + let old = gc_root_meta_ts - Duration::days(4); + let policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; + let before_root = uuid_from_date_time(gc_root_timestamp - Duration::milliseconds(1)); + let at_root = uuid_from_date_time(gc_root_timestamp); - assert!(is_bloom_index_gc_candidate( - "1/2/_i_b_v2/0191114d30fc70000000000000000000_v4.parquet", + assert!(is_vacuum_object_gc_candidate( + &format!("1/2/_i_b_v2/{}_v4.parquet", before_root.as_simple()), recent, - gc_root_timestamp_prefix, gc_root_meta_ts, + policy, )); - assert!(!is_bloom_index_gc_candidate( - "1/2/_i_b_v2/0191114d30fd70000000000000000000_v4.parquet", + assert!(!is_vacuum_object_gc_candidate( + &format!("1/2/_i_b_v2/{}_v4.parquet", at_root.as_simple()), old, - gc_root_timestamp_prefix, gc_root_meta_ts, + policy, )); - assert!(!is_bloom_index_gc_candidate( - "1/2/_i_b_v2/0191114D30FC70000000000000000000_v4.parquet", + assert!(!is_vacuum_object_gc_candidate( + &format!( + "1/2/_i_b_v2/{}_v4.parquet", + before_root.as_simple().to_string().to_ascii_uppercase() + ), recent, - gc_root_timestamp_prefix, gc_root_meta_ts, + policy, )); - assert!(is_bloom_index_gc_candidate( + assert!(is_vacuum_object_gc_candidate( "1/2/_i_b_v2/0123456789ab4def8123456789abcdef_v4.parquet", old, - gc_root_timestamp_prefix, gc_root_meta_ts, + policy, )); } } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader.rs b/src/query/storages/fuse/src/io/read/block/block_reader.rs index 01dc040fab256..56f7ed6a8342c 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader.rs @@ -241,6 +241,10 @@ impl BlockReader { } impl BlockReadContext { + pub(crate) fn table_context(&self) -> &Arc { + &self.ctx + } + pub fn operator(&self) -> &Operator { &self.operator } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs index 592ae9a8b0e46..cfe3fd3208b89 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs @@ -33,6 +33,11 @@ use crate::FuseColumnGroupPartInfo; use crate::io::BlockReadContext; use crate::io::BlockReader; +fn column_group_read_concurrency(max_threads: usize, max_storage_io_requests: usize) -> usize { + let outer_readers = max_threads.min(max_storage_io_requests).max(1); + max_storage_io_requests.max(1).div_ceil(outer_readers) +} + impl BlockReader { #[async_backtrace::framed] pub(crate) async fn read_column_groups_data_by_merge_io( @@ -68,17 +73,24 @@ impl BlockReadContext { column_groups: &[FuseColumnGroupPartInfo], ignore_column_ids: &Option>, ) -> Result { - let reads = column_groups.iter().map(|group| { - self.read_columns_data_by_merge_io( - settings, - &group.location, - &group.columns_meta, - ignore_column_ids, - ) - }); - Ok(BlockReadResult::merge( - futures::future::try_join_all(reads).await?, - )) + let query_settings = self.table_context().get_settings(); + let concurrency = column_group_read_concurrency( + query_settings.get_max_threads()? as usize, + query_settings.get_max_storage_io_requests()? as usize, + ); + let mut results = Vec::with_capacity(column_groups.len()); + for groups in column_groups.chunks(concurrency) { + let reads = groups.iter().map(|group| { + self.read_columns_data_by_merge_io( + settings, + &group.location, + &group.columns_meta, + ignore_column_ids, + ) + }); + results.extend(futures::future::try_join_all(reads).await?); + } + Ok(BlockReadResult::merge(results)) } #[async_backtrace::framed] @@ -205,3 +217,17 @@ impl<'a> ColumnCacheKeyBuilder<'a> { TableDataCacheKey::new(self.block_path, *column_id, offset, len) } } + +#[cfg(test)] +mod tests { + use super::column_group_read_concurrency; + + #[test] + fn test_column_group_read_concurrency() { + assert_eq!(column_group_read_concurrency(8, 64), 8); + assert_eq!(column_group_read_concurrency(8, 10), 2); + assert_eq!(column_group_read_concurrency(64, 8), 1); + assert_eq!(column_group_read_concurrency(0, 0), 1); + assert_eq!(column_group_read_concurrency(0, 8), 8); + } +} diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 42a95f88584c9..4bd269aae5902 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -59,6 +59,7 @@ use databend_storages_common_table_meta::meta::ClusterStatistics; use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; +use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::encode_column_hll; @@ -167,6 +168,133 @@ pub struct BlockBuilder { pub serialize_hll: bool, } +struct ColumnGroupUpdate { + active_column_ids: Vec, + location: Location, + file_size: u64, + uncompressed_size: u64, + column_metas: HashMap, + column_stats: StatisticsOfColumns, +} + +enum BloomLayoutMerge { + Preserve, + Canonical { + files: Vec, + file_size: u64, + }, +} + +fn merge_column_group_metadata( + origin: &BlockMeta, + current_column_ids: &HashSet, + update: ColumnGroupUpdate, +) -> BlockMeta { + let updated_column_ids = update + .active_column_ids + .iter() + .copied() + .collect::>(); + let mut column_groups = origin.physical_column_groups().into_owned(); + for group in &mut column_groups { + group.active_column_ids.retain(|column_id| { + current_column_ids.contains(column_id) && !updated_column_ids.contains(column_id) + }); + } + column_groups.retain(|group| !group.active_column_ids.is_empty()); + column_groups.push(ColumnGroupFileMeta { + active_column_ids: update.active_column_ids, + location: update.location.clone(), + format_version: update.location.1, + file_size: update.file_size, + uncompressed_size: update.uncompressed_size, + leaf_column_metas: update.column_metas.clone(), + }); + + let mut block_meta = origin.clone(); + block_meta.location = update.location; + block_meta.file_size = column_groups.iter().map(|group| group.file_size).sum(); + block_meta.block_size = column_groups + .iter() + .map(|group| group.uncompressed_size) + .sum(); + block_meta.column_groups = column_groups; + block_meta + .col_metas + .retain(|column_id, _| current_column_ids.contains(column_id)); + block_meta.col_metas.extend(update.column_metas); + block_meta + .col_stats + .retain(|column_id, _| current_column_ids.contains(column_id)); + block_meta.col_stats.extend(update.column_stats); + block_meta +} + +fn merge_partial_update_bloom_layout( + origin: &BlockMeta, + legacy_bloom_column_ids: Option<&[ColumnId]>, + current_bloom_column_ids: &HashSet, + invalidated_bloom_column_ids: &HashSet, + updated_bloom_column_ids: &HashSet, + bloom_index_state: Option<&BloomIndexState>, +) -> Result { + let layout = origin.bloom_index_layout(); + if matches!(layout, Some(BloomIndexLayout::Legacy { .. })) { + let active_column_ids = legacy_bloom_column_ids + .ok_or_else(|| ErrorCode::Internal("legacy Bloom file columns were not loaded"))?; + let keep_legacy_layout = !active_column_ids.is_empty() + && bloom_index_state.is_none() + && invalidated_bloom_column_ids.is_empty() + && active_column_ids + .iter() + .all(|column_id| current_bloom_column_ids.contains(column_id)); + if keep_legacy_layout { + return Ok(BloomLayoutMerge::Preserve); + } + } + + let mut files = match layout { + None => Vec::new(), + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { + let active_column_ids = legacy_bloom_column_ids + .ok_or_else(|| ErrorCode::Internal("legacy Bloom file columns were not loaded"))?; + if active_column_ids.is_empty() { + Vec::new() + } else { + vec![BloomIndexFileMeta { + active_column_ids: active_column_ids.to_vec(), + location: location.clone(), + format_version: location.1, + file_size, + }] + } + } + Some(BloomIndexLayout::Split { files }) => files.to_vec(), + }; + for file in &mut files { + file.active_column_ids.retain(|column_id| { + current_bloom_column_ids.contains(column_id) + && !invalidated_bloom_column_ids.contains(column_id) + }); + } + files.retain(|file| !file.active_column_ids.is_empty()); + if let Some(state) = bloom_index_state { + let mut active_column_ids = updated_bloom_column_ids.iter().copied().collect::>(); + active_column_ids.sort_unstable(); + files.push(BloomIndexFileMeta { + active_column_ids, + location: state.location.clone(), + format_version: state.location.1, + file_size: state.size, + }); + } + let file_size = files.iter().map(|file| file.file_size).sum(); + Ok(BloomLayoutMerge::Canonical { files, file_size }) +} + impl BlockBuilder { fn add_hll_distinct_counts( column_distinct_count: &mut HashMap, @@ -372,7 +500,6 @@ impl BlockBuilder { } let updated_block = data_block; let updated_column_ids = updated_schema.to_leaf_column_ids(); - let updated_column_id_set = updated_column_ids.iter().copied().collect::>(); let (data_location, block_id) = self .meta_locations @@ -458,38 +585,15 @@ impl BlockBuilder { let file_size = buffer.len() as u64; let current_column_ids = self.source_schema.to_leaf_column_id_set(); - let mut column_groups = origin.physical_column_groups().into_owned(); - for group in &mut column_groups { - group.active_column_ids.retain(|column_id| { - current_column_ids.contains(column_id) && !updated_column_id_set.contains(column_id) + let mut block_meta = + merge_column_group_metadata(origin, ¤t_column_ids, ColumnGroupUpdate { + active_column_ids: updated_column_ids, + location: data_location, + file_size, + uncompressed_size, + column_metas: updated_col_metas, + column_stats: updated_col_stats, }); - } - column_groups.retain(|group| !group.active_column_ids.is_empty()); - column_groups.push(ColumnGroupFileMeta { - active_column_ids: updated_column_ids, - location: data_location.clone(), - format_version: data_location.1, - file_size, - uncompressed_size, - leaf_column_metas: updated_col_metas.clone(), - }); - - let mut block_meta = origin.clone(); - block_meta.location = data_location.clone(); - block_meta.file_size = column_groups.iter().map(|group| group.file_size).sum(); - block_meta.block_size = column_groups - .iter() - .map(|group| group.uncompressed_size) - .sum(); - block_meta.column_groups = column_groups; - block_meta - .col_metas - .retain(|column_id, _| current_column_ids.contains(column_id)); - block_meta.col_metas.extend(updated_col_metas); - block_meta - .col_stats - .retain(|column_id, _| current_column_ids.contains(column_id)); - block_meta.col_stats.extend(updated_col_stats); block_meta.create_on = Some(Utc::now()); let current_bloom_column_ids = self @@ -499,67 +603,17 @@ impl BlockBuilder { .filter(|field| BloomIndex::supported_type(field.data_type())) .map(TableField::column_id) .collect::>(); - let keep_legacy_bloom_layout = if matches!( - origin.bloom_index_layout(), - Some(BloomIndexLayout::Legacy { .. }) - ) { - let active_column_ids = legacy_bloom_column_ids - .ok_or_else(|| ErrorCode::Internal("legacy Bloom file columns were not loaded"))?; - !active_column_ids.is_empty() - && bloom_index_state.is_none() - && invalidated_bloom_column_ids.is_empty() - && active_column_ids - .iter() - .all(|column_id| current_bloom_column_ids.contains(column_id)) - } else { - false - }; - - if !keep_legacy_bloom_layout { - let mut bloom_index_files = match origin.bloom_index_layout() { - None => Vec::new(), - Some(BloomIndexLayout::Legacy { - location, - file_size, - }) => { - let active_column_ids = legacy_bloom_column_ids.ok_or_else(|| { - ErrorCode::Internal("legacy Bloom file columns were not loaded") - })?; - if !active_column_ids.is_empty() { - vec![BloomIndexFileMeta { - active_column_ids: active_column_ids.to_vec(), - location: location.clone(), - format_version: location.1, - file_size, - }] - } else { - Vec::new() - } - } - Some(BloomIndexLayout::Split { files }) => files.to_vec(), - }; - for file in &mut bloom_index_files { - file.active_column_ids.retain(|column_id| { - current_bloom_column_ids.contains(column_id) - && !invalidated_bloom_column_ids.contains(column_id) - }); - } - bloom_index_files.retain(|file| !file.active_column_ids.is_empty()); - if let Some(state) = &bloom_index_state { - let mut active_column_ids = - updated_bloom_column_ids.iter().copied().collect::>(); - active_column_ids.sort_unstable(); - bloom_index_files.push(BloomIndexFileMeta { - active_column_ids, - location: state.location.clone(), - format_version: state.location.1, - file_size: state.size, - }); - } + if let BloomLayoutMerge::Canonical { files, file_size } = merge_partial_update_bloom_layout( + origin, + legacy_bloom_column_ids, + ¤t_bloom_column_ids, + &invalidated_bloom_column_ids, + &updated_bloom_column_ids, + bloom_index_state.as_ref(), + )? { block_meta.bloom_filter_index_location = None; - block_meta.bloom_filter_index_size = - bloom_index_files.iter().map(|file| file.file_size).sum(); - block_meta.bloom_index_files = bloom_index_files; + block_meta.bloom_filter_index_size = file_size; + block_meta.bloom_index_files = files; block_meta.ngram_filter_index_size = None; } if invalidate_virtual_columns { diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index efae3c6420539..3fac39a65dd0f 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -60,19 +60,21 @@ use crate::operations::mutation::ClusterStatsGenType; use crate::operations::mutation::SerializeDataMeta; use crate::statistics::ClusterStatsGenerator; +struct PendingSerialization { + block: DataBlock, + stats_type: ClusterStatsGenType, + index: Option, +} + #[allow(clippy::large_enum_variant)] enum State { Consume, NeedLegacyBloomMeta { - block: DataBlock, - stats_type: ClusterStatsGenType, - index: Option, + pending: PendingSerialization, origin_block_meta: Arc, }, NeedSerialize { - block: DataBlock, - stats_type: ClusterStatsGenType, - index: Option, + pending: PendingSerialization, origin_block_meta: Option>, legacy_bloom_column_ids: Option>, }, @@ -405,17 +407,21 @@ impl Processor for TransformSerializeBlock { .is_some_and(|origin| self.needs_legacy_bloom_meta(origin)) { self.state = State::NeedLegacyBloomMeta { - block: input_data, - stats_type: serialize_block.stats_type, - index: Some(serialize_block.index), + pending: PendingSerialization { + block: input_data, + stats_type: serialize_block.stats_type, + index: Some(serialize_block.index), + }, origin_block_meta: origin_block_meta.unwrap(), }; Ok(Event::Async) } else { self.state = State::NeedSerialize { - block: input_data, - stats_type: serialize_block.stats_type, - index: Some(serialize_block.index), + pending: PendingSerialization { + block: input_data, + stats_type: serialize_block.stats_type, + index: Some(serialize_block.index), + }, origin_block_meta, legacy_bloom_column_ids: None, }; @@ -447,9 +453,11 @@ impl Processor for TransformSerializeBlock { 0 }; self.state = State::NeedSerialize { - block: input_data, - stats_type: ClusterStatsGenType::Generally, - index: None, + pending: PendingSerialization { + block: input_data, + stats_type: ClusterStatsGenType::Generally, + index: None, + }, origin_block_meta: None, legacy_bloom_column_ids: None, }; @@ -460,9 +468,12 @@ impl Processor for TransformSerializeBlock { fn process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { State::NeedSerialize { - block, - stats_type, - index, + pending: + PendingSerialization { + block, + stats_type, + index, + }, origin_block_meta, legacy_bloom_column_ids, } => { @@ -501,18 +512,14 @@ impl Processor for TransformSerializeBlock { async fn async_process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { State::NeedLegacyBloomMeta { - block, - stats_type, - index, + pending, origin_block_meta, } => { let legacy_bloom_column_ids = self .load_legacy_bloom_column_ids(&origin_block_meta) .await?; self.state = State::NeedSerialize { - block, - stats_type, - index, + pending, origin_block_meta: Some(origin_block_meta), legacy_bloom_column_ids: Some(legacy_bloom_column_ids), }; diff --git a/src/query/storages/fuse/src/operations/mod.rs b/src/query/storages/fuse/src/operations/mod.rs index 455b55c3941d6..d4d9f91c2f2d5 100644 --- a/src/query/storages/fuse/src/operations/mod.rs +++ b/src/query/storages/fuse/src/operations/mod.rs @@ -60,6 +60,8 @@ pub use snapshot_hint::*; pub use table_index::do_refresh_table_index; pub use util::*; pub use vacuum::ASSUMPTION_MAX_TXN_DURATION; +pub use vacuum::VacuumObjectKeyPolicy; +pub use vacuum::is_vacuum_object_gc_candidate; pub use vacuum::vacuum_tables_from_info; pub use virtual_column::VirtualColumnVacuumResult; pub use virtual_column::cleanup_vacuum_virtual_column_files; diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 11b52c6419e79..910ed55c33d2a 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -41,7 +41,7 @@ use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_cache::LoadParams; use databend_storages_common_index::BloomIndex; -use databend_storages_common_index::filters::BlockFilter; +use databend_storages_common_index::filters::BlockBloomFilterIndexVersion; use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_io::ReadSettings; @@ -51,7 +51,6 @@ use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; -use databend_storages_common_table_meta::meta::Versioned; use log::info; use log::warn; use opendal::Operator; @@ -79,6 +78,13 @@ use crate::operations::replace_into::mutator::DeletionAccumulator; use crate::operations::replace_into::mutator::row_hash_of_columns; use crate::pruning::read_multi_file_block_filter; +fn replace_digest_pruning_supported(format_version: u64) -> bool { + matches!( + BlockBloomFilterIndexVersion::try_from(format_version), + Ok(BlockBloomFilterIndexVersion::V3(_)) | Ok(BlockBloomFilterIndexVersion::V4(_)) + ) +} + struct AggregationContext { segment_locations: AHashMap, block_slots_in_charge: Option, @@ -714,6 +720,12 @@ impl AggregationContext { location, file_size, }) => { + // REPLACE carries current digest hashes instead of scalar values. V2 filters were + // built from legacy scalar values, so testing current digests against them can + // produce false negatives and incorrectly prune a conflicting block. + if !replace_digest_pruning_supported(location.1) { + return Ok(None); + } let col_names = bloom_on_conflict_field_index .iter() .map(|idx| { @@ -751,7 +763,7 @@ impl AggregationContext { }; // REPLACE carries current digest hashes instead of scalar values, so it cannot // safely evaluate legacy V2 filters. Keep the block in that compatibility case. - if filter.format_version != BlockFilter::VERSION { + if !replace_digest_pruning_supported(filter.format_version) { return Ok(None); } let col_names = fields @@ -790,9 +802,29 @@ mod tests { use databend_common_expression::TableSchema; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; + use databend_storages_common_index::filters::BlockFilter; + use databend_storages_common_table_meta::meta::Versioned; use super::*; + #[test] + fn test_replace_digest_pruning_version_eligibility() { + assert!( + !replace_digest_pruning_supported(2), + "V2 filters contain legacy scalar hashes, not current digests" + ); + assert!( + replace_digest_pruning_supported(3), + "V3 filters use the digest encoding" + ); + assert!( + replace_digest_pruning_supported(BlockFilter::VERSION), + "the current filter format uses the digest encoding" + ); + assert!(!replace_digest_pruning_supported(0)); + assert!(!replace_digest_pruning_supported(u64::MAX)); + } + #[test] fn test_check_overlap() -> Result<()> { // setup : diff --git a/src/query/storages/fuse/src/operations/vacuum.rs b/src/query/storages/fuse/src/operations/vacuum.rs index ab45f91e4205f..ef40c120307be 100644 --- a/src/query/storages/fuse/src/operations/vacuum.rs +++ b/src/query/storages/fuse/src/operations/vacuum.rs @@ -36,6 +36,7 @@ use databend_storages_common_cache::TableSnapshot; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::VACUUM2_OBJECT_KEY_PREFIX; use databend_storages_common_table_meta::meta::is_uuid_v7; +use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; use databend_storages_common_table_meta::meta::uuid_from_date_time; use futures_util::TryStreamExt; use log::info; @@ -44,6 +45,7 @@ use opendal::Entry; use opendal::ErrorKind; use opendal::Operator; use opendal::Scheme; +use uuid::Uuid; use crate::FuseTable; use crate::RetentionPolicy; @@ -84,6 +86,61 @@ use crate::io::TableMetaLocationGenerator; /// the above risks will not exist. pub const ASSUMPTION_MAX_TXN_DURATION: Duration = Duration::days(3); +/// Object-key policy used when deciding whether an unreferenced Vacuum2 object predates the GC +/// root. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VacuumObjectKeyPolicy { + /// The listing has already applied the lexicographic cutoff for `h`-prefixed Vacuum2 keys. + Vacuum2PrefixedBeforeRoot, + /// Current objects use a canonical, prefixless UUID V7 as the first file-name component. + PrefixlessUuidV7 { gc_root_timestamp: DateTime }, +} + +impl VacuumObjectKeyPolicy { + /// Return a decision when the object key itself is trustworthy. `None` selects the legacy + /// transaction-window fallback. + fn trusted_gc_candidate(self, path: &str) -> Option { + match self { + Self::Vacuum2PrefixedBeforeRoot => path + .rsplit('/') + .next() + .is_some_and(|name| name.starts_with(VACUUM2_OBJECT_KEY_PREFIX)) + .then_some(true), + Self::PrefixlessUuidV7 { gc_root_timestamp } => { + let uuid_str = try_extract_uuid_str_from_path(path).ok()?; + let file_name = path.rsplit('/').next()?; + if !file_name.starts_with(uuid_str) { + return None; + } + + let uuid = Uuid::parse_str(uuid_str).ok()?; + if !is_uuid_v7(&uuid) || uuid.as_simple().to_string() != uuid_str { + return None; + } + + let (seconds, nanos) = uuid.get_timestamp()?.to_unix(); + let created_at = DateTime::::from_timestamp(seconds.try_into().ok()?, nanos)?; + Some(created_at.timestamp_millis() < gc_root_timestamp.timestamp_millis()) + } + } + } +} + +/// Decide whether an unreferenced object is eligible for Vacuum2 collection. +/// +/// Trusted current keys use their embedded creation time and a strict GC-root cutoff. Legacy or +/// malformed keys use the compatibility window based on object modification time. +pub fn is_vacuum_object_gc_candidate( + path: &str, + modified: DateTime, + gc_root_meta_ts: DateTime, + key_policy: VacuumObjectKeyPolicy, +) -> bool { + key_policy + .trusted_gc_candidate(path) + .unwrap_or(modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts) +} + pub struct SnapshotGcSelection { pub gc_root: Arc, pub snapshots_to_gc: Vec, @@ -171,9 +228,9 @@ async fn is_gc_candidate_segment_block( gc_root_meta_ts: DateTime, ) -> Result { let path = entry.path(); - let last_part = path.rsplit('/').next().unwrap(); - if last_part.starts_with(VACUUM2_OBJECT_KEY_PREFIX) { - return Ok(true); + let key_policy = VacuumObjectKeyPolicy::Vacuum2PrefixedBeforeRoot; + if let Some(candidate) = key_policy.trusted_gc_candidate(path) { + return Ok(candidate); } let last_modified = if let Some(v) = entry.metadata().last_modified() { v @@ -188,7 +245,12 @@ async fn is_gc_candidate_segment_block( })? }; - Ok(last_modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts) + Ok(is_vacuum_object_gc_candidate( + path, + last_modified, + gc_root_meta_ts, + key_policy, + )) } impl FuseTable { @@ -721,3 +783,106 @@ pub fn slice_summary(s: &[T]) -> String { format!("{:?}", s) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn bloom_path(uuid: Uuid) -> String { + format!("1/2/_i_b_v2/{}_v4.parquet", uuid.as_simple()) + } + + #[test] + fn test_prefixless_uuid_v7_gc_classification() { + let gc_root_timestamp = + DateTime::::from_timestamp(1_700_000_000, 500_000_000).unwrap(); + let gc_root_meta_ts = gc_root_timestamp + Duration::days(10); + let recent = gc_root_meta_ts - Duration::hours(1); + let old = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION - Duration::hours(1); + let policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; + + let before_root = bloom_path(uuid_from_date_time( + gc_root_timestamp - Duration::milliseconds(1), + )); + let at_root = bloom_path(uuid_from_date_time(gc_root_timestamp)); + let after_root = bloom_path(uuid_from_date_time( + gc_root_timestamp + Duration::milliseconds(1), + )); + + assert!(is_vacuum_object_gc_candidate( + &before_root, + recent, + gc_root_meta_ts, + policy, + )); + assert!(!is_vacuum_object_gc_candidate( + &at_root, + old, + gc_root_meta_ts, + policy, + )); + assert!(!is_vacuum_object_gc_candidate( + &after_root, + old, + gc_root_meta_ts, + policy, + )); + } + + #[test] + fn test_legacy_or_malformed_gc_classification_uses_transaction_window() { + let gc_root_timestamp = DateTime::::from_timestamp(1_700_000_000, 0).unwrap(); + let gc_root_meta_ts = gc_root_timestamp + Duration::days(10); + let policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; + let valid_v7 = bloom_path(uuid_from_date_time( + gc_root_timestamp - Duration::milliseconds(1), + )); + let uppercase_v7 = valid_v7.to_ascii_uppercase(); + let v4 = bloom_path(Uuid::new_v4()); + let malformed = "1/2/_i_b_v2/not-a-uuid_v4.parquet"; + let at_window = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION; + let before_window = at_window - Duration::milliseconds(1); + + for path in [&uppercase_v7, &v4, malformed] { + assert!(!is_vacuum_object_gc_candidate( + path, + at_window, + gc_root_meta_ts, + policy, + )); + assert!(is_vacuum_object_gc_candidate( + path, + before_window, + gc_root_meta_ts, + policy, + )); + } + } + + #[test] + fn test_prefixed_policy_preserves_existing_listing_classification() { + let gc_root_meta_ts = Utc::now(); + let recent = gc_root_meta_ts - Duration::hours(1); + let old = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION - Duration::hours(1); + let policy = VacuumObjectKeyPolicy::Vacuum2PrefixedBeforeRoot; + + assert!(is_vacuum_object_gc_candidate( + "1/2/_b/h-malformed.parquet", + recent, + gc_root_meta_ts, + policy, + )); + assert!(!is_vacuum_object_gc_candidate( + "1/2/_b/legacy.parquet", + recent, + gc_root_meta_ts, + policy, + )); + assert!(is_vacuum_object_gc_candidate( + "1/2/_b/legacy.parquet", + old, + gc_root_meta_ts, + policy, + )); + } +} diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index cc4c67a3f861e..3c208ee934e99 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -320,6 +320,14 @@ impl<'a> FuseEncodingImpl<'a> { } let row_group = &file_meta.row_groups()[0]; let columns = row_group.columns(); + let mut columns_by_range = HashMap::with_capacity(columns.len()); + for column in columns { + // Keep the first match to preserve the previous `iter().find()` behavior for + // malformed metadata containing duplicate byte ranges. + columns_by_range + .entry(column.byte_range()) + .or_insert(column); + } let mut block_rows = Vec::new(); for field in fields.iter() { @@ -334,10 +342,7 @@ impl<'a> FuseEncodingImpl<'a> { continue; }; let column_range = column_meta.offset_length(); - let Some(column_chunk) = columns - .iter() - .find(|column| column.byte_range() == column_range) - else { + let Some(column_chunk) = columns_by_range.get(&column_range).copied() else { continue; }; let compressed_size = u64::try_from(column_chunk.compressed_size()).map_err(|_| { From d2d797d55af76950fc0303d2220dc691a68465f3 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:44:40 +0800 Subject: [PATCH 11/30] refactor(storage): pair bloom indexes with column groups --- src/meta/api/src/api_impl/index_api.rs | 9 + src/meta/api/src/api_impl/table_api.rs | 31 +- src/meta/app/src/schema/mod.rs | 1 + src/meta/app/src/schema/table/mod.rs | 141 ++++++++ .../fuse/operations/vacuum_table_v2.rs | 20 +- .../fuse/operations/computed_columns.rs | 141 -------- .../it/storages/fuse/operations/vacuum2.rs | 134 ++----- .../src/catalogs/default/session_catalog.rs | 3 + .../interpreter_table_add_column.rs | 4 + .../interpreter_table_modify_column.rs | 6 + .../interpreter_table_set_options.rs | 26 +- .../it/indexes/ngram_index/index_refresh.rs | 121 +++---- .../it/storages/fuse/bloom_index_meta_size.rs | 1 - .../fuse/operations/mutation/update.rs | 332 ++---------------- .../it/storages/fuse/operations/prewhere.rs | 2 +- .../storages/fuse/operations/replace_into.rs | 11 +- .../storages/common/cache/src/manager.rs | 1 - .../storages/common/session/src/temp_table.rs | 26 +- .../common/table_meta/src/meta/current/mod.rs | 3 +- .../common/table_meta/src/meta/v2/mod.rs | 3 +- .../common/table_meta/src/meta/v2/segment.rs | 121 ++----- .../src/meta/v3/frozen/block_meta.rs | 1 - .../common/table_meta/src/meta/v4/segment.rs | 1 - src/query/storages/fuse/src/constants.rs | 2 +- src/query/storages/fuse/src/fuse_part.rs | 108 +++++- src/query/storages/fuse/src/io/locations.rs | 9 +- .../fuse/src/io/write/block_writer.rs | 146 ++------ .../fuse/src/io/write/bloom_index_writer.rs | 51 +-- .../fuse/src/io/write/stream/block_builder.rs | 1 - src/query/storages/fuse/src/lib.rs | 1 + .../processors/transform_serialize_block.rs | 108 +----- src/query/storages/fuse/src/operations/gc.rs | 7 +- .../fuse/src/operations/read_partitions.rs | 7 +- .../mutator/replace_into_operation_agg.rs | 9 +- .../fuse/src/operations/table_index.rs | 43 ++- .../storages/fuse/src/pruning/block_pruner.rs | 3 +- .../storages/fuse/src/pruning/bloom_pruner.rs | 226 +++--------- .../column_oriented_block_prune.rs | 13 +- .../fuse/src/table_functions/fuse_block.rs | 19 +- .../09_0054_partial_update.test | 16 +- ...001_partial_update_cluster_dependency.test | 45 +-- 41 files changed, 667 insertions(+), 1286 deletions(-) diff --git a/src/meta/api/src/api_impl/index_api.rs b/src/meta/api/src/api_impl/index_api.rs index 2e17991c75f50..ef2abe43f636f 100644 --- a/src/meta/api/src/api_impl/index_api.rs +++ b/src/meta/api/src/api_impl/index_api.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use databend_common_meta_app::KeyExistsBuilder; use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; +use databend_common_meta_app::app_error::CommitTableMetaError; use databend_common_meta_app::app_error::DuplicatedIndexColumnId; use databend_common_meta_app::app_error::IndexColumnIdNotFound; use databend_common_meta_app::app_error::UnknownTableId; @@ -347,6 +348,14 @@ where options: req.options.clone(), }; indexes.insert(req.name.clone(), index); + table_meta + .validate_column_group_compatibility() + .map_err(|error| { + KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( + req.table_id.to_string(), + error, + ))) + })?; let mut txn_req = TxnRequest::new( // diff --git a/src/meta/api/src/api_impl/table_api.rs b/src/meta/api/src/api_impl/table_api.rs index 02b43f42f0db0..96e7220ebc696 100644 --- a/src/meta/api/src/api_impl/table_api.rs +++ b/src/meta/api/src/api_impl/table_api.rs @@ -198,6 +198,13 @@ fn validate_create_table_request(req: &CreateTableReq) -> Result<(), KVAppError> AppError::CreateAsDropTableWithoutDropTime(CreateAsDropTableWithoutDropTime::new(name)), )); } + req.table_meta + .validate_column_group_compatibility() + .map_err(|error| { + KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( + name, error, + ))) + })?; Ok(()) } @@ -1210,12 +1217,24 @@ where } let mut new_table_meta_map: BTreeMap = BTreeMap::new(); - for ((req, _), (tb_meta_seq, _)) in update_table_metas.iter_mut().zip(tb_meta_vec.iter()) { + for ((req, _), (tb_meta_seq, table_meta)) in + update_table_metas.iter_mut().zip(tb_meta_vec.iter()) + { let tbid = TableId { table_id: req.table_id, }; let new_table_meta = req.new_table_meta.clone(); + table_meta + .as_ref() + .unwrap() + .validate_column_group_transition(&new_table_meta) + .map_err(|error| { + KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( + req.table_id.to_string(), + error, + ))) + })?; tbl_seqs.insert(req.table_id, *tb_meta_seq); txn.condition.push(txn_cond_seq(&tbid, Eq, *tb_meta_seq)); @@ -1431,6 +1450,7 @@ where } let mut table_meta = seq_meta.data; + let old_table_meta = table_meta.clone(); for (k, opt_v) in &req.options { match opt_v { @@ -1443,6 +1463,15 @@ where } } + old_table_meta + .validate_column_group_transition(&table_meta) + .map_err(|error| { + AppError::CommitTableMetaError(CommitTableMetaError::new( + req.table_id.to_string(), + error, + )) + })?; + Ok(Some(table_meta)) }) .await??; diff --git a/src/meta/app/src/schema/mod.rs b/src/meta/app/src/schema/mod.rs index 0213e827e9358..e98938468c3b7 100644 --- a/src/meta/app/src/schema/mod.rs +++ b/src/meta/app/src/schema/mod.rs @@ -125,6 +125,7 @@ pub use table::ListDroppedTableResp; pub use table::ListTableCopiedFileReply; pub use table::ListTableReq; pub use table::ListTableTagsReq; +pub use table::OPT_KEY_ENABLE_PARTIAL_UPDATE; pub use table::RenameTableReply; pub use table::RenameTableReq; pub use table::SecurityPolicyColumnMap; diff --git a/src/meta/app/src/schema/table/mod.rs b/src/meta/app/src/schema/table/mod.rs index 67574b2044943..fbd7d037c615a 100644 --- a/src/meta/app/src/schema/table/mod.rs +++ b/src/meta/app/src/schema/table/mod.rs @@ -203,7 +203,63 @@ pub struct TableMeta { pub constraints: BTreeMap, } +pub const OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; +const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; +const COLUMN_ORIENTED_SEGMENT_FORMAT: &str = "column_oriented"; + impl TableMeta { + pub fn column_group_layout_enabled(&self) -> bool { + self.options + .get(OPT_KEY_ENABLE_PARTIAL_UPDATE) + .is_some_and(|value| value.eq_ignore_ascii_case("true")) + } + + /// Validate the persisted table features that may coexist with column-group blocks. + pub fn validate_column_group_compatibility(&self) -> std::result::Result<(), String> { + if !self.column_group_layout_enabled() { + return Ok(()); + } + if !self.engine.eq_ignore_ascii_case("FUSE") { + return Err(format!( + "column-group layout requires FUSE engine, but found {}", + self.engine + )); + } + if self + .options + .get(OPT_KEY_SEGMENT_FORMAT) + .is_some_and(|format| format.eq_ignore_ascii_case(COLUMN_ORIENTED_SEGMENT_FORMAT)) + { + return Err("column-group layout is incompatible with column-oriented segments".into()); + } + if self + .schema + .fields() + .iter() + .any(|field| field.computed_expr().is_some()) + { + return Err("column-group layout is incompatible with computed columns".into()); + } + if !self.indexes.is_empty() { + let mut indexes = self.indexes.keys().cloned().collect::>(); + indexes.sort(); + return Err(format!( + "column-group layout is incompatible with table indexes: {}", + indexes.join(", ") + )); + } + Ok(()) + } + + pub fn validate_column_group_transition(&self, next: &Self) -> std::result::Result<(), String> { + if self.column_group_layout_enabled() && !next.column_group_layout_enabled() { + return Err( + "column-group layout cannot be disabled without a full layout migration".into(), + ); + } + next.validate_column_group_compatibility() + } + /// Returns the cluster key defined on the main branch, if any. pub fn cluster_key_meta(&self) -> Option<(u32, String)> { // - Prefer `cluster_key_v2` if present (branch-aware) @@ -1251,14 +1307,99 @@ mod kvapi_key_impl { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + + use databend_common_expression::ComputedExpr; + use databend_common_expression::TableDataType; + use databend_common_expression::TableField; + use databend_common_expression::TableSchema; + use databend_common_expression::types::NumberDataType; use databend_meta_client::kvapi; use databend_meta_client::kvapi::StructKey; use databend_meta_client::kvapi::testing::assert_round_trip; use crate::schema::TableCopiedFileNameIdent; use crate::schema::TableIdToName; + use crate::schema::TableIndex; + use crate::schema::TableIndexType; use crate::schema::TableMeta; + fn column_group_table_meta() -> TableMeta { + TableMeta { + schema: Arc::new(TableSchema::new(vec![TableField::new( + "a", + TableDataType::Number(NumberDataType::Int32), + )])), + engine: "FUSE".to_string(), + options: BTreeMap::from([("enable_partial_update".to_string(), "true".to_string())]), + ..Default::default() + } + } + + #[test] + fn test_column_group_compatibility_and_sticky_transition() { + let meta = column_group_table_meta(); + assert!(meta.validate_column_group_compatibility().is_ok()); + assert!(meta.validate_column_group_transition(&meta).is_ok()); + + let mut non_fuse = meta.clone(); + non_fuse.engine = "MEMORY".to_string(); + assert!( + non_fuse + .validate_column_group_compatibility() + .unwrap_err() + .contains("requires FUSE") + ); + + let mut column_oriented = meta.clone(); + column_oriented + .options + .insert("segment_format".to_string(), "column_oriented".to_string()); + assert!( + column_oriented + .validate_column_group_compatibility() + .unwrap_err() + .contains("column-oriented") + ); + + let mut computed = meta.clone(); + computed.schema = Arc::new(TableSchema::new(vec![ + TableField::new("a", TableDataType::Number(NumberDataType::Int32)) + .with_computed_expr(Some(ComputedExpr::Virtual("a + 1".to_string()))), + ])); + assert!( + computed + .validate_column_group_compatibility() + .unwrap_err() + .contains("computed columns") + ); + + let mut indexed = meta.clone(); + indexed.indexes.insert("idx".to_string(), TableIndex { + index_type: TableIndexType::Inverted, + name: "idx".to_string(), + column_ids: vec![0], + sync_creation: false, + version: "v1".to_string(), + options: BTreeMap::new(), + }); + assert!( + indexed + .validate_column_group_compatibility() + .unwrap_err() + .contains("table indexes: idx") + ); + + let mut disabled = meta.clone(); + disabled.options.remove("enable_partial_update"); + assert!( + meta.validate_column_group_transition(&disabled) + .unwrap_err() + .contains("cannot be disabled") + ); + } + #[test] fn test_table_id_to_name_key_format() { assert_round_trip(TableIdToName { table_id: 9 }, "__fd_table_id_to_name/9"); diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index 7fc4342f7905e..ef845a10838bd 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -27,6 +27,7 @@ use databend_common_exception::Result; use databend_common_meta_app::schema::ListIndexesByIdReq; use databend_common_meta_app::schema::TableIndex; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::block_bloom_index_locations; use databend_common_storages_fuse::io::SegmentsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::operations::VacuumObjectKeyPolicy; @@ -145,9 +146,9 @@ pub async fn do_vacuum2( .map(|location| location.0.clone()), ); gc_root_indexes.extend( - block - .bloom_index_file_locations() - .map(|location| location.0.clone()), + block_bloom_index_locations(&block) + .into_iter() + .map(|location| location.0), ); } } @@ -192,10 +193,9 @@ pub async fn do_vacuum2( slice_summary(&blocks_to_gc) )); - // Bloom files are immutable, but their lifetime is not tied to the data file whose UUID was - // originally used for the index path. Ngram refresh, for example, consolidates a split Bloom - // layout into a fresh file while retaining the active data groups. List the Bloom directory - // independently so those superseded files do not leak forever. + // Column-group Bloom paths are derived from their owning data-group paths. The files live in a + // separate directory, so scan that directory and retain exactly the files referenced by the + // protected groups. This also removes orphan Bloom files left by interrupted writes. let start = std::time::Instant::now(); let bloom_indexes_before_gc_root = list_bloom_indexes_before_gc_root(fuse_table, gc_root_timestamp, gc_root_meta_ts).await?; @@ -386,7 +386,7 @@ fn collect_block_index_locations( inverted_indexes: &BTreeMap, ) -> Vec { let mut indexes_to_gc = Vec::with_capacity( - blocks_to_gc.len() * (table_agg_index_ids.len() + inverted_indexes.len() + 1), + blocks_to_gc.len() * (table_agg_index_ids.len() + inverted_indexes.len()), ); for loc in blocks_to_gc { for index_id in table_agg_index_ids { @@ -405,8 +405,6 @@ fn collect_block_index_locations( ), ); } - indexes_to_gc - .push(TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(loc)); } indexes_to_gc } @@ -499,14 +497,12 @@ mod tests { "idx", "123456789", ), - TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(&blocks[0]), TableMetaLocationGenerator::gen_agg_index_location_from_block_location(&blocks[1], 7), TableMetaLocationGenerator::gen_inverted_index_location_from_block_location( &blocks[1], "idx", "123456789", ), - TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(&blocks[1]), ]); } diff --git a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs index bd11215955733..1733ecfc3acde 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/computed_columns.rs @@ -153,144 +153,3 @@ async fn test_computed_column() -> anyhow::Result<()> { Ok(()) } - -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_rejects_virtual_column_with_change_tracking() -> anyhow::Result<()> { - let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} \ - (id int, value int, virtual_value bigint as (value + 1) virtual) engine=fuse \ - enable_partial_update=true change_tracking=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name}(id, value) values (1, 10), (2, 20)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set value = value + 1 where id = 1" - )) - .await?; - - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and value = 11 and virtual_value = 12) \ - or (id = 2 and value = 20 and virtual_value = 21)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - - let rows = fixture - .execute_query(&format!( - "select count(*) from fuse_block('{db}', '{table_name}') \ - where column_groups = parse_json('[]')" - )) - .await?; - assert_eq!(query_count(rows).await?, 1); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_rejects_stored_computed_column_table() -> anyhow::Result<()> { - let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} \ - (id int, value int, stored_value bigint as (value + 1) stored) engine=fuse \ - enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name}(id, value) values (1, 10), (2, 20)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set value = value + 1 where id = 1" - )) - .await?; - - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and value = 11 and stored_value = 12) \ - or (id = 2 and value = 20 and stored_value = 21)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - - let rows = fixture - .execute_query(&format!( - "select count(*) from fuse_block('{db}', '{table_name}') \ - where column_groups = parse_json('[]')" - )) - .await?; - assert_eq!(query_count(rows).await?, 1); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_rejects_computed_column_table() -> anyhow::Result<()> { - let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} \ - (id int, a int, k bigint as (a + 1) virtual) engine=fuse \ - enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name}(id, a) values (1, 10), (2, 20)" - )) - .await?; - fixture - .execute_command(&format!("alter table {db}.{table_name} cluster by(k)")) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set a = a + 1 where id = 1" - )) - .await?; - - let updated = latest_default_block_meta(&fixture).await?; - assert!(updated.column_groups.is_empty()); - assert!(updated.cluster_stats.is_some()); - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and a = 11 and k = 12) or (id = 2 and a = 20 and k = 21)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - - Ok(()) -} diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs index cf9c4a314ffcf..4b508cfbea72c 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs @@ -17,6 +17,7 @@ use std::path::Path; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_enterprise_query::test_kits::context::EESetup; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContextTableAccess; @@ -151,11 +152,21 @@ async fn test_vacuum2_preserves_active_partial_update_files() -> anyhow::Result< "update {db}.{table_name} set a = a + 1 where id = 1" )) .await?; - let obsolete_group = latest_default_block_meta(&fixture) - .await? - .location - .0 - .clone(); + let first_update = latest_default_block_meta(&fixture).await?; + let obsolete_group = first_update.location.0.clone(); + let obsolete_bloom = first_update + .column_groups + .iter() + .find(|group| group.location.0 == obsolete_group) + .and_then(|group| { + group.bloom.as_ref().map(|bloom| { + TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &group.location.0, + bloom.format_version, + ) + }) + }) + .expect("updated group should have an ordinary Bloom file"); fixture .execute_command(&format!( @@ -185,11 +196,16 @@ async fn test_vacuum2_preserves_active_partial_update_files() -> anyhow::Result< let operator = fuse_table.get_operator(); for group in ¤t.column_groups { operator.stat(&group.location.0).await?; - } - for file in ¤t.bloom_index_files { - operator.stat(&file.location.0).await?; + if let Some(bloom) = &group.bloom { + let bloom_location = TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &group.location.0, + bloom.format_version, + ); + operator.stat(&bloom_location).await?; + } } assert!(operator.stat(&obsolete_group).await.is_err()); + assert!(operator.stat(&obsolete_bloom).await.is_err()); let rows = fixture .execute_query(&format!( @@ -203,108 +219,6 @@ async fn test_vacuum2_preserves_active_partial_update_files() -> anyhow::Result< Ok(()) } -#[tokio::test(flavor = "multi_thread")] -async fn test_vacuum2_removes_superseded_split_bloom_files() -> anyhow::Result<()> { - let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; - fixture - .default_session() - .get_settings() - .set_data_retention_time_in_days(0)?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, a int, content string) engine=fuse \ - bloom_index_columns='id,a' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values \ - (1, 10, 'aaaaaaaaaa'), (2, 20, 'bbbbbbbbbb')" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set a = a + 1 where id = 1" - )) - .await?; - - let split_meta = latest_default_block_meta(&fixture).await?; - assert_eq!(split_meta.bloom_index_files.len(), 2); - let superseded_bloom_files = split_meta - .bloom_index_files - .iter() - .map(|file| file.location.0.clone()) - .collect::>(); - - fixture - .execute_command(&format!( - "create ngram index idx_content on {db}.{table_name}(content)" - )) - .await?; - fixture - .execute_command(&format!( - "refresh ngram index idx_content on {db}.{table_name}" - )) - .await?; - - let current = latest_default_block_meta(&fixture).await?; - assert!(current.bloom_index_files.is_empty()); - let current_bloom_file = current - .bloom_filter_index_location - .as_ref() - .expect("consolidated Bloom file") - .0 - .clone(); - assert!( - superseded_bloom_files - .iter() - .all(|path| path != ¤t_bloom_file) - ); - let active_data_files = current - .data_file_locations() - .map(|location| location.0.clone()) - .collect::>(); - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let operator = fuse_table.get_operator(); - for path in &superseded_bloom_files { - operator.stat(path).await?; - } - - fixture - .execute_command(&format!("call system$fuse_vacuum2('{db}', '{table_name}')")) - .await?; - - for path in superseded_bloom_files { - assert!( - operator.stat(&path).await.is_err(), - "{path} was not removed" - ); - } - operator.stat(¤t_bloom_file).await?; - for path in active_data_files { - operator.stat(&path).await?; - } - - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and a = 11 and content = 'aaaaaaaaaa') \ - or (id = 2 and a = 20 and content = 'bbbbbbbbbb')" - )) - .await?; - assert_eq!(databend_query::test_kits::query_count(rows).await?, 2); - - Ok(()) -} - /// Verifies that dedup_file_locations correctly removes duplicates and reports samples. #[test] fn test_dedup_file_locations() { diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/service/src/catalogs/default/session_catalog.rs index 717254c984ba2..bbc3cb296130e 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/service/src/catalogs/default/session_catalog.rs @@ -611,6 +611,9 @@ impl Catalog for SessionCatalog { match state { TxnState::AutoCommit => { let update_temp_tables = std::mem::take(&mut req.update_temp_tables); + self.temp_tbl_mgr + .lock() + .validate_multi_table_meta(&update_temp_tables)?; let reply = if req.is_empty() { Ok(Default::default()) } else { diff --git a/src/query/service/src/interpreters/interpreter_table_add_column.rs b/src/query/service/src/interpreters/interpreter_table_add_column.rs index c956d03291b3a..2c77a39899515 100644 --- a/src/query/service/src/interpreters/interpreter_table_add_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_add_column.rs @@ -147,6 +147,10 @@ impl Interpreter for AddTableColumnInterpreter { table_info .meta .add_column(&field, &self.plan.comment, index)?; + table_info + .meta + .validate_column_group_compatibility() + .map_err(ErrorCode::TableOptionInvalid)?; // if the new column is a stored computed field and table is non-empty, // need rebuild the table to generate stored computed column. diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 050ed2665970e..16bc3c034b36d 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -253,6 +253,12 @@ impl ModifyTableColumnInterpreter { let fuse_table = FuseTable::try_from_table(table.as_ref())?; let new_schema = Arc::new(new_schema); + let mut next_meta = table_info.meta.clone(); + next_meta.schema = new_schema.clone(); + table_info + .meta + .validate_column_group_transition(&next_meta) + .map_err(ErrorCode::TableOptionInvalid)?; if !modified_cols.is_empty() { // Only need to validate the data types of modified columns that are referenced // by the cluster key. The cluster key expression itself is already validated diff --git a/src/query/service/src/interpreters/interpreter_table_set_options.rs b/src/query/service/src/interpreters/interpreter_table_set_options.rs index 1fe5cae09b4a0..c7fd524cfdc3e 100644 --- a/src/query/service/src/interpreters/interpreter_table_set_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_set_options.rs @@ -191,6 +191,24 @@ impl Interpreter for SetOptionsInterpreter { // check enable_virtual_column is_valid_fuse_virtual_column_opt(&self.plan.set_options)?; + // Reject incompatible layout transitions before analyze or segment conversion writes + // auxiliary data. The meta service repeats this check at the atomic commit point. + let current_meta = &table.get_table_info().meta; + let mut next_meta = current_meta.clone(); + for (key, value) in &options_map { + match value { + Some(value) => { + next_meta.options.insert(key.clone(), value.clone()); + } + None => { + next_meta.options.remove(key); + } + } + } + current_meta + .validate_column_group_transition(&next_meta) + .map_err(ErrorCode::TableOptionInvalid)?; + let table = analyze_table(self.ctx.clone(), table, &self.plan.set_options).await?; let table_version = table.get_table_info().ident.seq; @@ -289,9 +307,11 @@ async fn set_segment_format( .await?; for segment in segments { let segment = segment?; - if segment.blocks.iter().any(|block| { - !block.column_groups.is_empty() || !block.bloom_index_files.is_empty() - }) { + if segment + .blocks + .iter() + .any(|block| !block.column_groups.is_empty()) + { return Err(ErrorCode::TableOptionInvalid( "cannot convert segments containing partial-update files to the column-oriented format; rewrite the blocks to the single-file layout first", )); diff --git a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs index 17812bb1035ca..9985db99bf2fb 100644 --- a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; use std::sync::Arc; use databend_common_exception::Result; @@ -22,8 +21,8 @@ use databend_query::sessions::TableContext; use databend_query::storages::index::filters::BlockFilter; use databend_query::test_kits::TestFixture; use databend_query::test_kits::latest_default_block_meta; +use databend_query::test_kits::query_count; use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::Versioned; use futures_util::StreamExt; #[tokio::test(flavor = "multi_thread")] @@ -111,95 +110,97 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { } #[tokio::test(flavor = "multi_thread")] -async fn test_refresh_ngram_preserves_split_bloom_files() -> anyhow::Result<()> { +async fn test_ngram_index_is_incompatible_with_partial_update() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; let db = fixture.default_db_name(); let table_name = fixture.default_table_name(); fixture.create_default_database().await?; + fixture .execute_command(&format!( - "create table {db}.{table_name} (id int, a int, d string, e string) engine=fuse \ - bloom_index_columns='id,a' enable_partial_update=true" + "create table {db}.{table_name} (id int, content string) engine=fuse \ + enable_partial_update=true" )) .await?; + assert!( + fixture + .execute_command(&format!( + "create ngram index idx_content on {db}.{table_name}(content)" + )) + .await + .is_err() + ); + fixture + .execute_command(&format!("drop table {db}.{table_name}")) + .await?; + fixture .execute_command(&format!( - "insert into {db}.{table_name} values \ - (1, 10, 'aaaaaaaaaa', 'xxxxxxxxxx'), \ - (2, 20, 'bbbbbbbbbb', 'yyyyyyyyyy')" + "create table {db}.{table_name} (id int, content string) engine=fuse" )) .await?; fixture - .execute_command("set enable_partial_update = 1") + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 'before'), (2, 'unchanged')" + )) .await?; fixture .execute_command(&format!( - "update {db}.{table_name} set a = a + 1 where id = 1" + "create ngram index idx_content on {db}.{table_name}(content)" )) .await?; - - let split_meta = latest_default_block_meta(&fixture).await?; - assert_eq!(split_meta.bloom_index_files.len(), 2); - let table = fixture.latest_default_table().await?; - let id_column_id = table.schema().field(0).column_id(); - let a_column_id = table.schema().field(1).column_id(); - let d_column_id = table.schema().field(2).column_id(); - let e_column_id = table.schema().field(3).column_id(); - let active_column_ids = split_meta - .bloom_index_files - .iter() - .flat_map(|file| file.active_column_ids.iter().copied()) - .collect::>(); - assert_eq!( - active_column_ids, - HashSet::from([id_column_id, a_column_id]) + fixture + .execute_command(&format!( + "refresh ngram index idx_content on {db}.{table_name}" + )) + .await?; + assert!( + fixture + .execute_command(&format!( + "alter table {db}.{table_name} \ + set options(enable_partial_update=true)" + )) + .await + .is_err() ); - let operator = DataOperator::instance().operator(); - let split_files = split_meta - .bloom_index_files - .iter() - .map(|file| (file.location.0.clone(), file.file_size)) - .collect::>(); fixture .execute_command(&format!( - "create ngram index idx_e on {db}.{table_name}(e) gram_size=4" + "drop ngram index idx_content on {db}.{table_name}" )) .await?; fixture - .execute_command(&format!("create ngram index idx_d on {db}.{table_name}(d)")) + .execute_command(&format!( + "alter table {db}.{table_name} set options(enable_partial_update=true)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") .await?; fixture - .execute_command(&format!("refresh ngram index idx_d on {db}.{table_name}")) + .execute_command(&format!( + "update {db}.{table_name} set content = 'after' where id = 1" + )) .await?; - let refreshed_meta = latest_default_block_meta(&fixture).await?; - assert!(refreshed_meta.bloom_index_files.is_empty()); - let refreshed_location = refreshed_meta - .bloom_filter_index_location - .as_ref() - .expect("refreshed Bloom index location"); - assert_eq!(refreshed_location.1, BlockFilter::VERSION); - assert!( - split_files + let block_meta = latest_default_block_meta(&fixture).await?; + assert_eq!(block_meta.column_groups.len(), 2); + assert!(block_meta.bloom_filter_index_location.is_none()); + assert_eq!( + block_meta + .column_groups .iter() - .all(|(path, _)| path != &refreshed_location.0) + .filter(|group| group.bloom.is_some()) + .count(), + 1 ); - for (path, size) in &split_files { - assert_eq!(operator.stat(path).await?.content_length(), *size); - } - - let columns = [ - format!("Bloom({id_column_id})"), - format!("Bloom({a_column_id})"), - format!("Ngram({d_column_id})_3_1048576"), - format!("Ngram({e_column_id})_4_1048576"), - ]; - let block_filter = get_block_filter(&fixture, &db, &table_name, &columns).await?; - assert_eq!(block_filter.filters.len(), columns.len()); - for column in columns { - assert!(block_filter.filter_schema.index_of(&column).is_ok()); - } + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and content = 'after') or (id = 2 and content = 'unchanged')" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); Ok(()) } diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index 943550667f962..4bfc02314476c 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -334,7 +334,6 @@ fn build_test_segment_info( cluster_stats: None, location: block_location, bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)), - bloom_index_files: vec![], bloom_filter_index_size: 0, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 81468c7220613..0d72c08870c7e 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -93,13 +93,21 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { origin.col_stats.get(&id_column_id) ); assert!(updated.bloom_filter_index_location.is_none()); - assert_eq!(updated.bloom_index_files.len(), 2); + assert_eq!( + updated + .column_groups + .iter() + .filter(|group| group.bloom.is_some()) + .count(), + 2 + ); assert_eq!( updated.bloom_filter_index_size, updated - .bloom_index_files + .column_groups .iter() - .map(|file| file.file_size) + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) .sum::() ); let updated_hll = latest_block_hll(&fixture).await?; @@ -160,7 +168,14 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { .all(|group| group.location != updated.location) ); assert!(updated_again.bloom_filter_index_location.is_none()); - assert_eq!(updated_again.bloom_index_files.len(), 1); + assert_eq!( + updated_again + .column_groups + .iter() + .filter(|group| group.bloom.is_some()) + .count(), + 1 + ); let rows = fixture .execute_query(&format!( @@ -187,7 +202,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { .await?; let fully_rewritten = latest_default_block_meta(&fixture).await?; assert!(fully_rewritten.column_groups.is_empty()); - assert!(fully_rewritten.bloom_index_files.is_empty()); + assert!(fully_rewritten.bloom_filter_index_location.is_some()); let rows = fixture .execute_query(&format!( @@ -254,313 +269,6 @@ async fn test_partial_update_preserves_block_compression() -> anyhow::Result<()> Ok(()) } -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_reads_legacy_bloom_schema() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, value int, other int) engine=fuse \ - bloom_index_columns='id' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 10, 20), (2, 30, 40)" - )) - .await?; - let table = fixture.latest_default_table().await?; - let id_column_id = table.schema().field_with_name("id")?.column_id(); - - fixture - .execute_command(&format!( - "alter table {db}.{table_name} set options(bloom_index_columns='value')" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set other = other + 1 where id = 1" - )) - .await?; - - let updated = latest_default_block_meta(&fixture).await?; - assert!(updated.bloom_filter_index_location.is_none()); - assert_eq!(updated.bloom_index_files.len(), 1); - assert_eq!(updated.bloom_index_files[0].active_column_ids, vec![ - id_column_id - ]); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_drops_obsolete_split_bloom_file_after_drop_column() --> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, value int, flag boolean) engine=fuse \ - bloom_index_columns='id,value' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 10, true), (2, 20, false)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set value = value + 1 where id = 1" - )) - .await?; - - let table = fixture.latest_default_table().await?; - let value_column_id = table.schema().field_with_name("value")?.column_id(); - let split_meta = latest_default_block_meta(&fixture).await?; - let obsolete_bloom_location = split_meta - .bloom_index_files - .iter() - .find(|file| file.active_column_ids.contains(&value_column_id)) - .unwrap() - .location - .clone(); - - fixture - .execute_command(&format!("alter table {db}.{table_name} drop column value")) - .await?; - // Boolean is not supported by the ordinary Bloom filter. This exercises metadata cleanup - // even when the current UPDATE neither invalidates nor writes a Bloom filter. - fixture - .execute_command(&format!( - "update {db}.{table_name} set flag = not flag where id = 1" - )) - .await?; - - let updated = latest_default_block_meta(&fixture).await?; - assert!( - updated - .bloom_index_files - .iter() - .all(|file| file.location != obsolete_bloom_location - && !file.active_column_ids.contains(&value_column_id)) - ); - assert_eq!( - updated.bloom_filter_index_size, - updated - .bloom_index_files - .iter() - .map(|file| file.file_size) - .sum::() - ); - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and not flag) or (id = 2 and not flag)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_ignores_missing_legacy_bloom_file() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, value int, flag boolean) engine=fuse \ - bloom_index_columns='id,value' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 10, true), (2, 20, false)" - )) - .await?; - - let table = fixture.latest_default_table().await?; - let value_column_id = table.schema().field_with_name("value")?.column_id(); - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let operator = fuse_table.get_operator(); - let origin = latest_default_block_meta(&fixture).await?; - let missing_location = origin.bloom_filter_index_location.as_ref().unwrap().clone(); - operator.delete(&missing_location.0).await?; - - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set value = value + 1 where id = 1" - )) - .await?; - - let updated = latest_default_block_meta(&fixture).await?; - assert!(updated.bloom_filter_index_location.is_none()); - assert_eq!(updated.bloom_index_files.len(), 1); - assert_eq!(updated.bloom_index_files[0].active_column_ids, vec![ - value_column_id - ]); - assert_ne!(updated.bloom_index_files[0].location, missing_location); - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and value = 11 and flag) \ - or (id = 2 and value = 20 and not flag)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_auto_fix_missing_split_bloom_index() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, value int) engine=fuse \ - bloom_index_columns='id,value' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 10), (3, 30)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set value = value + 1 where id = 1" - )) - .await?; - - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let operator = fuse_table.get_operator(); - let value_column_id = table.schema().field_with_name("value")?.column_id(); - let block_meta = latest_default_block_meta(&fixture).await?; - assert_eq!(block_meta.bloom_index_files.len(), 2); - let missing_file = block_meta - .bloom_index_files - .iter() - .find(|file| file.active_column_ids.contains(&value_column_id)) - .unwrap(); - let existing_file = block_meta - .bloom_index_files - .iter() - .find(|file| file.location != missing_file.location) - .unwrap(); - let existing_data = operator.read(&existing_file.location.0).await?.to_bytes(); - operator.delete(&missing_file.location.0).await?; - assert!(!operator.exists(&missing_file.location.0).await?); - - fixture - .execute_command("set enable_auto_fix_missing_bloom_index = 1") - .await?; - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} where value = 20" - )) - .await?; - assert_eq!(query_count(rows).await?, 0); - assert!(operator.exists(&missing_file.location.0).await?); - assert_eq!( - operator.read(&existing_file.location.0).await?.to_bytes(), - existing_data - ); - - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} where id = 1 and value = 11" - )) - .await?; - assert_eq!(query_count(rows).await?, 1); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_auto_fix_missing_legacy_bloom_index_from_split_data() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, flag boolean) engine=fuse \ - bloom_index_columns='id' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, true), (3, false)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set flag = false where flag" - )) - .await?; - - let block_meta = latest_default_block_meta(&fixture).await?; - assert_eq!(block_meta.column_groups.len(), 2); - assert!(block_meta.bloom_index_files.is_empty()); - let bloom_location = block_meta.bloom_filter_index_location.as_ref().unwrap(); - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let operator = fuse_table.get_operator(); - operator.delete(&bloom_location.0).await?; - assert!(!operator.exists(&bloom_location.0).await?); - - fixture - .execute_command("set enable_auto_fix_missing_bloom_index = 1") - .await?; - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} where id = 2" - )) - .await?; - assert_eq!(query_count(rows).await?, 0); - assert!(operator.exists(&bloom_location.0).await?); - - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} where id = 1 and not flag" - )) - .await?; - assert_eq!(query_count(rows).await?, 1); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread")] async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; diff --git a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs index f5ee8d7659980..77a8e88d6b532 100644 --- a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs +++ b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs @@ -331,7 +331,7 @@ async fn prepare_prewhere_data() -> Result { location: "test_block".to_string(), bloom_filter_index_location: None, bloom_filter_index_size: 0, - bloom_index_files: vec![], + column_group_bloom_files: vec![], create_on: None, nums_rows: num_rows, column_groups: vec![FuseColumnGroupPartInfo { diff --git a/src/query/service/tests/it/storages/fuse/operations/replace_into.rs b/src/query/service/tests/it/storages/fuse/operations/replace_into.rs index af17c3abb44ed..0ec89c5bd15c1 100644 --- a/src/query/service/tests/it/storages/fuse/operations/replace_into.rs +++ b/src/query/service/tests/it/storages/fuse/operations/replace_into.rs @@ -88,11 +88,18 @@ async fn test_replace_bloom_prunes_partial_update_block() -> anyhow::Result<()> )) .await?; let block_meta = latest_default_block_meta(&fixture).await?; - assert_eq!(block_meta.bloom_index_files.len(), 2); + assert_eq!( + block_meta + .column_groups + .iter() + .filter(|group| group.bloom.is_some()) + .count(), + 2 + ); // Make the active data files unreadable while leaving both Bloom files intact. A key that // might conflict must still try to read the block, proving that the missing data is visible. - // The absent key below can succeed only if REPLACE reads and combines the split Bloom files. + // The absent key below can succeed only if REPLACE reads and combines the paired Bloom files. let table = fixture.latest_default_table().await?; let fuse_table = FuseTable::try_from_table(table.as_ref())?; let operator = fuse_table.get_operator(); diff --git a/src/query/storages/common/cache/src/manager.rs b/src/query/storages/common/cache/src/manager.rs index c98c19a7c4016..b9244c16bc48d 100644 --- a/src/query/storages/common/cache/src/manager.rs +++ b/src/query/storages/common/cache/src/manager.rs @@ -1301,7 +1301,6 @@ mod tests { cluster_stats: None, location: ("".to_string(), 0), bloom_filter_index_location: None, - bloom_index_files: vec![], bloom_filter_index_size: 0, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/storages/common/session/src/temp_table.rs b/src/query/storages/common/session/src/temp_table.rs index 5dbfe89bb6f35..4cd1df0f7e5ae 100644 --- a/src/query/storages/common/session/src/temp_table.rs +++ b/src/query/storages/common/session/src/temp_table.rs @@ -105,6 +105,9 @@ impl TempTblMgr { as_dropped, .. } = req; + table_meta + .validate_column_group_compatibility() + .map_err(ErrorCode::TableOptionInvalid)?; let orphan_table_name = as_dropped.then(|| format!("orphan@{}", name_ident.table_name)); let Some(db_id) = table_meta.options.get(OPT_KEY_DATABASE_ID) else { @@ -265,6 +268,19 @@ impl TempTblMgr { .collect()) } + pub fn validate_multi_table_meta(&self, req: &[UpdateTempTableReq]) -> Result<()> { + for r in req { + let table = self.id_to_table.get(&r.table_id).ok_or_else(|| { + ErrorCode::UnknownTable(format!("Temporary table id {} not found", r.table_id)) + })?; + table + .meta + .validate_column_group_transition(&r.new_table_meta) + .map_err(ErrorCode::TableOptionInvalid)?; + } + Ok(()) + } + pub fn update_multi_table_meta(&mut self, req: Vec) { for r in req { let UpdateTempTableReq { @@ -293,13 +309,19 @@ impl TempTblMgr { table_id ))); }; + let mut new_meta = table.meta.clone(); for (k, v) in options { if let Some(v) = v { - table.meta.options.insert(k, v); + new_meta.options.insert(k, v); } else { - table.meta.options.remove(&k); + new_meta.options.remove(&k); } } + table + .meta + .validate_column_group_transition(&new_meta) + .map_err(ErrorCode::TableOptionInvalid)?; + table.meta = new_meta; Ok(UpsertTableOptionReply {}) } diff --git a/src/query/storages/common/table_meta/src/meta/current/mod.rs b/src/query/storages/common/table_meta/src/meta/current/mod.rs index 7ea5f05c4e522..530969c3641e9 100644 --- a/src/query/storages/common/table_meta/src/meta/current/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/current/mod.rs @@ -15,9 +15,8 @@ pub use v0::ColumnMeta as SingleColumnMeta; pub use v2::AdditionalStatsMeta; pub use v2::BlockMeta; -pub use v2::BloomIndexFileMeta; -pub use v2::BloomIndexLayout; pub use v2::ClusterStatistics; +pub use v2::ColumnGroupBloomMeta; pub use v2::ColumnGroupFileMeta; pub use v2::ColumnMeta; pub use v2::ColumnStatistics; diff --git a/src/query/storages/common/table_meta/src/meta/v2/mod.rs b/src/query/storages/common/table_meta/src/meta/v2/mod.rs index bfafc7f1ab1de..c7b8f54e8391b 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/mod.rs @@ -19,8 +19,7 @@ pub mod statistics; mod table_snapshot_statistics; pub use segment::BlockMeta; -pub use segment::BloomIndexFileMeta; -pub use segment::BloomIndexLayout; +pub use segment::ColumnGroupBloomMeta; pub use segment::ColumnGroupFileMeta; pub use segment::ColumnMeta; pub use segment::DraftVirtualBlockMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index bded2ab487dc2..f792f4c243bb4 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -185,6 +185,9 @@ pub struct ColumnGroupFileMeta { pub file_size: u64, pub uncompressed_size: u64, pub leaf_column_metas: HashMap, + /// Ordinary Bloom file paired with this data file. Its location is derived from `location`. + #[serde(default)] + pub bloom: Option, } impl ColumnGroupFileMeta { @@ -198,48 +201,16 @@ impl ColumnGroupFileMeta { } } -/// Metadata of one physical Bloom index file referenced by a logical block. +/// Metadata of the ordinary Bloom file paired with a physical column-group file. /// -/// A file may still contain filters that are no longer active. Readers must only use the filters -/// whose column ids are listed in [`Self::active_column_ids`]. +/// The Bloom file is self-describing. A stored filter is active only while its column id remains in +/// the owning [`ColumnGroupFileMeta::active_column_ids`]. #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] -pub struct BloomIndexFileMeta { - pub active_column_ids: Vec, - pub location: Location, +pub struct ColumnGroupBloomMeta { pub format_version: FormatVersion, pub file_size: u64, } -/// Borrowed view of the physical Bloom-index layout used by a logical block. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BloomIndexLayout<'a> { - Legacy { - location: &'a Location, - file_size: u64, - }, - Split { - files: &'a [BloomIndexFileMeta], - }, -} - -impl<'a> BloomIndexLayout<'a> { - /// Normalize optional legacy and split Bloom metadata into one physical-layout view. - pub fn from_metadata( - legacy_location: Option<&'a Location>, - legacy_file_size: u64, - split_files: &'a [BloomIndexFileMeta], - ) -> Option { - if split_files.is_empty() { - legacy_location.map(|location| Self::Legacy { - location, - file_size: legacy_file_size, - }) - } else { - Some(Self::Split { files: split_files }) - } - } -} - /// Meta information of a block /// Part of and kept inside the [SegmentInfo] #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -266,13 +237,6 @@ pub struct BlockMeta { /// location of bloom filter index pub bloom_filter_index_location: Option, - /// Physical files that contain the active Bloom filters of this logical block. - /// - /// An empty vector is the legacy single-file representation described by - /// `bloom_filter_index_location` and `bloom_filter_index_size`. - #[serde(default)] - pub bloom_index_files: Vec, - #[serde(default)] pub bloom_filter_index_size: u64, pub inverted_index_size: Option, @@ -324,7 +288,6 @@ impl BlockMeta { cluster_stats, location, bloom_filter_index_location, - bloom_index_files: vec![], bloom_filter_index_size, inverted_index_size, ngram_filter_index_size, @@ -343,15 +306,6 @@ impl BlockMeta { pub fn compression(&self) -> Compression { self.compression } - /// Normalize optional legacy and split Bloom metadata into one physical-layout view. - pub fn bloom_index_layout(&self) -> Option> { - BloomIndexLayout::from_metadata( - self.bloom_filter_index_location.as_ref(), - self.bloom_filter_index_size, - &self.bloom_index_files, - ) - } - /// Active physical data files referenced by this logical block. pub fn data_file_locations(&self) -> impl Iterator { self.column_groups @@ -361,16 +315,6 @@ impl BlockMeta { .chain(self.column_groups.iter().map(|group| &group.location)) } - /// Active physical Bloom files referenced by this logical block. - pub fn bloom_index_file_locations(&self) -> impl Iterator { - self.bloom_index_files - .is_empty() - .then_some(self.bloom_filter_index_location.as_ref()) - .flatten() - .into_iter() - .chain(self.bloom_index_files.iter().map(|file| &file.location)) - } - fn legacy_column_group( &self, projected_column_ids: Option<&HashSet>, @@ -395,6 +339,13 @@ impl BlockMeta { file_size: self.file_size, uncompressed_size: self.block_size, leaf_column_metas, + bloom: self + .bloom_filter_index_location + .as_ref() + .map(|location| ColumnGroupBloomMeta { + format_version: location.1, + file_size: self.bloom_filter_index_size, + }), } } @@ -439,6 +390,7 @@ impl BlockMeta { file_size, uncompressed_size, leaf_column_metas, + bloom: None, }) }; @@ -461,6 +413,10 @@ impl BlockMeta { group.uncompressed_size, &group.leaf_column_metas, ) + .map(|mut projected| { + projected.bloom = group.bloom.clone(); + projected + }) }) .collect() } @@ -581,7 +537,6 @@ impl BlockMeta { cluster_stats: None, location: (s.location.path.clone(), 0), bloom_filter_index_location: None, - bloom_index_files: vec![], bloom_filter_index_size: 0, compression: Compression::Lz4, inverted_index_size: None, @@ -615,7 +570,6 @@ impl BlockMeta { cluster_stats: None, location: s.location.clone(), bloom_filter_index_location: s.bloom_filter_index_location.clone(), - bloom_index_files: vec![], bloom_filter_index_size: s.bloom_filter_index_size, compression: s.compression, inverted_index_size: None, @@ -648,31 +602,6 @@ impl From<(v0::SegmentInfo, &[TableField])> for SegmentInfo { mod tests { use super::*; - #[test] - fn test_bloom_index_layout_normalizes_legacy_and_split_metadata() { - let legacy_location = ("legacy-bloom".to_string(), 2); - assert_eq!( - BloomIndexLayout::from_metadata(Some(&legacy_location), 10, &[]), - Some(BloomIndexLayout::Legacy { - location: &legacy_location, - file_size: 10, - }) - ); - - let split_files = vec![BloomIndexFileMeta { - active_column_ids: vec![1], - location: ("split-bloom".to_string(), 4), - format_version: 4, - file_size: 20, - }]; - assert_eq!( - BloomIndexLayout::from_metadata(Some(&legacy_location), 10, &split_files), - Some(BloomIndexLayout::Split { - files: &split_files, - }) - ); - } - #[test] fn test_active_leaf_column_metas_excludes_inactive_and_missing_columns() { let group = ColumnGroupFileMeta { @@ -699,6 +628,7 @@ mod tests { }), ), ]), + bloom: None, }; let active_metas = group @@ -709,7 +639,7 @@ mod tests { } #[test] - fn test_deserialize_legacy_block_meta_without_file_lists() { + fn test_deserialize_legacy_block_meta_without_column_groups() { let block_meta = BlockMeta::new( 10, 300, @@ -740,17 +670,8 @@ mod tests { .remove("column_groups") .is_some() ); - assert!( - value - .as_object_mut() - .unwrap() - .remove("bloom_index_files") - .is_some() - ); - let decoded: BlockMeta = serde_json::from_value(value).unwrap(); assert!(decoded.column_groups.is_empty()); - assert!(decoded.bloom_index_files.is_empty()); assert_eq!(decoded.location, location); } } diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index 0694bc8d6e8e8..7c365a51038b1 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -61,7 +61,6 @@ impl From for crate::meta::BlockMeta { cluster_stats: value.cluster_stats.map(|v| v.into()), location: value.location, bloom_filter_index_location: value.bloom_filter_index_location, - bloom_index_files: vec![], bloom_filter_index_size: value.bloom_filter_index_size, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/storages/common/table_meta/src/meta/v4/segment.rs b/src/query/storages/common/table_meta/src/meta/v4/segment.rs index 56d32604cb587..04a19acef6cb5 100644 --- a/src/query/storages/common/table_meta/src/meta/v4/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v4/segment.rs @@ -549,7 +549,6 @@ mod tests { cluster_stats: None, location: ("block.parquet".to_string(), 0), bloom_filter_index_location: None, - bloom_index_files: vec![], bloom_filter_index_size: 0, inverted_index_size: None, ngram_filter_index_size: None, diff --git a/src/query/storages/fuse/src/constants.rs b/src/query/storages/fuse/src/constants.rs index d5da27468466b..d6cb43644ee66 100644 --- a/src/query/storages/fuse/src/constants.rs +++ b/src/query/storages/fuse/src/constants.rs @@ -24,7 +24,7 @@ pub const FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP: &str = "data_retention_num_snapshots_to_keep"; pub const FUSE_OPT_KEY_ENABLE_AUTO_VACUUM: &str = "enable_auto_vacuum"; pub const FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE: &str = "enable_auto_analyze"; -pub const FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; +pub use databend_common_meta_app::schema::OPT_KEY_ENABLE_PARTIAL_UPDATE as FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; pub const FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN: &str = "enable_virtual_column"; pub const FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD: &str = "auto_compaction_imperfect_blocks_threshold"; diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 68d3c8574c67d..011892f7f8882 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::any::Any; +use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; use std::collections::hash_map::DefaultHasher; @@ -32,13 +33,14 @@ use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_table_meta::meta::BlockMeta; -use databend_storages_common_table_meta::meta::BloomIndexFileMeta; -use databend_storages_common_table_meta::meta::BloomIndexLayout; +use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; use databend_storages_common_table_meta::meta::Location; +use crate::io::TableMetaLocationGenerator; + /// Projected column chunks to read from one physical column-group file. #[derive(Clone, serde::Serialize, serde::Deserialize, PartialEq, Debug)] pub struct FuseColumnGroupPartInfo { @@ -46,6 +48,79 @@ pub struct FuseColumnGroupPartInfo { pub columns_meta: HashMap, } +/// Runtime description of a Bloom file paired with one physical data group. +#[derive(Clone, serde::Serialize, serde::Deserialize, PartialEq, Debug)] +pub struct FuseBloomIndexFileInfo { + pub active_column_ids: Vec, + pub location: Location, + pub file_size: u64, +} + +#[derive(Clone, Debug)] +pub enum BloomIndexLayout<'a> { + Legacy { + location: &'a Location, + file_size: u64, + }, + ColumnGroups { + files: Cow<'a, [FuseBloomIndexFileInfo]>, + }, +} + +fn column_group_bloom_location(group: &ColumnGroupFileMeta) -> Option { + group.bloom.as_ref().map(|bloom| { + ( + TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &group.location.0, + bloom.format_version, + ), + bloom.format_version, + ) + }) +} + +pub(crate) fn column_group_bloom_files(meta: &BlockMeta) -> Vec { + meta.column_groups + .iter() + .filter_map(|group| { + let bloom = group.bloom.as_ref()?; + Some(FuseBloomIndexFileInfo { + active_column_ids: group.active_column_ids.clone(), + location: column_group_bloom_location(group)?, + file_size: bloom.file_size, + }) + }) + .collect() +} + +/// Physical ordinary Bloom files referenced by a logical block. +pub fn block_bloom_index_locations(meta: &BlockMeta) -> Vec { + if meta.column_groups.is_empty() { + return meta.bloom_filter_index_location.iter().cloned().collect(); + } + + meta.column_groups + .iter() + .filter_map(column_group_bloom_location) + .collect() +} + +pub(crate) fn bloom_index_layout(meta: &BlockMeta) -> Option> { + if meta.column_groups.is_empty() { + return meta.bloom_filter_index_location.as_ref().map(|location| { + BloomIndexLayout::Legacy { + location, + file_size: meta.bloom_filter_index_size, + } + }); + } + + let files = column_group_bloom_files(meta); + (!files.is_empty()).then_some(BloomIndexLayout::ColumnGroups { + files: Cow::Owned(files), + }) +} + pub(crate) fn project_column_groups( meta: &BlockMeta, projected_column_ids: &HashSet, @@ -66,8 +141,8 @@ pub struct FuseBlockPartInfo { pub bloom_filter_index_location: Option, pub bloom_filter_index_size: u64, - #[serde(default)] - pub bloom_index_files: Vec, + #[serde(default, alias = "bloom_index_files")] + pub column_group_bloom_files: Vec, pub create_on: Option>, pub nums_rows: usize, @@ -103,13 +178,22 @@ impl PartInfo for FuseBlockPartInfo { } impl FuseBlockPartInfo { - /// Normalize optional legacy and split Bloom metadata into one physical-layout view. + /// Normalize optional legacy and column-group Bloom metadata into one physical-layout view. pub fn bloom_index_layout(&self) -> Option> { - BloomIndexLayout::from_metadata( - self.bloom_filter_index_location.as_ref(), - self.bloom_filter_index_size, - &self.bloom_index_files, - ) + if !self.column_groups.is_empty() { + return (!self.column_group_bloom_files.is_empty()).then(|| { + BloomIndexLayout::ColumnGroups { + files: Cow::Borrowed(&self.column_group_bloom_files), + } + }); + } + + self.bloom_filter_index_location + .as_ref() + .map(|location| BloomIndexLayout::Legacy { + location, + file_size: self.bloom_filter_index_size, + }) } #[allow(clippy::too_many_arguments)] @@ -117,7 +201,7 @@ impl FuseBlockPartInfo { location: String, bloom_filter_index_location: Option, bloom_filter_index_size: u64, - bloom_index_files: Vec, + column_group_bloom_files: Vec, rows_count: u64, column_groups: Vec, columns_stat: Option>, @@ -130,7 +214,7 @@ impl FuseBlockPartInfo { location, bloom_filter_index_location, bloom_filter_index_size, - bloom_index_files, + column_group_bloom_files, create_on, column_groups, nums_rows: rows_count as usize, diff --git a/src/query/storages/fuse/src/io/locations.rs b/src/query/storages/fuse/src/io/locations.rs index d18ba2c13168f..57bec4f32dcac 100644 --- a/src/query/storages/fuse/src/io/locations.rs +++ b/src/query/storages/fuse/src/io/locations.rs @@ -346,6 +346,10 @@ impl TableMetaLocationGenerator { } pub fn gen_bloom_index_location_from_block_location(loc: &str) -> String { + Self::gen_bloom_index_location_with_version(loc, BlockFilter::VERSION) + } + + pub fn gen_bloom_index_location_with_version(loc: &str, format_version: u64) -> String { let splits = loc.split('/').collect::>(); let len = splits.len(); let prefix = splits[..len - 2].join("/"); @@ -353,10 +357,7 @@ impl TableMetaLocationGenerator { let id: String = block_name.chars().take(32).collect(); format!( "{}/{}/{}_v{}.parquet", - prefix, - FUSE_TBL_XOR_BLOOM_INDEX_PREFIX, - id, - BlockFilter::VERSION, + prefix, FUSE_TBL_XOR_BLOOM_INDEX_PREFIX, id, format_version, ) } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 6b1c0f8fdfd1e..88914f84ceb11 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -49,15 +49,13 @@ use databend_common_metrics::storage::metrics_inc_block_write_milliseconds; use databend_common_metrics::storage::metrics_inc_block_write_nums; use databend_storages_common_blocks::SerializedParquet; use databend_storages_common_blocks::blocks_to_parquet_with_stats; -use databend_storages_common_index::BloomIndex; use databend_storages_common_index::NgramArgs; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BlockTopN; -use databend_storages_common_table_meta::meta::BloomIndexFileMeta; -use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ClusterStatistics; +use databend_storages_common_table_meta::meta::ColumnGroupBloomMeta; use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; @@ -180,14 +178,7 @@ struct ColumnGroupUpdate { uncompressed_size: u64, column_metas: HashMap, column_stats: StatisticsOfColumns, -} - -enum BloomLayoutMerge { - Preserve, - Canonical { - files: Vec, - file_size: u64, - }, + bloom: Option, } fn merge_column_group_metadata( @@ -201,6 +192,26 @@ fn merge_column_group_metadata( .copied() .collect::>(); let mut column_groups = origin.physical_column_groups().into_owned(); + if origin.column_groups.is_empty() { + let legacy_bloom_is_paired = + origin + .bloom_filter_index_location + .as_ref() + .is_none_or(|location| { + let expected = + TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &origin.location.0, + location.1, + ); + expected == location.0 + }); + if !legacy_bloom_is_paired { + // Ngram refresh historically wrote a new random Bloom path. After the Ngram index is + // dropped the table may enable column groups, but that unpaired file cannot be + // represented by ColumnGroupBloomMeta. Drop the auxiliary reference and fail open. + column_groups[0].bloom = None; + } + } for group in &mut column_groups { group.active_column_ids.retain(|column_id| { current_column_ids.contains(column_id) && !updated_column_ids.contains(column_id) @@ -214,6 +225,7 @@ fn merge_column_group_metadata( file_size: update.file_size, uncompressed_size: update.uncompressed_size, leaf_column_metas: update.column_metas.clone(), + bloom: update.bloom, }); let mut block_meta = origin.clone(); @@ -232,74 +244,17 @@ fn merge_column_group_metadata( .col_stats .retain(|column_id, _| current_column_ids.contains(column_id)); block_meta.col_stats.extend(update.column_stats); + block_meta.bloom_filter_index_location = None; + block_meta.bloom_filter_index_size = block_meta + .column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum(); + block_meta.ngram_filter_index_size = None; block_meta } -fn merge_partial_update_bloom_layout( - origin: &BlockMeta, - legacy_bloom_column_ids: Option<&[ColumnId]>, - current_bloom_column_ids: &HashSet, - invalidated_bloom_column_ids: &HashSet, - updated_bloom_column_ids: &HashSet, - bloom_index_state: Option<&BloomIndexState>, -) -> Result { - let layout = origin.bloom_index_layout(); - if matches!(layout, Some(BloomIndexLayout::Legacy { .. })) { - let active_column_ids = legacy_bloom_column_ids - .ok_or_else(|| ErrorCode::Internal("legacy Bloom file columns were not loaded"))?; - let keep_legacy_layout = !active_column_ids.is_empty() - && bloom_index_state.is_none() - && invalidated_bloom_column_ids.is_empty() - && active_column_ids - .iter() - .all(|column_id| current_bloom_column_ids.contains(column_id)); - if keep_legacy_layout { - return Ok(BloomLayoutMerge::Preserve); - } - } - - let mut files = match layout { - None => Vec::new(), - Some(BloomIndexLayout::Legacy { - location, - file_size, - }) => { - let active_column_ids = legacy_bloom_column_ids - .ok_or_else(|| ErrorCode::Internal("legacy Bloom file columns were not loaded"))?; - if active_column_ids.is_empty() { - Vec::new() - } else { - vec![BloomIndexFileMeta { - active_column_ids: active_column_ids.to_vec(), - location: location.clone(), - format_version: location.1, - file_size, - }] - } - } - Some(BloomIndexLayout::Split { files }) => files.to_vec(), - }; - for file in &mut files { - file.active_column_ids.retain(|column_id| { - current_bloom_column_ids.contains(column_id) - && !invalidated_bloom_column_ids.contains(column_id) - }); - } - files.retain(|file| !file.active_column_ids.is_empty()); - if let Some(state) = bloom_index_state { - let mut active_column_ids = updated_bloom_column_ids.iter().copied().collect::>(); - active_column_ids.sort_unstable(); - files.push(BloomIndexFileMeta { - active_column_ids, - location: state.location.clone(), - format_version: state.location.1, - file_size: state.size, - }); - } - let file_size = files.iter().map(|file| file.file_size).sum(); - Ok(BloomLayoutMerge::Canonical { files, file_size }) -} - impl BlockBuilder { fn add_hll_distinct_counts( column_distinct_count: &mut HashMap, @@ -450,7 +405,6 @@ impl BlockBuilder { cluster_stats, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), - bloom_index_files: vec![], bloom_filter_index_size: bloom_index_state .as_ref() .map(|v| v.size) @@ -493,7 +447,6 @@ impl BlockBuilder { data_block: DataBlock, origin: &BlockMeta, updated_field_indices: &[FieldIndex], - legacy_bloom_column_ids: Option<&[ColumnId]>, ) -> Result { if data_block.num_rows() as u64 != origin.row_count { return Err(ErrorCode::Internal( @@ -541,16 +494,6 @@ impl BlockBuilder { .map(|field| (updated_index, field)) }) .collect::>(); - let updated_bloom_column_ids = updated_bloom_columns_map - .values() - .map(|field| field.column_id()) - .collect::>(); - let invalidated_bloom_column_ids = updated_field_indices - .iter() - .map(|index| self.source_schema.field(*index)) - .filter(|field| BloomIndex::supported_type(field.data_type())) - .map(TableField::column_id) - .collect::>(); let rebuild_bloom_index = !updated_bloom_columns_map.is_empty(); let bloom_index_state = if rebuild_bloom_index { let location = self.meta_locations.block_bloom_index_location(&block_id); @@ -619,29 +562,14 @@ impl BlockBuilder { uncompressed_size, column_metas: updated_col_metas, column_stats: updated_col_stats, + bloom: bloom_index_state + .as_ref() + .map(|state| ColumnGroupBloomMeta { + format_version: state.location.1, + file_size: state.size, + }), }); block_meta.create_on = Some(Utc::now()); - - let current_bloom_column_ids = self - .source_schema - .fields() - .iter() - .filter(|field| BloomIndex::supported_type(field.data_type())) - .map(TableField::column_id) - .collect::>(); - if let BloomLayoutMerge::Canonical { files, file_size } = merge_partial_update_bloom_layout( - origin, - legacy_bloom_column_ids, - ¤t_bloom_column_ids, - &invalidated_bloom_column_ids, - &updated_bloom_column_ids, - bloom_index_state.as_ref(), - )? { - block_meta.bloom_filter_index_location = None; - block_meta.bloom_filter_index_size = file_size; - block_meta.bloom_index_files = files; - block_meta.ngram_filter_index_size = None; - } if invalidate_virtual_columns { block_meta.virtual_block_meta = None; } diff --git a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs index c0e14b822e446..1c408c0db35bd 100644 --- a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs @@ -14,7 +14,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; -use std::collections::HashSet; use std::sync::Arc; use databend_common_catalog::plan::Projection; @@ -33,7 +32,6 @@ use databend_storages_common_index::BloomIndexType; use databend_storages_common_index::NgramArgs; use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::BloomIndexFileMeta; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; @@ -209,30 +207,13 @@ impl BloomIndexRebuilder { &self, bloom_index_location: &Location, data_block: &DataBlock, - active_column_ids: Option<&HashSet>, ) -> Result> { Self::validate_rebuild_version(bloom_index_location)?; - let bloom_columns_map = self - .bloom_columns_map - .iter() - .filter(|(_, field)| { - active_column_ids.is_none_or(|ids| ids.contains(&field.column_id())) - }) - .map(|(index, field)| (*index, field.clone())) - .collect(); - let ngram_args = self - .ngram_args - .iter() - .filter(|arg| { - active_column_ids.is_none_or(|ids| ids.contains(&arg.field().column_id())) - }) - .cloned() - .collect::>(); let mut builder = BloomIndexBuilder::create( self.table_ctx.get_function_context()?, self.bloom_index_type, - bloom_columns_map, - &ngram_args, + self.bloom_columns_map.clone(), + &self.ngram_args, )?; builder.add_block(data_block)?; let maybe_bloom_index = builder.finalize()?; @@ -252,33 +233,7 @@ impl BloomIndexRebuilder { block_read_info: &BlockReadInfo, ) -> Result> { let data_block = self.read_data_block(block_read_info).await?; - self.bloom_index_state_from_data_block(bloom_index_location, &data_block, None) - } - - pub(crate) async fn bloom_index_states_from_block_meta( - &self, - bloom_index_files: &[BloomIndexFileMeta], - block_read_info: &BlockReadInfo, - ) -> Result>> { - let data_block = self.read_data_block(block_read_info).await?; - let mut states = Vec::with_capacity(bloom_index_files.len()); - for file in bloom_index_files { - let active_column_ids = file - .active_column_ids - .iter() - .copied() - .collect::>(); - let Some((state, _)) = self.bloom_index_state_from_data_block( - &file.location, - &data_block, - Some(&active_column_ids), - )? - else { - return Ok(None); - }; - states.push(state); - } - Ok(Some(states)) + self.bloom_index_state_from_data_block(bloom_index_location, &data_block) } } diff --git a/src/query/storages/fuse/src/io/write/stream/block_builder.rs b/src/query/storages/fuse/src/io/write/stream/block_builder.rs index 236520fefcb1a..b9cb3fafa321e 100644 --- a/src/query/storages/fuse/src/io/write/stream/block_builder.rs +++ b/src/query/storages/fuse/src/io/write/stream/block_builder.rs @@ -409,7 +409,6 @@ impl StreamBlockBuilder { cluster_stats: None, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), - bloom_index_files: vec![], bloom_filter_index_size: bloom_index_state .as_ref() .map(|v| v.size) diff --git a/src/query/storages/fuse/src/lib.rs b/src/query/storages/fuse/src/lib.rs index 75641f43bd6ac..edb35dd810ac0 100644 --- a/src/query/storages/fuse/src/lib.rs +++ b/src/query/storages/fuse/src/lib.rs @@ -74,6 +74,7 @@ pub use fuse_column::FuseTableColumnStatisticsProvider; pub use fuse_part::FuseBlockPartInfo; pub use fuse_part::FuseColumnGroupPartInfo; pub use fuse_part::FuseLazyPartInfo; +pub use fuse_part::block_bloom_index_locations; pub use fuse_table::FuseTable; pub use fuse_table::RetentionPolicy; pub use fuse_type::FuseSegmentFormat; diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index 782b7c5f91be0..9fee04cbd1638 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::any::Any; -use std::collections::HashSet; use std::sync::Arc; use databend_common_base::base::ProgressValues; @@ -22,7 +21,6 @@ use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoDowncast; -use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; @@ -39,9 +37,7 @@ use databend_common_storage::MutationStatus; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::BlockMeta; -use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::TableMetaTimestamps; -use log::warn; use opendal::Operator; use crate::FuseTable; @@ -52,7 +48,6 @@ use crate::io::SpatialIndexBuilder; use crate::io::VectorIndexBuilder; use crate::io::VirtualColumnBuilder; use crate::io::create_inverted_index_builders; -use crate::io::read::bloom::block_filter_reader::load_index_meta; use crate::operations::common::BlockMetaIndex; use crate::operations::common::MutationLogEntry; use crate::operations::common::MutationLogs; @@ -69,14 +64,9 @@ struct PendingSerialization { #[allow(clippy::large_enum_variant)] enum State { Consume, - NeedLegacyBloomMeta { - pending: PendingSerialization, - origin_block_meta: Arc, - }, NeedSerialize { pending: PendingSerialization, origin_block_meta: Option>, - legacy_bloom_column_ids: Option>, }, Serialized { serialized: BlockSerialization, @@ -286,50 +276,6 @@ impl TransformSerializeBlock { self.block_builder.clone() } - fn needs_legacy_bloom_meta(&self, origin: &BlockMeta) -> bool { - self.updated_field_indices.is_some() - && matches!( - origin.bloom_index_layout(), - Some(BloomIndexLayout::Legacy { .. }) - ) - } - - async fn load_legacy_bloom_column_ids(&self, origin: &BlockMeta) -> Result> { - let Some(BloomIndexLayout::Legacy { - location, - file_size, - }) = origin.bloom_index_layout() - else { - return Err(ErrorCode::Internal("legacy Bloom location is missing")); - }; - let meta = match load_index_meta(self.dal.clone(), &location.0, file_size, None).await { - Ok(meta) => meta, - Err(error) => { - warn!( - "failed to load legacy Bloom metadata at {:?}; dropping its reference from the partial-update output: {}", - location, error - ); - return Ok(vec![]); - } - }; - let stored_filter_names = meta - .columns - .iter() - .map(|(name, _)| name.as_str()) - .collect::>(); - let mut active_column_ids = Vec::new(); - for field in self.block_builder.source_schema.fields() { - if !BloomIndex::supported_type(field.data_type()) { - continue; - } - let filter_name = BloomIndex::build_filter_bloom_name(location.1, field)?; - if stored_filter_names.contains(filter_name.as_str()) { - active_column_ids.push(field.column_id()); - } - } - Ok(active_column_ids) - } - fn mutation_logs( entry: MutationLogEntry, logical_updated_rows: u64, @@ -355,9 +301,6 @@ impl Processor for TransformSerializeBlock { } fn event(&mut self) -> Result { - if matches!(self.state, State::NeedLegacyBloomMeta { .. }) { - return Ok(Event::Async); - } if matches!(self.state, State::NeedSerialize { .. }) { return Ok(Event::Sync); } @@ -424,32 +367,15 @@ impl Processor for TransformSerializeBlock { serialize_block.logical_updated_rows, serialize_block.logical_deleted_rows, ); - let origin_block_meta = serialize_block.origin_block_meta; - if origin_block_meta - .as_deref() - .is_some_and(|origin| self.needs_legacy_bloom_meta(origin)) - { - self.state = State::NeedLegacyBloomMeta { - pending: PendingSerialization { - block: input_data, - stats_type: serialize_block.stats_type, - index: Some(serialize_block.index), - }, - origin_block_meta: origin_block_meta.unwrap(), - }; - Ok(Event::Async) - } else { - self.state = State::NeedSerialize { - pending: PendingSerialization { - block: input_data, - stats_type: serialize_block.stats_type, - index: Some(serialize_block.index), - }, - origin_block_meta, - legacy_bloom_column_ids: None, - }; - Ok(Event::Sync) - } + self.state = State::NeedSerialize { + pending: PendingSerialization { + block: input_data, + stats_type: serialize_block.stats_type, + index: Some(serialize_block.index), + }, + origin_block_meta: serialize_block.origin_block_meta, + }; + Ok(Event::Sync) } } SerializeDataMeta::CompactExtras(compact_extras) => { @@ -478,7 +404,6 @@ impl Processor for TransformSerializeBlock { index: None, }, origin_block_meta: None, - legacy_bloom_column_ids: None, }; Ok(Event::Sync) } @@ -494,7 +419,6 @@ impl Processor for TransformSerializeBlock { index, }, origin_block_meta, - legacy_bloom_column_ids, } => { // Check if the datablock is valid, this is needed to ensure data is correct block.check_valid()?; @@ -506,7 +430,6 @@ impl Processor for TransformSerializeBlock { block, &origin_block_meta, updated_field_indices, - legacy_bloom_column_ids.as_deref(), )? } else { self.block_builder @@ -530,19 +453,6 @@ impl Processor for TransformSerializeBlock { #[async_backtrace::framed] async fn async_process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { - State::NeedLegacyBloomMeta { - pending, - origin_block_meta, - } => { - let legacy_bloom_column_ids = self - .load_legacy_bloom_column_ids(&origin_block_meta) - .await?; - self.state = State::NeedSerialize { - pending, - origin_block_meta: Some(origin_block_meta), - legacy_bloom_column_ids: Some(legacy_bloom_column_ids), - }; - } State::Serialized { serialized, index } => { let (logical_updated_rows, logical_deleted_rows) = std::mem::take(&mut self.pending_logical_change); diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs index 33ae1d0d0738f..af79e9c4de591 100644 --- a/src/query/storages/fuse/src/operations/gc.rs +++ b/src/query/storages/fuse/src/operations/gc.rs @@ -49,6 +49,7 @@ use log::warn; use crate::FUSE_TBL_SNAPSHOT_PREFIX; use crate::FuseTable; +use crate::block_bloom_index_locations; use crate::index::InvertedIndexFile; use crate::io::InvertedIndexReader; use crate::io::SegmentsIO; @@ -809,9 +810,9 @@ impl TryFrom> for LocationTuple { .map(|location| location.0.clone()), ); bloom_location.extend( - block_meta - .bloom_index_file_locations() - .map(|location| location.0.clone()), + block_bloom_index_locations(&block_meta) + .into_iter() + .map(|location| location.0), ); } if let Some(loc) = value.as_ref().summary.additional_stats_loc() { diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 620cdfda9bfa8..7e7914b666804 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -87,6 +87,7 @@ use crate::FuseLazyPartInfo; use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; +use crate::fuse_part::column_group_bloom_files; use crate::fuse_part::project_column_groups; use crate::io::BloomIndexRebuilder; use crate::pruning::BlockPruner; @@ -1408,7 +1409,7 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, - meta.bloom_index_files.clone(), + column_group_bloom_files(meta), rows_count, column_groups, Some(columns_stats), @@ -1465,7 +1466,7 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, - meta.bloom_index_files.clone(), + column_group_bloom_files(meta), rows_count, column_groups, Some(columns_stat), @@ -1558,6 +1559,7 @@ mod tests { file_size: 10, uncompressed_size: 10, leaf_column_metas: HashMap::from([(1, column_1_meta), (2, stale_column_2_meta)]), + bloom: None, }, ColumnGroupFileMeta { active_column_ids: vec![2], @@ -1566,6 +1568,7 @@ mod tests { file_size: 12, uncompressed_size: 12, leaf_column_metas: HashMap::from([(2, active_column_2_meta.clone())]), + bloom: None, }, ]; diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index ba323bef24ede..14aaa7ca71830 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -47,7 +47,6 @@ use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BlockSlotDescription; -use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; @@ -57,6 +56,8 @@ use opendal::Operator; use tokio::sync::Semaphore; use crate::FuseTable; +use crate::fuse_part::BloomIndexLayout; +use crate::fuse_part::bloom_index_layout; use crate::io::BlockBuilder; use crate::io::BlockReader; use crate::io::BlockWriter; @@ -718,7 +719,7 @@ impl AggregationContext { bloom_on_conflict_field_index: &[FieldIndex], ) -> Result>>>> { // different block may have different version of bloom filter index - let (block_filter, col_names) = match block_meta.bloom_index_layout() { + let (block_filter, col_names) = match bloom_index_layout(block_meta) { None => return Ok(None), Some(BloomIndexLayout::Legacy { location, @@ -749,7 +750,7 @@ impl AggregationContext { .await?; (block_filter, col_names) } - Some(BloomIndexLayout::Split { files }) => { + Some(BloomIndexLayout::ColumnGroups { files }) => { let fields = bloom_on_conflict_field_index .iter() .map(|idx| self.on_conflict_fields[*idx].table_field.clone()) @@ -759,7 +760,7 @@ impl AggregationContext { &self.read_settings, &fields, &[], - files, + &files, ) .await? else { diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index 41f10fed08bb8..d0cd84f9ef72c 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -438,27 +438,27 @@ async fn check_ngram_index_generated( stats: Option>, ngram_index_arg: &RefreshNgramIndexArg, ) -> Result> { - // A partial update may split active Bloom filters across several immutable files. Such a - // block must be consolidated when adding Ngram filters, so do not mistake the Bloom file - // derived from the newest data group for the complete logical index. - if block_meta.bloom_index_files.is_empty() { - if let Some((index_path, _)) = &block_meta.bloom_filter_index_location { - if let Ok(content_length) = operator - .stat(index_path) - .await - .map(|meta| meta.content_length()) - { - let bloom_index_meta = - load_index_meta(operator.clone(), index_path, content_length, None).await?; - - if ngram_index_arg.ngram_index_names.iter().all(|name| { - bloom_index_meta - .columns - .iter() - .any(|(column_name, _)| column_name == name) - }) { - return Ok(None); - } + if !block_meta.column_groups.is_empty() { + return Err(ErrorCode::RefreshIndexError( + "Ngram index is incompatible with column-group layout".to_string(), + )); + } + if let Some((index_path, _)) = &block_meta.bloom_filter_index_location { + if let Ok(content_length) = operator + .stat(index_path) + .await + .map(|meta| meta.content_length()) + { + let bloom_index_meta = + load_index_meta(operator.clone(), index_path, content_length, None).await?; + + if ngram_index_arg.ngram_index_names.iter().all(|name| { + bloom_index_meta + .columns + .iter() + .any(|(column_name, _)| column_name == name) + }) { + return Ok(None); } } } @@ -755,7 +755,6 @@ impl AsyncTransform for NgramIndexTransform { let state = BloomIndexState::from_bloom_index(&bloom_index, index_location)?; new_block_meta.bloom_filter_index_location = Some(state.location.clone()); - new_block_meta.bloom_index_files.clear(); new_block_meta.bloom_filter_index_size = state.size(); new_block_meta.ngram_filter_index_size = state.ngram_size(); BlockWriter::write_down_bloom_index_state(&self.operator, Some(state)).await?; diff --git a/src/query/storages/fuse/src/pruning/block_pruner.rs b/src/query/storages/fuse/src/pruning/block_pruner.rs index 9ce9ea0932963..e16036014631a 100644 --- a/src/query/storages/fuse/src/pruning/block_pruner.rs +++ b/src/query/storages/fuse/src/pruning/block_pruner.rs @@ -33,6 +33,7 @@ use log::debug; use tokio::sync::OwnedSemaphorePermit; use super::SegmentLocation; +use crate::fuse_part::bloom_index_layout; use crate::pruning::PruningContext; use crate::pruning::PruningCostKind; use crate::pruning::RuntimeStatsPruner; @@ -339,7 +340,7 @@ impl BlockPruner { .measure_async( PruningCostKind::BlocksBloom, bloom_pruner.should_keep( - block_meta.bloom_index_layout(), + bloom_index_layout(&block_meta), &block_meta.col_stats, column_ids, &block_meta.as_ref().into(), diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 09cfbe9075569..50ae981e1e4ce 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -34,8 +34,6 @@ use databend_storages_common_index::FilterEvalResult; use databend_storages_common_index::NgramArgs; use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::BloomIndexFileMeta; -use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::Versioned; @@ -45,6 +43,8 @@ use log::warn; use opendal::Operator; use crate::FuseBlockPartInfo; +use crate::fuse_part::BloomIndexLayout; +use crate::fuse_part::FuseBloomIndexFileInfo; use crate::io::BlockWriter; use crate::io::BloomBlockFilterReader; use crate::io::BloomIndexRebuilder; @@ -115,9 +115,9 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( location.1, )? } - Some(BloomIndexLayout::Split { files }) => { + Some(BloomIndexLayout::ColumnGroups { files }) => { let Some(filter) = - read_multi_file_block_filter(dal, settings, &result.bloom_fields, &[], files) + read_multi_file_block_filter(dal, settings, &result.bloom_fields, &[], &files) .await? else { return Ok(false); @@ -149,20 +149,11 @@ pub(crate) struct MultiFileBlockFilter { pub format_version: u64, } -fn multi_file_filter_version( - index_fields: &[TableField], - index_files: &[BloomIndexFileMeta], -) -> Option { +fn merged_filter_version(versions: impl IntoIterator) -> Option { let mut has_v2 = false; let mut has_digest_encoding = false; - for file in index_files { - if !index_fields - .iter() - .any(|field| file.active_column_ids.contains(&field.column_id())) - { - continue; - } - if file.format_version == 2 { + for version in versions { + if version == 2 { has_v2 = true; } else { has_digest_encoding = true; @@ -180,18 +171,15 @@ pub(crate) async fn read_multi_file_block_filter( settings: &ReadSettings, index_fields: &[TableField], ngram_args: &[NgramArgs], - index_files: &[BloomIndexFileMeta], + index_files: &[FuseBloomIndexFileInfo], ) -> Result> { - let Some(format_version) = multi_file_filter_version(index_fields, index_files) else { - // V2 filters use legacy scalar encoding and cannot be evaluated together with the digest - // encoding of newer files. Keeping the block is the safe compatibility behavior. - return Ok(None); - }; - let mut merged_fields = Vec::new(); - let mut merged_filters = Vec::new(); + // The owning data group may contain active columns for which its Bloom file has no filter. + // Decide encoding compatibility from the filters actually loaded, not merely from the data + // group's active ids. + let mut loaded_filters = Vec::new(); for file in index_files { - let mut rename = HashMap::new(); + let mut ordinary_fields_by_filter_name = HashMap::new(); let mut columns = Vec::new(); for field in index_fields { if !file.active_column_ids.contains(&field.column_id()) { @@ -203,12 +191,10 @@ pub(crate) async fn read_multi_file_block_filter( arg.gram_size(), arg.bloom_size(), ); - rename.insert(name.clone(), name.clone()); columns.push(name); } else { - let physical = BloomIndex::build_filter_bloom_name(file.format_version, field)?; - let normalized = BloomIndex::build_filter_bloom_name(format_version, field)?; - rename.insert(physical.clone(), normalized); + let physical = BloomIndex::build_filter_bloom_name(file.location.1, field)?; + ordinary_fields_by_filter_name.insert(physical.clone(), field.clone()); columns.push(physical); } } @@ -226,14 +212,38 @@ pub(crate) async fn read_multi_file_block_filter( .iter() .zip(filter.filters.into_iter()) { - let Some(name) = rename.get(field.name()) else { - continue; - }; - merged_fields.push(TableField::new(name, TableDataType::Binary)); - merged_filters.push(value); + loaded_filters.push(( + field.name().clone(), + ordinary_fields_by_filter_name.get(field.name()).cloned(), + file.location.1, + value, + )); } } + if loaded_filters.is_empty() { + return Ok(None); + } + let Some(format_version) = merged_filter_version( + loaded_filters + .iter() + .filter_map(|(_, field, version, _)| field.as_ref().map(|_| *version)), + ) else { + // V2 filters use legacy scalar encoding and cannot be evaluated together with the digest + // encoding of newer files. Keeping the block is the safe compatibility behavior. + return Ok(None); + }; + let mut merged_fields = Vec::with_capacity(loaded_filters.len()); + let mut merged_filters = Vec::with_capacity(loaded_filters.len()); + for (physical_name, bloom_field, _, filter) in loaded_filters { + let name = match bloom_field { + Some(field) => BloomIndex::build_filter_bloom_name(format_version, &field)?, + None => physical_name, + }; + merged_fields.push(TableField::new(&name, TableDataType::Binary)); + merged_filters.push(filter); + } + Ok(Some(MultiFileBlockFilter { block_filter: BlockFilter { filter_schema: Arc::new(TableSchema::new(merged_fields)), @@ -445,9 +455,9 @@ impl BloomPrunerCreator { async fn apply_files( &self, - index_files: &[BloomIndexFileMeta], + index_files: &[FuseBloomIndexFileInfo], column_stats: &StatisticsOfColumns, - block_meta: &BlockReadInfo, + _block_meta: &BlockReadInfo, ) -> Result { let maybe_filter = read_multi_file_block_filter( &self.dal, @@ -457,35 +467,6 @@ impl BloomPrunerCreator { index_files, ) .await; - let maybe_filter = match (&maybe_filter, &self.bloom_index_builder) { - (Err(_), Some(bloom_index_builder)) => { - match self - .try_rebuild_missing_bloom_index_files( - index_files, - bloom_index_builder, - block_meta, - ) - .await - { - Ok(true) => { - read_multi_file_block_filter( - &self.dal, - &self.settings, - &self.index_fields, - &self.ngram_args, - index_files, - ) - .await - } - Ok(false) => maybe_filter, - Err(e) => { - info!("failed to re-build missing split Bloom indexes: {}", e); - maybe_filter - } - } - } - _ => maybe_filter, - }; let Some(filter) = maybe_filter? else { return Ok(true); }; @@ -505,64 +486,6 @@ impl BloomPrunerCreator { )? != FilterEvalResult::MustFalse) } - async fn try_rebuild_missing_bloom_index_files( - &self, - index_files: &[BloomIndexFileMeta], - bloom_index_builder: &BloomIndexRebuilder, - block_read_info: &BlockReadInfo, - ) -> Result { - let needed_column_ids = self - .index_fields - .iter() - .map(TableField::column_id) - .collect::>(); - let mut missing_files = Vec::new(); - for file in index_files { - if !file - .active_column_ids - .iter() - .any(|column_id| needed_column_ids.contains(column_id)) - { - continue; - } - if self.dal.exists(&file.location.0).await? { - continue; - } - if file.format_version != BlockFilter::VERSION { - info!( - "cannot rebuild missing split Bloom index {:?} with legacy format {}", - file.location, file.format_version - ); - return Ok(false); - } - missing_files.push(file.clone()); - } - if missing_files.is_empty() { - return Ok(false); - } - - let Some(states) = bloom_index_builder - .bloom_index_states_from_block_meta(&missing_files, block_read_info) - .await? - else { - return Ok(false); - }; - if states - .iter() - .zip(&missing_files) - .any(|(state, file)| state.size() != file.file_size) - { - info!("cannot rebuild missing split Bloom indexes because serialized sizes changed"); - return Ok(false); - } - for (state, file) in states.into_iter().zip(&missing_files) { - BlockWriter::write_down_bloom_index_state(&bloom_index_builder.table_dal, Some(state)) - .await?; - info!("re-created missing split Bloom index {:?}", file.location); - } - Ok(true) - } - async fn try_rebuild_missing_bloom_index( &self, bloom_index_location: &Location, @@ -633,8 +556,8 @@ impl BloomPruner for BloomPrunerCreator { block_meta: &BlockReadInfo, ) -> bool { match index_layout { - Some(BloomIndexLayout::Split { files }) => { - match self.apply_files(files, column_stats, block_meta).await { + Some(BloomIndexLayout::ColumnGroups { files }) => { + match self.apply_files(&files, column_stats, block_meta).await { Ok(value) => value, Err(e) => { warn!( @@ -669,57 +592,16 @@ impl BloomPruner for BloomPrunerCreator { #[cfg(test)] mod tests { - use databend_common_expression::TableDataType; - use databend_common_expression::TableField; - use databend_common_expression::types::NumberDataType; use databend_storages_common_index::filters::BlockFilter; - use databend_storages_common_table_meta::meta::BloomIndexFileMeta; use databend_storages_common_table_meta::meta::Versioned; - use super::multi_file_filter_version; - - fn field(column_id: u32) -> TableField { - let mut field = TableField::new( - &format!("c{column_id}"), - TableDataType::Number(NumberDataType::UInt64), - ); - field.column_id = column_id; - field - } - - fn file(active_column_ids: Vec, format_version: u64) -> BloomIndexFileMeta { - BloomIndexFileMeta { - active_column_ids, - location: (format!("bloom-{format_version}"), format_version), - format_version, - file_size: 1, - } - } + use super::merged_filter_version; #[test] - fn test_multi_file_filter_version() { - let fields = vec![field(1)]; - assert_eq!( - multi_file_filter_version(&fields, &[file(vec![1], 2)]), - Some(2) - ); - assert_eq!( - multi_file_filter_version(&fields, &[file(vec![1], 2), file(vec![2], 4)]), - Some(2), - "an irrelevant current file does not create a mixed-codec read" - ); - assert_eq!( - multi_file_filter_version(&fields, &[file(vec![1], 2), file(vec![1], 4)]), - None, - "V2 and digest encodings for the requested field cannot be merged" - ); - assert_eq!( - multi_file_filter_version(&fields, &[file(vec![1], 3), file(vec![1], 4)]), - Some(BlockFilter::VERSION) - ); - assert_eq!( - multi_file_filter_version(&fields, &[file(vec![2], 2)]), - Some(BlockFilter::VERSION) - ); + fn test_merged_filter_version() { + assert_eq!(merged_filter_version([2]), Some(2)); + assert_eq!(merged_filter_version([2, 4]), None); + assert_eq!(merged_filter_version([3, 4]), Some(BlockFilter::VERSION)); + assert_eq!(merged_filter_version([]), Some(BlockFilter::VERSION)); } } diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index 7223edd921e35..d12e15edc7879 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -32,7 +32,6 @@ use databend_common_pipeline::sinks::AsyncSink; use databend_common_pipeline::sinks::AsyncSinker; use databend_storages_common_pruner::BlockMetaIndex; use databend_storages_common_pruner::RangeIndexInput; -use databend_storages_common_table_meta::meta::BloomIndexLayout; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnMetaV0; use databend_storages_common_table_meta::meta::ColumnStatistics; @@ -44,6 +43,7 @@ use tokio::sync::OwnedSemaphorePermit; use super::PrunedColumnOrientedSegmentMeta; use crate::FuseBlockPartInfo; use crate::FuseColumnGroupPartInfo; +use crate::fuse_part::BloomIndexLayout; use crate::pruning::BlockPruner; use crate::pruning_pipeline::RuntimeFilterPruneContext; @@ -223,11 +223,12 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { if !bloom_pruner .should_keep( - BloomIndexLayout::from_metadata( - bloom_filter_index_location.as_ref(), - bloom_filter_index_size, - &[], - ), + bloom_filter_index_location.as_ref().map(|location| { + BloomIndexLayout::Legacy { + location, + file_size: bloom_filter_index_size, + } + }), &columns_stat, column_ids.clone(), &block_read_info, diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index 857f8cadb4546..903ec0585dfba 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -90,7 +90,6 @@ impl TableMetaFunc for FuseBlock { TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))), ), TableField::new("column_groups", TableDataType::Variant), - TableField::new("bloom_index_files", TableDataType::Variant), ]) } @@ -117,7 +116,6 @@ impl TableMetaFunc for FuseBlock { let mut spatial_index_size = Vec::with_capacity(len); let mut virtual_column_size = Vec::with_capacity(len); let mut column_groups = Vec::with_capacity(len); - let mut bloom_index_files = Vec::with_capacity(len); let segments_io = SegmentsIO::create(ctx.clone(), tbl.operator.clone(), tbl.schema()); @@ -165,25 +163,11 @@ impl TableMetaFunc for FuseBlock { "format_version": group.format_version, "file_size": group.file_size, "uncompressed_size": group.uncompressed_size, + "bloom": group.bloom, }) }) .collect::>(), )?); - bloom_index_files.push(serialize_variant( - &block - .bloom_index_files - .iter() - .map(|file| { - serde_json::json!({ - "active_column_ids": file.active_column_ids, - "location": file.location.0, - "format_version": file.format_version, - "file_size": file.file_size, - }) - }) - .collect::>(), - )?); - num_rows += 1; if num_rows >= limit { break 'FOR; @@ -208,7 +192,6 @@ impl TableMetaFunc for FuseBlock { UInt64Type::from_opt_data(spatial_index_size).into(), UInt64Type::from_opt_data(virtual_column_size).into(), VariantType::from_data(column_groups).into(), - VariantType::from_data(bloom_index_files).into(), ], num_rows, )) diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test index c338a0f736b88..4db7c943c1e3e 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -33,11 +33,11 @@ select * from t_partial_update order by id 1 11 2 21 -query BB -select column_groups != parse_json('[]'), bloom_index_files != parse_json('[]') +query B +select column_groups != parse_json('[]') from fuse_block('default', 't_partial_update') ---- -1 1 +1 query I select count(distinct column_name) @@ -67,11 +67,15 @@ set enable_partial_update = 0 statement ok update t_partial_update set value = value + 1 where id = 1 -query BB -select column_groups = parse_json('[]'), bloom_index_files = parse_json('[]') +query B +select column_groups = parse_json('[]') from fuse_block('default', 't_partial_update') ---- -1 1 +1 + +# The table capability is sticky; only the session setting can disable new partial writes. +statement error +alter table t_partial_update set options(enable_partial_update = false) statement ok drop table t_partial_update diff --git a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test index cbac9fc64cad1..33200c23120cc 100644 --- a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test +++ b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test @@ -13,42 +13,35 @@ ## limitations under the License. statement ok -create or replace database partial_update_cluster_dependency +create or replace database partial_update_computed_column statement ok -use partial_update_cluster_dependency +use partial_update_computed_column -# A table with a computed column must use the full-update path. The cluster key also verifies that -# the fallback pipeline uses the persisted physical schema and refreshes its cluster statistics. -statement ok -create table t( +# The incompatibility is enforced in both directions when table metadata is changed. +statement error 2322 +create table invalid_create( id int, - a int, - k bigint as (a + 1) virtual + value int, + computed_value bigint as (value + 1) virtual ) enable_partial_update = true statement ok -insert into t(id, a) values (1, 10), (2, 20) - -statement ok -alter table t cluster by(k) +create table computed_first( + id int, + value int, + computed_value bigint as (value + 1) virtual +) -statement ok -set enable_partial_update = 1 +statement error +alter table computed_first set options(enable_partial_update = true) statement ok -update t set a = a + 1 where id = 1 - -query B -select column_groups = parse_json('[]') from fuse_block('partial_update_cluster_dependency', 't') ----- -1 +create table partial_update_first(id int, value int) enable_partial_update = true -query III -select id, a, k from t order by id ----- -1 11 12 -2 20 21 +statement error +alter table partial_update_first + add column computed_value bigint as (value + 1) virtual statement ok -drop database partial_update_cluster_dependency +drop database partial_update_computed_column From d9f9b7a31afcdb56aaa3820361d1248473cd1804 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:19:30 +0800 Subject: [PATCH 12/30] fix(storage): address paired bloom review findings --- src/meta/api/src/api_impl/index_api.rs | 2 +- src/meta/api/src/api_impl/table_api.rs | 7 +- src/meta/app/src/schema/mod.rs | 2 + src/meta/app/src/schema/table/mod.rs | 87 ++++++++++--------- .../fuse/operations/vacuum_table_v2.rs | 13 +-- .../common/table_option_validation.rs | 2 +- .../interpreter_table_add_column.rs | 2 +- .../interpreters/interpreter_table_create.rs | 2 +- .../interpreter_table_modify_column.rs | 2 +- .../interpreter_table_set_options.rs | 4 +- .../fuse/operations/mutation/update.rs | 22 +++++ .../fuse/pruning_column_oriented_segment.rs | 2 +- .../storages/common/session/src/temp_table.rs | 6 +- .../common/table_meta/src/meta/v2/segment.rs | 28 ++++++ .../common/table_meta/src/table/table_keys.rs | 1 - src/query/storages/fuse/src/fuse_part.rs | 58 +++++++------ src/query/storages/fuse/src/fuse_table.rs | 2 +- .../fuse/src/table_functions/fuse_block.rs | 26 ++++-- 18 files changed, 169 insertions(+), 99 deletions(-) diff --git a/src/meta/api/src/api_impl/index_api.rs b/src/meta/api/src/api_impl/index_api.rs index ef2abe43f636f..1ee555dc127c9 100644 --- a/src/meta/api/src/api_impl/index_api.rs +++ b/src/meta/api/src/api_impl/index_api.rs @@ -353,7 +353,7 @@ where .map_err(|error| { KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( req.table_id.to_string(), - error, + error.to_string(), ))) })?; diff --git a/src/meta/api/src/api_impl/table_api.rs b/src/meta/api/src/api_impl/table_api.rs index 96e7220ebc696..1eb6fc6359da4 100644 --- a/src/meta/api/src/api_impl/table_api.rs +++ b/src/meta/api/src/api_impl/table_api.rs @@ -202,7 +202,8 @@ fn validate_create_table_request(req: &CreateTableReq) -> Result<(), KVAppError> .validate_column_group_compatibility() .map_err(|error| { KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( - name, error, + name, + error.to_string(), ))) })?; Ok(()) @@ -1232,7 +1233,7 @@ where .map_err(|error| { KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( req.table_id.to_string(), - error, + error.to_string(), ))) })?; @@ -1468,7 +1469,7 @@ where .map_err(|error| { AppError::CommitTableMetaError(CommitTableMetaError::new( req.table_id.to_string(), - error, + error.to_string(), )) })?; diff --git a/src/meta/app/src/schema/mod.rs b/src/meta/app/src/schema/mod.rs index e98938468c3b7..b0ad2c2883fe2 100644 --- a/src/meta/app/src/schema/mod.rs +++ b/src/meta/app/src/schema/mod.rs @@ -100,6 +100,7 @@ pub use lock::LockMeta; pub use lock::LockType; pub use ownership::Ownership; pub use sequence::*; +pub use table::ColumnGroupCompatibilityError; pub use table::CommitTableMetaReply; pub use table::CommitTableMetaReq; pub use table::CreateTableIndexReq; @@ -126,6 +127,7 @@ pub use table::ListTableCopiedFileReply; pub use table::ListTableReq; pub use table::ListTableTagsReq; pub use table::OPT_KEY_ENABLE_PARTIAL_UPDATE; +pub use table::OPT_KEY_SEGMENT_FORMAT; pub use table::RenameTableReply; pub use table::RenameTableReq; pub use table::SecurityPolicyColumnMap; diff --git a/src/meta/app/src/schema/table/mod.rs b/src/meta/app/src/schema/table/mod.rs index fbd7d037c615a..d3341898f325c 100644 --- a/src/meta/app/src/schema/table/mod.rs +++ b/src/meta/app/src/schema/table/mod.rs @@ -204,9 +204,23 @@ pub struct TableMeta { } pub const OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; -const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; +pub const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; const COLUMN_ORIENTED_SEGMENT_FORMAT: &str = "column_oriented"; +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +pub enum ColumnGroupCompatibilityError { + #[error("column-group layout requires FUSE engine, but found {0}")] + NonFuseEngine(String), + #[error("column-group layout is incompatible with column-oriented segments")] + ColumnOrientedSegment, + #[error("column-group layout is incompatible with computed columns")] + ComputedColumns, + #[error("column-group layout is incompatible with table indexes: {}", .0.join(", "))] + TableIndexes(Vec), + #[error("column-group layout cannot be disabled without a full layout migration")] + DisableRequiresMigration, +} + impl TableMeta { pub fn column_group_layout_enabled(&self) -> bool { self.options @@ -215,14 +229,15 @@ impl TableMeta { } /// Validate the persisted table features that may coexist with column-group blocks. - pub fn validate_column_group_compatibility(&self) -> std::result::Result<(), String> { + pub fn validate_column_group_compatibility( + &self, + ) -> std::result::Result<(), ColumnGroupCompatibilityError> { if !self.column_group_layout_enabled() { return Ok(()); } if !self.engine.eq_ignore_ascii_case("FUSE") { - return Err(format!( - "column-group layout requires FUSE engine, but found {}", - self.engine + return Err(ColumnGroupCompatibilityError::NonFuseEngine( + self.engine.clone(), )); } if self @@ -230,7 +245,7 @@ impl TableMeta { .get(OPT_KEY_SEGMENT_FORMAT) .is_some_and(|format| format.eq_ignore_ascii_case(COLUMN_ORIENTED_SEGMENT_FORMAT)) { - return Err("column-group layout is incompatible with column-oriented segments".into()); + return Err(ColumnGroupCompatibilityError::ColumnOrientedSegment); } if self .schema @@ -238,24 +253,22 @@ impl TableMeta { .iter() .any(|field| field.computed_expr().is_some()) { - return Err("column-group layout is incompatible with computed columns".into()); + return Err(ColumnGroupCompatibilityError::ComputedColumns); } if !self.indexes.is_empty() { let mut indexes = self.indexes.keys().cloned().collect::>(); indexes.sort(); - return Err(format!( - "column-group layout is incompatible with table indexes: {}", - indexes.join(", ") - )); + return Err(ColumnGroupCompatibilityError::TableIndexes(indexes)); } Ok(()) } - pub fn validate_column_group_transition(&self, next: &Self) -> std::result::Result<(), String> { + pub fn validate_column_group_transition( + &self, + next: &Self, + ) -> std::result::Result<(), ColumnGroupCompatibilityError> { if self.column_group_layout_enabled() && !next.column_group_layout_enabled() { - return Err( - "column-group layout cannot be disabled without a full layout migration".into(), - ); + return Err(ColumnGroupCompatibilityError::DisableRequiresMigration); } next.validate_column_group_compatibility() } @@ -1319,6 +1332,7 @@ mod tests { use databend_meta_client::kvapi::StructKey; use databend_meta_client::kvapi::testing::assert_round_trip; + use crate::schema::ColumnGroupCompatibilityError; use crate::schema::TableCopiedFileNameIdent; use crate::schema::TableIdToName; use crate::schema::TableIndex; @@ -1345,22 +1359,20 @@ mod tests { let mut non_fuse = meta.clone(); non_fuse.engine = "MEMORY".to_string(); - assert!( - non_fuse - .validate_column_group_compatibility() - .unwrap_err() - .contains("requires FUSE") + assert_eq!( + non_fuse.validate_column_group_compatibility(), + Err(ColumnGroupCompatibilityError::NonFuseEngine( + "MEMORY".to_string() + )) ); let mut column_oriented = meta.clone(); column_oriented .options .insert("segment_format".to_string(), "column_oriented".to_string()); - assert!( - column_oriented - .validate_column_group_compatibility() - .unwrap_err() - .contains("column-oriented") + assert_eq!( + column_oriented.validate_column_group_compatibility(), + Err(ColumnGroupCompatibilityError::ColumnOrientedSegment) ); let mut computed = meta.clone(); @@ -1368,11 +1380,9 @@ mod tests { TableField::new("a", TableDataType::Number(NumberDataType::Int32)) .with_computed_expr(Some(ComputedExpr::Virtual("a + 1".to_string()))), ])); - assert!( - computed - .validate_column_group_compatibility() - .unwrap_err() - .contains("computed columns") + assert_eq!( + computed.validate_column_group_compatibility(), + Err(ColumnGroupCompatibilityError::ComputedColumns) ); let mut indexed = meta.clone(); @@ -1384,19 +1394,18 @@ mod tests { version: "v1".to_string(), options: BTreeMap::new(), }); - assert!( - indexed - .validate_column_group_compatibility() - .unwrap_err() - .contains("table indexes: idx") + assert_eq!( + indexed.validate_column_group_compatibility(), + Err(ColumnGroupCompatibilityError::TableIndexes(vec![ + "idx".to_string() + ])) ); let mut disabled = meta.clone(); disabled.options.remove("enable_partial_update"); - assert!( - meta.validate_column_group_transition(&disabled) - .unwrap_err() - .contains("cannot be disabled") + assert_eq!( + meta.validate_column_group_transition(&disabled), + Err(ColumnGroupCompatibilityError::DisableRequiresMigration) ); } diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index ef845a10838bd..acd35f279d37d 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -229,13 +229,11 @@ pub async fn do_vacuum2( + bloom_indexes_to_gc.len(), ); - // Bloom indexes must be removed before their data blocks. Keep the removed paths so the - // block-derived compatibility cleanup below does not issue duplicate deletes. + // Bloom indexes must be removed before their data blocks. if !bloom_indexes_to_gc.is_empty() { op.remove_file_in_batch(&bloom_indexes_to_gc).await?; files_to_gc.extend(bloom_indexes_to_gc.iter().cloned()); } - let removed_bloom_indexes = bloom_indexes_to_gc.into_iter().collect::>(); // order is important // indexes should be removed before their blocks, because index locations to gc are generated from block locations. @@ -244,8 +242,6 @@ pub async fn do_vacuum2( &ctx, table_info.desc.as_str(), &blocks_to_gc, - &gc_root_indexes, - &removed_bloom_indexes, &table_agg_index_ids, inverted_indexes, &mut files_to_gc, @@ -305,8 +301,6 @@ async fn purge_block_chunks( ctx: &Arc, table_desc: &str, blocks_to_gc: &[String], - protected_index_paths: &HashSet, - removed_bloom_index_paths: &HashSet, table_agg_index_ids: &[u64], inverted_indexes: &BTreeMap, files_to_gc: &mut Vec, @@ -325,11 +319,8 @@ async fn purge_block_chunks( return Err(err.with_context("failed to vacuum block chunk")); } - let mut indexes_to_gc = + let indexes_to_gc = collect_block_index_locations(block_chunk, table_agg_index_ids, inverted_indexes); - indexes_to_gc.retain(|path| { - !protected_index_paths.contains(path) && !removed_bloom_index_paths.contains(path) - }); ctx.set_status_info(&format!( "Collected indexes_to_gc for table {}, elapsed: {:?}, block chunk: {}/{}, blocks in chunk: {}, indexes_to_gc: {:?}", table_desc, diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index fd93d722a4b3c..8b4cfe5886a88 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -23,6 +23,7 @@ use databend_common_ast::ast::Engine; use databend_common_exception::ErrorCode; use databend_common_expression::TableSchemaRef; use databend_common_io::constants::DEFAULT_BLOCK_ROW_COUNT; +use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_settings::Settings; use databend_common_sql::ApproxDistinctColumns; use databend_common_sql::BloomIndexColumns; @@ -69,7 +70,6 @@ use databend_storages_common_table_meta::table::OPT_KEY_RANDOM_MAX_STRING_LEN; use databend_storages_common_table_meta::table::OPT_KEY_RANDOM_MIN_STRING_LEN; use databend_storages_common_table_meta::table::OPT_KEY_RANDOM_SEED; use databend_storages_common_table_meta::table::OPT_KEY_RECURSIVE_CTE; -use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_COMPRESSION; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; diff --git a/src/query/service/src/interpreters/interpreter_table_add_column.rs b/src/query/service/src/interpreters/interpreter_table_add_column.rs index 2c77a39899515..e60ceb1ca04a7 100644 --- a/src/query/service/src/interpreters/interpreter_table_add_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_add_column.rs @@ -150,7 +150,7 @@ impl Interpreter for AddTableColumnInterpreter { table_info .meta .validate_column_group_compatibility() - .map_err(ErrorCode::TableOptionInvalid)?; + .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; // if the new column is a stored computed field and table is non-empty, // need rebuild the table to generate stored computed column. diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index 43f127815e719..e3800101498c7 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -32,6 +32,7 @@ use databend_common_meta_app::schema::CommitTableMetaReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableReply; use databend_common_meta_app::schema::CreateTableReq; +use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; @@ -60,7 +61,6 @@ use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::table::OPT_KEY_COMMENT; use databend_storages_common_table_meta::table::OPT_KEY_ENABLE_COPY_DEDUP_FULL_PATH; -use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_PREFIX; diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 16bc3c034b36d..7ff4cdf114aef 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -258,7 +258,7 @@ impl ModifyTableColumnInterpreter { table_info .meta .validate_column_group_transition(&next_meta) - .map_err(ErrorCode::TableOptionInvalid)?; + .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; if !modified_cols.is_empty() { // Only need to validate the data types of modified columns that are referenced // by the cluster key. The cluster key expression itself is already validated diff --git a/src/query/service/src/interpreters/interpreter_table_set_options.rs b/src/query/service/src/interpreters/interpreter_table_set_options.rs index c7fd524cfdc3e..928a31edd0bda 100644 --- a/src/query/service/src/interpreters/interpreter_table_set_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_set_options.rs @@ -22,6 +22,7 @@ use databend_common_ast::ast::Engine; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_pipeline::core::Pipeline; use databend_common_sql::plans::SetOptionsPlan; @@ -49,7 +50,6 @@ use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING; use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING_BEGIN_VER; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; -use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; @@ -207,7 +207,7 @@ impl Interpreter for SetOptionsInterpreter { } current_meta .validate_column_group_transition(&next_meta) - .map_err(ErrorCode::TableOptionInvalid)?; + .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; let table = analyze_table(self.ctx.clone(), table, &self.plan.set_options).await?; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 0d72c08870c7e..2ae370b2f535e 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -14,6 +14,7 @@ use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; +use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::BlockHLL; @@ -57,6 +58,7 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { let table = fixture.latest_default_table().await?; let schema = table.schema(); + let operator = FuseTable::try_from_table(table.as_ref())?.get_operator(); let id_column_id = schema.field(0).column_id(); let value_column_id = schema.field(1).column_id(); let origin = latest_default_block_meta(&fixture).await?; @@ -110,6 +112,26 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { .map(|bloom| bloom.file_size) .sum::() ); + + // Paired Bloom failures must keep the block without invoking the legacy auto-fix path. + let changed_bloom = changed_group.bloom.as_ref().unwrap(); + let missing_bloom_location = TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &changed_group.location.0, + changed_bloom.format_version, + ); + operator.delete(&missing_bloom_location).await?; + fixture + .execute_command("set enable_auto_fix_missing_bloom_index = 1") + .await?; + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11) or (id = 2 and value = 20)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + assert!(!operator.exists(&missing_bloom_location).await?); + let updated_hll = latest_block_hll(&fixture).await?; assert_eq!( updated_hll.get(&id_column_id), diff --git a/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs b/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs index 06401e6dcd23f..2b7f920a6ab0c 100644 --- a/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs +++ b/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs @@ -34,6 +34,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::number::Int64Type; use databend_common_expression::types::number::UInt64Type; use databend_common_meta_app::schema::CreateOption; +use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_pipeline::core::Pipeline; use databend_common_sql::BloomIndexColumns; use databend_common_sql::parse_to_filters; @@ -57,7 +58,6 @@ use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; -use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use opendal::Operator; diff --git a/src/query/storages/common/session/src/temp_table.rs b/src/query/storages/common/session/src/temp_table.rs index 4cd1df0f7e5ae..04a67e7ce6063 100644 --- a/src/query/storages/common/session/src/temp_table.rs +++ b/src/query/storages/common/session/src/temp_table.rs @@ -107,7 +107,7 @@ impl TempTblMgr { } = req; table_meta .validate_column_group_compatibility() - .map_err(ErrorCode::TableOptionInvalid)?; + .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; let orphan_table_name = as_dropped.then(|| format!("orphan@{}", name_ident.table_name)); let Some(db_id) = table_meta.options.get(OPT_KEY_DATABASE_ID) else { @@ -276,7 +276,7 @@ impl TempTblMgr { table .meta .validate_column_group_transition(&r.new_table_meta) - .map_err(ErrorCode::TableOptionInvalid)?; + .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; } Ok(()) } @@ -320,7 +320,7 @@ impl TempTblMgr { table .meta .validate_column_group_transition(&new_meta) - .map_err(ErrorCode::TableOptionInvalid)?; + .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; table.meta = new_meta; Ok(UpsertTableOptionReply {}) } diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index f792f4c243bb4..7c92a400f4486 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -638,6 +638,34 @@ mod tests { assert_eq!(active_metas, vec![(1, (10, 11))]); } + #[test] + fn test_deserialize_pre_paired_column_group_meta() { + #[derive(Serialize)] + struct PrePairedColumnGroupFileMeta { + active_column_ids: Vec, + location: Location, + format_version: FormatVersion, + file_size: u64, + uncompressed_size: u64, + leaf_column_metas: HashMap, + } + + let group = PrePairedColumnGroupFileMeta { + active_column_ids: vec![1], + location: ("data.parquet".to_string(), 2), + format_version: 2, + file_size: 10, + uncompressed_size: 20, + leaf_column_metas: HashMap::new(), + }; + let bytes = rmp_serde::to_vec_named(&group).unwrap(); + let decoded: ColumnGroupFileMeta = rmp_serde::from_slice(&bytes).unwrap(); + + assert_eq!(decoded.bloom, None); + assert_eq!(decoded.location, group.location); + assert_eq!(decoded.active_column_ids, group.active_column_ids); + } + #[test] fn test_deserialize_legacy_block_meta_without_column_groups() { let block_meta = BlockMeta::new( diff --git a/src/query/storages/common/table_meta/src/table/table_keys.rs b/src/query/storages/common/table_meta/src/table/table_keys.rs index 6df391599ce42..bfdda175efd79 100644 --- a/src/query/storages/common/table_meta/src/table/table_keys.rs +++ b/src/query/storages/common/table_meta/src/table/table_keys.rs @@ -29,7 +29,6 @@ pub const OPT_KEY_RECURSIVE_CTE: &str = "recursive_cte"; pub const OPT_KEY_SNAPSHOT_LOCATION: &str = "snapshot_location"; pub const OPT_KEY_SNAPSHOT_LOCATION_FIXED_FLAG: &str = "snapshot_location_fixed"; pub const OPT_KEY_STORAGE_FORMAT: &str = "storage_format"; -pub const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; pub const OPT_KEY_TABLE_COMPRESSION: &str = "compression"; pub const OPT_KEY_COMMENT: &str = "comment"; pub const OPT_KEY_ENGINE: &str = "engine"; diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 011892f7f8882..eb32a7c57160e 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -67,6 +67,26 @@ pub enum BloomIndexLayout<'a> { }, } +impl<'a> BloomIndexLayout<'a> { + fn from_metadata( + has_column_groups: bool, + legacy_location: Option<&'a Location>, + legacy_file_size: u64, + column_group_files: Cow<'a, [FuseBloomIndexFileInfo]>, + ) -> Option { + if has_column_groups { + return (!column_group_files.is_empty()).then_some(Self::ColumnGroups { + files: column_group_files, + }); + } + + legacy_location.map(|location| Self::Legacy { + location, + file_size: legacy_file_size, + }) + } +} + fn column_group_bloom_location(group: &ColumnGroupFileMeta) -> Option { group.bloom.as_ref().map(|bloom| { ( @@ -106,19 +126,13 @@ pub fn block_bloom_index_locations(meta: &BlockMeta) -> Vec { } pub(crate) fn bloom_index_layout(meta: &BlockMeta) -> Option> { - if meta.column_groups.is_empty() { - return meta.bloom_filter_index_location.as_ref().map(|location| { - BloomIndexLayout::Legacy { - location, - file_size: meta.bloom_filter_index_size, - } - }); - } - let files = column_group_bloom_files(meta); - (!files.is_empty()).then_some(BloomIndexLayout::ColumnGroups { - files: Cow::Owned(files), - }) + BloomIndexLayout::from_metadata( + !meta.column_groups.is_empty(), + meta.bloom_filter_index_location.as_ref(), + meta.bloom_filter_index_size, + Cow::Owned(files), + ) } pub(crate) fn project_column_groups( @@ -180,20 +194,12 @@ impl PartInfo for FuseBlockPartInfo { impl FuseBlockPartInfo { /// Normalize optional legacy and column-group Bloom metadata into one physical-layout view. pub fn bloom_index_layout(&self) -> Option> { - if !self.column_groups.is_empty() { - return (!self.column_group_bloom_files.is_empty()).then(|| { - BloomIndexLayout::ColumnGroups { - files: Cow::Borrowed(&self.column_group_bloom_files), - } - }); - } - - self.bloom_filter_index_location - .as_ref() - .map(|location| BloomIndexLayout::Legacy { - location, - file_size: self.bloom_filter_index_size, - }) + BloomIndexLayout::from_metadata( + !self.column_groups.is_empty(), + self.bloom_filter_index_location.as_ref(), + self.bloom_filter_index_size, + Cow::Borrowed(&self.column_group_bloom_files), + ) } #[allow(clippy::too_many_arguments)] diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index 0eb78700c3376..788b27a673ce6 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -64,6 +64,7 @@ use databend_common_io::constants::DEFAULT_BLOCK_COMPRESSED_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_PER_SEGMENT; use databend_common_io::constants::DEFAULT_BLOCK_ROW_COUNT; use databend_common_meta_app::schema::DatabaseType; +use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; @@ -105,7 +106,6 @@ use databend_storages_common_table_meta::table::OPT_KEY_BLOOM_INDEX_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_LEGACY_SNAPSHOT_LOC; -use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION_FIXED_FLAG; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index 903ec0585dfba..ef5cb60489d33 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -135,13 +135,25 @@ impl TableMetaFunc for FuseBlock { block_size.push(block.block_size); file_size.push(block.file_size); row_count.push(block.row_count); - bloom_filter_location.push( - block - .bloom_filter_index_location - .as_ref() - .map(|s| s.0.clone()), - ); - bloom_filter_size.push(block.bloom_filter_index_size); + if block.column_groups.is_empty() { + bloom_filter_location.push( + block + .bloom_filter_index_location + .as_ref() + .map(|s| s.0.clone()), + ); + bloom_filter_size.push(block.bloom_filter_index_size); + } else { + bloom_filter_location.push(None); + bloom_filter_size.push( + block + .column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum(), + ); + } inverted_index_size.push(block.inverted_index_size); ngram_index_size.push(block.ngram_filter_index_size); vector_index_size.push(block.vector_index_size); From bcca08de5b44e2899e15915b088c536cda67eef5 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:37:23 +0800 Subject: [PATCH 13/30] fix(storage): close paired bloom layout gaps --- src/meta/app/src/schema/table/mod.rs | 3 +- .../it/indexes/ngram_index/index_refresh.rs | 55 +++++++++++++++++-- .../it/storages/fuse/operations/prewhere.rs | 1 + src/query/storages/fuse/src/fuse_part.rs | 6 +- .../fuse/src/io/write/block_writer.rs | 13 +++-- .../src/operations/read/read_block_context.rs | 1 + .../fuse/src/operations/read_partitions.rs | 2 + .../storages/fuse/src/pruning/bloom_pruner.rs | 3 +- .../fuse/src/pruning/expr_runtime_pruner.rs | 2 + .../column_oriented_block_prune.rs | 1 + 10 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/meta/app/src/schema/table/mod.rs b/src/meta/app/src/schema/table/mod.rs index d3341898f325c..2346c6af913ee 100644 --- a/src/meta/app/src/schema/table/mod.rs +++ b/src/meta/app/src/schema/table/mod.rs @@ -256,8 +256,7 @@ impl TableMeta { return Err(ColumnGroupCompatibilityError::ComputedColumns); } if !self.indexes.is_empty() { - let mut indexes = self.indexes.keys().cloned().collect::>(); - indexes.sort(); + let indexes = self.indexes.keys().cloned().collect::>(); return Err(ColumnGroupCompatibilityError::TableIndexes(indexes)); } Ok(()) diff --git a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs index 9985db99bf2fb..5a000d48266d3 100644 --- a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs @@ -163,15 +163,59 @@ async fn test_ngram_index_is_incompatible_with_partial_update() -> anyhow::Resul .await .is_err() ); + drop_ngram_enable_partial_update_and_check(&fixture, &db, &table_name).await?; + fixture + .execute_command(&format!("drop table {db}.{table_name}")) + .await?; fixture .execute_command(&format!( - "drop ngram index idx_content on {db}.{table_name}" + "create table {db}.{table_name} (id int, content string) engine=fuse" )) .await?; fixture .execute_command(&format!( - "alter table {db}.{table_name} set options(enable_partial_update=true)" + "create ngram index idx_content on {db}.{table_name}(content)" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 'before'), (2, 'unchanged')" + )) + .await?; + assert!( + latest_default_block_meta(&fixture) + .await? + .ngram_filter_index_size + .is_some() + ); + assert!( + fixture + .execute_command(&format!( + "alter table {db}.{table_name} \ + set options(enable_partial_update=true)" + )) + .await + .is_err() + ); + drop_ngram_enable_partial_update_and_check(&fixture, &db, &table_name).await?; + + Ok(()) +} + +async fn drop_ngram_enable_partial_update_and_check( + fixture: &TestFixture, + database: &str, + table: &str, +) -> anyhow::Result<()> { + fixture + .execute_command(&format!( + "drop ngram index idx_content on {database}.{table}" + )) + .await?; + fixture + .execute_command(&format!( + "alter table {database}.{table} set options(enable_partial_update=true)" )) .await?; fixture @@ -179,11 +223,11 @@ async fn test_ngram_index_is_incompatible_with_partial_update() -> anyhow::Resul .await?; fixture .execute_command(&format!( - "update {db}.{table_name} set content = 'after' where id = 1" + "update {database}.{table} set content = 'after' where id = 1" )) .await?; - let block_meta = latest_default_block_meta(&fixture).await?; + let block_meta = latest_default_block_meta(fixture).await?; assert_eq!(block_meta.column_groups.len(), 2); assert!(block_meta.bloom_filter_index_location.is_none()); assert_eq!( @@ -196,12 +240,11 @@ async fn test_ngram_index_is_incompatible_with_partial_update() -> anyhow::Resul ); let rows = fixture .execute_query(&format!( - "select count(*) from {db}.{table_name} \ + "select count(*) from {database}.{table} \ where (id = 1 and content = 'after') or (id = 2 and content = 'unchanged')" )) .await?; assert_eq!(query_count(rows).await?, 2); - Ok(()) } diff --git a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs index 77a8e88d6b532..7d48df9073170 100644 --- a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs +++ b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs @@ -331,6 +331,7 @@ async fn prepare_prewhere_data() -> Result { location: "test_block".to_string(), bloom_filter_index_location: None, bloom_filter_index_size: 0, + column_group_layout: false, column_group_bloom_files: vec![], create_on: None, nums_rows: num_rows, diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index eb32a7c57160e..60a019eb7de68 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -155,6 +155,8 @@ pub struct FuseBlockPartInfo { pub bloom_filter_index_location: Option, pub bloom_filter_index_size: u64, + #[serde(default)] + pub column_group_layout: bool, #[serde(default, alias = "bloom_index_files")] pub column_group_bloom_files: Vec, @@ -195,7 +197,7 @@ impl FuseBlockPartInfo { /// Normalize optional legacy and column-group Bloom metadata into one physical-layout view. pub fn bloom_index_layout(&self) -> Option> { BloomIndexLayout::from_metadata( - !self.column_groups.is_empty(), + self.column_group_layout || !self.column_group_bloom_files.is_empty(), self.bloom_filter_index_location.as_ref(), self.bloom_filter_index_size, Cow::Borrowed(&self.column_group_bloom_files), @@ -207,6 +209,7 @@ impl FuseBlockPartInfo { location: String, bloom_filter_index_location: Option, bloom_filter_index_size: u64, + column_group_layout: bool, column_group_bloom_files: Vec, rows_count: u64, column_groups: Vec, @@ -220,6 +223,7 @@ impl FuseBlockPartInfo { location, bloom_filter_index_location, bloom_filter_index_size, + column_group_layout, column_group_bloom_files, create_on, column_groups, diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 88914f84ceb11..4ed828f7a69f7 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -193,8 +193,8 @@ fn merge_column_group_metadata( .collect::>(); let mut column_groups = origin.physical_column_groups().into_owned(); if origin.column_groups.is_empty() { - let legacy_bloom_is_paired = - origin + let legacy_bloom_is_ordinary_and_paired = origin.ngram_filter_index_size.is_none() + && origin .bloom_filter_index_location .as_ref() .is_none_or(|location| { @@ -205,10 +205,11 @@ fn merge_column_group_metadata( ); expected == location.0 }); - if !legacy_bloom_is_paired { - // Ngram refresh historically wrote a new random Bloom path. After the Ngram index is - // dropped the table may enable column groups, but that unpaired file cannot be - // represented by ColumnGroupBloomMeta. Drop the auxiliary reference and fail open. + if !legacy_bloom_is_ordinary_and_paired { + // A legacy Bloom may have an unpaired path after Ngram refresh, or a paired path but + // contain Ngram filters when the index existed at INSERT time. Neither file can be + // represented by ColumnGroupBloomMeta after the index is dropped. Drop the auxiliary + // reference and fail open. column_groups[0].bloom = None; } } diff --git a/src/query/storages/fuse/src/operations/read/read_block_context.rs b/src/query/storages/fuse/src/operations/read/read_block_context.rs index b6e912a30253b..377f9a1490c7c 100644 --- a/src/query/storages/fuse/src/operations/read/read_block_context.rs +++ b/src/query/storages/fuse/src/operations/read/read_block_context.rs @@ -131,6 +131,7 @@ impl ReadBlockContext { location, None, 0, + false, vec![], num_rows, column_groups, diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 7e7914b666804..aa8579a4c5318 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -1409,6 +1409,7 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, + !meta.column_groups.is_empty(), column_group_bloom_files(meta), rows_count, column_groups, @@ -1466,6 +1467,7 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, + !meta.column_groups.is_empty(), column_group_bloom_files(meta), rows_count, column_groups, diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 50ae981e1e4ce..50a3b28c136c9 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -457,7 +457,6 @@ impl BloomPrunerCreator { &self, index_files: &[FuseBloomIndexFileInfo], column_stats: &StatisticsOfColumns, - _block_meta: &BlockReadInfo, ) -> Result { let maybe_filter = read_multi_file_block_filter( &self.dal, @@ -557,7 +556,7 @@ impl BloomPruner for BloomPrunerCreator { ) -> bool { match index_layout { Some(BloomIndexLayout::ColumnGroups { files }) => { - match self.apply_files(&files, column_stats, block_meta).await { + match self.apply_files(&files, column_stats).await { Ok(value) => value, Err(e) => { warn!( diff --git a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs index f4b65db5a0f48..99ded650a2bcf 100644 --- a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs +++ b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs @@ -537,6 +537,7 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, + false, vec![], 4, vec![], @@ -728,6 +729,7 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, + false, vec![], 1000, vec![], diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index d12e15edc7879..d47ad895c4685 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -291,6 +291,7 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { location_path, bloom_filter_index_location, bloom_filter_index_size, + false, vec![], row_count, column_groups, From e12c93baf932638234204ec1b98f948d242adfc3 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:02:47 +0800 Subject: [PATCH 14/30] refactor(storage): tighten paired bloom normalization --- .../fuse/operations/vacuum_table_v2.rs | 6 +- .../common/table_meta/src/meta/v2/segment.rs | 8 +- src/query/storages/fuse/src/fuse_part.rs | 21 ++-- .../fuse/src/io/write/block_writer.rs | 19 +++- src/query/storages/fuse/src/operations/gc.rs | 7 +- .../fuse/src/table_functions/fuse_block.rs | 104 ++++++++++++++---- 6 files changed, 113 insertions(+), 52 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index acd35f279d37d..b4bfe0659e647 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -145,11 +145,7 @@ pub async fn do_vacuum2( .data_file_locations() .map(|location| location.0.clone()), ); - gc_root_indexes.extend( - block_bloom_index_locations(&block) - .into_iter() - .map(|location| location.0), - ); + gc_root_indexes.extend(block_bloom_index_locations(&block).map(|location| location.0)); } } ctx.set_status_info(&format!( diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 7c92a400f4486..878b9d6d27afd 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -339,13 +339,7 @@ impl BlockMeta { file_size: self.file_size, uncompressed_size: self.block_size, leaf_column_metas, - bloom: self - .bloom_filter_index_location - .as_ref() - .map(|location| ColumnGroupBloomMeta { - format_version: location.1, - file_size: self.bloom_filter_index_size, - }), + bloom: None, } } diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 60a019eb7de68..c309346701c6c 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -114,15 +114,18 @@ pub(crate) fn column_group_bloom_files(meta: &BlockMeta) -> Vec Vec { - if meta.column_groups.is_empty() { - return meta.bloom_filter_index_location.iter().cloned().collect(); - } - - meta.column_groups - .iter() - .filter_map(column_group_bloom_location) - .collect() +pub fn block_bloom_index_locations(meta: &BlockMeta) -> impl Iterator + '_ { + let legacy = meta + .column_groups + .is_empty() + .then_some(meta.bloom_filter_index_location.as_ref()) + .flatten() + .cloned(); + legacy.into_iter().chain( + meta.column_groups + .iter() + .filter_map(column_group_bloom_location), + ) } pub(crate) fn bloom_index_layout(meta: &BlockMeta) -> Option> { diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 4ed828f7a69f7..5cf1435d750ba 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -205,12 +205,19 @@ fn merge_column_group_metadata( ); expected == location.0 }); - if !legacy_bloom_is_ordinary_and_paired { - // A legacy Bloom may have an unpaired path after Ngram refresh, or a paired path but - // contain Ngram filters when the index existed at INSERT time. Neither file can be - // represented by ColumnGroupBloomMeta after the index is dropped. Drop the auxiliary - // reference and fail open. - column_groups[0].bloom = None; + // A legacy Bloom may have an unpaired path after Ngram refresh, or a paired path but + // contain Ngram filters when the index existed at INSERT time. Neither file can be + // represented by ColumnGroupBloomMeta after the index is dropped, so adopt only an + // ordinary paired file and otherwise fail open. + if legacy_bloom_is_ordinary_and_paired { + column_groups[0].bloom = + origin + .bloom_filter_index_location + .as_ref() + .map(|location| ColumnGroupBloomMeta { + format_version: location.1, + file_size: origin.bloom_filter_index_size, + }); } } for group in &mut column_groups { diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs index af79e9c4de591..5a19c794cf9c4 100644 --- a/src/query/storages/fuse/src/operations/gc.rs +++ b/src/query/storages/fuse/src/operations/gc.rs @@ -809,11 +809,8 @@ impl TryFrom> for LocationTuple { .data_file_locations() .map(|location| location.0.clone()), ); - bloom_location.extend( - block_bloom_index_locations(&block_meta) - .into_iter() - .map(|location| location.0), - ); + bloom_location + .extend(block_bloom_index_locations(&block_meta).map(|location| location.0)); } if let Some(loc) = value.as_ref().summary.additional_stats_loc() { hll_location.insert(loc.0); diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index ef5cb60489d33..1299cef30537a 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -31,6 +31,7 @@ use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::VariantType; use databend_common_expression::types::string::StringColumnBuilder; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use serde::Serialize; @@ -51,6 +52,30 @@ fn serialize_variant(value: &impl Serialize) -> Result> { .to_vec()) } +fn observable_bloom_and_ngram_meta(block: &BlockMeta) -> (Option, u64, Option) { + if block.column_groups.is_empty() { + return ( + block + .bloom_filter_index_location + .as_ref() + .map(|location| location.0.clone()), + block.bloom_filter_index_size, + block.ngram_filter_index_size, + ); + } + + ( + None, + block + .column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum(), + None, + ) +} + #[async_trait::async_trait] impl TableMetaFunc for FuseBlock { fn schema() -> Arc { @@ -135,27 +160,12 @@ impl TableMetaFunc for FuseBlock { block_size.push(block.block_size); file_size.push(block.file_size); row_count.push(block.row_count); - if block.column_groups.is_empty() { - bloom_filter_location.push( - block - .bloom_filter_index_location - .as_ref() - .map(|s| s.0.clone()), - ); - bloom_filter_size.push(block.bloom_filter_index_size); - } else { - bloom_filter_location.push(None); - bloom_filter_size.push( - block - .column_groups - .iter() - .filter_map(|group| group.bloom.as_ref()) - .map(|bloom| bloom.file_size) - .sum(), - ); - } + let (bloom_location, bloom_size, ngram_size) = + observable_bloom_and_ngram_meta(block); + bloom_filter_location.push(bloom_location); + bloom_filter_size.push(bloom_size); inverted_index_size.push(block.inverted_index_size); - ngram_index_size.push(block.ngram_filter_index_size); + ngram_index_size.push(ngram_size); vector_index_size.push(block.vector_index_size); spatial_index_size.push(block.spatial_index_size); virtual_column_size.push( @@ -209,3 +219,57 @@ impl TableMetaFunc for FuseBlock { )) } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use databend_storages_common_table_meta::meta::ColumnGroupBloomMeta; + use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; + use databend_storages_common_table_meta::meta::Compression; + + use super::*; + + #[test] + fn test_column_group_layout_hides_stale_top_level_bloom_and_ngram_meta() { + let mut block = BlockMeta::new( + 0, + 0, + 0, + HashMap::new(), + HashMap::new(), + None, + ("data.parquet".to_string(), 1), + Some(("stale-bloom".to_string(), 1)), + 11, + None, + Some(7), + None, + None, + None, + None, + None, + None, + Compression::Lz4Raw, + None, + ); + assert_eq!( + observable_bloom_and_ngram_meta(&block), + (Some("stale-bloom".to_string()), 11, Some(7)) + ); + + block.column_groups.push(ColumnGroupFileMeta { + active_column_ids: vec![], + location: ("data.parquet".to_string(), 1), + format_version: 1, + file_size: 0, + uncompressed_size: 0, + leaf_column_metas: HashMap::new(), + bloom: Some(ColumnGroupBloomMeta { + format_version: 1, + file_size: 5, + }), + }); + assert_eq!(observable_bloom_and_ngram_meta(&block), (None, 5, None)); + } +} From 8338fd8dc35f152cd3d6f3189aea9022d894caa1 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:23:33 +0800 Subject: [PATCH 15/30] refactor(storage): remove split bloom ngram path --- .../mutator/replace_into_operation_agg.rs | 1 - .../storages/fuse/src/pruning/bloom_pruner.rs | 49 ++++++------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 14aaa7ca71830..10c6082e692cd 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -759,7 +759,6 @@ impl AggregationContext { &self.data_accessor, &self.read_settings, &fields, - &[], &files, ) .await? diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 50a3b28c136c9..ff6f7f0d9a5d6 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -117,8 +117,7 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( } Some(BloomIndexLayout::ColumnGroups { files }) => { let Some(filter) = - read_multi_file_block_filter(dal, settings, &result.bloom_fields, &[], &files) - .await? + read_multi_file_block_filter(dal, settings, &result.bloom_fields, &files).await? else { return Ok(false); }; @@ -170,7 +169,6 @@ pub(crate) async fn read_multi_file_block_filter( dal: &Operator, settings: &ReadSettings, index_fields: &[TableField], - ngram_args: &[NgramArgs], index_files: &[FuseBloomIndexFileInfo], ) -> Result> { // The owning data group may contain active columns for which its Bloom file has no filter. @@ -179,24 +177,15 @@ pub(crate) async fn read_multi_file_block_filter( let mut loaded_filters = Vec::new(); for file in index_files { - let mut ordinary_fields_by_filter_name = HashMap::new(); + let mut fields_by_filter_name = HashMap::new(); let mut columns = Vec::new(); for field in index_fields { if !file.active_column_ids.contains(&field.column_id()) { continue; } - if let Some(arg) = ngram_args.iter().find(|arg| arg.field() == field) { - let name = BloomIndex::build_filter_ngram_name( - field.column_id(), - arg.gram_size(), - arg.bloom_size(), - ); - columns.push(name); - } else { - let physical = BloomIndex::build_filter_bloom_name(file.location.1, field)?; - ordinary_fields_by_filter_name.insert(physical.clone(), field.clone()); - columns.push(physical); - } + let physical = BloomIndex::build_filter_bloom_name(file.location.1, field)?; + fields_by_filter_name.insert(physical.clone(), field); + columns.push(physical); } if columns.is_empty() { continue; @@ -212,34 +201,26 @@ pub(crate) async fn read_multi_file_block_filter( .iter() .zip(filter.filters.into_iter()) { - loaded_filters.push(( - field.name().clone(), - ordinary_fields_by_filter_name.get(field.name()).cloned(), - file.location.1, - value, - )); + if let Some(index_field) = fields_by_filter_name.get(field.name()) { + loaded_filters.push((*index_field, file.location.1, value)); + } } } if loaded_filters.is_empty() { return Ok(None); } - let Some(format_version) = merged_filter_version( - loaded_filters - .iter() - .filter_map(|(_, field, version, _)| field.as_ref().map(|_| *version)), - ) else { + let Some(format_version) = + merged_filter_version(loaded_filters.iter().map(|(_, version, _)| *version)) + else { // V2 filters use legacy scalar encoding and cannot be evaluated together with the digest // encoding of newer files. Keeping the block is the safe compatibility behavior. return Ok(None); }; let mut merged_fields = Vec::with_capacity(loaded_filters.len()); let mut merged_filters = Vec::with_capacity(loaded_filters.len()); - for (physical_name, bloom_field, _, filter) in loaded_filters { - let name = match bloom_field { - Some(field) => BloomIndex::build_filter_bloom_name(format_version, &field)?, - None => physical_name, - }; + for (field, _, filter) in loaded_filters { + let name = BloomIndex::build_filter_bloom_name(format_version, field)?; merged_fields.push(TableField::new(&name, TableDataType::Binary)); merged_filters.push(filter); } @@ -458,11 +439,13 @@ impl BloomPrunerCreator { index_files: &[FuseBloomIndexFileInfo], column_stats: &StatisticsOfColumns, ) -> Result { + if !self.ngram_args.is_empty() { + return Ok(true); + } let maybe_filter = read_multi_file_block_filter( &self.dal, &self.settings, &self.index_fields, - &self.ngram_args, index_files, ) .await; From 7674c8f812a3aa89587aa182d285ad1e01454d0c Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:18:51 +0800 Subject: [PATCH 16/30] refactor(storage): narrow partial update feature scope --- src/meta/api/src/api_impl/index_api.rs | 9 - src/meta/api/src/api_impl/table_api.rs | 32 +-- src/meta/app/src/schema/mod.rs | 4 - src/meta/app/src/schema/table/mod.rs | 165 ----------- .../src/catalogs/default/session_catalog.rs | 3 - .../common/table_option_validation.rs | 2 +- .../interpreter_table_add_column.rs | 4 - .../interpreters/interpreter_table_create.rs | 2 +- .../interpreter_table_modify_column.rs | 6 - .../interpreter_table_set_options.rs | 29 +- src/query/service/src/physical_plans/mod.rs | 15 - .../physical_column_mutation.rs | 58 ++-- .../src/physical_plans/physical_mutation.rs | 15 +- .../physical_mutation_source.rs | 30 +- .../it/indexes/ngram_index/index_refresh.rs | 161 +---------- .../fuse/operations/mutation/update.rs | 56 ---- .../it/storages/fuse/operations/prewhere.rs | 1 - .../storages/fuse/operations/replace_into.rs | 76 ----- .../fuse/pruning_column_oriented_segment.rs | 2 +- .../storages/common/session/src/temp_table.rs | 26 +- .../block_read_info.rs | 15 - .../common/table_meta/src/table/table_keys.rs | 5 +- src/query/storages/fuse/src/constants.rs | 2 +- src/query/storages/fuse/src/fuse_part.rs | 11 +- src/query/storages/fuse/src/fuse_table.rs | 2 +- .../fuse/src/io/write/block_writer.rs | 44 +-- .../fuse/src/io/write/bloom_index_writer.rs | 66 ++--- .../src/operations/read/read_block_context.rs | 1 - .../fuse/src/operations/read_partitions.rs | 2 - .../mutator/replace_into_operation_agg.rs | 260 ++++++++---------- .../fuse/src/operations/table_index.rs | 239 +++++++++------- .../storages/fuse/src/pruning/bloom_pruner.rs | 5 +- .../fuse/src/pruning/expr_runtime_pruner.rs | 2 - src/query/storages/fuse/src/pruning/mod.rs | 1 - .../column_oriented_block_prune.rs | 2 - .../fuse/src/table_functions/fuse_block.rs | 92 +------ .../09_0054_partial_update.test | 19 -- ...001_partial_update_cluster_dependency.test | 47 ---- 38 files changed, 348 insertions(+), 1163 deletions(-) delete mode 100644 tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test diff --git a/src/meta/api/src/api_impl/index_api.rs b/src/meta/api/src/api_impl/index_api.rs index 1ee555dc127c9..2e17991c75f50 100644 --- a/src/meta/api/src/api_impl/index_api.rs +++ b/src/meta/api/src/api_impl/index_api.rs @@ -17,7 +17,6 @@ use std::collections::HashMap; use databend_common_meta_app::KeyExistsBuilder; use databend_common_meta_app::KeyWithTenant; use databend_common_meta_app::app_error::AppError; -use databend_common_meta_app::app_error::CommitTableMetaError; use databend_common_meta_app::app_error::DuplicatedIndexColumnId; use databend_common_meta_app::app_error::IndexColumnIdNotFound; use databend_common_meta_app::app_error::UnknownTableId; @@ -348,14 +347,6 @@ where options: req.options.clone(), }; indexes.insert(req.name.clone(), index); - table_meta - .validate_column_group_compatibility() - .map_err(|error| { - KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( - req.table_id.to_string(), - error.to_string(), - ))) - })?; let mut txn_req = TxnRequest::new( // diff --git a/src/meta/api/src/api_impl/table_api.rs b/src/meta/api/src/api_impl/table_api.rs index ed321daf92a95..9b524c965f987 100644 --- a/src/meta/api/src/api_impl/table_api.rs +++ b/src/meta/api/src/api_impl/table_api.rs @@ -208,14 +208,6 @@ fn validate_create_table_request(req: &CreateTableReq) -> Result<(), KVAppError> AppError::CreateAsDropTableWithoutDropTime(CreateAsDropTableWithoutDropTime::new(name)), )); } - req.table_meta - .validate_column_group_compatibility() - .map_err(|error| { - KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( - name, - error.to_string(), - ))) - })?; let is_mv = is_materialized_view_engine(&req.table_meta.engine); if is_mv != req.materialized_view.is_some() || (is_mv && req.as_dropped) { return Err(KVAppError::AppError( @@ -1289,24 +1281,12 @@ where } let mut new_table_meta_map: BTreeMap = BTreeMap::new(); - for ((req, _), (tb_meta_seq, table_meta)) in - update_table_metas.iter_mut().zip(tb_meta_vec.iter()) - { + for ((req, _), (tb_meta_seq, _)) in update_table_metas.iter_mut().zip(tb_meta_vec.iter()) { let tbid = TableId { table_id: req.table_id, }; let new_table_meta = req.new_table_meta.clone(); - table_meta - .as_ref() - .unwrap() - .validate_column_group_transition(&new_table_meta) - .map_err(|error| { - KVAppError::AppError(AppError::CommitTableMetaError(CommitTableMetaError::new( - req.table_id.to_string(), - error.to_string(), - ))) - })?; tbl_seqs.insert(req.table_id, *tb_meta_seq); txn.condition.push(txn_cond_seq(&tbid, Eq, *tb_meta_seq)); @@ -1522,7 +1502,6 @@ where } let mut table_meta = seq_meta.data; - let old_table_meta = table_meta.clone(); for (k, opt_v) in &req.options { match opt_v { @@ -1535,15 +1514,6 @@ where } } - old_table_meta - .validate_column_group_transition(&table_meta) - .map_err(|error| { - AppError::CommitTableMetaError(CommitTableMetaError::new( - req.table_id.to_string(), - error.to_string(), - )) - })?; - Ok(Some(table_meta)) }) .await??; diff --git a/src/meta/app/src/schema/mod.rs b/src/meta/app/src/schema/mod.rs index 67b7cae8f12af..e975f3c03452b 100644 --- a/src/meta/app/src/schema/mod.rs +++ b/src/meta/app/src/schema/mod.rs @@ -116,7 +116,6 @@ pub use materialized_view::SourceTableMVResource; pub use materialized_view::is_materialized_view_engine; pub use ownership::Ownership; pub use sequence::*; -pub use table::ColumnGroupCompatibilityError; pub use table::CommitTableMetaReply; pub use table::CommitTableMetaReq; pub use table::CreateTableIndexReq; @@ -142,9 +141,6 @@ pub use table::ListDroppedTableResp; pub use table::ListTableCopiedFileReply; pub use table::ListTableReq; pub use table::ListTableTagsReq; -pub use table::OPT_KEY_ENABLE_PARTIAL_UPDATE; -pub use table::OPT_KEY_PARTITION_BY; -pub use table::OPT_KEY_SEGMENT_FORMAT; pub use table::RenameTableReply; pub use table::RenameTableReq; pub use table::SecurityPolicyColumnMap; diff --git a/src/meta/app/src/schema/table/mod.rs b/src/meta/app/src/schema/table/mod.rs index 60d23e1b3d201..0f824702e7c8a 100644 --- a/src/meta/app/src/schema/table/mod.rs +++ b/src/meta/app/src/schema/table/mod.rs @@ -204,82 +204,7 @@ pub struct TableMeta { pub constraints: BTreeMap, } -pub const OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; -/// Normalized Fuse partition expressions set by `CREATE TABLE PARTITION BY`. -pub const OPT_KEY_PARTITION_BY: &str = "partition_by"; -pub const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; -const COLUMN_ORIENTED_SEGMENT_FORMAT: &str = "column_oriented"; - -#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] -pub enum ColumnGroupCompatibilityError { - #[error("column-group layout requires FUSE engine, but found {0}")] - NonFuseEngine(String), - #[error("column-group layout is incompatible with column-oriented segments")] - ColumnOrientedSegment, - #[error("column-group layout is incompatible with partitioned tables")] - PartitionedTable, - #[error("column-group layout is incompatible with computed columns")] - ComputedColumns, - #[error("column-group layout is incompatible with table indexes: {}", .0.join(", "))] - TableIndexes(Vec), - #[error("column-group layout cannot be disabled without a full layout migration")] - DisableRequiresMigration, -} - impl TableMeta { - pub fn column_group_layout_enabled(&self) -> bool { - self.options - .get(OPT_KEY_ENABLE_PARTIAL_UPDATE) - .is_some_and(|value| value.eq_ignore_ascii_case("true")) - } - - /// Validate the persisted table features that may coexist with column-group blocks. - pub fn validate_column_group_compatibility( - &self, - ) -> std::result::Result<(), ColumnGroupCompatibilityError> { - if !self.column_group_layout_enabled() { - return Ok(()); - } - if !self.engine.eq_ignore_ascii_case("FUSE") { - return Err(ColumnGroupCompatibilityError::NonFuseEngine( - self.engine.clone(), - )); - } - if self - .options - .get(OPT_KEY_SEGMENT_FORMAT) - .is_some_and(|format| format.eq_ignore_ascii_case(COLUMN_ORIENTED_SEGMENT_FORMAT)) - { - return Err(ColumnGroupCompatibilityError::ColumnOrientedSegment); - } - if self.options.contains_key(OPT_KEY_PARTITION_BY) { - return Err(ColumnGroupCompatibilityError::PartitionedTable); - } - if self - .schema - .fields() - .iter() - .any(|field| field.computed_expr().is_some()) - { - return Err(ColumnGroupCompatibilityError::ComputedColumns); - } - if !self.indexes.is_empty() { - let indexes = self.indexes.keys().cloned().collect::>(); - return Err(ColumnGroupCompatibilityError::TableIndexes(indexes)); - } - Ok(()) - } - - pub fn validate_column_group_transition( - &self, - next: &Self, - ) -> std::result::Result<(), ColumnGroupCompatibilityError> { - if self.column_group_layout_enabled() && !next.column_group_layout_enabled() { - return Err(ColumnGroupCompatibilityError::DisableRequiresMigration); - } - next.validate_column_group_compatibility() - } - /// Returns the cluster key defined on the main branch, if any. pub fn cluster_key_meta(&self) -> Option<(u32, String)> { // - Prefer `cluster_key_v2` if present (branch-aware) @@ -1334,104 +1259,14 @@ mod kvapi_key_impl { #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use std::sync::Arc; - - use databend_common_expression::ComputedExpr; - use databend_common_expression::TableDataType; - use databend_common_expression::TableField; - use databend_common_expression::TableSchema; - use databend_common_expression::types::NumberDataType; use databend_meta_client::kvapi; use databend_meta_client::kvapi::StructKey; use databend_meta_client::kvapi::testing::assert_round_trip; - use crate::schema::ColumnGroupCompatibilityError; use crate::schema::TableCopiedFileNameIdent; use crate::schema::TableIdToName; - use crate::schema::TableIndex; - use crate::schema::TableIndexType; use crate::schema::TableMeta; - fn column_group_table_meta() -> TableMeta { - TableMeta { - schema: Arc::new(TableSchema::new(vec![TableField::new( - "a", - TableDataType::Number(NumberDataType::Int32), - )])), - engine: "FUSE".to_string(), - options: BTreeMap::from([("enable_partial_update".to_string(), "true".to_string())]), - ..Default::default() - } - } - - #[test] - fn test_column_group_compatibility_and_sticky_transition() { - let meta = column_group_table_meta(); - assert!(meta.validate_column_group_compatibility().is_ok()); - assert!(meta.validate_column_group_transition(&meta).is_ok()); - - let mut non_fuse = meta.clone(); - non_fuse.engine = "MEMORY".to_string(); - assert_eq!( - non_fuse.validate_column_group_compatibility(), - Err(ColumnGroupCompatibilityError::NonFuseEngine( - "MEMORY".to_string() - )) - ); - - let mut column_oriented = meta.clone(); - column_oriented - .options - .insert("segment_format".to_string(), "column_oriented".to_string()); - assert_eq!( - column_oriented.validate_column_group_compatibility(), - Err(ColumnGroupCompatibilityError::ColumnOrientedSegment) - ); - - let mut partitioned = meta.clone(); - partitioned - .options - .insert("partition_by".to_string(), "a".to_string()); - assert_eq!( - partitioned.validate_column_group_compatibility(), - Err(ColumnGroupCompatibilityError::PartitionedTable) - ); - - let mut computed = meta.clone(); - computed.schema = Arc::new(TableSchema::new(vec![ - TableField::new("a", TableDataType::Number(NumberDataType::Int32)) - .with_computed_expr(Some(ComputedExpr::Virtual("a + 1".to_string()))), - ])); - assert_eq!( - computed.validate_column_group_compatibility(), - Err(ColumnGroupCompatibilityError::ComputedColumns) - ); - - let mut indexed = meta.clone(); - indexed.indexes.insert("idx".to_string(), TableIndex { - index_type: TableIndexType::Inverted, - name: "idx".to_string(), - column_ids: vec![0], - sync_creation: false, - version: "v1".to_string(), - options: BTreeMap::new(), - }); - assert_eq!( - indexed.validate_column_group_compatibility(), - Err(ColumnGroupCompatibilityError::TableIndexes(vec![ - "idx".to_string() - ])) - ); - - let mut disabled = meta.clone(); - disabled.options.remove("enable_partial_update"); - assert_eq!( - meta.validate_column_group_transition(&disabled), - Err(ColumnGroupCompatibilityError::DisableRequiresMigration) - ); - } - #[test] fn test_table_id_to_name_key_format() { assert_round_trip(TableIdToName { table_id: 9 }, "__fd_table_id_to_name/9"); diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/service/src/catalogs/default/session_catalog.rs index bbc3cb296130e..717254c984ba2 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/service/src/catalogs/default/session_catalog.rs @@ -611,9 +611,6 @@ impl Catalog for SessionCatalog { match state { TxnState::AutoCommit => { let update_temp_tables = std::mem::take(&mut req.update_temp_tables); - self.temp_tbl_mgr - .lock() - .validate_multi_table_meta(&update_temp_tables)?; let reply = if req.is_empty() { Ok(Default::default()) } else { diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index 8b4cfe5886a88..fd93d722a4b3c 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -23,7 +23,6 @@ use databend_common_ast::ast::Engine; use databend_common_exception::ErrorCode; use databend_common_expression::TableSchemaRef; use databend_common_io::constants::DEFAULT_BLOCK_ROW_COUNT; -use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_settings::Settings; use databend_common_sql::ApproxDistinctColumns; use databend_common_sql::BloomIndexColumns; @@ -70,6 +69,7 @@ use databend_storages_common_table_meta::table::OPT_KEY_RANDOM_MAX_STRING_LEN; use databend_storages_common_table_meta::table::OPT_KEY_RANDOM_MIN_STRING_LEN; use databend_storages_common_table_meta::table::OPT_KEY_RANDOM_SEED; use databend_storages_common_table_meta::table::OPT_KEY_RECURSIVE_CTE; +use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TABLE_COMPRESSION; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; diff --git a/src/query/service/src/interpreters/interpreter_table_add_column.rs b/src/query/service/src/interpreters/interpreter_table_add_column.rs index e60ceb1ca04a7..c956d03291b3a 100644 --- a/src/query/service/src/interpreters/interpreter_table_add_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_add_column.rs @@ -147,10 +147,6 @@ impl Interpreter for AddTableColumnInterpreter { table_info .meta .add_column(&field, &self.plan.comment, index)?; - table_info - .meta - .validate_column_group_compatibility() - .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; // if the new column is a stored computed field and table is non-empty, // need rebuild the table to generate stored computed column. diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index 1cb691c6d37ef..6a0d416c20124 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -32,7 +32,6 @@ use databend_common_meta_app::schema::CommitTableMetaReq; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::CreateTableReply; use databend_common_meta_app::schema::CreateTableReq; -use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; @@ -62,6 +61,7 @@ use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::table::OPT_KEY_COMMENT; use databend_storages_common_table_meta::table::OPT_KEY_ENABLE_COPY_DEDUP_FULL_PATH; use databend_storages_common_table_meta::table::OPT_KEY_PARTITION_BY; +use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_PREFIX; diff --git a/src/query/service/src/interpreters/interpreter_table_modify_column.rs b/src/query/service/src/interpreters/interpreter_table_modify_column.rs index 92df414a92624..857fa1b402ce4 100644 --- a/src/query/service/src/interpreters/interpreter_table_modify_column.rs +++ b/src/query/service/src/interpreters/interpreter_table_modify_column.rs @@ -254,12 +254,6 @@ impl ModifyTableColumnInterpreter { let fuse_table = FuseTable::try_from_table(table.as_ref())?; let new_schema = Arc::new(new_schema); - let mut next_meta = table_info.meta.clone(); - next_meta.schema = new_schema.clone(); - table_info - .meta - .validate_column_group_transition(&next_meta) - .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; if !modified_cols.is_empty() { // Only need to validate the data types of modified columns that are referenced // by the cluster key. The cluster key expression itself is already validated diff --git a/src/query/service/src/interpreters/interpreter_table_set_options.rs b/src/query/service/src/interpreters/interpreter_table_set_options.rs index 6fcbf1bb02d72..04cae8ba6d26b 100644 --- a/src/query/service/src/interpreters/interpreter_table_set_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_set_options.rs @@ -22,7 +22,6 @@ use databend_common_ast::ast::Engine; use databend_common_catalog::table::TableExt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_meta_app::schema::UpsertTableOptionReq; use databend_common_pipeline::core::Pipeline; use databend_common_sql::plans::SetOptionsPlan; @@ -51,6 +50,7 @@ use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING_BEGIN_VE use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; use databend_storages_common_table_meta::table::OPT_KEY_PARTITION_BY; +use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; @@ -199,24 +199,6 @@ impl Interpreter for SetOptionsInterpreter { // check enable_virtual_column is_valid_fuse_virtual_column_opt(&self.plan.set_options)?; - // Reject incompatible layout transitions before analyze or segment conversion writes - // auxiliary data. The meta service repeats this check at the atomic commit point. - let current_meta = &table.get_table_info().meta; - let mut next_meta = current_meta.clone(); - for (key, value) in &options_map { - match value { - Some(value) => { - next_meta.options.insert(key.clone(), value.clone()); - } - None => { - next_meta.options.remove(key); - } - } - } - current_meta - .validate_column_group_transition(&next_meta) - .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; - let table = analyze_table(self.ctx.clone(), table, &self.plan.set_options).await?; let table_version = table.get_table_info().ident.seq; @@ -315,15 +297,6 @@ async fn set_segment_format( .await?; for segment in segments { let segment = segment?; - if segment - .blocks - .iter() - .any(|block| !block.column_groups.is_empty()) - { - return Err(ErrorCode::TableOptionInvalid( - "cannot convert segments containing partial-update files to the column-oriented format; rewrite the blocks to the single-file layout first", - )); - } for block in segment.blocks { segment_builder.add_block(block.as_ref().clone())?; } diff --git a/src/query/service/src/physical_plans/mod.rs b/src/query/service/src/physical_plans/mod.rs index cf8ff07810b37..caafefcb67581 100644 --- a/src/query/service/src/physical_plans/mod.rs +++ b/src/query/service/src/physical_plans/mod.rs @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_expression::FieldIndex; -use databend_common_expression::TableSchema; - mod physical_add_stream_column; mod physical_aggregate_expand; mod physical_aggregate_final; @@ -120,18 +117,6 @@ pub use physical_window_partition::*; pub use runtime_filter::PhysicalRuntimeFilter; pub use runtime_filter::PhysicalRuntimeFilters; -pub(crate) fn physical_field_to_storage_index( - physical_schema: &TableSchema, - storage_schema: &TableSchema, - physical_index: FieldIndex, -) -> Option { - let column_id = physical_schema.field(physical_index).column_id(); - storage_schema - .fields() - .iter() - .position(|field| field.column_id() == column_id) -} - pub mod explain; mod format; mod physical_cte_consumer; diff --git a/src/query/service/src/physical_plans/physical_column_mutation.rs b/src/query/service/src/physical_plans/physical_column_mutation.rs index 47687820fa06e..82abb25dea269 100644 --- a/src/query/service/src/physical_plans/physical_column_mutation.rs +++ b/src/query/service/src/physical_plans/physical_column_mutation.rs @@ -35,7 +35,6 @@ use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ColumnMutationFormatter; use crate::physical_plans::format::PhysicalFormat; -use crate::physical_plans::physical_field_to_storage_index; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; @@ -54,7 +53,7 @@ pub struct ColumnMutation { pub has_filter_column: bool, pub table_meta_timestamps: TableMetaTimestamps, pub udf_col_num: usize, - /// Field indices in the physical table schema after virtual computed fields are removed. + /// Field indices written by a partial update. pub partial_update_fields: Option>, } @@ -120,10 +119,11 @@ impl IPhysicalPlan for ColumnMutation { let mut exprs = Vec::with_capacity(mutation_expr.len()); for (id, remote_expr) in mutation_expr { let expr = remote_expr.as_expr(&BUILTIN_FUNCTIONS); - if let Some(schema_index) = field_id_to_schema_index.get(id) { - schema_offset_to_new_offset.insert(*schema_index, next_column_offset); - } - field_id_to_schema_index.insert(*id, next_column_offset); + let schema_index = field_id_to_schema_index.get(id).unwrap(); + schema_offset_to_new_offset.insert(*schema_index, next_column_offset); + field_id_to_schema_index + .entry(*id) + .and_modify(|e| *e = next_column_offset); next_column_offset += 1; exprs.push(expr); } @@ -144,15 +144,21 @@ impl IPhysicalPlan for ColumnMutation { remote_expr .as_expr(&BUILTIN_FUNCTIONS) .project_column_ref(|index| { - Ok(schema_offset_to_new_offset + schema_offset_to_new_offset .get(index) + .ok_or_else(|| { + ErrorCode::BadArguments(format!( + "Unable to get field named \"{}\"", + index + )) + }) .copied() - .unwrap_or(*index)) })?; - if let Some(schema_index) = field_id_to_schema_index.get(id) { - schema_offset_to_new_offset.insert(*schema_index, next_column_offset); - } - field_id_to_schema_index.insert(*id, next_column_offset); + let schema_index = field_id_to_schema_index.get(id).unwrap(); + schema_offset_to_new_offset.insert(*schema_index, next_column_offset); + field_id_to_schema_index + .entry(*id) + .and_modify(|e| *e = next_column_offset); next_column_offset += 1; exprs.push(expr); } @@ -166,23 +172,11 @@ impl IPhysicalPlan for ColumnMutation { // carry derived UDF argument/result columns that must not be // serialized back into the table. let projection = if let Some(updated_fields) = &self.partial_update_fields { - let table = builder - .ctx - .build_table_by_table_info(&self.table_info, None)?; - let table = FuseTable::try_from_table(table.as_ref())?; - let schema_with_stream = table.schema_with_stream(); - let source_schema = schema_with_stream.remove_virtual_computed_fields(); updated_fields .iter() - .map(|source_index| { - let field_id = physical_field_to_storage_index( - &source_schema, - &schema_with_stream, - *source_index, - ) - .ok_or_else(|| ErrorCode::Internal("updated field is not in schema"))?; + .map(|field_id| { field_id_to_schema_index - .get(&field_id) + .get(field_id) .copied() .ok_or_else(|| { ErrorCode::Internal("updated field is not in mutation output") @@ -216,23 +210,17 @@ impl IPhysicalPlan for ColumnMutation { let write_column_group = self.partial_update_fields.is_some(); let block_thresholds = table.get_block_thresholds(); - let physical_table_schema = table.schema_with_stream().remove_virtual_computed_fields(); - let physical_input_schema = DataSchema::from(&physical_table_schema).into(); let cluster_stats_gen = if write_column_group { ClusterStatsGenerator::default() } else if matches!(self.mutation_kind, MutationKind::Delete) { - table.get_cluster_stats_gen( - builder.ctx.clone(), - 0, - block_thresholds, - physical_input_schema, - )? + let input_schema = DataSchema::from(table.schema_with_stream()).into(); + table.get_cluster_stats_gen(builder.ctx.clone(), 0, block_thresholds, input_schema)? } else { table.cluster_gen_for_update( builder.ctx.clone(), &mut builder.main_pipeline, block_thresholds, - Some(physical_input_schema), + None, )? }; diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 0056a1e60a338..85e02f3eebc4b 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -110,15 +110,7 @@ fn build_partial_update_info( ) -> Result> { if !ctx.get_settings().get_enable_partial_update()? || !table.get_option(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, false) - || table.is_column_oriented() - || table.partition_key_count() != 0 - || !table.get_table_info().meta.indexes.is_empty() || update_list.is_empty() - || table - .schema() - .fields() - .iter() - .any(|field| field.computed_expr().is_some()) { return Ok(None); } @@ -146,8 +138,7 @@ fn build_partial_update_info( } } - let source_schema = schema_with_stream.remove_virtual_computed_fields(); - let updated_field_indices = source_schema + let updated_field_indices = schema_with_stream .fields() .iter() .enumerate() @@ -158,7 +149,7 @@ fn build_partial_update_info( }) .collect::>(); if updated_field_indices.is_empty() - || updated_field_indices.len() >= source_schema.fields().len() + || updated_field_indices.len() >= schema_with_stream.fields().len() { return Ok(None); } @@ -186,7 +177,7 @@ fn build_partial_update_info( required_columns.insert(symbol); } for (field_index, field) in schema_with_stream.fields().iter().enumerate() { - if updated_column_ids.contains(&field.column_id()) && field.computed_expr().is_none() { + if updated_column_ids.contains(&field.column_id()) { let Some(symbol) = find_symbol(field_index) else { return Ok(None); }; diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index cbb558a04b022..0a772e2c2b8cb 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -55,7 +55,6 @@ use databend_common_storages_fuse::operations::VirtualSchemaMode; use crate::physical_plans::PhysicalPlanBuilder; use crate::physical_plans::format::MutationSourceFormatter; use crate::physical_plans::format::PhysicalFormat; -use crate::physical_plans::physical_field_to_storage_index; use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; @@ -75,8 +74,8 @@ pub struct MutationSource { pub output_schema: DataSchemaRef, pub input_type: MutationType, pub read_partition_columns: ColumnSet, - /// Physical schema projection used by partial updates. Metadata Symbols remain in - /// `read_partition_columns` for expression/output naming and must not be used as field indices. + /// Table schema projection used by partial updates. Metadata Symbols remain in + /// `read_partition_columns` for expression and output naming. #[serde(default)] pub partial_update_projection: Option>, pub truncate_table: bool, @@ -161,25 +160,14 @@ impl IPhysicalPlan for MutationSource { ); } - let physical_schema = table.schema_with_stream().remove_virtual_computed_fields(); let partial_update = self.partial_update_projection.is_some(); - let read_partition_field_indices: Vec = if let Some(projection) = - &self.partial_update_projection - { - let storage_schema = table.schema_with_stream(); - projection - .iter() - .map(|field_index| { - physical_field_to_storage_index(&physical_schema, &storage_schema, *field_index) - .expect("physical field must exist in storage schema") - }) - .collect() - } else { - self.read_partition_columns - .iter() - .map(|idx| idx.as_field_index()) - .collect() - }; + let read_partition_field_indices: Vec = + self.partial_update_projection.clone().unwrap_or_else(|| { + self.read_partition_columns + .iter() + .map(|idx| idx.as_field_index()) + .collect() + }); let stream_projection = self .partial_update_projection .as_deref() diff --git a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs index 5a000d48266d3..4f208d640c523 100644 --- a/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs +++ b/src/query/service/tests/it/indexes/ngram_index/index_refresh.rs @@ -20,8 +20,6 @@ use databend_common_storages_fuse::io::read::bloom::block_filter_reader::load_bl use databend_query::sessions::TableContext; use databend_query::storages::index::filters::BlockFilter; use databend_query::test_kits::TestFixture; -use databend_query::test_kits::latest_default_block_meta; -use databend_query::test_kits::query_count; use databend_storages_common_io::ReadSettings; use futures_util::StreamExt; @@ -46,7 +44,7 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { .execute_command("CREATE NGRAM INDEX idx2 ON default.t3(d);") .await?; - let block_filter_0 = get_block_filter(&fixture, "default", "t3", &[ + let block_filter_0 = get_block_filter(&fixture, &[ "Bloom(0)".to_string(), "Bloom(1)".to_string(), "Bloom(2)".to_string(), @@ -59,7 +57,7 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { fixture .execute_command("REFRESH NGRAM INDEX idx2 ON default.t3;") .await?; - let block_filter_1 = get_block_filter(&fixture, "default", "t3", &[ + let block_filter_1 = get_block_filter(&fixture, &[ "Bloom(0)".to_string(), "Bloom(1)".to_string(), "Bloom(2)".to_string(), @@ -90,7 +88,7 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { fixture .execute_command("REFRESH NGRAM INDEX idx2 ON default.t3;") .await?; - let block_filter_2 = get_block_filter(&fixture, "default", "t3", &[ + let block_filter_2 = get_block_filter(&fixture, &[ "Bloom(0)".to_string(), "Bloom(1)".to_string(), "Bloom(2)".to_string(), @@ -109,156 +107,11 @@ async fn test_fuse_do_refresh_ngram_index() -> anyhow::Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread")] -async fn test_ngram_index_is_incompatible_with_partial_update() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - fixture.create_default_database().await?; - - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, content string) engine=fuse \ - enable_partial_update=true" - )) - .await?; - assert!( - fixture - .execute_command(&format!( - "create ngram index idx_content on {db}.{table_name}(content)" - )) - .await - .is_err() - ); - fixture - .execute_command(&format!("drop table {db}.{table_name}")) - .await?; - - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, content string) engine=fuse" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 'before'), (2, 'unchanged')" - )) - .await?; - fixture - .execute_command(&format!( - "create ngram index idx_content on {db}.{table_name}(content)" - )) - .await?; - fixture - .execute_command(&format!( - "refresh ngram index idx_content on {db}.{table_name}" - )) - .await?; - assert!( - fixture - .execute_command(&format!( - "alter table {db}.{table_name} \ - set options(enable_partial_update=true)" - )) - .await - .is_err() - ); - drop_ngram_enable_partial_update_and_check(&fixture, &db, &table_name).await?; - - fixture - .execute_command(&format!("drop table {db}.{table_name}")) - .await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, content string) engine=fuse" - )) - .await?; - fixture - .execute_command(&format!( - "create ngram index idx_content on {db}.{table_name}(content)" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 'before'), (2, 'unchanged')" - )) - .await?; - assert!( - latest_default_block_meta(&fixture) - .await? - .ngram_filter_index_size - .is_some() - ); - assert!( - fixture - .execute_command(&format!( - "alter table {db}.{table_name} \ - set options(enable_partial_update=true)" - )) - .await - .is_err() - ); - drop_ngram_enable_partial_update_and_check(&fixture, &db, &table_name).await?; - - Ok(()) -} - -async fn drop_ngram_enable_partial_update_and_check( - fixture: &TestFixture, - database: &str, - table: &str, -) -> anyhow::Result<()> { - fixture - .execute_command(&format!( - "drop ngram index idx_content on {database}.{table}" - )) - .await?; - fixture - .execute_command(&format!( - "alter table {database}.{table} set options(enable_partial_update=true)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {database}.{table} set content = 'after' where id = 1" - )) - .await?; - - let block_meta = latest_default_block_meta(fixture).await?; - assert_eq!(block_meta.column_groups.len(), 2); - assert!(block_meta.bloom_filter_index_location.is_none()); - assert_eq!( - block_meta - .column_groups - .iter() - .filter(|group| group.bloom.is_some()) - .count(), - 1 - ); - let rows = fixture - .execute_query(&format!( - "select count(*) from {database}.{table} \ - where (id = 1 and content = 'after') or (id = 2 and content = 'unchanged')" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - Ok(()) -} - -async fn get_block_filter( - fixture: &TestFixture, - database: &str, - table: &str, - columns: &[String], -) -> Result { +async fn get_block_filter(fixture: &TestFixture, columns: &[String]) -> Result { let block = fixture - .execute_query(&format!( - "select bloom_filter_location, bloom_filter_size \ - from fuse_block('{database}', '{table}')" - )) + .execute_query( + "select bloom_filter_location, bloom_filter_size from fuse_block('default', 't3');", + ) .await? .next() .await diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 2ae370b2f535e..d94361ae021ea 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -290,59 +290,3 @@ async fn test_partial_update_preserves_block_compression() -> anyhow::Result<()> Ok(()) } - -#[tokio::test(flavor = "multi_thread")] -async fn test_partial_update_invalidates_virtual_columns_when_disabled() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, payload variant) engine=fuse \ - enable_partial_update=true enable_virtual_column=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values \ - (1, parse_json('{{\"x\": 1}}')), (2, parse_json('{{\"x\": 2}}'))" - )) - .await?; - assert!( - latest_default_block_meta(&fixture) - .await? - .virtual_block_meta - .is_some() - ); - - fixture - .execute_command(&format!( - "alter table {db}.{table_name} set options(enable_virtual_column=false)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set payload = parse_json('{{\"x\": 3}}') where id = 1" - )) - .await?; - - assert!( - latest_default_block_meta(&fixture) - .await? - .virtual_block_meta - .is_none() - ); - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} where payload:x::int in (2, 3)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - - Ok(()) -} diff --git a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs index 7d48df9073170..77a8e88d6b532 100644 --- a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs +++ b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs @@ -331,7 +331,6 @@ async fn prepare_prewhere_data() -> Result { location: "test_block".to_string(), bloom_filter_index_location: None, bloom_filter_index_size: 0, - column_group_layout: false, column_group_bloom_files: vec![], create_on: None, nums_rows: num_rows, diff --git a/src/query/service/tests/it/storages/fuse/operations/replace_into.rs b/src/query/service/tests/it/storages/fuse/operations/replace_into.rs index 0ec89c5bd15c1..4288c6c3fc006 100644 --- a/src/query/service/tests/it/storages/fuse/operations/replace_into.rs +++ b/src/query/service/tests/it/storages/fuse/operations/replace_into.rs @@ -13,8 +13,6 @@ // limitations under the License. use databend_common_storages_fuse::FuseTable; -use databend_query::test_kits::TestFixture; -use databend_query::test_kits::latest_default_block_meta; use itertools::Itertools; #[test] @@ -60,77 +58,3 @@ fn test_partition() -> anyhow::Result<()> { } Ok(()) } - -#[tokio::test(flavor = "multi_thread")] -async fn test_replace_bloom_prunes_partial_update_block() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let table_name = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture - .execute_command(&format!( - "create table {db}.{table_name} (id int, a int, b int) engine=fuse \ - cluster by(id) bloom_index_columns='id,a,b' enable_partial_update=true" - )) - .await?; - fixture - .execute_command(&format!( - "insert into {db}.{table_name} values (1, 10, 100), (100, 30, 200)" - )) - .await?; - fixture - .execute_command("set enable_partial_update = 1") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set a = a + 1 where id = 1" - )) - .await?; - let block_meta = latest_default_block_meta(&fixture).await?; - assert_eq!( - block_meta - .column_groups - .iter() - .filter(|group| group.bloom.is_some()) - .count(), - 2 - ); - - // Make the active data files unreadable while leaving both Bloom files intact. A key that - // might conflict must still try to read the block, proving that the missing data is visible. - // The absent key below can succeed only if REPLACE reads and combines the paired Bloom files. - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let operator = fuse_table.get_operator(); - for group in block_meta.physical_column_groups().iter() { - operator.delete(&group.location.0).await?; - } - assert!( - fixture - .execute_command(&format!( - "replace into {db}.{table_name} on(id, a) values (1, 11, 999)" - )) - .await - .is_err() - ); - - fixture - .execute_command(&format!( - "replace into {db}.{table_name} on(id, a) values (50, 20, 999)" - )) - .await?; - let latest_table = fixture.latest_default_table().await?; - let latest_fuse = FuseTable::try_from_table(latest_table.as_ref())?; - assert_eq!( - latest_fuse - .read_table_snapshot() - .await? - .unwrap() - .summary - .row_count, - 3 - ); - - Ok(()) -} diff --git a/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs b/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs index ba7a12bb2c049..723cd1773c5cf 100644 --- a/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs +++ b/src/query/service/tests/it/storages/fuse/pruning_column_oriented_segment.rs @@ -34,7 +34,6 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::number::Int64Type; use databend_common_expression::types::number::UInt64Type; use databend_common_meta_app::schema::CreateOption; -use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_pipeline::core::Pipeline; use databend_common_sql::BloomIndexColumns; use databend_common_sql::parse_to_filters; @@ -58,6 +57,7 @@ use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; +use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use opendal::Operator; diff --git a/src/query/storages/common/session/src/temp_table.rs b/src/query/storages/common/session/src/temp_table.rs index 04a67e7ce6063..5dbfe89bb6f35 100644 --- a/src/query/storages/common/session/src/temp_table.rs +++ b/src/query/storages/common/session/src/temp_table.rs @@ -105,9 +105,6 @@ impl TempTblMgr { as_dropped, .. } = req; - table_meta - .validate_column_group_compatibility() - .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; let orphan_table_name = as_dropped.then(|| format!("orphan@{}", name_ident.table_name)); let Some(db_id) = table_meta.options.get(OPT_KEY_DATABASE_ID) else { @@ -268,19 +265,6 @@ impl TempTblMgr { .collect()) } - pub fn validate_multi_table_meta(&self, req: &[UpdateTempTableReq]) -> Result<()> { - for r in req { - let table = self.id_to_table.get(&r.table_id).ok_or_else(|| { - ErrorCode::UnknownTable(format!("Temporary table id {} not found", r.table_id)) - })?; - table - .meta - .validate_column_group_transition(&r.new_table_meta) - .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; - } - Ok(()) - } - pub fn update_multi_table_meta(&mut self, req: Vec) { for r in req { let UpdateTempTableReq { @@ -309,19 +293,13 @@ impl TempTblMgr { table_id ))); }; - let mut new_meta = table.meta.clone(); for (k, v) in options { if let Some(v) = v { - new_meta.options.insert(k, v); + table.meta.options.insert(k, v); } else { - new_meta.options.remove(&k); + table.meta.options.remove(&k); } } - table - .meta - .validate_column_group_transition(&new_meta) - .map_err(|error| ErrorCode::TableOptionInvalid(error.to_string()))?; - table.meta = new_meta; Ok(UpsertTableOptionReply {}) } diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs index 45e7265b2eba7..a5422703484d4 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_read_info.rs @@ -13,12 +13,10 @@ // limitations under the License. use std::collections::HashMap; -use std::collections::HashSet; use databend_common_expression::ColumnId; use crate::meta::BlockMeta; -use crate::meta::ColumnGroupFileMeta; use crate::meta::ColumnMeta; use crate::meta::Compression; @@ -27,29 +25,16 @@ pub struct BlockReadInfo { pub location: String, pub row_count: u64, pub col_metas: HashMap, - /// Active physical files that make up this logical block. - /// - /// An empty vector is the legacy single-file representation described by `location` and - /// `col_metas`. - #[serde(default)] - pub column_groups: Vec, pub compression: Compression, pub block_size: u64, } impl From<&BlockMeta> for BlockReadInfo { fn from(meta: &BlockMeta) -> Self { - let column_groups = if meta.column_groups.is_empty() { - vec![] - } else { - let projected_column_ids = meta.col_metas.keys().copied().collect::>(); - meta.project_column_groups(&projected_column_ids) - }; BlockReadInfo { location: meta.location.0.clone(), row_count: meta.row_count, col_metas: meta.col_metas.clone(), - column_groups, compression: meta.compression, block_size: meta.block_size, } diff --git a/src/query/storages/common/table_meta/src/table/table_keys.rs b/src/query/storages/common/table_meta/src/table/table_keys.rs index 998d20201d72a..284f1f33eafd3 100644 --- a/src/query/storages/common/table_meta/src/table/table_keys.rs +++ b/src/query/storages/common/table_meta/src/table/table_keys.rs @@ -21,7 +21,6 @@ use std::sync::LazyLock; use databend_common_exception::ErrorCode; use databend_common_frozen_api::FrozenAPI; use databend_common_meta_app::schema::OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID; -pub use databend_common_meta_app::schema::OPT_KEY_PARTITION_BY; use crate::meta::ColumnCountMinSketch; pub const OPT_KEY_DATABASE_ID: &str = "database_id"; @@ -31,6 +30,7 @@ pub const OPT_KEY_RECURSIVE_CTE: &str = "recursive_cte"; pub const OPT_KEY_SNAPSHOT_LOCATION: &str = "snapshot_location"; pub const OPT_KEY_SNAPSHOT_LOCATION_FIXED_FLAG: &str = "snapshot_location_fixed"; pub const OPT_KEY_STORAGE_FORMAT: &str = "storage_format"; +pub const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; pub const OPT_KEY_TABLE_COMPRESSION: &str = "compression"; pub const OPT_KEY_COMMENT: &str = "comment"; pub const OPT_KEY_ENGINE: &str = "engine"; @@ -74,6 +74,9 @@ pub const OPT_KEY_RANDOM_MAX_STRING_LEN: &str = "max_string_len"; pub const OPT_KEY_RANDOM_MAX_ARRAY_LEN: &str = "max_array_len"; pub const OPT_KEY_CLUSTER_TYPE: &str = "cluster_type"; +/// The normalized Fuse partition expressions. This is set by CREATE TABLE +/// PARTITION BY and is not a user-settable table option. +pub const OPT_KEY_PARTITION_BY: &str = "partition_by"; pub const OPT_KEY_ENABLE_COPY_DEDUP_FULL_PATH: &str = "copy_dedup_full_path"; pub const LINEAR_CLUSTER_TYPE: &str = "linear"; pub const HILBERT_CLUSTER_TYPE: &str = "hilbert"; diff --git a/src/query/storages/fuse/src/constants.rs b/src/query/storages/fuse/src/constants.rs index d6cb43644ee66..d5da27468466b 100644 --- a/src/query/storages/fuse/src/constants.rs +++ b/src/query/storages/fuse/src/constants.rs @@ -24,7 +24,7 @@ pub const FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP: &str = "data_retention_num_snapshots_to_keep"; pub const FUSE_OPT_KEY_ENABLE_AUTO_VACUUM: &str = "enable_auto_vacuum"; pub const FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE: &str = "enable_auto_analyze"; -pub use databend_common_meta_app::schema::OPT_KEY_ENABLE_PARTIAL_UPDATE as FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; +pub const FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; pub const FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN: &str = "enable_virtual_column"; pub const FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD: &str = "auto_compaction_imperfect_blocks_threshold"; diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index c309346701c6c..ae9845f39bb5b 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -69,13 +69,12 @@ pub enum BloomIndexLayout<'a> { impl<'a> BloomIndexLayout<'a> { fn from_metadata( - has_column_groups: bool, legacy_location: Option<&'a Location>, legacy_file_size: u64, column_group_files: Cow<'a, [FuseBloomIndexFileInfo]>, ) -> Option { - if has_column_groups { - return (!column_group_files.is_empty()).then_some(Self::ColumnGroups { + if !column_group_files.is_empty() { + return Some(Self::ColumnGroups { files: column_group_files, }); } @@ -131,7 +130,6 @@ pub fn block_bloom_index_locations(meta: &BlockMeta) -> impl Iterator Option> { let files = column_group_bloom_files(meta); BloomIndexLayout::from_metadata( - !meta.column_groups.is_empty(), meta.bloom_filter_index_location.as_ref(), meta.bloom_filter_index_size, Cow::Owned(files), @@ -158,8 +156,6 @@ pub struct FuseBlockPartInfo { pub bloom_filter_index_location: Option, pub bloom_filter_index_size: u64, - #[serde(default)] - pub column_group_layout: bool, #[serde(default, alias = "bloom_index_files")] pub column_group_bloom_files: Vec, @@ -200,7 +196,6 @@ impl FuseBlockPartInfo { /// Normalize optional legacy and column-group Bloom metadata into one physical-layout view. pub fn bloom_index_layout(&self) -> Option> { BloomIndexLayout::from_metadata( - self.column_group_layout || !self.column_group_bloom_files.is_empty(), self.bloom_filter_index_location.as_ref(), self.bloom_filter_index_size, Cow::Borrowed(&self.column_group_bloom_files), @@ -212,7 +207,6 @@ impl FuseBlockPartInfo { location: String, bloom_filter_index_location: Option, bloom_filter_index_size: u64, - column_group_layout: bool, column_group_bloom_files: Vec, rows_count: u64, column_groups: Vec, @@ -226,7 +220,6 @@ impl FuseBlockPartInfo { location, bloom_filter_index_location, bloom_filter_index_size, - column_group_layout, column_group_bloom_files, create_on, column_groups, diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index 105e7d5d8666b..3141f8c31af6b 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -66,7 +66,6 @@ use databend_common_io::constants::DEFAULT_BLOCK_COMPRESSED_SIZE; use databend_common_io::constants::DEFAULT_BLOCK_PER_SEGMENT; use databend_common_io::constants::DEFAULT_BLOCK_ROW_COUNT; use databend_common_meta_app::schema::DatabaseType; -use databend_common_meta_app::schema::OPT_KEY_SEGMENT_FORMAT; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; @@ -109,6 +108,7 @@ use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING; use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_LEGACY_SNAPSHOT_LOC; use databend_storages_common_table_meta::table::OPT_KEY_PARTITION_BY; +use databend_storages_common_table_meta::table::OPT_KEY_SEGMENT_FORMAT; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION_FIXED_FLAG; use databend_storages_common_table_meta::table::OPT_KEY_STORAGE_FORMAT; diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 5cf1435d750ba..0f4711f5dcb7f 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -27,7 +27,6 @@ use databend_common_expression::BlockMetaInfo; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; -use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchemaRef; use databend_common_expression::local_block_meta_serde; @@ -193,32 +192,14 @@ fn merge_column_group_metadata( .collect::>(); let mut column_groups = origin.physical_column_groups().into_owned(); if origin.column_groups.is_empty() { - let legacy_bloom_is_ordinary_and_paired = origin.ngram_filter_index_size.is_none() - && origin + column_groups[0].bloom = + origin .bloom_filter_index_location .as_ref() - .is_none_or(|location| { - let expected = - TableMetaLocationGenerator::gen_bloom_index_location_with_version( - &origin.location.0, - location.1, - ); - expected == location.0 + .map(|location| ColumnGroupBloomMeta { + format_version: location.1, + file_size: origin.bloom_filter_index_size, }); - // A legacy Bloom may have an unpaired path after Ngram refresh, or a paired path but - // contain Ngram filters when the index existed at INSERT time. Neither file can be - // represented by ColumnGroupBloomMeta after the index is dropped, so adopt only an - // ordinary paired file and otherwise fail open. - if legacy_bloom_is_ordinary_and_paired { - column_groups[0].bloom = - origin - .bloom_filter_index_location - .as_ref() - .map(|location| ColumnGroupBloomMeta { - format_version: location.1, - file_size: origin.bloom_filter_index_size, - }); - } } for group in &mut column_groups { group.active_column_ids.retain(|column_id| { @@ -259,7 +240,6 @@ fn merge_column_group_metadata( .filter_map(|group| group.bloom.as_ref()) .map(|bloom| bloom.file_size) .sum(); - block_meta.ngram_filter_index_size = None; block_meta } @@ -534,15 +514,6 @@ impl BlockBuilder { let column_hlls = build_column_hlls(&updated_block, &updated_ndv_columns_map)?; Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); - let invalidate_virtual_columns = updated_field_indices.iter().any(|index| { - self.source_schema - .field(*index) - .data_type() - .remove_nullable() - == TableDataType::Variant - }); - let virtual_column_state = None; - let updated_col_stats = gen_columns_statistics( &updated_block, Some(column_distinct_count), @@ -578,9 +549,6 @@ impl BlockBuilder { }), }); block_meta.create_on = Some(Utc::now()); - if invalidate_virtual_columns { - block_meta.virtual_block_meta = None; - } let column_hlls = self.finalize_column_hlls(column_hlls)?; @@ -589,7 +557,7 @@ impl BlockBuilder { block_meta, bloom_index_state, inverted_index_states: vec![], - virtual_column_state, + virtual_column_state: None, vector_index_state: None, spatial_index_state: None, column_hlls, diff --git a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs index 1c408c0db35bd..36bf112287081 100644 --- a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs @@ -39,7 +39,6 @@ use databend_storages_common_table_meta::table::TableCompression; use opendal::Buffer; use opendal::Operator; -use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; use crate::io::BlockReader; @@ -153,7 +152,11 @@ impl BloomIndexRebuilder { Ok(()) } - async fn read_data_block(&self, block_read_info: &BlockReadInfo) -> Result { + pub async fn bloom_index_state_from_block_meta( + &self, + bloom_index_location: &Location, + block_read_info: &BlockReadInfo, + ) -> Result> { let ctx = self.table_ctx.clone(); let projection = @@ -168,46 +171,20 @@ impl BloomIndexRebuilder { )?; let settings = ReadSettings::from_ctx(&self.table_ctx)?; - let column_groups = if block_read_info.column_groups.is_empty() { - vec![FuseColumnGroupPartInfo { - location: block_read_info.location.clone(), - columns_meta: block_read_info.col_metas.clone(), - }] - } else { - block_read_info - .column_groups - .iter() - .map(|group| FuseColumnGroupPartInfo { - location: group.location.0.clone(), - columns_meta: group - .active_leaf_column_metas() - .map(|(column_id, column_meta)| (column_id, column_meta.clone())) - .collect(), - }) - .collect() - }; - let merge_io_read_result = block_reader - .read_column_groups_data_by_merge_io(&settings, &column_groups, &None) + .read_columns_data_by_merge_io( + &settings, + &block_read_info.location, + &block_read_info.col_metas, + &None, + ) .await?; - let column_chunks = merge_io_read_result.columns_chunks()?; - match self.storage_format { - FuseStorageFormat::Parquet => block_reader.deserialize_column_groups( - block_read_info.row_count as usize, - &column_groups, - column_chunks, - &block_read_info.compression, - None, - ), - FuseStorageFormat::Unsupported => Err(crate::unsupported_storage_format_error()), - } - } + let data_block = block_reader.deserialize_chunks_with_meta( + block_read_info, + &self.storage_format, + merge_io_read_result, + )?; - fn bloom_index_state_from_data_block( - &self, - bloom_index_location: &Location, - data_block: &DataBlock, - ) -> Result> { Self::validate_rebuild_version(bloom_index_location)?; let mut builder = BloomIndexBuilder::create( self.table_ctx.get_function_context()?, @@ -215,7 +192,7 @@ impl BloomIndexRebuilder { self.bloom_columns_map.clone(), &self.ngram_args, )?; - builder.add_block(data_block)?; + builder.add_block(&data_block)?; let maybe_bloom_index = builder.finalize()?; match maybe_bloom_index { @@ -226,15 +203,6 @@ impl BloomIndexRebuilder { ))), } } - - pub async fn bloom_index_state_from_block_meta( - &self, - bloom_index_location: &Location, - block_read_info: &BlockReadInfo, - ) -> Result> { - let data_block = self.read_data_block(block_read_info).await?; - self.bloom_index_state_from_data_block(bloom_index_location, &data_block) - } } #[cfg(test)] diff --git a/src/query/storages/fuse/src/operations/read/read_block_context.rs b/src/query/storages/fuse/src/operations/read/read_block_context.rs index 377f9a1490c7c..b6e912a30253b 100644 --- a/src/query/storages/fuse/src/operations/read/read_block_context.rs +++ b/src/query/storages/fuse/src/operations/read/read_block_context.rs @@ -131,7 +131,6 @@ impl ReadBlockContext { location, None, 0, - false, vec![], num_rows, column_groups, diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 43f5700c8b63b..2dada57e0e471 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -1410,7 +1410,6 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, - !meta.column_groups.is_empty(), column_group_bloom_files(meta), rows_count, column_groups, @@ -1468,7 +1467,6 @@ impl FuseTable { location, meta.bloom_filter_index_location.clone(), meta.bloom_filter_index_size, - !meta.column_groups.is_empty(), column_group_bloom_files(meta), rows_count, column_groups, diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 10c6082e692cd..831d87b7a9b88 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -27,6 +27,7 @@ use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; +use databend_common_expression::DataBlock; use databend_common_expression::FieldIndex; use databend_common_expression::FromData; use databend_common_expression::Scalar; @@ -41,7 +42,6 @@ use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_cache::LoadParams; use databend_storages_common_index::BloomIndex; -use databend_storages_common_index::filters::BlockBloomFilterIndexVersion; use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::FilterImpl; use databend_storages_common_io::ReadSettings; @@ -56,8 +56,6 @@ use opendal::Operator; use tokio::sync::Semaphore; use crate::FuseTable; -use crate::fuse_part::BloomIndexLayout; -use crate::fuse_part::bloom_index_layout; use crate::io::BlockBuilder; use crate::io::BlockReader; use crate::io::BlockWriter; @@ -77,14 +75,6 @@ use crate::operations::replace_into::meta::ReplaceIntoOperation; use crate::operations::replace_into::meta::UniqueKeyDigest; use crate::operations::replace_into::mutator::DeletionAccumulator; use crate::operations::replace_into::mutator::row_hash_of_columns; -use crate::pruning::read_multi_file_block_filter; - -fn replace_digest_pruning_supported(format_version: u64) -> bool { - matches!( - BlockBloomFilterIndexVersion::try_from(format_version), - Ok(BlockBloomFilterIndexVersion::V3(_)) | Ok(BlockBloomFilterIndexVersion::V4(_)) - ) -} struct AggregationContext { segment_locations: AHashMap, @@ -526,13 +516,8 @@ impl AggregationContext { metrics_inc_replace_row_number_totally_loaded(block_meta.row_count); // read the remaining columns - let remain_columns_data = read_block( - self.write_settings.storage_format, - remain_columns_reader, - block_meta, - &self.read_settings, - ) - .await?; + let remain_columns_data = + self.read_block(remain_columns_reader, block_meta).await?; // remove the deleted rows let remain_columns_data_after_deletion = @@ -651,6 +636,42 @@ impl AggregationContext { } } + async fn read_block(&self, reader: &BlockReader, block_meta: &BlockMeta) -> Result { + let merged_io_read_result = reader + .read_columns_data_by_merge_io( + &self.read_settings, + &block_meta.location.0, + &block_meta.col_metas, + &None, + ) + .await?; + + // deserialize block data + // cpu intensive task, send them to dedicated thread pool + let storage_format = self.write_settings.storage_format; + let block_meta_ptr = block_meta.clone(); + let reader = reader.clone(); + GlobalIORuntime::instance() + .spawn(async move { + let column_chunks = merged_io_read_result.columns_chunks()?; + reader.deserialize_chunks( + block_meta_ptr.location.0.as_str(), + block_meta_ptr.row_count as usize, + &block_meta_ptr.compression, + &block_meta_ptr.col_metas, + column_chunks, + &storage_format, + ) + }) + .await + .map_err(|e| { + ErrorCode::Internal( + "unexpected, failed to join aggregation context read block tasks for replace into.", + ) + .add_message_back(e.to_string()) + })? + } + // return true if the block is pruned, otherwise false async fn apply_bloom_pruning( &self, @@ -661,122 +682,93 @@ impl AggregationContext { if bloom_on_conflict_field_index.is_empty() { return false; } - match self - .load_bloom_filter(block_meta, bloom_on_conflict_field_index) - .await - { - Ok(Some(filters)) => { - // the caller ensures that the input_hashes is not empty - let row_count = input_hashes[0].len(); - - // let assume that the target block is prunable - let mut block_pruned = true; - for row in 0..row_count { - // for each row, by default, assume that columns of this row do have conflict with the target block. - let mut row_not_prunable = true; - for (col_idx, col_hash) in input_hashes.iter().enumerate() { - // For each column of current row, check if the corresponding bloom - // filter contains the digest of the column. - // - // Any one of the columns NOT contains by the corresponding bloom filter, - // indicates that the row is prunable(thus, we do not stop on the first column that - // the bloom filter contains). - - // - if bloom filter presents, check if the column is contained - // - if bloom filter absents, do nothing(since by default, we assume that the row is not-prunable) - if let Some(col_filter) = &filters[col_idx] { - let hash = col_hash[row]; - if hash == 0 || !col_filter.contains_digest(hash) { - // - hash == 0 indicates that the column value is null, which equals nothing. - // - NOT `contains_digest`, indicates that this column of row does not match - row_not_prunable = false; - // if one column not match, we do not need to check other columns - break; + if let Some(loc) = &block_meta.bloom_filter_index_location { + match self + .load_bloom_filter( + loc, + block_meta.bloom_filter_index_size, + bloom_on_conflict_field_index, + ) + .await + { + Ok(filters) => { + // the caller ensures that the input_hashes is not empty + let row_count = input_hashes[0].len(); + + // let assume that the target block is prunable + let mut block_pruned = true; + for row in 0..row_count { + // for each row, by default, assume that columns of this row do have conflict with the target block. + let mut row_not_prunable = true; + for (col_idx, col_hash) in input_hashes.iter().enumerate() { + // For each column of current row, check if the corresponding bloom + // filter contains the digest of the column. + // + // Any one of the columns NOT contains by the corresponding bloom filter, + // indicates that the row is prunable(thus, we do not stop on the first column that + // the bloom filter contains). + + // - if bloom filter presents, check if the column is contained + // - if bloom filter absents, do nothing(since by default, we assume that the row is not-prunable) + if let Some(col_filter) = &filters[col_idx] { + let hash = col_hash[row]; + if hash == 0 || !col_filter.contains_digest(hash) { + // - hash == 0 indicates that the column value is null, which equals nothing. + // - NOT `contains_digest`, indicates that this column of row does not match + row_not_prunable = false; + // if one column not match, we do not need to check other columns + break; + } } } + if row_not_prunable { + // any row not prunable indicates that the target block is not prunable + block_pruned = false; + break; + } } - if row_not_prunable { - // any row not prunable indicates that the target block is not prunable - block_pruned = false; - break; - } + block_pruned + } + Err(e) => { + // broken index should not stop us: + warn!("failed to build bloom index column name: {}", e); + // failed to load bloom filter, do not prune + false } - block_pruned - } - Ok(None) => false, - Err(e) => { - // broken index should not stop us: - warn!("failed to build bloom index column name: {}", e); - // failed to load bloom filter, do not prune - false } + } else { + // no bloom filter, no pruning + false } } async fn load_bloom_filter( &self, - block_meta: &BlockMeta, + location: &Location, + index_len: u64, bloom_on_conflict_field_index: &[FieldIndex], - ) -> Result>>>> { + ) -> Result>>> { // different block may have different version of bloom filter index - let (block_filter, col_names) = match bloom_index_layout(block_meta) { - None => return Ok(None), - Some(BloomIndexLayout::Legacy { - location, - file_size, - }) => { - // REPLACE carries current digest hashes instead of scalar values. V2 filters were - // built from legacy scalar values, so testing current digests against them can - // produce false negatives and incorrectly prune a conflicting block. - if !replace_digest_pruning_supported(location.1) { - return Ok(None); - } - let col_names = bloom_on_conflict_field_index - .iter() - .map(|idx| { - BloomIndex::build_filter_bloom_name( - location.1, - &self.on_conflict_fields[*idx].table_field, - ) - }) - .collect::>>()?; - let block_filter = location - .read_block_filter( - self.data_accessor.clone(), - &self.read_settings, - &col_names, - file_size, - ) - .await?; - (block_filter, col_names) - } - Some(BloomIndexLayout::ColumnGroups { files }) => { - let fields = bloom_on_conflict_field_index - .iter() - .map(|idx| self.on_conflict_fields[*idx].table_field.clone()) - .collect::>(); - let Some(filter) = read_multi_file_block_filter( - &self.data_accessor, - &self.read_settings, - &fields, - &files, - ) - .await? - else { - return Ok(None); - }; - // REPLACE carries current digest hashes instead of scalar values, so it cannot - // safely evaluate legacy V2 filters. Keep the block in that compatibility case. - if !replace_digest_pruning_supported(filter.format_version) { - return Ok(None); - } - let col_names = fields - .iter() - .map(|field| BloomIndex::build_filter_bloom_name(filter.format_version, field)) - .collect::>>()?; - (filter.block_filter, col_names) - } - }; + let mut col_names = Vec::with_capacity(bloom_on_conflict_field_index.len()); + + for idx in bloom_on_conflict_field_index { + let bloom_column_name = BloomIndex::build_filter_bloom_name( + location.1, + &self.on_conflict_fields[*idx].table_field, + )?; + col_names.push(bloom_column_name); + } + + // using load_bloom_filter_by_columns is attractive, + // but it do not care about the version of the bloom filter index + let block_filter = location + .read_block_filter( + self.data_accessor.clone(), + &self.read_settings, + &col_names, + index_len, + ) + .await?; // reorder the filter according to the order of bloom_on_conflict_field let mut filters = Vec::with_capacity(bloom_on_conflict_field_index.len()); @@ -788,14 +780,14 @@ impl AggregationContext { Err(_) => { info!( "bloom filter column {} not found for block {}", - filter_col_name, block_meta.location.0 + filter_col_name, location.0 ); filters.push(None); } } } - Ok(Some(filters)) + Ok(filters) } } @@ -806,29 +798,9 @@ mod tests { use databend_common_expression::TableSchema; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; - use databend_storages_common_index::filters::BlockFilter; - use databend_storages_common_table_meta::meta::Versioned; use super::*; - #[test] - fn test_replace_digest_pruning_version_eligibility() { - assert!( - !replace_digest_pruning_supported(2), - "V2 filters contain legacy scalar hashes, not current digests" - ); - assert!( - replace_digest_pruning_supported(3), - "V3 filters use the digest encoding" - ); - assert!( - replace_digest_pruning_supported(BlockFilter::VERSION), - "the current filter format uses the digest encoding" - ); - assert!(!replace_digest_pruning_supported(0)); - assert!(!replace_digest_pruning_supported(u64::MAX)); - } - #[test] fn test_check_overlap() -> Result<()> { // setup : diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index d0cd84f9ef72c..8ce35fd216b3b 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -28,8 +28,9 @@ use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; -use databend_common_expression::FieldIndex; +use databend_common_expression::TableDataType; use databend_common_expression::TableField; +use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; use databend_common_expression::local_block_meta_serde; use databend_common_meta_app::schema::TableIndex; @@ -41,8 +42,10 @@ use databend_common_pipeline::sources::AsyncSourcer; use databend_common_pipeline_transforms::AsyncTransform; use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_sql::executor::physical_plans::MutationKind; +use databend_storages_common_cache::CacheAccessor; +use databend_storages_common_cache::CacheManager; +use databend_storages_common_cache::FilterImpl; use databend_storages_common_cache::LoadParams; -use databend_storages_common_index::BloomIndexType; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; @@ -52,15 +55,17 @@ use databend_storages_common_table_meta::meta::RawBlockHLL; use databend_storages_common_table_meta::meta::SegmentStatistics; use databend_storages_common_table_meta::meta::SingleColumnMeta; use databend_storages_common_table_meta::meta::Statistics; +use databend_storages_common_table_meta::meta::Versioned; use log::info; use opendal::Operator; -use uuid::Uuid; use crate::FuseStorageFormat; use crate::FuseTable; use crate::index::BloomIndex; use crate::index::BloomIndexBuilder; use crate::index::NgramArgs; +use crate::index::filters::BlockFilter; +use crate::index::filters::Filter; use crate::io::BlockReader; use crate::io::BlockWriter; use crate::io::BloomIndexState; @@ -68,6 +73,7 @@ use crate::io::MetaReaders; use crate::io::SpatialIndexBuilder; use crate::io::TableMetaLocationGenerator; use crate::io::VectorIndexBuilder; +use crate::io::read::bloom::block_filter_reader::load_bloom_filter_by_columns; use crate::io::read::bloom::block_filter_reader::load_index_meta; use crate::io::read::load_spatial_index_meta; use crate::io::read::load_vector_index_meta; @@ -112,26 +118,13 @@ pub async fn do_refresh_table_index( info!("Start refresh {} index {}", index_type, index_name); let table_schema = fuse_table.schema(); - let table_meta = &fuse_table.get_table_info().meta; - let index_arg = build_refresh_index_arg( - fuse_table, - &index_name, - &index_type, - table_meta, - &index_schema, - &table_schema, - )?; - let field_indices = match &index_arg { - // A refresh publishes one new current-format Bloom file. Rebuild every configured Bloom - // and Ngram filter from the logical row so old V2/V3 encodings are never relabeled V4. - RefreshIndexArg::Ngram(arg) => arg.source_field_indices.clone(), - RefreshIndexArg::Vector(_) | RefreshIndexArg::Spatial(_) => index_schema - .fields - .iter() - .map(|field| table_schema.index_of(field.name())) - .collect::>>()?, - }; + // Collect field indices used by index. + let mut field_indices = Vec::with_capacity(index_schema.fields.len()); + for field in &index_schema.fields { + let field_index = table_schema.index_of(field.name())?; + field_indices.push(field_index); + } // Read data here to keep the order of blocks in segment. let projection = Projection::Columns(field_indices); @@ -139,14 +132,15 @@ pub async fn do_refresh_table_index( let block_reader = fuse_table.create_block_reader(ctx.clone(), projection, false)?; let meta_locations = fuse_table.meta_location_generator().clone(); - let segment_reader = - MetaReaders::segment_info_reader(fuse_table.get_operator(), table_schema.clone()); + let segment_reader = MetaReaders::segment_info_reader(fuse_table.get_operator(), table_schema); if snapshot.segments.is_empty() { return Ok(0); } let operator = fuse_table.get_operator_ref(); + let table_meta = &fuse_table.get_table_info().meta; + let index_arg = build_refresh_index_arg(&index_name, &index_type, table_meta, &index_schema)?; let target_segments = segment_locs.map(|locs| locs.into_iter().collect::>()); // Read the segment infos and collect the block metas that need to generate the index. @@ -225,10 +219,10 @@ pub async fn do_refresh_table_index( NgramIndexTransform::new( ctx.clone(), operator.clone(), - ngram_index_arg.bloom_index_type, - ngram_index_arg.bloom_columns_map.clone(), - ngram_index_arg.ngram_args.clone(), - meta_locations.clone(), + settings, + ngram_index_arg.index_ngram_args.clone(), + ngram_index_arg.ngram_index_names.clone(), + ngram_index_arg.existing_names_prefix.clone(), ) }); } @@ -301,18 +295,21 @@ pub async fn do_refresh_table_index( // build the index arguments used for refresh fn build_refresh_index_arg( - fuse_table: &FuseTable, index_name: &String, index_type: &TableIndexType, table_meta: &TableMeta, index_schema: &TableSchemaRef, - table_schema: &TableSchemaRef, ) -> Result { match index_type { TableIndexType::Ngram => { let index_ngram_args = FuseTable::create_ngram_index_args(&table_meta.indexes, index_schema, false)?; + let existing_names_prefix = index_ngram_args + .iter() + .map(|arg| format!("Ngram({})", arg.column_id())) + .collect::>(); + let ngram_index_names = index_ngram_args .iter() .map(|arg| { @@ -324,26 +321,10 @@ fn build_refresh_index_arg( }) .collect::>(); - let source_schema: TableSchemaRef = - table_schema.remove_virtual_computed_fields().into(); - let source_field_indices = source_schema - .fields() - .iter() - .map(|field| table_schema.index_of(field.name())) - .collect::>>()?; - let ngram_arg = RefreshNgramIndexArg { + index_ngram_args, ngram_index_names, - source_field_indices, - bloom_index_type: fuse_table.bloom_index_type(), - bloom_columns_map: fuse_table - .bloom_index_cols() - .bloom_index_fields(source_schema.clone(), BloomIndex::supported_type)?, - ngram_args: FuseTable::create_ngram_index_args( - &table_meta.indexes, - &source_schema, - false, - )?, + existing_names_prefix, }; Ok(RefreshIndexArg::Ngram(ngram_arg)) } @@ -438,30 +419,37 @@ async fn check_ngram_index_generated( stats: Option>, ngram_index_arg: &RefreshNgramIndexArg, ) -> Result> { - if !block_meta.column_groups.is_empty() { - return Err(ErrorCode::RefreshIndexError( - "Ngram index is incompatible with column-group layout".to_string(), - )); - } - if let Some((index_path, _)) = &block_meta.bloom_filter_index_location { - if let Ok(content_length) = operator - .stat(index_path) - .await - .map(|meta| meta.content_length()) - { - let bloom_index_meta = - load_index_meta(operator.clone(), index_path, content_length, None).await?; - - if ngram_index_arg.ngram_index_names.iter().all(|name| { - bloom_index_meta - .columns - .iter() - .any(|(column_name, _)| column_name == name) - }) { - return Ok(None); + let index_location = TableMetaLocationGenerator::gen_bloom_index_location_from_block_location( + &block_meta.location.0, + ); + // only generate bloom index if it is not exist. + let index_columns = if let Ok(content_length) = operator + .stat(&index_location) + .await + .map(|meta| meta.content_length()) + { + let bloom_index_meta = + load_index_meta(operator.clone(), &index_location, content_length, None).await?; + + let mut all_generated = true; + for ngram_index_name in &ngram_index_arg.ngram_index_names { + if !bloom_index_meta + .columns + .iter() + .any(|(column_name, _)| column_name == ngram_index_name) + { + all_generated = false; + break; } } - } + if all_generated { + return Ok(None); + } + + Some(bloom_index_meta.columns.clone()) + } else { + None + }; let ngram_index_meta = RefreshIndexMeta { index: BlockMetaIndex { segment_idx, @@ -472,7 +460,7 @@ async fn check_ngram_index_generated( .as_ref() .and_then(|v| v.block_hlls.get(block_idx)) .cloned(), - index_columns: None, + index_columns, index_meta: None, }; Ok(Some(ngram_index_meta)) @@ -695,28 +683,28 @@ impl AsyncSource for IndexSource { pub struct NgramIndexTransform { ctx: Arc, operator: Operator, - bloom_index_type: BloomIndexType, - bloom_columns_map: BTreeMap, - ngram_args: Vec, - meta_locations: TableMetaLocationGenerator, + settings: ReadSettings, + index_ngram_args: Vec, + ngram_index_names: Vec, + existing_names_prefix: Vec, } impl NgramIndexTransform { pub fn new( ctx: Arc, operator: Operator, - bloom_index_type: BloomIndexType, - bloom_columns_map: BTreeMap, - ngram_args: Vec, - meta_locations: TableMetaLocationGenerator, + settings: ReadSettings, + index_ngram_args: Vec, + ngram_index_names: Vec, + existing_names_prefix: Vec, ) -> Self { Self { ctx, operator, - bloom_index_type, - bloom_columns_map, - ngram_args, - meta_locations, + settings, + index_ngram_args, + ngram_index_names, + existing_names_prefix, } } } @@ -731,33 +719,94 @@ impl AsyncTransform for NgramIndexTransform { index, block_meta, column_hlls, - index_columns: _, + index_columns, index_meta: _index_meta, } = data_block .get_meta() .and_then(RefreshIndexMeta::downcast_ref_from) .unwrap(); + let index_path = TableMetaLocationGenerator::gen_bloom_index_location_from_block_location( + &block_meta.location.0, + ); + let mut new_block_meta = Arc::unwrap_or_clone(block_meta.clone()); + let index_location = (index_path.clone(), BlockFilter::VERSION); let mut builder = BloomIndexBuilder::create( self.ctx.get_function_context()?, - self.bloom_index_type, - self.bloom_columns_map.clone(), - &self.ngram_args, + databend_storages_common_index::BloomIndexType::default(), + BTreeMap::new(), + &self.index_ngram_args, )?; builder.add_block(&data_block)?; - if let Some(bloom_index) = builder.finalize()? { - let index_location = self - .meta_locations - .block_bloom_index_location(&Uuid::now_v7()); + if let Some(new_ngram_index) = builder.finalize()? { + let mut old_ngram_names = Vec::new(); + let bloom_index = if let Some(old_index_columns) = index_columns { + let mut index_columns = Vec::with_capacity(old_index_columns.len()); + for column in old_index_columns { + let name = column.0.to_string(); + if self + .existing_names_prefix + .iter() + .any(|name_prefix| name.starts_with(name_prefix)) + { + old_ngram_names.push(name); + continue; + } + index_columns.push(name); + } + + let mut block_filter = load_bloom_filter_by_columns( + self.operator.clone(), + &self.settings, + &index_columns, + &index_path, + block_meta.bloom_filter_index_size, + ) + .await?; + let new_ngram_columns = new_ngram_index.serialize_to_data_block()?.take_columns(); + for new_ngram_column in new_ngram_columns { + let (new_filter, _) = FilterImpl::from_bytes( + unsafe { new_ngram_column.index_unchecked(0) } + .as_binary() + .unwrap(), + )?; + block_filter.filters.push(Arc::new(new_filter)); + } + + let mut new_filter_schema = TableSchema::clone(&block_filter.filter_schema); + for ngram_index_name in &self.ngram_index_names { + new_filter_schema + .add_columns(&[TableField::new(ngram_index_name, TableDataType::Binary)])?; + } + block_filter.filter_schema = Arc::new(new_filter_schema); + + BloomIndex::from_filter_block( + self.ctx.get_function_context()?, + block_filter.filter_schema, + block_filter.filters, + index_location.1, + )? + } else { + new_ngram_index + }; let state = BloomIndexState::from_bloom_index(&bloom_index, index_location)?; - new_block_meta.bloom_filter_index_location = Some(state.location.clone()); new_block_meta.bloom_filter_index_size = state.size(); new_block_meta.ngram_filter_index_size = state.ngram_size(); BlockWriter::write_down_bloom_index_state(&self.operator, Some(state)).await?; + + // remove old bloom index meta and filter + if let Some(cache) = CacheManager::instance().get_bloom_index_meta_cache() { + cache.evict(&index_path); + } + if let Some(cache) = CacheManager::instance().get_bloom_index_filter_cache() { + for old_ngram_name in old_ngram_names { + cache.evict(&format!("{index_path}-{}", old_ngram_name)); + } + } } else { return Err(ErrorCode::RefreshIndexError( "Refresh Ngram index failed".to_string(), @@ -1012,11 +1061,9 @@ enum RefreshIndexArg { } struct RefreshNgramIndexArg { + index_ngram_args: Vec, ngram_index_names: Vec, - source_field_indices: Vec, - bloom_index_type: BloomIndexType, - bloom_columns_map: BTreeMap, - ngram_args: Vec, + existing_names_prefix: Vec, } struct RefreshVectorIndexArg { diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index ff6f7f0d9a5d6..d806b1c9fbc14 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -165,7 +165,7 @@ fn merged_filter_version(versions: impl IntoIterator) -> Option } } -pub(crate) async fn read_multi_file_block_filter( +async fn read_multi_file_block_filter( dal: &Operator, settings: &ReadSettings, index_fields: &[TableField], @@ -439,9 +439,6 @@ impl BloomPrunerCreator { index_files: &[FuseBloomIndexFileInfo], column_stats: &StatisticsOfColumns, ) -> Result { - if !self.ngram_args.is_empty() { - return Ok(true); - } let maybe_filter = read_multi_file_block_filter( &self.dal, &self.settings, diff --git a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs index 99ded650a2bcf..f4b65db5a0f48 100644 --- a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs +++ b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs @@ -537,7 +537,6 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, - false, vec![], 4, vec![], @@ -729,7 +728,6 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, - false, vec![], 1000, vec![], diff --git a/src/query/storages/fuse/src/pruning/mod.rs b/src/query/storages/fuse/src/pruning/mod.rs index f9b638ace046b..0ada116e90c23 100644 --- a/src/query/storages/fuse/src/pruning/mod.rs +++ b/src/query/storages/fuse/src/pruning/mod.rs @@ -30,7 +30,6 @@ mod virtual_column_pruner; pub use block_pruner::BlockPruner; pub use bloom_pruner::BloomPruner; pub use bloom_pruner::BloomPrunerCreator; -pub(crate) use bloom_pruner::read_multi_file_block_filter; pub use expr_bloom_filter::ExprBloomFilter; pub use expr_runtime_pruner::ExprRuntimePruner; pub use expr_runtime_pruner::RuntimeFilterExpr; diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index d47ad895c4685..3f793fe1175e5 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -216,7 +216,6 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { location: location_path.clone(), row_count, col_metas: columns_meta.clone(), - column_groups: vec![], compression, block_size, }; @@ -291,7 +290,6 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { location_path, bloom_filter_index_location, bloom_filter_index_size, - false, vec![], row_count, column_groups, diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index 1299cef30537a..903ec0585dfba 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -31,7 +31,6 @@ use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::VariantType; use databend_common_expression::types::string::StringColumnBuilder; -use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use serde::Serialize; @@ -52,30 +51,6 @@ fn serialize_variant(value: &impl Serialize) -> Result> { .to_vec()) } -fn observable_bloom_and_ngram_meta(block: &BlockMeta) -> (Option, u64, Option) { - if block.column_groups.is_empty() { - return ( - block - .bloom_filter_index_location - .as_ref() - .map(|location| location.0.clone()), - block.bloom_filter_index_size, - block.ngram_filter_index_size, - ); - } - - ( - None, - block - .column_groups - .iter() - .filter_map(|group| group.bloom.as_ref()) - .map(|bloom| bloom.file_size) - .sum(), - None, - ) -} - #[async_trait::async_trait] impl TableMetaFunc for FuseBlock { fn schema() -> Arc { @@ -160,12 +135,15 @@ impl TableMetaFunc for FuseBlock { block_size.push(block.block_size); file_size.push(block.file_size); row_count.push(block.row_count); - let (bloom_location, bloom_size, ngram_size) = - observable_bloom_and_ngram_meta(block); - bloom_filter_location.push(bloom_location); - bloom_filter_size.push(bloom_size); + bloom_filter_location.push( + block + .bloom_filter_index_location + .as_ref() + .map(|s| s.0.clone()), + ); + bloom_filter_size.push(block.bloom_filter_index_size); inverted_index_size.push(block.inverted_index_size); - ngram_index_size.push(ngram_size); + ngram_index_size.push(block.ngram_filter_index_size); vector_index_size.push(block.vector_index_size); spatial_index_size.push(block.spatial_index_size); virtual_column_size.push( @@ -219,57 +197,3 @@ impl TableMetaFunc for FuseBlock { )) } } - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use databend_storages_common_table_meta::meta::ColumnGroupBloomMeta; - use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; - use databend_storages_common_table_meta::meta::Compression; - - use super::*; - - #[test] - fn test_column_group_layout_hides_stale_top_level_bloom_and_ngram_meta() { - let mut block = BlockMeta::new( - 0, - 0, - 0, - HashMap::new(), - HashMap::new(), - None, - ("data.parquet".to_string(), 1), - Some(("stale-bloom".to_string(), 1)), - 11, - None, - Some(7), - None, - None, - None, - None, - None, - None, - Compression::Lz4Raw, - None, - ); - assert_eq!( - observable_bloom_and_ngram_meta(&block), - (Some("stale-bloom".to_string()), 11, Some(7)) - ); - - block.column_groups.push(ColumnGroupFileMeta { - active_column_ids: vec![], - location: ("data.parquet".to_string(), 1), - format_version: 1, - file_size: 0, - uncompressed_size: 0, - leaf_column_metas: HashMap::new(), - bloom: Some(ColumnGroupBloomMeta { - format_version: 1, - file_size: 5, - }), - }); - assert_eq!(observable_bloom_and_ngram_meta(&block), (None, 5, None)); - } -} diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test index 3d094d3b1d2c2..681377105761c 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -73,10 +73,6 @@ from fuse_block('default', 't_partial_update') ---- 1 -# The table capability is sticky; only the session setting can disable new partial writes. -statement error -alter table t_partial_update set options(enable_partial_update = false) - statement ok drop table t_partial_update @@ -109,18 +105,3 @@ select * from t_partial_update_compression order by id statement ok drop table t_partial_update_compression - -# Partitioned and column-group layouts are mutually exclusive. -statement error -create table t_partial_update_partitioned(id int, value int) - partition by (id) - enable_partial_update = true - -statement ok -create table t_partial_update_partitioned(id int, value int) partition by (id) - -statement error -alter table t_partial_update_partitioned set options(enable_partial_update = true) - -statement ok -drop table t_partial_update_partitioned diff --git a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test b/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test deleted file mode 100644 index 33200c23120cc..0000000000000 --- a/tests/sqllogictests/suites/ee/02_computed_column/02_0001_partial_update_cluster_dependency.test +++ /dev/null @@ -1,47 +0,0 @@ -## Copyright 2023 Databend Cloud -## -## Licensed under the Elastic License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## https://www.elastic.co/licensing/elastic-license -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - -statement ok -create or replace database partial_update_computed_column - -statement ok -use partial_update_computed_column - -# The incompatibility is enforced in both directions when table metadata is changed. -statement error 2322 -create table invalid_create( - id int, - value int, - computed_value bigint as (value + 1) virtual -) enable_partial_update = true - -statement ok -create table computed_first( - id int, - value int, - computed_value bigint as (value + 1) virtual -) - -statement error -alter table computed_first set options(enable_partial_update = true) - -statement ok -create table partial_update_first(id int, value int) enable_partial_update = true - -statement error -alter table partial_update_first - add column computed_value bigint as (value + 1) virtual - -statement ok -drop database partial_update_computed_column From 5ad77e89073dd147c108cb34fa510db9207a8775 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:56:00 +0800 Subject: [PATCH 17/30] fix(storage): address partial update review findings --- .../fuse/operations/vacuum_table_v2.rs | 41 ----------- .../fuse/operations/mutation/update.rs | 44 +----------- .../common/table_meta/src/meta/v2/segment.rs | 8 --- src/query/storages/fuse/src/fuse_part.rs | 16 +++-- .../fuse/src/io/write/block_writer.rs | 10 +-- .../processors/transform_serialize_block.rs | 71 +++++++++++++------ .../fuse/src/operations/read_partitions.rs | 14 ++-- .../fuse/src/table_functions/fuse_block.rs | 2 +- .../09_0054_partial_update.test | 30 -------- 9 files changed, 71 insertions(+), 165 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index b4bfe0659e647..a293e01b6c9a3 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -453,9 +453,7 @@ fn slice_summary(s: &[T]) -> String { #[cfg(test)] mod tests { - use chrono::Duration; use databend_common_meta_app::schema::TableIndexType; - use databend_storages_common_table_meta::meta::uuid_from_date_time; use super::*; @@ -492,43 +490,4 @@ mod tests { ), ]); } - - #[test] - fn test_bloom_index_gc_candidate_timestamp_safety() { - let gc_root_timestamp = Utc::now() - Duration::days(1); - let gc_root_meta_ts = Utc::now(); - let recent = gc_root_meta_ts - Duration::hours(1); - let old = gc_root_meta_ts - Duration::days(4); - let policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; - let before_root = uuid_from_date_time(gc_root_timestamp - Duration::milliseconds(1)); - let at_root = uuid_from_date_time(gc_root_timestamp); - - assert!(is_vacuum_object_gc_candidate( - &format!("1/2/_i_b_v2/{}_v4.parquet", before_root.as_simple()), - recent, - gc_root_meta_ts, - policy, - )); - assert!(!is_vacuum_object_gc_candidate( - &format!("1/2/_i_b_v2/{}_v4.parquet", at_root.as_simple()), - old, - gc_root_meta_ts, - policy, - )); - assert!(!is_vacuum_object_gc_candidate( - &format!( - "1/2/_i_b_v2/{}_v4.parquet", - before_root.as_simple().to_string().to_ascii_uppercase() - ), - recent, - gc_root_meta_ts, - policy, - )); - assert!(is_vacuum_object_gc_candidate( - "1/2/_i_b_v2/0123456789ab4def8123456789abcdef_v4.parquet", - old, - gc_root_meta_ts, - policy, - )); - } } diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index d94361ae021ea..48ce1b71c3134 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -37,7 +37,7 @@ async fn latest_block_hll(fixture: &TestFixture) -> anyhow::Result { } #[tokio::test(flavor = "multi_thread")] -async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { +async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; let db = fixture.default_db_name(); let table_name = fixture.default_table_name(); @@ -142,28 +142,6 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { origin_hll.get(&value_column_id) ); - let rows = fixture - .execute_query(&format!( - "select count(distinct column_name) from fuse_page('{db}', '{table_name}') \ - where column_name in ('id', 'value')" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - let rows = fixture - .execute_query(&format!( - "select count(distinct block_location) from fuse_column('{db}', '{table_name}') \ - where column_name in ('id', 'value')" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - let rows = fixture - .execute_query(&format!( - "select count(distinct column_name) from fuse_encoding('{db}', '{table_name}') \ - where column_name in ('id', 'value')" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - fixture .execute_command(&format!( "alter table {db}.{table_name} set options(bloom_index_columns='id')" @@ -214,26 +192,6 @@ async fn test_update_writes_changed_column_group() -> anyhow::Result<()> { .await?; assert_eq!(query_count(rows).await?, 2); - fixture - .execute_command("set enable_partial_update = 0") - .await?; - fixture - .execute_command(&format!( - "update {db}.{table_name} set value = value + 1 where id = 1" - )) - .await?; - let fully_rewritten = latest_default_block_meta(&fixture).await?; - assert!(fully_rewritten.column_groups.is_empty()); - assert!(fully_rewritten.bloom_filter_index_location.is_some()); - - let rows = fixture - .execute_query(&format!( - "select count(*) from {db}.{table_name} \ - where (id = 1 and value = 12) or (id = 2 and value = 21)" - )) - .await?; - assert_eq!(query_count(rows).await?, 2); - Ok(()) } diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 878b9d6d27afd..5d00aee1f79a7 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -181,7 +181,6 @@ pub struct DraftVirtualBlockMeta { pub struct ColumnGroupFileMeta { pub active_column_ids: Vec, pub location: Location, - pub format_version: FormatVersion, pub file_size: u64, pub uncompressed_size: u64, pub leaf_column_metas: HashMap, @@ -335,7 +334,6 @@ impl BlockMeta { ColumnGroupFileMeta { active_column_ids, location: self.location.clone(), - format_version: self.location.1, file_size: self.file_size, uncompressed_size: self.block_size, leaf_column_metas, @@ -360,7 +358,6 @@ impl BlockMeta { let project_group = |active_column_ids: &[ColumnId], location: &Location, - format_version: FormatVersion, file_size: u64, uncompressed_size: u64, leaf_column_metas: &HashMap| { @@ -380,7 +377,6 @@ impl BlockMeta { Some(ColumnGroupFileMeta { active_column_ids, location: location.clone(), - format_version, file_size, uncompressed_size, leaf_column_metas, @@ -402,7 +398,6 @@ impl BlockMeta { project_group( &group.active_column_ids, &group.location, - group.format_version, group.file_size, group.uncompressed_size, &group.leaf_column_metas, @@ -601,7 +596,6 @@ mod tests { let group = ColumnGroupFileMeta { active_column_ids: vec![1, 3], location: ("group.parquet".to_string(), 4), - format_version: 4, file_size: 20, uncompressed_size: 40, leaf_column_metas: HashMap::from([ @@ -638,7 +632,6 @@ mod tests { struct PrePairedColumnGroupFileMeta { active_column_ids: Vec, location: Location, - format_version: FormatVersion, file_size: u64, uncompressed_size: u64, leaf_column_metas: HashMap, @@ -647,7 +640,6 @@ mod tests { let group = PrePairedColumnGroupFileMeta { active_column_ids: vec![1], location: ("data.parquet".to_string(), 2), - format_version: 2, file_size: 10, uncompressed_size: 20, leaf_column_metas: HashMap::new(), diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index ae9845f39bb5b..7f0bc03c9b39f 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -112,14 +112,16 @@ pub(crate) fn column_group_bloom_files(meta: &BlockMeta) -> Vec impl Iterator + '_ { - let legacy = meta - .column_groups +pub(crate) fn legacy_bloom_index_location(meta: &BlockMeta) -> Option<&Location> { + meta.column_groups .is_empty() .then_some(meta.bloom_filter_index_location.as_ref()) .flatten() - .cloned(); +} + +/// Physical ordinary Bloom files referenced by a logical block. +pub fn block_bloom_index_locations(meta: &BlockMeta) -> impl Iterator + '_ { + let legacy = legacy_bloom_index_location(meta).cloned(); legacy.into_iter().chain( meta.column_groups .iter() @@ -130,7 +132,7 @@ pub fn block_bloom_index_locations(meta: &BlockMeta) -> impl Iterator Option> { let files = column_group_bloom_files(meta); BloomIndexLayout::from_metadata( - meta.bloom_filter_index_location.as_ref(), + legacy_bloom_index_location(meta), meta.bloom_filter_index_size, Cow::Owned(files), ) @@ -156,7 +158,7 @@ pub struct FuseBlockPartInfo { pub bloom_filter_index_location: Option, pub bloom_filter_index_size: u64, - #[serde(default, alias = "bloom_index_files")] + #[serde(default)] pub column_group_bloom_files: Vec, pub create_on: Option>, diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 0f4711f5dcb7f..d1ff92bf29646 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; @@ -210,7 +209,6 @@ fn merge_column_group_metadata( column_groups.push(ColumnGroupFileMeta { active_column_ids: update.active_column_ids, location: update.location.clone(), - format_version: update.location.1, file_size: update.file_size, uncompressed_size: update.uncompressed_size, leaf_column_metas: update.column_metas.clone(), @@ -306,13 +304,7 @@ impl BlockBuilder { } else { (None, None) }; - if let Some(hlls) = &column_hlls { - for (key, val) in hlls { - if let Entry::Vacant(entry) = column_distinct_count.entry(*key) { - entry.insert(val.count()); - } - } - } + Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); let mut inverted_index_states = Vec::with_capacity(self.inverted_index_builders.len()); for inverted_index_builder in &self.inverted_index_builders { diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index fc67bc977ef6e..62ec7c27b2e36 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -71,6 +71,7 @@ enum State { Serialized { serialized: BlockSerialization, index: Option, + written_uncompressed_size: Option, }, } @@ -437,27 +438,47 @@ impl Processor for TransformSerializeBlock { // Check if the datablock is valid, this is needed to ensure data is correct block.check_valid()?; - let serialized = if let (Some(updated_field_indices), Some(origin_block_meta)) = - (&self.updated_field_indices, origin_block_meta) - { - self.block_builder.build_column_group( - block, - &origin_block_meta, - updated_field_indices, - )? - } else { - self.block_builder - .build(block, |block, generator| match &stats_type { - ClusterStatsGenType::Generally => generator.gen_stats_for_append(block), - ClusterStatsGenType::WithOrigin(origin_stats) => { - let cluster_stats = generator - .gen_with_origin_stats(&block, origin_stats.clone())?; - Ok((cluster_stats, block)) - } - })? - }; + let (serialized, written_uncompressed_size) = + if let (Some(updated_field_indices), Some(origin_block_meta)) = + (&self.updated_field_indices, origin_block_meta) + { + let serialized = self.block_builder.build_column_group( + block, + &origin_block_meta, + updated_field_indices, + )?; + let written_uncompressed_size = serialized + .block_meta + .column_groups + .last() + .ok_or_else(|| { + ErrorCode::Internal("column-group update produced no column group") + })? + .uncompressed_size; + (serialized, Some(written_uncompressed_size)) + } else { + let serialized = + self.block_builder.build( + block, + |block, generator| match &stats_type { + ClusterStatsGenType::Generally => { + generator.gen_stats_for_append(block) + } + ClusterStatsGenType::WithOrigin(origin_stats) => { + let cluster_stats = generator + .gen_with_origin_stats(&block, origin_stats.clone())?; + Ok((cluster_stats, block)) + } + }, + )?; + (serialized, None) + }; - self.state = State::Serialized { serialized, index }; + self.state = State::Serialized { + serialized, + index, + written_uncompressed_size, + }; } _ => return Err(ErrorCode::Internal("It's a bug.")), } @@ -467,13 +488,19 @@ impl Processor for TransformSerializeBlock { #[async_backtrace::framed] async fn async_process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { - State::Serialized { serialized, index } => { + State::Serialized { + serialized, + index, + written_uncompressed_size, + } => { let merge_hll = std::mem::take(&mut self.pending_merge_hll); let (logical_updated_rows, logical_deleted_rows) = std::mem::take(&mut self.pending_logical_change); let extended_block_meta = BlockWriter::write_down(&self.dal, serialized).await?; - let bytes = if let Some(draft_virtual_block_meta) = + let bytes = if let Some(written_uncompressed_size) = written_uncompressed_size { + written_uncompressed_size as usize + } else if let Some(draft_virtual_block_meta) = &extended_block_meta.draft_virtual_block_meta { (extended_block_meta.block_meta.block_size diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index 2dada57e0e471..dfde1904516f8 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -88,6 +88,7 @@ use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; use crate::fuse_part::column_group_bloom_files; +use crate::fuse_part::legacy_bloom_index_location; use crate::fuse_part::project_column_groups; use crate::io::BloomIndexRebuilder; use crate::pruning::BlockPruner; @@ -1408,7 +1409,7 @@ impl FuseTable { FuseBlockPartInfo::create( location, - meta.bloom_filter_index_location.clone(), + legacy_bloom_index_location(meta).cloned(), meta.bloom_filter_index_size, column_group_bloom_files(meta), rows_count, @@ -1465,7 +1466,7 @@ impl FuseTable { // not the count the rows in this partition FuseBlockPartInfo::create( location, - meta.bloom_filter_index_location.clone(), + legacy_bloom_index_location(meta).cloned(), meta.bloom_filter_index_size, column_group_bloom_files(meta), rows_count, @@ -1552,11 +1553,12 @@ mod tests { HashMap::from([(2, active_column_2_meta.clone())]) ); + block_meta.bloom_filter_index_location = Some(("stale-bloom.parquet".to_string(), 4)); + block_meta.bloom_filter_index_size = 10; block_meta.column_groups = vec![ ColumnGroupFileMeta { active_column_ids: vec![1], location: ("group-1.parquet".to_string(), 2), - format_version: 2, file_size: 10, uncompressed_size: 10, leaf_column_metas: HashMap::from([(1, column_1_meta), (2, stale_column_2_meta)]), @@ -1565,7 +1567,6 @@ mod tests { ColumnGroupFileMeta { active_column_ids: vec![2], location: block_meta.location.clone(), - format_version: 2, file_size: 12, uncompressed_size: 12, leaf_column_metas: HashMap::from([(2, active_column_2_meta.clone())]), @@ -1581,5 +1582,10 @@ mod tests { column_groups[0].columns_meta, HashMap::from([(2, active_column_2_meta)]) ); + + let part = FuseTable::all_columns_part(None, &None, &None, &block_meta); + let part = FuseBlockPartInfo::from_part(&part).unwrap(); + assert!(part.bloom_filter_index_location.is_none()); + assert!(part.bloom_index_layout().is_none()); } } diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index 903ec0585dfba..cc1150b03c1c2 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -160,7 +160,7 @@ impl TableMetaFunc for FuseBlock { serde_json::json!({ "active_column_ids": group.active_column_ids, "location": group.location.0, - "format_version": group.format_version, + "format_version": group.location.1, "file_size": group.file_size, "uncompressed_size": group.uncompressed_size, "bloom": group.bloom, diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test index 681377105761c..4e8f5d1d92706 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -75,33 +75,3 @@ from fuse_block('default', 't_partial_update') statement ok drop table t_partial_update - -# A partial group keeps the logical block codec even if the table option changes. -statement ok -create table t_partial_update_compression(id int, a int, b int) - compression = 'zstd' - enable_partial_update = true - -statement ok -insert into t_partial_update_compression values (1, 10, 20), (2, 30, 40) - -statement ok -alter table t_partial_update_compression set options(compression = 'snappy') - -statement ok -set enable_partial_update = 1 - -statement ok -update t_partial_update_compression set a = a + 1 where id = 1 - -statement ok -update t_partial_update_compression set b = b + 1 where id = 2 - -query III -select * from t_partial_update_compression order by id ----- -1 11 20 -2 30 41 - -statement ok -drop table t_partial_update_compression From 39076947aa06f53dd7992ff206055c474691c14e Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:14:55 +0800 Subject: [PATCH 18/30] fix(storage): close partial update review gaps --- .../common/table_meta/src/meta/v2/segment.rs | 27 ----------- src/query/storages/fuse/src/fuse_part.rs | 14 +++++- .../processors/transform_serialize_block.rs | 48 ++++++++----------- .../fuse/src/operations/read_partitions.rs | 6 ++- .../fuse/src/table_functions/fuse_block.rs | 9 ++-- 5 files changed, 40 insertions(+), 64 deletions(-) diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index 5d00aee1f79a7..4facf41cd55f3 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -185,7 +185,6 @@ pub struct ColumnGroupFileMeta { pub uncompressed_size: u64, pub leaf_column_metas: HashMap, /// Ordinary Bloom file paired with this data file. Its location is derived from `location`. - #[serde(default)] pub bloom: Option, } @@ -626,32 +625,6 @@ mod tests { assert_eq!(active_metas, vec![(1, (10, 11))]); } - #[test] - fn test_deserialize_pre_paired_column_group_meta() { - #[derive(Serialize)] - struct PrePairedColumnGroupFileMeta { - active_column_ids: Vec, - location: Location, - file_size: u64, - uncompressed_size: u64, - leaf_column_metas: HashMap, - } - - let group = PrePairedColumnGroupFileMeta { - active_column_ids: vec![1], - location: ("data.parquet".to_string(), 2), - file_size: 10, - uncompressed_size: 20, - leaf_column_metas: HashMap::new(), - }; - let bytes = rmp_serde::to_vec_named(&group).unwrap(); - let decoded: ColumnGroupFileMeta = rmp_serde::from_slice(&bytes).unwrap(); - - assert_eq!(decoded.bloom, None); - assert_eq!(decoded.location, group.location); - assert_eq!(decoded.active_column_ids, group.active_column_ids); - } - #[test] fn test_deserialize_legacy_block_meta_without_column_groups() { let block_meta = BlockMeta::new( diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 7f0bc03c9b39f..4742e678c7255 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -119,6 +119,18 @@ pub(crate) fn legacy_bloom_index_location(meta: &BlockMeta) -> Option<&Location> .flatten() } +pub(crate) fn block_bloom_index_size(meta: &BlockMeta) -> u64 { + if meta.column_groups.is_empty() { + meta.bloom_filter_index_size + } else { + meta.column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum() + } +} + /// Physical ordinary Bloom files referenced by a logical block. pub fn block_bloom_index_locations(meta: &BlockMeta) -> impl Iterator + '_ { let legacy = legacy_bloom_index_location(meta).cloned(); @@ -133,7 +145,7 @@ pub(crate) fn bloom_index_layout(meta: &BlockMeta) -> Option, - written_uncompressed_size: Option, + write_progress: ProgressValues, }, } @@ -438,24 +438,17 @@ impl Processor for TransformSerializeBlock { // Check if the datablock is valid, this is needed to ensure data is correct block.check_valid()?; - let (serialized, written_uncompressed_size) = + let (serialized, write_progress_bytes) = if let (Some(updated_field_indices), Some(origin_block_meta)) = (&self.updated_field_indices, origin_block_meta) { + let write_progress_bytes = block.estimate_block_size(block.num_columns()); let serialized = self.block_builder.build_column_group( block, &origin_block_meta, updated_field_indices, )?; - let written_uncompressed_size = serialized - .block_meta - .column_groups - .last() - .ok_or_else(|| { - ErrorCode::Internal("column-group update produced no column group") - })? - .uncompressed_size; - (serialized, Some(written_uncompressed_size)) + (serialized, write_progress_bytes) } else { let serialized = self.block_builder.build( @@ -471,13 +464,24 @@ impl Processor for TransformSerializeBlock { } }, )?; - (serialized, None) + let virtual_column_size = serialized + .virtual_column_state + .as_ref() + .map(|state| state.draft_virtual_block_meta.virtual_column_size) + .unwrap_or_default(); + let write_progress_bytes = + (serialized.block_meta.block_size + virtual_column_size) as usize; + (serialized, write_progress_bytes) }; + let write_progress = ProgressValues { + rows: serialized.block_meta.row_count as usize, + bytes: write_progress_bytes, + }; self.state = State::Serialized { serialized, index, - written_uncompressed_size, + write_progress, }; } _ => return Err(ErrorCode::Internal("It's a bug.")), @@ -491,31 +495,17 @@ impl Processor for TransformSerializeBlock { State::Serialized { serialized, index, - written_uncompressed_size, + write_progress, } => { let merge_hll = std::mem::take(&mut self.pending_merge_hll); let (logical_updated_rows, logical_deleted_rows) = std::mem::take(&mut self.pending_logical_change); let extended_block_meta = BlockWriter::write_down(&self.dal, serialized).await?; - let bytes = if let Some(written_uncompressed_size) = written_uncompressed_size { - written_uncompressed_size as usize - } else if let Some(draft_virtual_block_meta) = - &extended_block_meta.draft_virtual_block_meta - { - (extended_block_meta.block_meta.block_size - + draft_virtual_block_meta.virtual_column_size) as usize - } else { - extended_block_meta.block_meta.block_size as usize - }; - let progress_values = ProgressValues { - rows: extended_block_meta.block_meta.row_count as usize, - bytes, - }; self.block_builder .ctx .get_write_progress() - .incr(&progress_values); + .incr(&write_progress); let mutation_log_data_block = if let Some(index) = index { // we are replacing the block represented by the `index` diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index dfde1904516f8..dff999961af3e 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -87,6 +87,7 @@ use crate::FuseLazyPartInfo; use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; +use crate::fuse_part::block_bloom_index_size; use crate::fuse_part::column_group_bloom_files; use crate::fuse_part::legacy_bloom_index_location; use crate::fuse_part::project_column_groups; @@ -1410,7 +1411,7 @@ impl FuseTable { FuseBlockPartInfo::create( location, legacy_bloom_index_location(meta).cloned(), - meta.bloom_filter_index_size, + block_bloom_index_size(meta), column_group_bloom_files(meta), rows_count, column_groups, @@ -1467,7 +1468,7 @@ impl FuseTable { FuseBlockPartInfo::create( location, legacy_bloom_index_location(meta).cloned(), - meta.bloom_filter_index_size, + block_bloom_index_size(meta), column_group_bloom_files(meta), rows_count, column_groups, @@ -1586,6 +1587,7 @@ mod tests { let part = FuseTable::all_columns_part(None, &None, &None, &block_meta); let part = FuseBlockPartInfo::from_part(&part).unwrap(); assert!(part.bloom_filter_index_location.is_none()); + assert_eq!(part.bloom_filter_index_size, 0); assert!(part.bloom_index_layout().is_none()); } } diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index cc1150b03c1c2..4d6a2eb4671d0 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -36,6 +36,8 @@ use databend_storages_common_table_meta::meta::TableSnapshot; use serde::Serialize; use crate::FuseTable; +use crate::fuse_part::block_bloom_index_size; +use crate::fuse_part::legacy_bloom_index_location; use crate::io::SegmentsIO; use crate::sessions::TableContext; use crate::table_functions::TableMetaFuncTemplate; @@ -136,12 +138,9 @@ impl TableMetaFunc for FuseBlock { file_size.push(block.file_size); row_count.push(block.row_count); bloom_filter_location.push( - block - .bloom_filter_index_location - .as_ref() - .map(|s| s.0.clone()), + legacy_bloom_index_location(block).map(|location| location.0.clone()), ); - bloom_filter_size.push(block.bloom_filter_index_size); + bloom_filter_size.push(block_bloom_index_size(block)); inverted_index_size.push(block.inverted_index_size); ngram_index_size.push(block.ngram_filter_index_size); vector_index_size.push(block.vector_index_size); From 8352d9677b52964c95a47ce1b7da87da27a99916 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:29:27 +0800 Subject: [PATCH 19/30] fix(storage): finish partial update review cleanup --- .../src/interpreters/common/table_option_validation.rs | 1 + src/query/storages/fuse/src/operations/append.rs | 2 +- src/query/storages/fuse/src/operations/changes.rs | 1 + src/query/storages/fuse/src/pruning/bloom_pruner.rs | 6 +++--- .../suites/base/09_fuse_engine/09_0054_partial_update.test | 4 ++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index fd93d722a4b3c..44e9e8648e880 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -175,6 +175,7 @@ pub static UNSET_TABLE_OPTIONS_WHITE_LIST: LazyLock> = Laz r.insert(FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP); r.insert(FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD); r.insert(FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN); + r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(OPT_KEY_ENABLE_COPY_DEDUP_FULL_PATH); r.insert(FUSE_OPT_KEY_DATA_PAGE_ROWS); r.insert(FUSE_OPT_KEY_DATA_PAGE_BYTES); diff --git a/src/query/storages/fuse/src/operations/append.rs b/src/query/storages/fuse/src/operations/append.rs index f1d0d37909d35..c65d75b5f556d 100644 --- a/src/query/storages/fuse/src/operations/append.rs +++ b/src/query/storages/fuse/src/operations/append.rs @@ -223,12 +223,12 @@ impl FuseTable { ) -> Result { let input_schema = modified_schema.unwrap_or(DataSchema::from(self.schema_with_stream()).into()); - let num_input_columns = input_schema.num_fields(); let cluster_stats_gen = self.get_cluster_stats_gen(ctx.clone(), 0, block_thresholds, input_schema)?; let operators = cluster_stats_gen.operators.clone(); if !operators.is_empty() { + let num_input_columns = self.schema().fields().len(); let func_ctx2 = cluster_stats_gen.func_ctx.clone(); pipeline.add_transformer(move || { diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index 5dea07828bda9..78687945a559f 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -60,6 +60,7 @@ pub struct ChangesDesc { pub location: Option, pub desc: String, } + #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct StreamBacklog { // Physical rows in latest-only endpoint blocks. diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index d806b1c9fbc14..8608aec86e5db 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -143,9 +143,9 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( )? == FilterEvalResult::MustFalse) } -pub(crate) struct MultiFileBlockFilter { - pub block_filter: BlockFilter, - pub format_version: u64, +struct MultiFileBlockFilter { + block_filter: BlockFilter, + format_version: u64, } fn merged_filter_version(versions: impl IntoIterator) -> Option { diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test index 4e8f5d1d92706..2b23cac2a8512 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -60,9 +60,9 @@ where column_name in ('id', 'value') ---- 2 -# Disabling the setting forces the next rewrite back to the single-file layout. +# Unsetting the table option forces the next rewrite back to the single-file layout. statement ok -set enable_partial_update = 0 +alter table t_partial_update unset options(enable_partial_update) statement ok update t_partial_update set value = value + 1 where id = 1 From f7d1680b7b1be277cfa836beac98c0f2378329d8 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:43:46 +0800 Subject: [PATCH 20/30] refactor(storage): simplify partial update data flow --- .../src/physical_plans/physical_mutation.rs | 6 ----- .../operations/mutation/meta/mutation_meta.rs | 8 ++++--- .../mutation/processors/compact_source.rs | 24 +++++-------------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 85e02f3eebc4b..7475908163025 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -170,12 +170,6 @@ fn build_partial_update_info( .flat_map(ScalarExpr::used_columns) .chain(direct_filter.iter().flat_map(ScalarExpr::used_columns)) .collect::(); - for field_index in update_list.keys().copied() { - let Some(symbol) = find_symbol(field_index) else { - return Ok(None); - }; - required_columns.insert(symbol); - } for (field_index, field) in schema_with_stream.fields().iter().enumerate() { if updated_column_ids.contains(&field.column_id()) { let Some(symbol) = find_symbol(field_index) else { diff --git a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs index bcfba09d223fe..f5aaaf90419fa 100644 --- a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs +++ b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs @@ -84,9 +84,11 @@ impl SerializeBlock { pub enum CompactSourceMeta { Concat { - read_res: Vec, - metas: Vec>, - column_groups: Vec>, + blocks: Vec<( + BlockReadResult, + Arc, + Vec, + )>, index: BlockMetaIndex, }, Extras(CompactExtraInfo), diff --git a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs index eba5002f3e669..0bf81ec616230 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs @@ -98,7 +98,7 @@ impl PrefetchAsyncSource for CompactSource { &None, ) .await?; - Ok::<_, ErrorCode>((read_res, column_groups)) + Ok::<_, ErrorCode>((read_res, block, column_groups)) }) .await .unwrap() @@ -107,18 +107,13 @@ impl PrefetchAsyncSource for CompactSource { let start = Instant::now(); - let (read_res, column_groups) = futures::future::try_join_all(task_futures) - .await? - .into_iter() - .unzip(); + let blocks = futures::future::try_join_all(task_futures).await?; // Perf. { metrics_inc_compact_block_read_milliseconds(start.elapsed().as_millis() as u64); } Box::new(CompactSourceMeta::Concat { - read_res, - metas: task.blocks.clone(), - column_groups, + blocks, index: task.index.clone(), }) } @@ -160,17 +155,10 @@ impl BlockMetaTransform for CompactTransform { fn transform(&mut self, meta: CompactSourceMeta) -> Result> { match meta { - CompactSourceMeta::Concat { - read_res, - metas, - column_groups, - index, - } => { - let blocks = read_res + CompactSourceMeta::Concat { blocks, index } => { + let blocks = blocks .into_iter() - .zip(metas.into_iter()) - .zip(column_groups.into_iter()) - .map(|((data, meta), column_groups)| { + .map(|(data, meta, column_groups)| { let chunks = data.columns_chunks()?; let mut block = match self.storage_format { FuseStorageFormat::Parquet => { From 876c07040b273e314a6b4e2bfd5a9c7c79c58ce2 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:07:38 +0800 Subject: [PATCH 21/30] fix(storage): discard stale partial update HLLs --- .../physical_plans/physical_commit_sink.rs | 4 +-- .../physical_plans/physical_compact_source.rs | 4 +-- .../pipelines/builders/transform_builder.rs | 2 +- .../fuse/operations/mutation/update.rs | 10 +++++- .../storages/fuse/src/operations/commit.rs | 4 +-- .../transform_mutation_aggregator.rs | 34 ++++++++++++++++--- .../fuse/src/operations/table_index.rs | 4 +-- .../fuse/src/operations/virtual_column.rs | 8 ++--- 8 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/query/service/src/physical_plans/physical_commit_sink.rs b/src/query/service/src/physical_plans/physical_commit_sink.rs index fde0e57a43bf8..fc688ce3d3ad3 100644 --- a/src/query/service/src/physical_plans/physical_commit_sink.rs +++ b/src/query/service/src/physical_plans/physical_commit_sink.rs @@ -156,7 +156,7 @@ impl IPhysicalPlan for CommitSink { } else { builder .main_pipeline - .add_async_accumulating_transformer(|| { + .try_add_async_accumulating_transformer(|| { let base_segments = if matches!( kind, MutationKind::Compact @@ -194,7 +194,7 @@ impl IPhysicalPlan for CommitSink { *kind, self.table_meta_timestamps, ) - }); + })?; } let snapshot_gen = MutationGenerator::new(self.snapshot.clone(), *kind); diff --git a/src/query/service/src/physical_plans/physical_compact_source.rs b/src/query/service/src/physical_plans/physical_compact_source.rs index 53ffd233f46cd..9e36ec38a37da 100644 --- a/src/query/service/src/physical_plans/physical_compact_source.rs +++ b/src/query/service/src/physical_plans/physical_compact_source.rs @@ -206,7 +206,7 @@ impl IPhysicalPlan for CompactSource { builder.main_pipeline.try_resize(1)?; builder .main_pipeline - .add_async_accumulating_transformer(|| { + .try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( table, builder.ctx.clone(), @@ -217,7 +217,7 @@ impl IPhysicalPlan for CompactSource { MutationKind::Compact, self.table_meta_timestamps, ) - }); + })?; } Ok(()) } diff --git a/src/query/service/src/pipelines/builders/transform_builder.rs b/src/query/service/src/pipelines/builders/transform_builder.rs index fabc5cbeacc6b..ec52afbf081b7 100644 --- a/src/query/service/src/pipelines/builders/transform_builder.rs +++ b/src/query/service/src/pipelines/builders/transform_builder.rs @@ -172,7 +172,7 @@ impl PipelineBuilder { Statistics::default(), MutationKind::Insert, table_meta_timestamps, - ); + )?; Ok(ProcessorPtr::create(AsyncAccumulatingTransformer::create( input, output, aggregator, ))) diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index 48ce1b71c3134..b66c25937514e 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -144,7 +144,8 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { fixture .execute_command(&format!( - "alter table {db}.{table_name} set options(bloom_index_columns='id')" + "alter table {db}.{table_name} set options(\ + bloom_index_columns='id', approx_distinct_columns='id')" )) .await?; @@ -177,6 +178,13 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { 1 ); + let updated_again_hll = latest_block_hll(&fixture).await?; + assert_eq!( + updated_again_hll.get(&id_column_id), + updated_hll.get(&id_column_id) + ); + assert!(!updated_again_hll.contains_key(&value_column_id)); + let rows = fixture .execute_query(&format!( "select count(*) from {db}.{table_name} where value = 21" diff --git a/src/query/storages/fuse/src/operations/commit.rs b/src/query/storages/fuse/src/operations/commit.rs index 96908da07ddfa..7d71484329ffb 100644 --- a/src/query/storages/fuse/src/operations/commit.rs +++ b/src/query/storages/fuse/src/operations/commit.rs @@ -94,7 +94,7 @@ impl FuseTable { ) })?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( self, ctx.clone(), @@ -105,7 +105,7 @@ impl FuseTable { MutationKind::Insert, table_meta_timestamps, ) - }); + })?; let snapshot_gen = AppendGenerator::new(ctx.clone(), overwrite); pipeline.add_sink(|input| { diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs index de392b92c4fdf..9b7f5981a67a1 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs @@ -14,6 +14,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; @@ -36,6 +37,7 @@ use databend_common_pipeline_transforms::processors::AsyncAccumulatingTransform; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::parse_cluster_keys; use databend_storages_common_cache::SegmentStatistics; +use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; @@ -101,6 +103,7 @@ pub struct TableMutationAggregator { top_n: BlockTopN, logical_updated_rows: u64, logical_deleted_rows: u64, + current_hll_column_ids: Arc>, write_segment_ctx: WriteSegmentCtx, processed_log_entries: usize, @@ -199,9 +202,23 @@ impl TableMutationAggregator { removed_statistics: Statistics, kind: MutationKind, table_meta_timestamps: TableMetaTimestamps, - ) -> Self { + ) -> Result { let fill_missing_cluster_stats = table.resolve_physical_cluster_keys().is_some(); + let hll_schema = if matches!(kind, MutationKind::Insert | MutationKind::Replace) { + table.schema() + } else { + table.schema_with_stream() + }; + let current_hll_column_ids = Arc::new( + table + .approx_distinct_cols() + .distinct_column_fields(hll_schema, RangeIndex::supported_table_type)? + .into_values() + .map(|field| field.column_id()) + .collect(), + ); + let virtual_schema = table.table_info.meta.virtual_schema.clone(); let cluster_key_exprs = if fill_missing_cluster_stats { table @@ -227,7 +244,7 @@ impl TableMutationAggregator { table_meta_timestamps, fill_missing_cluster_stats, }; - TableMutationAggregator { + Ok(TableMutationAggregator { ctx, mutations: HashMap::new(), extended_mutations: HashMap::new(), @@ -243,10 +260,11 @@ impl TableMutationAggregator { top_n: HashMap::new(), logical_updated_rows: 0, logical_deleted_rows: 0, + current_hll_column_ids, write_segment_ctx, processed_log_entries: 0, table_id: table.get_id(), - } + }) } fn accumulate_top_n(&mut self, top_n: Option) -> Result<()> { @@ -547,6 +565,7 @@ impl TableMutationAggregator { let segment_mutation = self.mutations.remove(&index).unwrap(); let location = self.base_segments.get(index).cloned(); let write_segment_ctx = self.write_segment_ctx.clone(); + let current_hll_column_ids = self.current_hll_column_ids.clone(); tasks.push(async move { let mut force_all_blocks_perfect = false; @@ -589,7 +608,12 @@ impl TableMutationAggregator { { let previous_hll = block_editor.get(&idx).and_then(|(_, hll)| hll.as_ref()); - replace_partial_block_hll(previous_hll, new_hll, updated_column_ids)? + replace_partial_block_hll( + previous_hll, + new_hll, + updated_column_ids, + ¤t_hll_column_ids, + )? } else { new_hll }; @@ -993,12 +1017,14 @@ fn replace_partial_block_hll( previous: Option<&RawBlockHLL>, replacement: Option, updated_column_ids: &[ColumnId], + current_hll_column_ids: &HashSet, ) -> Result> { let mut merged = previous .map(decode_column_hll) .transpose()? .flatten() .unwrap_or_default(); + merged.retain(|column_id, _| current_hll_column_ids.contains(column_id)); for column_id in updated_column_ids { merged.remove(column_id); } diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index 8ce35fd216b3b..1bcabc2d9b69d 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -258,7 +258,7 @@ pub async fn do_refresh_table_index( pipeline.try_resize(1)?; let table_meta_timestamps = ctx.get_table_meta_timestamps(fuse_table, Some(snapshot.clone()))?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( fuse_table, ctx.clone(), @@ -269,7 +269,7 @@ pub async fn do_refresh_table_index( MutationKind::Refresh, table_meta_timestamps, ) - }); + })?; let prev_snapshot_id = snapshot.snapshot_id; let snapshot_gen = MutationGenerator::new(Some(snapshot), MutationKind::Refresh); diff --git a/src/query/storages/fuse/src/operations/virtual_column.rs b/src/query/storages/fuse/src/operations/virtual_column.rs index a4dd9848dd48c..b94537a7fb5f9 100644 --- a/src/query/storages/fuse/src/operations/virtual_column.rs +++ b/src/query/storages/fuse/src/operations/virtual_column.rs @@ -367,7 +367,7 @@ pub async fn commit_refresh_virtual_column( let table_meta_timestamps = ctx.get_table_meta_timestamps(fuse_table, Some(latest_snapshot.clone()))?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( fuse_table, ctx.clone(), @@ -378,7 +378,7 @@ pub async fn commit_refresh_virtual_column( MutationKind::Refresh, table_meta_timestamps, ) - }); + })?; let snapshot_gen = MutationGenerator::new(Some(latest_snapshot), MutationKind::Refresh); pipeline.add_sink(|input| { @@ -444,7 +444,7 @@ pub async fn do_vacuum_virtual_column( let table_meta_timestamps = ctx.get_table_meta_timestamps(fuse_table, Some(latest_snapshot.clone()))?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( fuse_table, ctx.clone(), @@ -455,7 +455,7 @@ pub async fn do_vacuum_virtual_column( MutationKind::Refresh, table_meta_timestamps, ) - }); + })?; let prev_snapshot_id = latest_snapshot.snapshot_id; let snapshot_gen = MutationGenerator::new(Some(latest_snapshot), MutationKind::Refresh); From e227560a870dd2917d83b70ece23007104f9dfca Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:25:50 +0800 Subject: [PATCH 22/30] fix(storage): filter carried partial update HLLs --- .../fuse/operations/mutation/update.rs | 58 +++++++++++++----- .../transform_mutation_aggregator.rs | 59 +++++++++++-------- 2 files changed, 79 insertions(+), 38 deletions(-) diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index b66c25937514e..cc0d8a115db7c 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -12,15 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_query::test_kits::*; use databend_storages_common_cache::LoadParams; use databend_storages_common_table_meta::meta::BlockHLL; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::decode_column_hll; -async fn latest_block_hll(fixture: &TestFixture) -> anyhow::Result { +async fn latest_block_metas(fixture: &TestFixture) -> anyhow::Result>> { + Ok(latest_default_segment(fixture).await?.block_metas()?) +} + +async fn latest_block_hlls(fixture: &TestFixture) -> anyhow::Result> { let table = fixture.latest_default_table().await?; let fuse_table = FuseTable::try_from_table(table.as_ref())?; let segment = latest_default_segment(fixture).await?; @@ -33,7 +40,11 @@ async fn latest_block_hll(fixture: &TestFixture) -> anyhow::Result { put_cache: false, }) .await?; - Ok(decode_column_hll(&stats.block_hlls[0])?.unwrap()) + stats + .block_hlls + .iter() + .map(|hll| Ok(decode_column_hll(hll)?.unwrap())) + .collect() } #[tokio::test(flavor = "multi_thread")] @@ -46,13 +57,14 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { fixture .execute_command(&format!( "create table {db}.{table_name} (id int, value int) engine=fuse \ + row_per_block=2 block_per_segment=1000 \ bloom_index_columns='id,value' approx_distinct_columns='id,value' \ enable_partial_update=true change_tracking=true" )) .await?; fixture .execute_command(&format!( - "insert into {db}.{table_name} values (1, 10), (2, 20)" + "insert into {db}.{table_name} values (1, 10), (3, 30), (2, 20), (4, 40)" )) .await?; @@ -61,8 +73,10 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { let operator = FuseTable::try_from_table(table.as_ref())?.get_operator(); let id_column_id = schema.field(0).column_id(); let value_column_id = schema.field(1).column_id(); - let origin = latest_default_block_meta(&fixture).await?; - let origin_hll = latest_block_hll(&fixture).await?; + let origins = latest_block_metas(&fixture).await?; + let origin_hlls = latest_block_hlls(&fixture).await?; + assert_eq!(origins.len(), 2); + assert_eq!(origin_hlls.len(), 2); fixture .execute_command("set enable_partial_update = 1") @@ -74,7 +88,13 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { )) .await?; - let updated = latest_default_block_meta(&fixture).await?; + let updated_blocks = latest_block_metas(&fixture).await?; + let updated_index = updated_blocks + .iter() + .position(|block| !block.column_groups.is_empty()) + .unwrap(); + let origin = &origins[updated_index]; + let updated = &updated_blocks[updated_index]; assert_eq!(updated.column_groups.len(), 2); let unchanged_group = updated @@ -132,7 +152,9 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { assert_eq!(query_count(rows).await?, 2); assert!(!operator.exists(&missing_bloom_location).await?); - let updated_hll = latest_block_hll(&fixture).await?; + let updated_hlls = latest_block_hlls(&fixture).await?; + let origin_hll = &origin_hlls[updated_index]; + let updated_hll = &updated_hlls[updated_index]; assert_eq!( updated_hll.get(&id_column_id), origin_hll.get(&id_column_id) @@ -154,13 +176,16 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { "update {db}.{table_name} set value = value + 1 where id = 2" )) .await?; - let updated_again = latest_default_block_meta(&fixture).await?; + let updated_again_blocks = latest_block_metas(&fixture).await?; + let updated_again_index = 1 - updated_index; + let updated_again_origin = &origins[updated_again_index]; + let updated_again = &updated_again_blocks[updated_again_index]; assert_eq!(updated_again.column_groups.len(), 2); assert!( updated_again .column_groups .iter() - .any(|group| group.location == origin.location) + .any(|group| group.location == updated_again_origin.location) ); assert!( updated_again @@ -178,12 +203,15 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { 1 ); - let updated_again_hll = latest_block_hll(&fixture).await?; - assert_eq!( - updated_again_hll.get(&id_column_id), - updated_hll.get(&id_column_id) - ); - assert!(!updated_again_hll.contains_key(&value_column_id)); + let updated_again_hlls = latest_block_hlls(&fixture).await?; + assert_eq!(updated_again_hlls.len(), 2); + for (updated_again_hll, updated_hll) in updated_again_hlls.iter().zip(&updated_hlls) { + assert_eq!( + updated_again_hll.get(&id_column_id), + updated_hll.get(&id_column_id) + ); + assert!(!updated_again_hll.contains_key(&value_column_id)); + } let rows = fixture .execute_query(&format!( diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs index 9b7f5981a67a1..0821a5b0c7ae4 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs @@ -592,13 +592,13 @@ impl TableMutationAggregator { .into_iter() .enumerate() .map(|(block_idx, block_meta)| { - let hll = stats - .as_ref() - .and_then(|v| v.block_hlls.get(block_idx)) - .cloned(); - (block_idx, (block_meta, hll)) + let hll = retain_current_block_hll( + stats.as_ref().and_then(|v| v.block_hlls.get(block_idx)), + ¤t_hll_column_ids, + )?; + Ok((block_idx, (block_meta, hll))) }) - .collect::>(); + .collect::>>()?; for (idx, (new_meta, new_hll)) in segment_mutation.replaced_blocks { let new_hll = if let Some(updated_column_ids) = new_meta @@ -608,14 +608,12 @@ impl TableMutationAggregator { { let previous_hll = block_editor.get(&idx).and_then(|(_, hll)| hll.as_ref()); - replace_partial_block_hll( - previous_hll, - new_hll, - updated_column_ids, - ¤t_hll_column_ids, - )? + replace_partial_block_hll(previous_hll, new_hll, updated_column_ids)? } else { new_hll + .map(|hll| decode_column_hll(&hll)) + .transpose()? + .flatten() }; block_editor.insert(idx, (new_meta, new_hll)); } @@ -632,7 +630,17 @@ impl TableMutationAggregator { } // assign back the mutated blocks to segment - let (new_blocks, new_hlls) = block_editor.into_values().unzip(); + let (new_blocks, new_hlls) = block_editor + .into_values() + .map(|(block_meta, hll)| { + Ok(( + block_meta, + hll.map(|hll| encode_column_hll(&hll)).transpose()?, + )) + }) + .collect::>>()? + .into_iter() + .unzip(); let stats = generate_segment_stats(new_hlls)?; (new_blocks, stats, Some(segment_info.summary)) } else { @@ -1014,17 +1022,11 @@ fn generate_segment_stats(hlls: Vec>) -> Result, + previous: Option<&BlockHLL>, replacement: Option, updated_column_ids: &[ColumnId], - current_hll_column_ids: &HashSet, -) -> Result> { - let mut merged = previous - .map(decode_column_hll) - .transpose()? - .flatten() - .unwrap_or_default(); - merged.retain(|column_id, _| current_hll_column_ids.contains(column_id)); +) -> Result> { + let mut merged = previous.cloned().unwrap_or_default(); for column_id in updated_column_ids { merged.remove(column_id); } @@ -1037,6 +1039,17 @@ fn replace_partial_block_hll( if merged.is_empty() { Ok(None) } else { - Ok(Some(encode_column_hll(&merged)?)) + Ok(Some(merged)) } } + +fn retain_current_block_hll( + hll: Option<&RawBlockHLL>, + current_hll_column_ids: &HashSet, +) -> Result> { + let Some(mut hll) = hll.map(decode_column_hll).transpose()?.flatten() else { + return Ok(None); + }; + hll.retain(|column_id, _| current_hll_column_ids.contains(column_id)); + Ok((!hll.is_empty()).then_some(hll)) +} From e6963ae16c5266d159d9e6b725d33bb62f9e7f36 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:41:38 +0800 Subject: [PATCH 23/30] fix(storage): normalize replacement HLL metadata --- .../fuse/operations/mutation/update.rs | 48 +++++++++-------- .../transform_mutation_aggregator.rs | 52 +++++++++++++++---- 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs index cc0d8a115db7c..eaf2af81b319a 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -23,14 +23,13 @@ use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::decode_column_hll; -async fn latest_block_metas(fixture: &TestFixture) -> anyhow::Result>> { - Ok(latest_default_segment(fixture).await?.block_metas()?) -} - -async fn latest_block_hlls(fixture: &TestFixture) -> anyhow::Result> { +async fn latest_blocks_with_hll( + fixture: &TestFixture, +) -> anyhow::Result, BlockHLL)>> { let table = fixture.latest_default_table().await?; let fuse_table = FuseTable::try_from_table(table.as_ref())?; let segment = latest_default_segment(fixture).await?; + let blocks = segment.block_metas()?; let (stats_location, stats_version) = segment.summary.additional_stats_loc().unwrap(); let stats = MetaReaders::segment_stats_reader(fuse_table.get_operator()) .read(&LoadParams { @@ -40,11 +39,18 @@ async fn latest_block_hlls(fixture: &TestFixture) -> anyhow::Result>>()?; + anyhow::ensure!( + blocks.len() == hlls.len(), + "block/HLL count mismatch: {} blocks, {} HLLs", + blocks.len(), + hlls.len() + ); + Ok(blocks.into_iter().zip(hlls).collect()) } #[tokio::test(flavor = "multi_thread")] @@ -73,10 +79,8 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { let operator = FuseTable::try_from_table(table.as_ref())?.get_operator(); let id_column_id = schema.field(0).column_id(); let value_column_id = schema.field(1).column_id(); - let origins = latest_block_metas(&fixture).await?; - let origin_hlls = latest_block_hlls(&fixture).await?; + let origins = latest_blocks_with_hll(&fixture).await?; assert_eq!(origins.len(), 2); - assert_eq!(origin_hlls.len(), 2); fixture .execute_command("set enable_partial_update = 1") @@ -88,13 +92,13 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { )) .await?; - let updated_blocks = latest_block_metas(&fixture).await?; + let updated_blocks = latest_blocks_with_hll(&fixture).await?; let updated_index = updated_blocks .iter() - .position(|block| !block.column_groups.is_empty()) + .position(|(block, _)| !block.column_groups.is_empty()) .unwrap(); - let origin = &origins[updated_index]; - let updated = &updated_blocks[updated_index]; + let (origin, origin_hll) = &origins[updated_index]; + let (updated, updated_hll) = &updated_blocks[updated_index]; assert_eq!(updated.column_groups.len(), 2); let unchanged_group = updated @@ -152,9 +156,6 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { assert_eq!(query_count(rows).await?, 2); assert!(!operator.exists(&missing_bloom_location).await?); - let updated_hlls = latest_block_hlls(&fixture).await?; - let origin_hll = &origin_hlls[updated_index]; - let updated_hll = &updated_hlls[updated_index]; assert_eq!( updated_hll.get(&id_column_id), origin_hll.get(&id_column_id) @@ -176,10 +177,10 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { "update {db}.{table_name} set value = value + 1 where id = 2" )) .await?; - let updated_again_blocks = latest_block_metas(&fixture).await?; + let updated_again_blocks = latest_blocks_with_hll(&fixture).await?; let updated_again_index = 1 - updated_index; - let updated_again_origin = &origins[updated_again_index]; - let updated_again = &updated_again_blocks[updated_again_index]; + let (updated_again_origin, _) = &origins[updated_again_index]; + let (updated_again, _) = &updated_again_blocks[updated_again_index]; assert_eq!(updated_again.column_groups.len(), 2); assert!( updated_again @@ -203,9 +204,10 @@ async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { 1 ); - let updated_again_hlls = latest_block_hlls(&fixture).await?; - assert_eq!(updated_again_hlls.len(), 2); - for (updated_again_hll, updated_hll) in updated_again_hlls.iter().zip(&updated_hlls) { + assert_eq!(updated_again_blocks.len(), updated_blocks.len()); + for ((_, updated_again_hll), (_, updated_hll)) in + updated_again_blocks.iter().zip(&updated_blocks) + { assert_eq!( updated_again_hll.get(&id_column_id), updated_hll.get(&id_column_id) diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs index 0821a5b0c7ae4..9fa5e6393a41c 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs @@ -592,7 +592,7 @@ impl TableMutationAggregator { .into_iter() .enumerate() .map(|(block_idx, block_meta)| { - let hll = retain_current_block_hll( + let hll = decode_current_block_hll( stats.as_ref().and_then(|v| v.block_hlls.get(block_idx)), ¤t_hll_column_ids, )?; @@ -601,13 +601,12 @@ impl TableMutationAggregator { .collect::>>()?; for (idx, (new_meta, new_hll)) in segment_mutation.replaced_blocks { + let previous_hll = block_editor.remove(&idx).and_then(|(_, hll)| hll); let new_hll = if let Some(updated_column_ids) = new_meta .column_groups .last() .map(|group| group.active_column_ids.as_slice()) { - let previous_hll = - block_editor.get(&idx).and_then(|(_, hll)| hll.as_ref()); replace_partial_block_hll(previous_hll, new_hll, updated_column_ids)? } else { new_hll @@ -635,7 +634,7 @@ impl TableMutationAggregator { .map(|(block_meta, hll)| { Ok(( block_meta, - hll.map(|hll| encode_column_hll(&hll)).transpose()?, + encode_current_block_hll(hll, ¤t_hll_column_ids)?, )) }) .collect::>>()? @@ -662,7 +661,14 @@ impl TableMutationAggregator { .replaced_blocks .into_iter() .sorted_by(|a, b| a.0.cmp(&b.0)) - .map(|(_, meta)| meta) + .map(|(_, (block_meta, hll))| { + Ok(( + block_meta, + normalize_current_block_hll(hll, ¤t_hll_column_ids)?, + )) + }) + .collect::>>()? + .into_iter() .unzip(); let stats = generate_segment_stats(new_hlls)?; (new_blocks, stats, None) @@ -1022,11 +1028,11 @@ fn generate_segment_stats(hlls: Vec>) -> Result, + previous: Option, replacement: Option, updated_column_ids: &[ColumnId], ) -> Result> { - let mut merged = previous.cloned().unwrap_or_default(); + let mut merged = previous.unwrap_or_default(); for column_id in updated_column_ids { merged.remove(column_id); } @@ -1043,13 +1049,37 @@ fn replace_partial_block_hll( } } -fn retain_current_block_hll( +fn retain_hll_columns( + mut hll: BlockHLL, + current_hll_column_ids: &HashSet, +) -> Option { + hll.retain(|column_id, _| current_hll_column_ids.contains(column_id)); + (!hll.is_empty()).then_some(hll) +} + +fn decode_current_block_hll( hll: Option<&RawBlockHLL>, current_hll_column_ids: &HashSet, ) -> Result> { - let Some(mut hll) = hll.map(decode_column_hll).transpose()?.flatten() else { + let Some(hll) = hll.map(decode_column_hll).transpose()?.flatten() else { return Ok(None); }; - hll.retain(|column_id, _| current_hll_column_ids.contains(column_id)); - Ok((!hll.is_empty()).then_some(hll)) + Ok(retain_hll_columns(hll, current_hll_column_ids)) +} + +fn encode_current_block_hll( + hll: Option, + current_hll_column_ids: &HashSet, +) -> Result> { + hll.and_then(|hll| retain_hll_columns(hll, current_hll_column_ids)) + .map(|hll| encode_column_hll(&hll)) + .transpose() +} + +fn normalize_current_block_hll( + hll: Option, + current_hll_column_ids: &HashSet, +) -> Result> { + let hll = hll.as_ref().map(decode_column_hll).transpose()?.flatten(); + encode_current_block_hll(hll, current_hll_column_ids) } From 10beca96658a36e89839cffdf9014f61888bdf4b Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:49:20 +0800 Subject: [PATCH 24/30] refactor(storage): share partial field remapping --- .../fuse/src/io/write/block_writer.rs | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index d1ff92bf29646..570c30c941cd0 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -464,16 +464,19 @@ impl BlockBuilder { .meta_locations .gen_block_location(self.table_meta_timestamps); - let updated_bloom_columns_map = updated_field_indices - .iter() - .enumerate() - .filter_map(|(updated_index, source_index)| { - self.bloom_columns_map - .get(source_index) - .cloned() - .map(|field| (updated_index, field)) - }) - .collect::>(); + let project_updated_fields = |fields: &BTreeMap| { + updated_field_indices + .iter() + .enumerate() + .filter_map(|(updated_index, source_index)| { + fields + .get(source_index) + .cloned() + .map(|field| (updated_index, field)) + }) + .collect::>() + }; + let updated_bloom_columns_map = project_updated_fields(&self.bloom_columns_map); let rebuild_bloom_index = !updated_bloom_columns_map.is_empty(); let bloom_index_state = if rebuild_bloom_index { let location = self.meta_locations.block_bloom_index_location(&block_id); @@ -493,16 +496,7 @@ impl BlockBuilder { .as_ref() .map(|index| index.column_distinct_count.clone()) .unwrap_or_default(); - let updated_ndv_columns_map = updated_field_indices - .iter() - .enumerate() - .filter_map(|(updated_index, source_index)| { - self.ndv_columns_map - .get(source_index) - .cloned() - .map(|field| (updated_index, field)) - }) - .collect::>(); + let updated_ndv_columns_map = project_updated_fields(&self.ndv_columns_map); let column_hlls = build_column_hlls(&updated_block, &updated_ndv_columns_map)?; Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); From c45150b49fffcc736a36a24b4f41b4445badfb7d Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:42:22 +0800 Subject: [PATCH 25/30] refactor(storage): simplify vacuum2 bloom cleanup --- .../fuse/operations/vacuum_table_v2.rs | 56 ++----------------- 1 file changed, 6 insertions(+), 50 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index a293e01b6c9a3..c2709aa78e4c5 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -27,11 +27,8 @@ use databend_common_exception::Result; use databend_common_meta_app::schema::ListIndexesByIdReq; use databend_common_meta_app::schema::TableIndex; use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::block_bloom_index_locations; use databend_common_storages_fuse::io::SegmentsIO; use databend_common_storages_fuse::io::TableMetaLocationGenerator; -use databend_common_storages_fuse::operations::VacuumObjectKeyPolicy; -use databend_common_storages_fuse::operations::is_vacuum_object_gc_candidate; use databend_storages_common_cache::CacheAccessor; use databend_storages_common_cache::CacheManager; use databend_storages_common_io::Files; @@ -137,7 +134,6 @@ pub async fn do_vacuum2( .read_segments::>(&protected_segments, false) .await?; let mut gc_root_blocks = HashSet::new(); - let mut gc_root_indexes = HashSet::new(); for segment in segments { for block in segment?.block_metas()? { gc_root_blocks.extend( @@ -145,7 +141,6 @@ pub async fn do_vacuum2( .data_file_locations() .map(|location| location.0.clone()), ); - gc_root_indexes.extend(block_bloom_index_locations(&block).map(|location| location.0)); } } ctx.set_status_info(&format!( @@ -189,23 +184,6 @@ pub async fn do_vacuum2( slice_summary(&blocks_to_gc) )); - // Column-group Bloom paths are derived from their owning data-group paths. The files live in a - // separate directory, so scan that directory and retain exactly the files referenced by the - // protected groups. This also removes orphan Bloom files left by interrupted writes. - let start = std::time::Instant::now(); - let bloom_indexes_before_gc_root = - list_bloom_indexes_before_gc_root(fuse_table, gc_root_timestamp, gc_root_meta_ts).await?; - let bloom_indexes_to_gc = bloom_indexes_before_gc_root - .into_iter() - .filter(|path| !gc_root_indexes.contains(path)) - .collect::>(); - ctx.set_status_info(&format!( - "Filtered bloom_indexes_to_gc for table {}, elapsed: {:?}, bloom_indexes_to_gc: {:?}", - table_info.desc, - start.elapsed(), - slice_summary(&bloom_indexes_to_gc) - )); - let start = std::time::Instant::now(); let catalog = ctx.get_default_catalog()?; let table_agg_index_ids = catalog @@ -221,16 +199,9 @@ pub async fn do_vacuum2( blocks_to_gc.len() * (table_agg_index_ids.len() + inverted_indexes.len() + 2) + stats_to_gc.len() + segments_to_gc.len() - + snapshots_to_gc.len() - + bloom_indexes_to_gc.len(), + + snapshots_to_gc.len(), ); - // Bloom indexes must be removed before their data blocks. - if !bloom_indexes_to_gc.is_empty() { - op.remove_file_in_batch(&bloom_indexes_to_gc).await?; - files_to_gc.extend(bloom_indexes_to_gc.iter().cloned()); - } - // order is important // indexes should be removed before their blocks, because index locations to gc are generated from block locations. purge_block_chunks( @@ -348,32 +319,13 @@ async fn purge_block_chunks( Ok(()) } -async fn list_bloom_indexes_before_gc_root( - fuse_table: &FuseTable, - gc_root_timestamp: DateTime, - gc_root_meta_ts: DateTime, -) -> Result> { - let key_policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; - fuse_table - .list_files( - fuse_table - .meta_location_generator() - .block_bloom_index_prefix() - .to_string(), - |path, modified| { - is_vacuum_object_gc_candidate(&path, modified, gc_root_meta_ts, key_policy) - }, - ) - .await -} - fn collect_block_index_locations( blocks_to_gc: &[String], table_agg_index_ids: &[u64], inverted_indexes: &BTreeMap, ) -> Vec { let mut indexes_to_gc = Vec::with_capacity( - blocks_to_gc.len() * (table_agg_index_ids.len() + inverted_indexes.len()), + blocks_to_gc.len() * (table_agg_index_ids.len() + inverted_indexes.len() + 1), ); for loc in blocks_to_gc { for index_id in table_agg_index_ids { @@ -392,6 +344,8 @@ fn collect_block_index_locations( ), ); } + indexes_to_gc + .push(TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(loc)); } indexes_to_gc } @@ -482,12 +436,14 @@ mod tests { "idx", "123456789", ), + TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(&blocks[0]), TableMetaLocationGenerator::gen_agg_index_location_from_block_location(&blocks[1], 7), TableMetaLocationGenerator::gen_inverted_index_location_from_block_location( &blocks[1], "idx", "123456789", ), + TableMetaLocationGenerator::gen_bloom_index_location_from_block_location(&blocks[1]), ]); } } From 07225d85e081b460e052a48bf0737e7499893348 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:51:56 +0800 Subject: [PATCH 26/30] docs(storage): document partial update limitations --- src/query/service/src/physical_plans/physical_mutation.rs | 3 +++ src/query/storages/fuse/src/io/write/block_writer.rs | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 7475908163025..bb470123edcd5 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -115,6 +115,9 @@ fn build_partial_update_info( return Ok(None); } + // Computed columns are not supported by partial updates yet. In particular, stored + // computed columns that depend on an updated field are not added here; supporting them + // requires dependency expansion or a full-rewrite fallback. let schema_with_stream = table.schema_with_stream(); let mut updated_column_ids = update_list .keys() diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 570c30c941cd0..615275a46bd97 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -215,6 +215,9 @@ fn merge_column_group_metadata( bloom: update.bloom, }); + // Partial updates do not support inverted, Ngram, vector, or spatial indexes yet. Their + // metadata is cloned from `origin` without rebuilding derived files for the new block + // anchor, so tables using those indexes must not enter this path. let mut block_meta = origin.clone(); block_meta.location = update.location; block_meta.file_size = column_groups.iter().map(|group| group.file_size).sum(); @@ -538,6 +541,9 @@ impl BlockBuilder { let column_hlls = self.finalize_column_hlls(column_hlls)?; + // Fuse virtual columns are not supported by partial updates yet. Supporting them + // requires rebuilding or invalidating `origin.virtual_block_meta` when source columns + // change. Ok(BlockSerialization { block_raw_data: buffer, block_meta, From 1db5ef36443ad35084b6f19059a6eb4f7b259aef Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:15:51 +0800 Subject: [PATCH 27/30] docs(storage): define partial update support contract --- .../src/physical_plans/physical_mutation.rs | 8 +++++--- .../storages/fuse/src/io/write/block_writer.rs | 15 +++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index bb470123edcd5..2ba49e87a4a21 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -108,6 +108,11 @@ fn build_partial_update_info( database: Option<&str>, table_name: &str, ) -> Result> { + // IMPORTANT REVIEW SCOPE: Partial UPDATE is an opt-in feature defined only for Fuse tables + // without partitioning, computed columns, column-oriented segments, derived table indexes + // (inverted, Ngram, vector, or spatial), or Fuse virtual columns. REPLACE INTO is also outside + // this feature. These exclusions are feature preconditions, intentionally not validated or + // handled by fallback here. Review this path assuming the preconditions hold. if !ctx.get_settings().get_enable_partial_update()? || !table.get_option(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, false) || update_list.is_empty() @@ -115,9 +120,6 @@ fn build_partial_update_info( return Ok(None); } - // Computed columns are not supported by partial updates yet. In particular, stored - // computed columns that depend on an updated field are not added here; supporting them - // requires dependency expansion or a full-rewrite fallback. let schema_with_stream = table.schema_with_stream(); let mut updated_column_ids = update_list .keys() diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 615275a46bd97..9c0ee5875c47a 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -215,9 +215,6 @@ fn merge_column_group_metadata( bloom: update.bloom, }); - // Partial updates do not support inverted, Ngram, vector, or spatial indexes yet. Their - // metadata is cloned from `origin` without rebuilding derived files for the new block - // anchor, so tables using those indexes must not enter this path. let mut block_meta = origin.clone(); block_meta.location = update.location; block_meta.file_size = column_groups.iter().map(|group| group.file_size).sum(); @@ -425,6 +422,15 @@ impl BlockBuilder { /// Serialize only the fields changed by an UPDATE and merge their physical /// metadata back into the original logical block. + /// + /// # Supported feature scope + /// + /// IMPORTANT REVIEW SCOPE: This method is defined only for the Partial UPDATE feature domain: + /// no partitioning, computed columns, column-oriented segments, derived table indexes + /// (inverted, Ngram, vector, or spatial), or Fuse virtual columns. These exclusions are + /// contract preconditions, intentionally not validated or handled by fallback here. Within + /// that contract, this method maintains data, ordinary Bloom, HLL, and column statistics only; + /// handling metadata for excluded features is out of scope. pub fn build_column_group( &self, data_block: DataBlock, @@ -541,9 +547,6 @@ impl BlockBuilder { let column_hlls = self.finalize_column_hlls(column_hlls)?; - // Fuse virtual columns are not supported by partial updates yet. Supporting them - // requires rebuilding or invalidating `origin.virtual_block_meta` when source columns - // change. Ok(BlockSerialization { block_raw_data: buffer, block_meta, From 866fe7c072c537c59146b62492085cba1c23c9eb Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:29:16 +0800 Subject: [PATCH 28/30] docs(storage): clarify partial update preconditions --- .../service/src/physical_plans/physical_mutation.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 2ba49e87a4a21..7304f396841b0 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -111,8 +111,10 @@ fn build_partial_update_info( // IMPORTANT REVIEW SCOPE: Partial UPDATE is an opt-in feature defined only for Fuse tables // without partitioning, computed columns, column-oriented segments, derived table indexes // (inverted, Ngram, vector, or spatial), or Fuse virtual columns. REPLACE INTO is also outside - // this feature. These exclusions are feature preconditions, intentionally not validated or - // handled by fallback here. Review this path assuming the preconditions hold. + // this feature. These exclusions are user-enforced feature preconditions, intentionally not + // validated or handled by fallback here. Unsupported tables therefore remain technically + // reachable if the options are misused, but behavior after violating the preconditions is + // outside the feature contract and is not a review finding for this path. if !ctx.get_settings().get_enable_partial_update()? || !table.get_option(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, false) || update_list.is_empty() @@ -121,6 +123,9 @@ fn build_partial_update_info( } let schema_with_stream = table.schema_with_stream(); + // CODE REVIEW NOTE: Do not add stored or virtual computed columns to this set and do not add a + // fallback for them here. A schema containing any computed column violates the Partial UPDATE + // feature precondition above, so computed-column expansion is deliberately outside this path. let mut updated_column_ids = update_list .keys() .map(|index| schema_with_stream.field(*index).column_id()) From 103ae5b508861ae42d7a937a2bc6706b836128d3 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:44:42 +0800 Subject: [PATCH 29/30] docs(storage): clarify replace into scope --- src/query/storages/fuse/src/io/write/block_writer.rs | 3 ++- .../replace_into/mutator/replace_into_operation_agg.rs | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 9c0ee5875c47a..87d52363b6dac 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -430,7 +430,8 @@ impl BlockBuilder { /// (inverted, Ngram, vector, or spatial), or Fuse virtual columns. These exclusions are /// contract preconditions, intentionally not validated or handled by fallback here. Within /// that contract, this method maintains data, ordinary Bloom, HLL, and column statistics only; - /// handling metadata for excluded features is out of scope. + /// handling metadata for excluded features is out of scope. REPLACE INTO must not consume + /// column-group blocks produced by this method; that operation is outside the same contract. pub fn build_column_group( &self, data_block: DataBlock, diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 831d87b7a9b88..71de0973d22a7 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -637,6 +637,12 @@ impl AggregationContext { } async fn read_block(&self, reader: &BlockReader, block_meta: &BlockMeta) -> Result { + // CODE REVIEW NOTE: REPLACE INTO is deliberately outside the Partial UPDATE feature + // contract. This method is defined only for legacy single-file blocks and intentionally + // does not read `block_meta.column_groups`. A column-group block remains technically + // reachable if unsupported features are combined, but behavior then is outside the + // contract and is not a review finding for this path; do not add group-aware reads or a + // fallback here as part of Partial UPDATE. let merged_io_read_result = reader .read_columns_data_by_merge_io( &self.read_settings, From 7c0f6343adb5f8f7289eed5375ee5fa3f8600207 Mon Sep 17 00:00:00 2001 From: sky <75521613+SkyFan2002@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:55:26 +0800 Subject: [PATCH 30/30] docs(query): add partial update review rules --- src/query/AGENTS.md | 14 ++++++++++++++ .../src/physical_plans/physical_mutation.rs | 17 +++++++---------- .../storages/fuse/src/io/write/block_writer.rs | 14 +++++++------- .../mutator/replace_into_operation_agg.rs | 9 +++------ 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/query/AGENTS.md b/src/query/AGENTS.md index bbea664530e56..8bbdce8c080c5 100644 --- a/src/query/AGENTS.md +++ b/src/query/AGENTS.md @@ -6,3 +6,17 @@ - Read [`sql/README.md`](sql/README.md) for planner structure and optimizer test-support entry points, [`functions/README.md`](functions/README.md) for function registration and tests, and [`ast/README.md`](ast/README.md) for parser-specific workflow. - Do not infer ownership from directory names alone when a nearby README already defines the boundary. - Query-side behavior changes that remain in the branch should usually add coverage close to the affected crate and add SQL regression tests under `tests/suites/` or `tests/sqllogictests/` when user-visible SQL behavior changes. + +## Code Review Rules + +### Fuse Partial UPDATE support boundary + +- Review Fuse Partial UPDATE and column-group code only within its documented supported domain. +- Do not report findings whose reproduction requires combining Partial UPDATE with column-oriented + segments, partitioned tables, stored or virtual computed columns, Inverted/Ngram/Vector/Spatial + indexes, Fuse virtual columns, or `REPLACE INTO`. These combinations are deliberately + unsupported; do not require runtime guards, fallbacks, group-aware reads, or derived-metadata + maintenance unless a change explicitly expands the feature scope. +- Continue to report issues reproducible with supported Partial UPDATE behavior, including ordinary + Bloom, change tracking, HLL, compression, cluster statistics, metadata compatibility, and + GC/Vacuum. diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index 7304f396841b0..ea2387dc86ecf 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -108,13 +108,11 @@ fn build_partial_update_info( database: Option<&str>, table_name: &str, ) -> Result> { - // IMPORTANT REVIEW SCOPE: Partial UPDATE is an opt-in feature defined only for Fuse tables - // without partitioning, computed columns, column-oriented segments, derived table indexes - // (inverted, Ngram, vector, or spatial), or Fuse virtual columns. REPLACE INTO is also outside - // this feature. These exclusions are user-enforced feature preconditions, intentionally not - // validated or handled by fallback here. Unsupported tables therefore remain technically - // reachable if the options are misused, but behavior after violating the preconditions is - // outside the feature contract and is not a review finding for this path. + // Partial UPDATE is an opt-in feature defined only for Fuse tables without partitioning, + // computed columns, column-oriented segments, derived table indexes (inverted, Ngram, vector, + // or spatial), or Fuse virtual columns. REPLACE INTO is also outside this feature. These + // exclusions are user-enforced preconditions, intentionally not validated or handled by + // fallback here; behavior after violating them is outside the feature contract. if !ctx.get_settings().get_enable_partial_update()? || !table.get_option(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, false) || update_list.is_empty() @@ -123,9 +121,8 @@ fn build_partial_update_info( } let schema_with_stream = table.schema_with_stream(); - // CODE REVIEW NOTE: Do not add stored or virtual computed columns to this set and do not add a - // fallback for them here. A schema containing any computed column violates the Partial UPDATE - // feature precondition above, so computed-column expansion is deliberately outside this path. + // Computed columns are deliberately omitted because a schema containing one is outside the + // Partial UPDATE feature contract. let mut updated_column_ids = update_list .keys() .map(|index| schema_with_stream.field(*index).column_id()) diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 87d52363b6dac..03eceb2faa99f 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -425,13 +425,13 @@ impl BlockBuilder { /// /// # Supported feature scope /// - /// IMPORTANT REVIEW SCOPE: This method is defined only for the Partial UPDATE feature domain: - /// no partitioning, computed columns, column-oriented segments, derived table indexes - /// (inverted, Ngram, vector, or spatial), or Fuse virtual columns. These exclusions are - /// contract preconditions, intentionally not validated or handled by fallback here. Within - /// that contract, this method maintains data, ordinary Bloom, HLL, and column statistics only; - /// handling metadata for excluded features is out of scope. REPLACE INTO must not consume - /// column-group blocks produced by this method; that operation is outside the same contract. + /// This method is defined only for the Partial UPDATE feature domain: no partitioning, computed + /// columns, column-oriented segments, derived table indexes (inverted, Ngram, vector, or + /// spatial), or Fuse virtual columns. These exclusions are contract preconditions, + /// intentionally not validated or handled by fallback here. Within that contract, this method + /// maintains data, ordinary Bloom, HLL, and column statistics only; handling metadata for + /// excluded features is out of scope. REPLACE INTO must not consume column-group blocks + /// produced by this method; that operation is outside the same contract. pub fn build_column_group( &self, data_block: DataBlock, diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 71de0973d22a7..b626dfb90279b 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -637,12 +637,9 @@ impl AggregationContext { } async fn read_block(&self, reader: &BlockReader, block_meta: &BlockMeta) -> Result { - // CODE REVIEW NOTE: REPLACE INTO is deliberately outside the Partial UPDATE feature - // contract. This method is defined only for legacy single-file blocks and intentionally - // does not read `block_meta.column_groups`. A column-group block remains technically - // reachable if unsupported features are combined, but behavior then is outside the - // contract and is not a review finding for this path; do not add group-aware reads or a - // fallback here as part of Partial UPDATE. + // REPLACE INTO is outside the Partial UPDATE feature contract. This method is defined only + // for legacy single-file blocks and intentionally does not read `block_meta.column_groups`; + // behavior for column-group blocks is outside its contract. let merged_io_read_result = reader .read_columns_data_by_merge_io( &self.read_settings,