diff --git a/src/query/AGENTS.md b/src/query/AGENTS.md index bbea664530e56..8bbdce8c080c5 100644 --- a/src/query/AGENTS.md +++ b/src/query/AGENTS.md @@ -6,3 +6,17 @@ - Read [`sql/README.md`](sql/README.md) for planner structure and optimizer test-support entry points, [`functions/README.md`](functions/README.md) for function registration and tests, and [`ast/README.md`](ast/README.md) for parser-specific workflow. - Do not infer ownership from directory names alone when a nearby README already defines the boundary. - Query-side behavior changes that remain in the branch should usually add coverage close to the affected crate and add SQL regression tests under `tests/suites/` or `tests/sqllogictests/` when user-visible SQL behavior changes. + +## Code Review Rules + +### Fuse Partial UPDATE support boundary + +- Review Fuse Partial UPDATE and column-group code only within its documented supported domain. +- Do not report findings whose reproduction requires combining Partial UPDATE with column-oriented + segments, partitioned tables, stored or virtual computed columns, Inverted/Ngram/Vector/Spatial + indexes, Fuse virtual columns, or `REPLACE INTO`. These combinations are deliberately + unsupported; do not require runtime guards, fallbacks, group-aware reads, or derived-metadata + maintenance unless a change explicitly expands the feature scope. +- Continue to report issues reproducible with supported Partial UPDATE behavior, including ordinary + Bloom, change tracking, HLL, compression, cluster statistics, metadata compatibility, and + GC/Vacuum. diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index 9c26393cea85f..c2709aa78e4c5 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -135,7 +135,13 @@ pub async fn do_vacuum2( .await?; let mut gc_root_blocks = HashSet::new(); for segment in segments { - gc_root_blocks.extend(segment?.block_metas()?.iter().map(|b| b.location.0.clone())); + for block in segment?.block_metas()? { + gc_root_blocks.extend( + block + .data_file_locations() + .map(|location| location.0.clone()), + ); + } } ctx.set_status_info(&format!( "Read segments for table {}, elapsed: {:?}, total protected blocks: {}", diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs index 971a7feaf9f88..4b508cfbea72c 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs @@ -16,10 +16,13 @@ use std::path::Path; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_enterprise_query::test_kits::context::EESetup; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContextTableAccess; use databend_query::test_kits::TestFixture; +use databend_query::test_kits::latest_default_block_meta; use databend_storages_common_io::dedup_file_locations; // TODO investigate this @@ -118,6 +121,104 @@ async fn test_vacuum2_all() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum2_preserves_active_partial_update_files() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, b int) engine=fuse \ + bloom_index_columns='id,a,b' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, 20), (2, 30, 40)" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + let first_update = latest_default_block_meta(&fixture).await?; + let obsolete_group = first_update.location.0.clone(); + let obsolete_bloom = first_update + .column_groups + .iter() + .find(|group| group.location.0 == obsolete_group) + .and_then(|group| { + group.bloom.as_ref().map(|bloom| { + TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &group.location.0, + bloom.format_version, + ) + }) + }) + .expect("updated group should have an ordinary Bloom file"); + + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set b = b + 1 where id = 1" + )) + .await?; + + let current = latest_default_block_meta(&fixture).await?; + assert!( + current + .column_groups + .iter() + .all(|group| group.location.0 != obsolete_group) + ); + + fixture + .execute_command(&format!("call system$fuse_vacuum2('{db}', '{table_name}')")) + .await?; + + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let operator = fuse_table.get_operator(); + for group in ¤t.column_groups { + operator.stat(&group.location.0).await?; + if let Some(bloom) = &group.bloom { + let bloom_location = TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &group.location.0, + bloom.format_version, + ); + operator.stat(&bloom_location).await?; + } + } + assert!(operator.stat(&obsolete_group).await.is_err()); + assert!(operator.stat(&obsolete_bloom).await.is_err()); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and a = 12 and b = 21) \ + or (id = 2 and a = 30 and b = 40)" + )) + .await?; + assert_eq!(databend_query::test_kits::query_count(rows).await?, 2); + + Ok(()) +} + /// Verifies that dedup_file_locations correctly removes duplicates and reports samples. #[test] fn test_dedup_file_locations() { diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index cf8e0b3386ff5..44e9e8648e880 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -37,6 +37,7 @@ use databend_common_storages_fuse::FUSE_OPT_KEY_DATA_RETENTION_PERIOD_IN_HOURS; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARQUET_DICTIONARY; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN; use databend_common_storages_fuse::FUSE_OPT_KEY_FILE_SIZE; use databend_common_storages_fuse::FUSE_OPT_KEY_RECLUSTER_DEPTH; @@ -90,6 +91,7 @@ pub static CREATE_FUSE_OPTIONS: LazyLock> = LazyLock::new( r.insert(FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP); r.insert(FUSE_OPT_KEY_ENABLE_AUTO_VACUUM); r.insert(FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE); + r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN); r.insert(FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD); @@ -173,6 +175,7 @@ pub static UNSET_TABLE_OPTIONS_WHITE_LIST: LazyLock> = Laz r.insert(FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP); r.insert(FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD); r.insert(FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN); + r.insert(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE); r.insert(OPT_KEY_ENABLE_COPY_DEDUP_FULL_PATH); r.insert(FUSE_OPT_KEY_DATA_PAGE_ROWS); r.insert(FUSE_OPT_KEY_DATA_PAGE_BYTES); diff --git a/src/query/service/src/interpreters/interpreter_mutation.rs b/src/query/service/src/interpreters/interpreter_mutation.rs index 1ac94e0c36abf..03e3430a7f3c4 100644 --- a/src/query/service/src/interpreters/interpreter_mutation.rs +++ b/src/query/service/src/interpreters/interpreter_mutation.rs @@ -246,6 +246,7 @@ pub async fn build_mutation_info( partitions, statistics, table_meta_timestamps, + partial_update: false, }) } diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index d0bb2dbdaef84..6a0d416c20124 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -47,6 +47,7 @@ use databend_common_storages_fuse::FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER; use databend_common_storages_fuse::FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FuseSegmentFormat; use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::io::MetaReaders; @@ -500,6 +501,7 @@ impl CreateTableInterpreter { is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_ENABLE_AUTO_VACUUM)?; // check enable auto analyze. is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE)?; + is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE)?; is_valid_option_of_type::(&table_meta.options, FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER)?; is_valid_option_of_type::( &table_meta.options, diff --git a/src/query/service/src/interpreters/interpreter_table_set_options.rs b/src/query/service/src/interpreters/interpreter_table_set_options.rs index b69db9d66d3bf..04cae8ba6d26b 100644 --- a/src/query/service/src/interpreters/interpreter_table_set_options.rs +++ b/src/query/service/src/interpreters/interpreter_table_set_options.rs @@ -30,6 +30,7 @@ use databend_common_storages_fuse::FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER; use databend_common_storages_fuse::FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE; use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_AUTO_VACUUM; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FuseSegmentFormat; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::SegmentsIO; @@ -167,6 +168,10 @@ impl Interpreter for SetOptionsInterpreter { // Same as settings of FUSE_OPT_KEY_ENABLE_AUTO_VACUUM, expect value type is unsigned integer is_valid_option_of_type::(&self.plan.set_options, FUSE_OPT_KEY_ENABLE_AUTO_VACUUM)?; is_valid_option_of_type::(&self.plan.set_options, FUSE_OPT_KEY_AGGRESSIVE_RECLUSTER)?; + is_valid_option_of_type::( + &self.plan.set_options, + FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, + )?; is_valid_option_of_type::( &self.plan.set_options, FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD, diff --git a/src/query/service/src/physical_plans/physical_column_mutation.rs b/src/query/service/src/physical_plans/physical_column_mutation.rs index af4fe752ec01f..82abb25dea269 100644 --- a/src/query/service/src/physical_plans/physical_column_mutation.rs +++ b/src/query/service/src/physical_plans/physical_column_mutation.rs @@ -20,6 +20,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; +use databend_common_expression::FieldIndex; use databend_common_expression::RemoteExpr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableInfo; @@ -29,6 +30,7 @@ use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::TransformSerializeBlock; +use databend_common_storages_fuse::statistics::ClusterStatsGenerator; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ColumnMutationFormatter; @@ -51,6 +53,8 @@ pub struct ColumnMutation { pub has_filter_column: bool, pub table_meta_timestamps: TableMetaTimestamps, pub udf_col_num: usize, + /// Field indices written by a partial update. + pub partial_update_fields: Option>, } #[typetag::serde] @@ -98,6 +102,7 @@ impl IPhysicalPlan for ColumnMutation { has_filter_column: self.has_filter_column, table_meta_timestamps: self.table_meta_timestamps, udf_col_num: self.udf_col_num, + partial_update_fields: self.partial_update_fields.clone(), }) } @@ -166,12 +171,26 @@ impl IPhysicalPlan for ColumnMutation { // Keep only table fields in their schema order. Mutation input may // carry derived UDF argument/result columns that must not be // serialized back into the table. - let mut projection = field_id_to_schema_index.iter().collect::>(); - projection.sort_by_key(|(field_id, _)| *field_id); - let projection = projection - .into_iter() - .map(|(_, schema_index)| *schema_index) - .collect::>(); + let projection = if let Some(updated_fields) = &self.partial_update_fields { + updated_fields + .iter() + .map(|field_id| { + field_id_to_schema_index + .get(field_id) + .copied() + .ok_or_else(|| { + ErrorCode::Internal("updated field is not in mutation output") + }) + }) + .collect::>>()? + } else { + let mut projection = field_id_to_schema_index.iter().collect::>(); + projection.sort_by_key(|(field_id, _)| *field_id); + projection + .into_iter() + .map(|(_, schema_index)| *schema_index) + .collect::>() + }; block_operators.push(BlockOperator::Project { projection }); builder.main_pipeline.add_transformer(|| { @@ -188,8 +207,12 @@ impl IPhysicalPlan for ColumnMutation { .build_table_by_table_info(&self.table_info, None)?; let table = FuseTable::try_from_table(table.as_ref())?; + let write_column_group = self.partial_update_fields.is_some(); + let block_thresholds = table.get_block_thresholds(); - let cluster_stats_gen = if matches!(self.mutation_kind, MutationKind::Delete) { + let cluster_stats_gen = if write_column_group { + ClusterStatsGenerator::default() + } else if matches!(self.mutation_kind, MutationKind::Delete) { let input_schema = DataSchema::from(table.schema_with_stream()).into(); table.get_cluster_stats_gen(builder.ctx.clone(), 0, block_thresholds, input_schema)? } else { @@ -202,15 +225,27 @@ impl IPhysicalPlan for ColumnMutation { }; builder.main_pipeline.add_transform(|input, output| { - let proc = TransformSerializeBlock::try_create( - builder.ctx.clone(), - input, - output, - table, - cluster_stats_gen.clone(), - self.mutation_kind, - self.table_meta_timestamps, - )?; + let proc = if write_column_group { + TransformSerializeBlock::try_create_for_update( + builder.ctx.clone(), + input, + output, + table, + cluster_stats_gen.clone(), + self.partial_update_fields.clone().unwrap(), + self.table_meta_timestamps, + )? + } else { + TransformSerializeBlock::try_create( + builder.ctx.clone(), + input, + output, + table, + cluster_stats_gen.clone(), + self.mutation_kind, + self.table_meta_timestamps, + )? + }; proc.into_processor() }) } diff --git a/src/query/service/src/physical_plans/physical_commit_sink.rs b/src/query/service/src/physical_plans/physical_commit_sink.rs index fde0e57a43bf8..fc688ce3d3ad3 100644 --- a/src/query/service/src/physical_plans/physical_commit_sink.rs +++ b/src/query/service/src/physical_plans/physical_commit_sink.rs @@ -156,7 +156,7 @@ impl IPhysicalPlan for CommitSink { } else { builder .main_pipeline - .add_async_accumulating_transformer(|| { + .try_add_async_accumulating_transformer(|| { let base_segments = if matches!( kind, MutationKind::Compact @@ -194,7 +194,7 @@ impl IPhysicalPlan for CommitSink { *kind, self.table_meta_timestamps, ) - }); + })?; } let snapshot_gen = MutationGenerator::new(self.snapshot.clone(), *kind); diff --git a/src/query/service/src/physical_plans/physical_compact_source.rs b/src/query/service/src/physical_plans/physical_compact_source.rs index 53ffd233f46cd..9e36ec38a37da 100644 --- a/src/query/service/src/physical_plans/physical_compact_source.rs +++ b/src/query/service/src/physical_plans/physical_compact_source.rs @@ -206,7 +206,7 @@ impl IPhysicalPlan for CompactSource { builder.main_pipeline.try_resize(1)?; builder .main_pipeline - .add_async_accumulating_transformer(|| { + .try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( table, builder.ctx.clone(), @@ -217,7 +217,7 @@ impl IPhysicalPlan for CompactSource { MutationKind::Compact, self.table_meta_timestamps, ) - }); + })?; } Ok(()) } diff --git a/src/query/service/src/physical_plans/physical_mutation.rs b/src/query/service/src/physical_plans/physical_mutation.rs index e37402097604b..ea2387dc86ecf 100644 --- a/src/query/service/src/physical_plans/physical_mutation.rs +++ b/src/query/service/src/physical_plans/physical_mutation.rs @@ -57,11 +57,13 @@ use databend_common_sql::binder::wrap_cast; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::parse_cluster_keys; use databend_common_sql::parse_computed_field_index_expr; use databend_common_sql::plans::BoundColumnRef; use databend_common_sql::plans::ConstantExpr; use databend_common_sql::plans::FunctionCall; use databend_common_sql::plans::TruncateMode; +use databend_common_storages_fuse::FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::operations::TransformSerializeBlock; use databend_storages_common_table_meta::meta::Location; @@ -91,6 +93,105 @@ use crate::sessions::TableContext; // The predicate column symbol should not conflict with update expr column bindings. pub const PREDICATE_COLUMN_INDEX: Symbol = Symbol::DUMMY_COLUMN; +struct PartialUpdateInfo { + required_columns: ColumnSet, + updated_field_indices: Vec, +} + +#[allow(clippy::too_many_arguments)] +fn build_partial_update_info( + ctx: Arc, + table: &FuseTable, + bind_context: &BindContext, + update_list: &HashMap, + direct_filter: &[ScalarExpr], + database: Option<&str>, + table_name: &str, +) -> Result> { + // Partial UPDATE is an opt-in feature defined only for Fuse tables without partitioning, + // computed columns, column-oriented segments, derived table indexes (inverted, Ngram, vector, + // or spatial), or Fuse virtual columns. REPLACE INTO is also outside this feature. These + // exclusions are user-enforced preconditions, intentionally not validated or handled by + // fallback here; behavior after violating them is outside the feature contract. + if !ctx.get_settings().get_enable_partial_update()? + || !table.get_option(FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE, false) + || update_list.is_empty() + { + return Ok(None); + } + + let schema_with_stream = table.schema_with_stream(); + // Computed columns are deliberately omitted because a schema containing one is outside the + // Partial UPDATE feature contract. + let mut updated_column_ids = update_list + .keys() + .map(|index| schema_with_stream.field(*index).column_id()) + .collect::>(); + + for stream_column in table.stream_columns() { + updated_column_ids.insert(stream_column.column_id()); + } + + if let Some(cluster_keys) = table.resolve_cluster_keys() { + let cluster_exprs = parse_cluster_keys(ctx.clone(), Arc::new(table.clone()), cluster_keys)?; + let updates_cluster_key = cluster_exprs + .iter() + .flat_map(|expr| expr.column_refs()) + .any(|(index, _)| { + updated_column_ids.contains(&schema_with_stream.field(index).column_id()) + }); + if updates_cluster_key { + return Ok(None); + } + } + + let updated_field_indices = schema_with_stream + .fields() + .iter() + .enumerate() + .filter_map(|(index, field)| { + updated_column_ids + .contains(&field.column_id()) + .then_some(index) + }) + .collect::>(); + if updated_field_indices.is_empty() + || updated_field_indices.len() >= schema_with_stream.fields().len() + { + return Ok(None); + } + + let find_symbol = |field_index: FieldIndex| { + let field = schema_with_stream.field(field_index); + bind_context + .columns + .iter() + .find(|binding| { + BindContext::match_column_binding(database, Some(table_name), field.name(), binding) + }) + .map(|binding| binding.index) + }; + + let mut required_columns = update_list + .values() + .flat_map(ScalarExpr::used_columns) + .chain(direct_filter.iter().flat_map(ScalarExpr::used_columns)) + .collect::(); + for (field_index, field) in schema_with_stream.fields().iter().enumerate() { + if updated_column_ids.contains(&field.column_id()) { + let Some(symbol) = find_symbol(field_index) else { + return Ok(None); + }; + required_columns.insert(symbol); + } + } + + Ok(Some(PartialUpdateInfo { + required_columns, + updated_field_indices, + })) +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Mutation { pub meta: PhysicalPlanMeta, @@ -269,6 +370,41 @@ impl PhysicalPlanBuilder { .. } = mutation; + let table = self + .ctx + .get_table(catalog_name, database_name, table_name) + .await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let partial_table_name = table_name_alias + .as_ref() + .map(|name| name.to_lowercase()) + .unwrap_or_else(|| table_name.clone()); + let partial_database = table_name_alias.is_none().then_some(database_name.as_str()); + let partial_update = if *strategy == MutationStrategy::Direct { + matched_evaluators + .first() + .and_then(|evaluator| evaluator.update.as_ref()) + .map(|update_list| { + build_partial_update_info( + self.ctx.clone(), + fuse_table, + bind_context, + update_list, + direct_filter, + partial_database, + &partial_table_name, + ) + }) + .transpose()? + .flatten() + } else { + None + }; + if let Some(info) = &partial_update { + required = info.required_columns.clone(); + } + self.mutation_build_info.as_mut().unwrap().partial_update = partial_update.is_some(); + let mut maybe_udfs = BTreeSet::new(); for matched_evaluator in matched_evaluators { if let Some(condition) = &matched_evaluator.condition { @@ -308,10 +444,6 @@ impl PhysicalPlanBuilder { return Ok(plan); } - let table = self - .ctx - .get_table(catalog_name, database_name, table_name) - .await?; let table_info = table.get_table_info(); let table_name = table_name.clone(); @@ -399,6 +531,7 @@ impl PhysicalPlanBuilder { has_filter_column: predicate_column_index.is_some(), table_meta_timestamps: mutation_build_info.table_meta_timestamps, udf_col_num, + partial_update_fields: partial_update.map(|info| info.updated_field_indices), }); if *distributed { @@ -1069,10 +1202,10 @@ fn build_field_id_to_schema_index( column_binding, ) { let column_index = column_binding.index; - let schema_index = mutation_input_schema - .index_of(&column_index.to_string()) - .unwrap(); - field_id_to_schema_index.insert(field_id, schema_index); + if let Ok(schema_index) = mutation_input_schema.index_of(&column_index.to_string()) + { + field_id_to_schema_index.insert(field_id, schema_index); + } break; } } diff --git a/src/query/service/src/physical_plans/physical_mutation_source.rs b/src/query/service/src/physical_plans/physical_mutation_source.rs index d0a6466ab78f5..0a772e2c2b8cb 100644 --- a/src/query/service/src/physical_plans/physical_mutation_source.rs +++ b/src/query/service/src/physical_plans/physical_mutation_source.rs @@ -28,6 +28,7 @@ use databend_common_expression::DataBlock; use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::FieldIndex; use databend_common_expression::FunctionContext; use databend_common_expression::type_check::check_function; use databend_common_expression::types::DataType; @@ -73,6 +74,10 @@ pub struct MutationSource { pub output_schema: DataSchemaRef, pub input_type: MutationType, pub read_partition_columns: ColumnSet, + /// Table schema projection used by partial updates. Metadata Symbols remain in + /// `read_partition_columns` for expression and output naming. + #[serde(default)] + pub partial_update_projection: Option>, pub truncate_table: bool, pub partitions: Partitions, @@ -117,6 +122,7 @@ impl IPhysicalPlan for MutationSource { output_schema: self.output_schema.clone(), input_type: self.input_type.clone(), read_partition_columns: self.read_partition_columns.clone(), + partial_update_projection: self.partial_update_projection.clone(), truncate_table: self.truncate_table, partitions: self.partitions.clone(), statistics: self.statistics.clone(), @@ -154,11 +160,18 @@ impl IPhysicalPlan for MutationSource { ); } - let read_partition_columns: Vec = self - .read_partition_columns - .iter() - .map(|idx| idx.as_field_index()) - .collect(); + let partial_update = self.partial_update_projection.is_some(); + let read_partition_field_indices: Vec = + self.partial_update_projection.clone().unwrap_or_else(|| { + self.read_partition_columns + .iter() + .map(|idx| idx.as_field_index()) + .collect() + }); + let stream_projection = self + .partial_update_projection + .as_deref() + .unwrap_or(&read_partition_field_indices); let is_lazy = self.partitions.partitions_type() == PartInfoType::LazyLevel && is_delete; if is_lazy { @@ -166,7 +179,7 @@ impl IPhysicalPlan for MutationSource { let table_clone = table.clone(); let ctx_clone = builder.ctx.clone(); let filters_clone = self.filters.clone(); - let projection = Projection::Columns(read_partition_columns.clone()); + let projection = Projection::Columns(read_partition_field_indices.clone()); let mut segment_locations = Vec::with_capacity(self.partitions.partitions.len()); for part in &self.partitions.partitions { // Safe to downcast because we know the partition is lazy @@ -205,29 +218,36 @@ impl IPhysicalPlan for MutationSource { } else { MutationAction::Update }; - let col_indices = self - .read_partition_columns - .iter() - .map(|idx| idx.as_field_index()) - .collect(); let update_mutation_with_filter = self.input_type == MutationType::Update && filter.is_some(); table.add_mutation_source( builder.ctx.clone(), filter, - col_indices, + read_partition_field_indices.clone(), &mut builder.main_pipeline, mutation_action, + partial_update, )?; if table.change_tracking_enabled() { - let stream_ctx = StreamContext::try_create( - builder.ctx.get_function_context()?, - table.schema_with_stream(), - table.get_table_info().ident.seq, - is_delete, - update_mutation_with_filter, - )?; + let stream_ctx = if partial_update { + StreamContext::try_create_projected( + builder.ctx.get_function_context()?, + table.schema_with_stream(), + stream_projection, + table.get_table_info().ident.seq, + is_delete, + update_mutation_with_filter, + )? + } else { + StreamContext::try_create( + builder.ctx.get_function_context()?, + table.schema_with_stream(), + table.get_table_info().ident.seq, + is_delete, + update_mutation_with_filter, + )? + }; builder .main_pipeline .add_transformer(|| TransformAddStreamColumns::new(stream_ctx.clone())); @@ -241,6 +261,7 @@ impl PhysicalPlanBuilder { pub async fn build_mutation_source( &mut self, mutation_source: &databend_common_sql::plans::MutationSource, + required: ColumnSet, ) -> Result { let all_predicates: Vec = mutation_source .secure_predicates @@ -266,9 +287,21 @@ impl PhysicalPlanBuilder { }; let mutation_info = self.mutation_build_info.as_ref().unwrap(); + let partial_update = self + .mutation_build_info + .as_ref() + .is_some_and(|info| info.partial_update); let metadata = self.metadata.read(); let mut fields = Vec::with_capacity(mutation_source.columns.len()); for column_index in mutation_source.columns.iter() { + if partial_update + && !required.contains(column_index) + && !mutation_source + .read_partition_columns + .contains(column_index) + { + continue; + } let column = metadata.column(*column_index); // Ignore virtual computed columns. if let Ok(column_id) = mutation_source.schema.index_of(&column.name()) { @@ -277,6 +310,19 @@ impl PhysicalPlanBuilder { } fields.sort_by_key(|(_, _, id)| *id); + let partial_update_projection = partial_update.then(|| { + fields + .iter() + .map(|(_, _, field_index)| *field_index) + .collect::>() + }); + + let read_partition_columns = if partial_update { + fields.iter().map(|(_, index, _)| *index).collect() + } else { + mutation_source.read_partition_columns.clone() + }; + let mut fields = fields .into_iter() .map(|(name, index, _)| { @@ -304,7 +350,8 @@ impl PhysicalPlanBuilder { display_filters, has_hidden_secure_filters: !mutation_source.secure_predicates.is_empty(), input_type: mutation_source.mutation_type.clone(), - read_partition_columns: mutation_source.read_partition_columns.clone(), + read_partition_columns, + partial_update_projection, truncate_table, meta: PhysicalPlanMeta::new("MutationSource"), partitions: mutation_info.partitions.clone(), diff --git a/src/query/service/src/physical_plans/physical_plan_builder.rs b/src/query/service/src/physical_plans/physical_plan_builder.rs index 8616c62826182..5e189e1598d50 100644 --- a/src/query/service/src/physical_plans/physical_plan_builder.rs +++ b/src/query/service/src/physical_plans/physical_plan_builder.rs @@ -169,7 +169,7 @@ impl PhysicalPlanBuilder { self.build_mutation(s_expr, mutation, required).await } RelOperator::MutationSource(mutation_source) => { - self.build_mutation_source(mutation_source).await + self.build_mutation_source(mutation_source, required).await } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, RelOperator::MaterializedCTE(materialized_cte) => { @@ -471,4 +471,5 @@ pub struct MutationBuildInfo { pub partitions: Partitions, pub statistics: PartStatistics, pub table_meta_timestamps: TableMetaTimestamps, + pub partial_update: bool, } diff --git a/src/query/service/src/pipelines/builders/transform_builder.rs b/src/query/service/src/pipelines/builders/transform_builder.rs index fabc5cbeacc6b..ec52afbf081b7 100644 --- a/src/query/service/src/pipelines/builders/transform_builder.rs +++ b/src/query/service/src/pipelines/builders/transform_builder.rs @@ -172,7 +172,7 @@ impl PipelineBuilder { Statistics::default(), MutationKind::Insert, table_meta_timestamps, - ); + )?; Ok(ProcessorPtr::create(AsyncAccumulatingTransformer::create( input, output, aggregator, ))) diff --git a/src/query/service/src/test_kits/fuse.rs b/src/query/service/src/test_kits/fuse.rs index 224ea991e61d9..98b2bf77a4e60 100644 --- a/src/query/service/src/test_kits/fuse.rs +++ b/src/query/service/src/test_kits/fuse.rs @@ -19,6 +19,7 @@ use std::vec; use chrono::DateTime; use chrono::Duration; use chrono::Utc; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockThresholds; use databend_common_expression::DataBlock; @@ -32,14 +33,17 @@ use databend_common_storages_factory::Table; use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; use databend_common_storages_fuse::FuseStorageFormat; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; use databend_common_storages_fuse::io::MetaWriter; use databend_common_storages_fuse::io::TableMetaLocationGenerator; use databend_common_storages_fuse::statistics::gen_columns_statistics; use databend_common_storages_fuse::statistics::merge_statistics; use databend_common_storages_fuse::statistics::reducers::reduce_block_metas; +use databend_storages_common_cache::LoadParams; use databend_storages_common_cache::SegmentStatistics; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::CompactSegmentInfo; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::Statistics; @@ -62,6 +66,39 @@ use crate::interpreters::MutationInterpreter; use crate::sessions::QueryContext; /// This file contains some helper functions for testing fuse table. +pub async fn latest_default_segment(fixture: &TestFixture) -> Result> { + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let snapshot = fuse_table + .read_table_snapshot() + .await? + .ok_or_else(|| ErrorCode::Internal("default test table has no snapshot"))?; + let (segment_location, segment_version) = snapshot + .segments + .first() + .ok_or_else(|| ErrorCode::Internal("default test table has no segment"))?; + MetaReaders::segment_info_reader(fuse_table.get_operator(), table.schema()) + .read(&LoadParams { + location: segment_location.clone(), + len_hint: None, + ver: *segment_version, + put_cache: false, + }) + .await +} + +pub async fn latest_default_block_meta(fixture: &TestFixture) -> Result> { + let segment = latest_default_segment(fixture).await?; + let blocks = segment.block_metas()?; + if blocks.len() != 1 { + return Err(ErrorCode::Internal(format!( + "expected one block in default test table, got {}", + blocks.len() + ))); + } + Ok(blocks[0].clone()) +} + pub async fn generate_snapshot_with_segments( fuse_table: &FuseTable, segment_locations: Vec, diff --git a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs index eff08a15cba53..4bfc02314476c 100644 --- a/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs +++ b/src/query/service/tests/it/storages/fuse/bloom_index_meta_size.rs @@ -330,6 +330,7 @@ fn build_test_segment_info( file_size: 0, col_stats: col_stats.clone(), col_metas, + column_groups: vec![], cluster_stats: None, location: block_location, bloom_filter_index_location: Some(location_gen.block_bloom_index_location(&block_uuid)), diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs index cdeb474f52580..9160af3163a83 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs @@ -16,6 +16,7 @@ mod block_compact_mutator; mod deletion; mod recluster_mutator; mod segments_compact_mutator; +mod update; pub use segments_compact_mutator::CompactSegmentTestFixture; pub use segments_compact_mutator::compact_segment; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs new file mode 100644 index 0000000000000..eaf2af81b319a --- /dev/null +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/update.rs @@ -0,0 +1,288 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::MetaReaders; +use databend_common_storages_fuse::io::TableMetaLocationGenerator; +use databend_query::test_kits::*; +use databend_storages_common_cache::LoadParams; +use databend_storages_common_table_meta::meta::BlockHLL; +use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::decode_column_hll; + +async fn latest_blocks_with_hll( + fixture: &TestFixture, +) -> anyhow::Result, BlockHLL)>> { + let table = fixture.latest_default_table().await?; + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + let segment = latest_default_segment(fixture).await?; + let blocks = segment.block_metas()?; + let (stats_location, stats_version) = segment.summary.additional_stats_loc().unwrap(); + let stats = MetaReaders::segment_stats_reader(fuse_table.get_operator()) + .read(&LoadParams { + location: stats_location, + len_hint: None, + ver: stats_version, + put_cache: false, + }) + .await?; + let hlls = stats + .block_hlls + .iter() + .map(|hll| Ok(decode_column_hll(hll)?.unwrap())) + .collect::>>()?; + anyhow::ensure!( + blocks.len() == hlls.len(), + "block/HLL count mismatch: {} blocks, {} HLLs", + blocks.len(), + hlls.len() + ); + Ok(blocks.into_iter().zip(hlls).collect()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_metadata_bloom_and_hll() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, value int) engine=fuse \ + row_per_block=2 block_per_segment=1000 \ + bloom_index_columns='id,value' approx_distinct_columns='id,value' \ + enable_partial_update=true change_tracking=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10), (3, 30), (2, 20), (4, 40)" + )) + .await?; + + let table = fixture.latest_default_table().await?; + let schema = table.schema(); + let operator = FuseTable::try_from_table(table.as_ref())?.get_operator(); + let id_column_id = schema.field(0).column_id(); + let value_column_id = schema.field(1).column_id(); + let origins = latest_blocks_with_hll(&fixture).await?; + assert_eq!(origins.len(), 2); + + fixture + .execute_command("set enable_partial_update = 1") + .await?; + + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 1" + )) + .await?; + + let updated_blocks = latest_blocks_with_hll(&fixture).await?; + let updated_index = updated_blocks + .iter() + .position(|(block, _)| !block.column_groups.is_empty()) + .unwrap(); + let (origin, origin_hll) = &origins[updated_index]; + let (updated, updated_hll) = &updated_blocks[updated_index]; + + assert_eq!(updated.column_groups.len(), 2); + let unchanged_group = updated + .column_groups + .iter() + .find(|group| group.location == origin.location) + .unwrap(); + assert_eq!(unchanged_group.active_column_ids, vec![id_column_id]); + let changed_group = updated + .column_groups + .iter() + .find(|group| group.location == updated.location) + .unwrap(); + assert!(changed_group.active_column_ids.contains(&value_column_id)); + assert_eq!(changed_group.active_column_ids.len(), 4); + assert_eq!( + updated.col_stats.get(&id_column_id), + origin.col_stats.get(&id_column_id) + ); + assert!(updated.bloom_filter_index_location.is_none()); + assert_eq!( + updated + .column_groups + .iter() + .filter(|group| group.bloom.is_some()) + .count(), + 2 + ); + assert_eq!( + updated.bloom_filter_index_size, + updated + .column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum::() + ); + + // Paired Bloom failures must keep the block without invoking the legacy auto-fix path. + let changed_bloom = changed_group.bloom.as_ref().unwrap(); + let missing_bloom_location = TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &changed_group.location.0, + changed_bloom.format_version, + ); + operator.delete(&missing_bloom_location).await?; + fixture + .execute_command("set enable_auto_fix_missing_bloom_index = 1") + .await?; + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11) or (id = 2 and value = 20)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + assert!(!operator.exists(&missing_bloom_location).await?); + + assert_eq!( + updated_hll.get(&id_column_id), + origin_hll.get(&id_column_id) + ); + assert_ne!( + updated_hll.get(&value_column_id), + origin_hll.get(&value_column_id) + ); + + fixture + .execute_command(&format!( + "alter table {db}.{table_name} set options(\ + bloom_index_columns='id', approx_distinct_columns='id')" + )) + .await?; + + fixture + .execute_command(&format!( + "update {db}.{table_name} set value = value + 1 where id = 2" + )) + .await?; + let updated_again_blocks = latest_blocks_with_hll(&fixture).await?; + let updated_again_index = 1 - updated_index; + let (updated_again_origin, _) = &origins[updated_again_index]; + let (updated_again, _) = &updated_again_blocks[updated_again_index]; + assert_eq!(updated_again.column_groups.len(), 2); + assert!( + updated_again + .column_groups + .iter() + .any(|group| group.location == updated_again_origin.location) + ); + assert!( + updated_again + .column_groups + .iter() + .all(|group| group.location != updated.location) + ); + assert!(updated_again.bloom_filter_index_location.is_none()); + assert_eq!( + updated_again + .column_groups + .iter() + .filter(|group| group.bloom.is_some()) + .count(), + 1 + ); + + assert_eq!(updated_again_blocks.len(), updated_blocks.len()); + for ((_, updated_again_hll), (_, updated_hll)) in + updated_again_blocks.iter().zip(&updated_blocks) + { + assert_eq!( + updated_again_hll.get(&id_column_id), + updated_hll.get(&id_column_id) + ); + assert!(!updated_again_hll.contains_key(&value_column_id)); + } + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} where value = 21" + )) + .await?; + assert_eq!(query_count(rows).await?, 1); + + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and value = 11) or (id = 2 and value = 21)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_partial_update_preserves_block_compression() -> anyhow::Result<()> { + let fixture = TestFixture::setup().await?; + let db = fixture.default_db_name(); + let table_name = fixture.default_table_name(); + + fixture.create_default_database().await?; + fixture + .execute_command(&format!( + "create table {db}.{table_name} (id int, a int, b int) engine=fuse \ + compression='zstd' enable_partial_update=true" + )) + .await?; + fixture + .execute_command(&format!( + "insert into {db}.{table_name} values (1, 10, 20), (2, 30, 40)" + )) + .await?; + let origin_compression = latest_default_block_meta(&fixture).await?.compression; + + fixture + .execute_command(&format!( + "alter table {db}.{table_name} set options(compression='snappy')" + )) + .await?; + fixture + .execute_command("set enable_partial_update = 1") + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set a = a + 1 where id = 1" + )) + .await?; + fixture + .execute_command(&format!( + "update {db}.{table_name} set b = b + 1 where id = 2" + )) + .await?; + + let updated = latest_default_block_meta(&fixture).await?; + assert_eq!(updated.compression, origin_compression); + assert_eq!(updated.column_groups.len(), 3); + let rows = fixture + .execute_query(&format!( + "select count(*) from {db}.{table_name} \ + where (id = 1 and a = 11 and b = 20) \ + or (id = 2 and a = 30 and b = 41)" + )) + .await?; + assert_eq!(query_count(rows).await?, 2); + + Ok(()) +} diff --git a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs index 0c089ee730095..77a8e88d6b532 100644 --- a/src/query/service/tests/it/storages/fuse/operations/prewhere.rs +++ b/src/query/service/tests/it/storages/fuse/operations/prewhere.rs @@ -44,6 +44,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_storages_fuse::FuseBlockPartInfo; +use databend_common_storages_fuse::FuseColumnGroupPartInfo; use databend_common_storages_fuse::io::BlockReader; use databend_common_storages_fuse::io::DataItem; use databend_common_storages_fuse::io::WriteSettings; @@ -330,9 +331,13 @@ async fn prepare_prewhere_data() -> Result { location: "test_block".to_string(), bloom_filter_index_location: None, bloom_filter_index_size: 0, + column_group_bloom_files: vec![], create_on: None, nums_rows: num_rows, - columns_meta: column_metas.clone(), + column_groups: vec![FuseColumnGroupPartInfo { + location: "test_block".to_string(), + columns_meta: column_metas.clone(), + }], columns_stat: None, compression, sort_min_max: None, diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 073d3722369ff..4bfc7cada4fdc 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -1591,6 +1591,13 @@ impl DefaultSettings { scope: SettingScope::Both, range: Some(SettingRange::Numeric(0..=1)), }), + ("enable_partial_update", DefaultSettingValue { + value: UserSettingValue::UInt64(0), + desc: "Enables eligible direct UPDATE statements to write column groups.", + mode: SettingMode::Both, + scope: SettingScope::Both, + range: Some(SettingRange::Numeric(0..=1)), + }), ("enable_auto_vacuum", DefaultSettingValue { value: UserSettingValue::UInt64(0), desc: "Whether to automatically trigger VACUUM operations on tables (using vacuum2)", diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index f10ffe80658c6..8a3b2f203e47d 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -1112,6 +1112,10 @@ impl Settings { Ok(self.try_get_u64("error_on_nondeterministic_update")? == 1) } + pub fn get_enable_partial_update(&self) -> Result { + Ok(self.try_get_u64("enable_partial_update")? == 1) + } + pub fn get_max_query_memory_usage(&self) -> Result { self.try_get_u64("max_query_memory_usage") } diff --git a/src/query/sql/src/planner/execution/stream_column.rs b/src/query/sql/src/planner/execution/stream_column.rs index e1b1f3f3c3484..d457b655b500b 100644 --- a/src/query/sql/src/planner/execution/stream_column.rs +++ b/src/query/sql/src/planner/execution/stream_column.rs @@ -52,9 +52,46 @@ impl StreamContext { table_version: u64, is_delete: bool, update_mutation_with_filter: bool, + ) -> Result { + Self::try_create_inner( + func_ctx, + schema, + None, + table_version, + is_delete, + update_mutation_with_filter, + ) + } + + pub fn try_create_projected( + func_ctx: FunctionContext, + schema: Arc, + projection: &[usize], + table_version: u64, + is_delete: bool, + update_mutation_with_filter: bool, + ) -> Result { + Self::try_create_inner( + func_ctx, + schema, + Some(projection), + table_version, + is_delete, + update_mutation_with_filter, + ) + } + + fn try_create_inner( + func_ctx: FunctionContext, + schema: Arc, + projection: Option<&[usize]>, + table_version: u64, + is_delete: bool, + update_mutation_with_filter: bool, ) -> Result { let input_schema = schema.remove_virtual_computed_fields(); - let num_fields = input_schema.fields().len() + update_mutation_with_filter as usize; + let projected_len = projection.map_or(input_schema.fields().len(), <[usize]>::len); + let num_fields = projected_len + update_mutation_with_filter as usize; let stream_columns = [ StreamColumn::new(ORIGIN_VERSION_COL_NAME, StreamColumnType::OriginVersion), @@ -69,12 +106,20 @@ impl StreamContext { let mut exprs = Vec::with_capacity(stream_columns.len()); for stream_column in stream_columns.iter() { let schema_index = input_schema.index_of(stream_column.column_name()).unwrap(); + let input_index = projection + .map(|projection| { + projection + .iter() + .position(|index| *index == schema_index) + .expect("partial update must read stream columns") + }) + .unwrap_or(schema_index); let origin_stream_column_scalar_expr = ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column: ColumnBindingBuilder::new( stream_column.column_name().to_string(), - Symbol::from_field_index(schema_index), + Symbol::from_field_index(input_index), Box::new(stream_column.data_type()), Visibility::Visible, ) @@ -83,14 +128,14 @@ impl StreamContext { let current_stream_column_scalar_expr = match stream_column.column_type() { StreamColumnType::OriginVersion => { - new_schema_index.insert(schema_index, num_fields + 2); + new_schema_index.insert(input_index, num_fields + 2); ScalarExpr::ConstantExpr(ConstantExpr { span: None, value: table_version.into(), }) } StreamColumnType::OriginBlockId => { - new_schema_index.insert(schema_index, num_fields + 3); + new_schema_index.insert(input_index, num_fields + 3); ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column: ColumnBindingBuilder::new( @@ -103,7 +148,7 @@ impl StreamContext { }) } StreamColumnType::OriginRowNum => { - new_schema_index.insert(schema_index, num_fields + 4); + new_schema_index.insert(input_index, num_fields + 4); ScalarExpr::BoundColumnRef(BoundColumnRef { span: None, column: ColumnBindingBuilder::new( diff --git a/src/query/storages/common/cache/src/manager.rs b/src/query/storages/common/cache/src/manager.rs index 39914f449644c..b9244c16bc48d 100644 --- a/src/query/storages/common/cache/src/manager.rs +++ b/src/query/storages/common/cache/src/manager.rs @@ -1297,6 +1297,7 @@ mod tests { file_size: 0, col_stats: Default::default(), col_metas: Default::default(), + column_groups: vec![], cluster_stats: None, location: ("".to_string(), 0), bloom_filter_index_location: None, diff --git a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs index b8e3d94d4f76c..fba09071c244a 100644 --- a/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/column_oriented_segment/block_meta.rs @@ -29,6 +29,12 @@ pub trait AbstractBlockMeta: Send + Sync + 'static + Sized { fn row_count(&self) -> u64; fn location_path(&self) -> String; fn col_metas(&self, col_ids: &HashSet) -> HashMap; + fn col_metas_by_location( + &self, + col_ids: &HashSet, + ) -> Vec<(String, HashMap)> { + vec![(self.location_path(), self.col_metas(col_ids))] + } fn virtual_block_meta(&self) -> Option; } @@ -54,6 +60,16 @@ impl AbstractBlockMeta for BlockMeta { col_metas } + fn col_metas_by_location( + &self, + col_ids: &HashSet, + ) -> Vec<(String, HashMap)> { + BlockMeta::project_column_groups(self, col_ids) + .into_iter() + .map(|group| (group.location.0, group.leaf_column_metas)) + .collect() + } + fn location_path(&self) -> String { self.location.0.to_string() } diff --git a/src/query/storages/common/table_meta/src/meta/current/mod.rs b/src/query/storages/common/table_meta/src/meta/current/mod.rs index 4e0719702e9e9..530969c3641e9 100644 --- a/src/query/storages/common/table_meta/src/meta/current/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/current/mod.rs @@ -16,6 +16,8 @@ pub use v0::ColumnMeta as SingleColumnMeta; pub use v2::AdditionalStatsMeta; pub use v2::BlockMeta; pub use v2::ClusterStatistics; +pub use v2::ColumnGroupBloomMeta; +pub use v2::ColumnGroupFileMeta; pub use v2::ColumnMeta; pub use v2::ColumnStatistics; pub use v2::DraftVirtualBlockMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/mod.rs b/src/query/storages/common/table_meta/src/meta/v2/mod.rs index 8db7b00c17bf3..c7b8f54e8391b 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/mod.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/mod.rs @@ -19,6 +19,8 @@ pub mod statistics; mod table_snapshot_statistics; pub use segment::BlockMeta; +pub use segment::ColumnGroupBloomMeta; +pub use segment::ColumnGroupFileMeta; pub use segment::ColumnMeta; pub use segment::DraftVirtualBlockMeta; pub use segment::DraftVirtualColumnMeta; diff --git a/src/query/storages/common/table_meta/src/meta/v2/segment.rs b/src/query/storages/common/table_meta/src/meta/v2/segment.rs index cd2f594366c46..4facf41cd55f3 100644 --- a/src/query/storages/common/table_meta/src/meta/v2/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v2/segment.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; use std::collections::HashMap; +use std::collections::HashSet; use std::ops::Range; use std::sync::Arc; @@ -171,6 +173,42 @@ pub struct DraftVirtualBlockMeta { pub virtual_location: Location, } +/// Metadata of one physical column-group file in a logical block. +/// +/// A file may still contain column chunks that are no longer active. Readers must only use the +/// chunks listed in [`Self::active_column_ids`]. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] +pub struct ColumnGroupFileMeta { + pub active_column_ids: Vec, + pub location: Location, + pub file_size: u64, + pub uncompressed_size: u64, + pub leaf_column_metas: HashMap, + /// Ordinary Bloom file paired with this data file. Its location is derived from `location`. + pub bloom: Option, +} + +impl ColumnGroupFileMeta { + /// Active leaf metadata in this physical file. + pub fn active_leaf_column_metas(&self) -> impl Iterator { + self.active_column_ids.iter().filter_map(|column_id| { + self.leaf_column_metas + .get(column_id) + .map(|column_meta| (*column_id, column_meta)) + }) + } +} + +/// Metadata of the ordinary Bloom file paired with a physical column-group file. +/// +/// The Bloom file is self-describing. A stored filter is active only while its column id remains in +/// the owning [`ColumnGroupFileMeta::active_column_ids`]. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] +pub struct ColumnGroupBloomMeta { + pub format_version: FormatVersion, + pub file_size: u64, +} + /// Meta information of a block /// Part of and kept inside the [SegmentInfo] #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -181,8 +219,18 @@ pub struct BlockMeta { #[serde(deserialize_with = "crate::meta::v2::statistics::deserialize_col_stats")] pub col_stats: HashMap, pub col_metas: HashMap, + /// Physical files that contain the active columns of this logical block. + /// + /// An empty vector is the legacy single-file representation described by `location`, + /// `file_size`, `block_size`, and `col_metas`. + #[serde(default)] + pub column_groups: Vec, pub cluster_stats: Option, - /// location of data block + /// Compatibility anchor for this logical block's data. + /// + /// In the legacy layout this is the only data-file location. In a split layout it identifies + /// the newest column-group file and does not cover the other active files; use + /// [`Self::physical_column_groups`] or [`Self::data_file_locations`] for physical reads. pub location: Location, /// location of bloom filter index pub bloom_filter_index_location: Option, @@ -234,6 +282,7 @@ impl BlockMeta { file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats, location, bloom_filter_index_location, @@ -255,6 +304,110 @@ impl BlockMeta { pub fn compression(&self) -> Compression { self.compression } + /// Active physical data files referenced by this logical block. + pub fn data_file_locations(&self) -> impl Iterator { + self.column_groups + .is_empty() + .then_some(&self.location) + .into_iter() + .chain(self.column_groups.iter().map(|group| &group.location)) + } + + fn legacy_column_group( + &self, + projected_column_ids: Option<&HashSet>, + ) -> ColumnGroupFileMeta { + let mut active_column_ids = self + .col_metas + .keys() + .filter(|column_id| { + projected_column_ids.is_none_or(|projected| projected.contains(column_id)) + }) + .copied() + .collect::>(); + active_column_ids.sort_unstable(); + let leaf_column_metas = active_column_ids + .iter() + .map(|column_id| (*column_id, self.col_metas[column_id].clone())) + .collect(); + ColumnGroupFileMeta { + active_column_ids, + location: self.location.clone(), + file_size: self.file_size, + uncompressed_size: self.block_size, + leaf_column_metas, + bloom: None, + } + } + + /// Normalize legacy and split layouts to the active physical data-file view. + pub fn physical_column_groups(&self) -> Cow<'_, [ColumnGroupFileMeta]> { + if !self.column_groups.is_empty() { + return Cow::Borrowed(&self.column_groups); + } + + Cow::Owned(vec![self.legacy_column_group(None)]) + } + + /// Project active leaf metadata while preserving each owning physical file. + pub fn project_column_groups( + &self, + projected_column_ids: &HashSet, + ) -> Vec { + let project_group = + |active_column_ids: &[ColumnId], + location: &Location, + file_size: u64, + uncompressed_size: u64, + leaf_column_metas: &HashMap| { + let active_column_ids = active_column_ids + .iter() + .filter(|column_id| projected_column_ids.contains(column_id)) + .filter(|column_id| leaf_column_metas.contains_key(column_id)) + .copied() + .collect::>(); + if active_column_ids.is_empty() { + return None; + } + let leaf_column_metas = active_column_ids + .iter() + .map(|column_id| (*column_id, leaf_column_metas[column_id].clone())) + .collect(); + Some(ColumnGroupFileMeta { + active_column_ids, + location: location.clone(), + file_size, + uncompressed_size, + leaf_column_metas, + bloom: None, + }) + }; + + if self.column_groups.is_empty() { + let group = self.legacy_column_group(Some(projected_column_ids)); + return (!group.active_column_ids.is_empty()) + .then_some(group) + .into_iter() + .collect(); + } + + self.column_groups + .iter() + .filter_map(|group| { + project_group( + &group.active_column_ids, + &group.location, + group.file_size, + group.uncompressed_size, + &group.leaf_column_metas, + ) + .map(|mut projected| { + projected.bloom = group.bloom.clone(); + projected + }) + }) + .collect() + } } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, FrozenAPI)] @@ -368,6 +521,7 @@ impl BlockMeta { file_size: s.file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats: None, location: (s.location.path.clone(), 0), bloom_filter_index_location: None, @@ -400,6 +554,7 @@ impl BlockMeta { file_size: s.file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats: None, location: s.location.clone(), bloom_filter_index_location: s.bloom_filter_index_location.clone(), @@ -430,3 +585,80 @@ impl From<(v0::SegmentInfo, &[TableField])> for SegmentInfo { SegmentInfo::from_v0(v, fields) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_active_leaf_column_metas_excludes_inactive_and_missing_columns() { + let group = ColumnGroupFileMeta { + active_column_ids: vec![1, 3], + location: ("group.parquet".to_string(), 4), + file_size: 20, + uncompressed_size: 40, + leaf_column_metas: HashMap::from([ + ( + 1, + ColumnMeta::Parquet(v0::ColumnMeta { + offset: 10, + len: 11, + num_values: 12, + }), + ), + ( + 2, + ColumnMeta::Parquet(v0::ColumnMeta { + offset: 20, + len: 21, + num_values: 22, + }), + ), + ]), + bloom: None, + }; + + let active_metas = group + .active_leaf_column_metas() + .map(|(column_id, column_meta)| (column_id, column_meta.offset_length())) + .collect::>(); + assert_eq!(active_metas, vec![(1, (10, 11))]); + } + + #[test] + fn test_deserialize_legacy_block_meta_without_column_groups() { + let block_meta = BlockMeta::new( + 10, + 300, + 30, + HashMap::new(), + HashMap::new(), + None, + ("old.parquet".to_string(), 2), + None, + 0, + None, + None, + None, + None, + None, + None, + None, + None, + Compression::Zstd, + None, + ); + let location = block_meta.location.clone(); + let mut value = serde_json::to_value(block_meta).unwrap(); + assert!( + value + .as_object_mut() + .unwrap() + .remove("column_groups") + .is_some() + ); + let decoded: BlockMeta = serde_json::from_value(value).unwrap(); + assert!(decoded.column_groups.is_empty()); + assert_eq!(decoded.location, location); + } +} diff --git a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs index f5034e796a578..7c365a51038b1 100644 --- a/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs +++ b/src/query/storages/common/table_meta/src/meta/v3/frozen/block_meta.rs @@ -57,6 +57,7 @@ impl From for crate::meta::BlockMeta { .into_iter() .map(|(k, v)| (k, v.into())) .collect(), + column_groups: vec![], cluster_stats: value.cluster_stats.map(|v| v.into()), location: value.location, bloom_filter_index_location: value.bloom_filter_index_location, diff --git a/src/query/storages/common/table_meta/src/meta/v4/segment.rs b/src/query/storages/common/table_meta/src/meta/v4/segment.rs index 65dbf75b4bfd9..04a19acef6cb5 100644 --- a/src/query/storages/common/table_meta/src/meta/v4/segment.rs +++ b/src/query/storages/common/table_meta/src/meta/v4/segment.rs @@ -545,6 +545,7 @@ mod tests { file_size: 16, col_stats: block_col_stats, col_metas, + column_groups: vec![], cluster_stats: None, location: ("block.parquet".to_string(), 0), bloom_filter_index_location: None, diff --git a/src/query/storages/common/table_meta/src/table/table_compression.rs b/src/query/storages/common/table_meta/src/table/table_compression.rs index f6c1f9eb31b4f..1485fd1ae20ee 100644 --- a/src/query/storages/common/table_meta/src/table/table_compression.rs +++ b/src/query/storages/common/table_meta/src/table/table_compression.rs @@ -24,6 +24,8 @@ pub enum TableCompression { None, LZ4, Snappy, + /// Kept for rewriting legacy blocks whose metadata records Gzip compression. + Gzip, #[default] Zstd, } @@ -55,11 +57,24 @@ impl From for meta::Compression { // Map to meta Lz4Raw. TableCompression::LZ4 => meta::Compression::Lz4Raw, TableCompression::Snappy => meta::Compression::Snappy, + TableCompression::Gzip => meta::Compression::Gzip, TableCompression::Zstd => meta::Compression::Zstd, } } } +impl From for TableCompression { + fn from(value: meta::Compression) -> Self { + match value { + meta::Compression::Lz4 | meta::Compression::Lz4Raw => TableCompression::LZ4, + meta::Compression::Snappy => TableCompression::Snappy, + meta::Compression::Zstd => TableCompression::Zstd, + meta::Compression::Gzip => TableCompression::Gzip, + meta::Compression::None => TableCompression::None, + } + } +} + /// Convert to parquet Compression. impl From for ParquetCompression { fn from(value: TableCompression) -> Self { @@ -67,6 +82,7 @@ impl From for ParquetCompression { TableCompression::None => ParquetCompression::UNCOMPRESSED, TableCompression::LZ4 => ParquetCompression::LZ4_RAW, TableCompression::Snappy => ParquetCompression::SNAPPY, + TableCompression::Gzip => ParquetCompression::GZIP(GzipLevel::default()), TableCompression::Zstd => ParquetCompression::ZSTD(ZstdLevel::default()), } } @@ -74,13 +90,6 @@ impl From for ParquetCompression { impl From for ParquetCompression { fn from(value: meta::Compression) -> Self { - match value { - meta::Compression::Lz4Raw => ParquetCompression::LZ4_RAW, - meta::Compression::Snappy => ParquetCompression::SNAPPY, - meta::Compression::Zstd => ParquetCompression::ZSTD(ZstdLevel::default()), - meta::Compression::None => ParquetCompression::UNCOMPRESSED, - meta::Compression::Lz4 => ParquetCompression::LZ4_RAW, - meta::Compression::Gzip => ParquetCompression::GZIP(GzipLevel::default()), - } + TableCompression::from(value).into() } } diff --git a/src/query/storages/fuse/src/constants.rs b/src/query/storages/fuse/src/constants.rs index a73e0667807d0..d5da27468466b 100644 --- a/src/query/storages/fuse/src/constants.rs +++ b/src/query/storages/fuse/src/constants.rs @@ -24,6 +24,7 @@ pub const FUSE_OPT_KEY_DATA_RETENTION_NUM_SNAPSHOTS_TO_KEEP: &str = "data_retention_num_snapshots_to_keep"; pub const FUSE_OPT_KEY_ENABLE_AUTO_VACUUM: &str = "enable_auto_vacuum"; pub const FUSE_OPT_KEY_ENABLE_AUTO_ANALYZE: &str = "enable_auto_analyze"; +pub const FUSE_OPT_KEY_ENABLE_PARTIAL_UPDATE: &str = "enable_partial_update"; pub const FUSE_OPT_KEY_ENABLE_VIRTUAL_COLUMN: &str = "enable_virtual_column"; pub const FUSE_OPT_KEY_AUTO_COMPACTION_IMPERFECT_BLOCKS_THRESHOLD: &str = "auto_compaction_imperfect_blocks_threshold"; diff --git a/src/query/storages/fuse/src/fuse_part.rs b/src/query/storages/fuse/src/fuse_part.rs index 6b171ed772f63..4742e678c7255 100644 --- a/src/query/storages/fuse/src/fuse_part.rs +++ b/src/query/storages/fuse/src/fuse_part.rs @@ -13,7 +13,9 @@ // limitations under the License. use std::any::Any; +use std::borrow::Cow; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; @@ -30,11 +32,137 @@ use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; use databend_storages_common_pruner::BlockMetaIndex; +use databend_storages_common_table_meta::meta::BlockMeta; +use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ColumnStatistics; use databend_storages_common_table_meta::meta::Compression; use databend_storages_common_table_meta::meta::Location; +use crate::io::TableMetaLocationGenerator; + +/// Projected column chunks to read from one physical column-group file. +#[derive(Clone, serde::Serialize, serde::Deserialize, PartialEq, Debug)] +pub struct FuseColumnGroupPartInfo { + pub location: String, + pub columns_meta: HashMap, +} + +/// Runtime description of a Bloom file paired with one physical data group. +#[derive(Clone, serde::Serialize, serde::Deserialize, PartialEq, Debug)] +pub struct FuseBloomIndexFileInfo { + pub active_column_ids: Vec, + pub location: Location, + pub file_size: u64, +} + +#[derive(Clone, Debug)] +pub enum BloomIndexLayout<'a> { + Legacy { + location: &'a Location, + file_size: u64, + }, + ColumnGroups { + files: Cow<'a, [FuseBloomIndexFileInfo]>, + }, +} + +impl<'a> BloomIndexLayout<'a> { + fn from_metadata( + legacy_location: Option<&'a Location>, + legacy_file_size: u64, + column_group_files: Cow<'a, [FuseBloomIndexFileInfo]>, + ) -> Option { + if !column_group_files.is_empty() { + return Some(Self::ColumnGroups { + files: column_group_files, + }); + } + + legacy_location.map(|location| Self::Legacy { + location, + file_size: legacy_file_size, + }) + } +} + +fn column_group_bloom_location(group: &ColumnGroupFileMeta) -> Option { + group.bloom.as_ref().map(|bloom| { + ( + TableMetaLocationGenerator::gen_bloom_index_location_with_version( + &group.location.0, + bloom.format_version, + ), + bloom.format_version, + ) + }) +} + +pub(crate) fn column_group_bloom_files(meta: &BlockMeta) -> Vec { + meta.column_groups + .iter() + .filter_map(|group| { + let bloom = group.bloom.as_ref()?; + Some(FuseBloomIndexFileInfo { + active_column_ids: group.active_column_ids.clone(), + location: column_group_bloom_location(group)?, + file_size: bloom.file_size, + }) + }) + .collect() +} + +pub(crate) fn legacy_bloom_index_location(meta: &BlockMeta) -> Option<&Location> { + meta.column_groups + .is_empty() + .then_some(meta.bloom_filter_index_location.as_ref()) + .flatten() +} + +pub(crate) fn block_bloom_index_size(meta: &BlockMeta) -> u64 { + if meta.column_groups.is_empty() { + meta.bloom_filter_index_size + } else { + meta.column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum() + } +} + +/// Physical ordinary Bloom files referenced by a logical block. +pub fn block_bloom_index_locations(meta: &BlockMeta) -> impl Iterator + '_ { + let legacy = legacy_bloom_index_location(meta).cloned(); + legacy.into_iter().chain( + meta.column_groups + .iter() + .filter_map(column_group_bloom_location), + ) +} + +pub(crate) fn bloom_index_layout(meta: &BlockMeta) -> Option> { + let files = column_group_bloom_files(meta); + BloomIndexLayout::from_metadata( + legacy_bloom_index_location(meta), + block_bloom_index_size(meta), + Cow::Owned(files), + ) +} + +pub(crate) fn project_column_groups( + meta: &BlockMeta, + projected_column_ids: &HashSet, +) -> Vec { + meta.project_column_groups(projected_column_ids) + .into_iter() + .map(|group| FuseColumnGroupPartInfo { + location: group.location.0, + columns_meta: group.leaf_column_metas, + }) + .collect() +} + /// Fuse table partition information. #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] pub struct FuseBlockPartInfo { @@ -42,10 +170,12 @@ pub struct FuseBlockPartInfo { pub bloom_filter_index_location: Option, pub bloom_filter_index_size: u64, + #[serde(default)] + pub column_group_bloom_files: Vec, pub create_on: Option>, pub nums_rows: usize, - pub columns_meta: HashMap, + pub column_groups: Vec, pub columns_stat: Option>, pub compression: Compression, @@ -77,13 +207,23 @@ impl PartInfo for FuseBlockPartInfo { } impl FuseBlockPartInfo { + /// Normalize optional legacy and column-group Bloom metadata into one physical-layout view. + pub fn bloom_index_layout(&self) -> Option> { + BloomIndexLayout::from_metadata( + self.bloom_filter_index_location.as_ref(), + self.bloom_filter_index_size, + Cow::Borrowed(&self.column_group_bloom_files), + ) + } + #[allow(clippy::too_many_arguments)] pub fn create( location: String, bloom_filter_index_location: Option, bloom_filter_index_size: u64, + column_group_bloom_files: Vec, rows_count: u64, - columns_meta: HashMap, + column_groups: Vec, columns_stat: Option>, compression: Compression, sort_min_max: Option<(Scalar, Scalar)>, @@ -94,8 +234,9 @@ impl FuseBlockPartInfo { location, bloom_filter_index_location, bloom_filter_index_size, + column_group_bloom_files, create_on, - columns_meta, + column_groups, nums_rows: rows_count as usize, compression, sort_min_max, diff --git a/src/query/storages/fuse/src/io/locations.rs b/src/query/storages/fuse/src/io/locations.rs index d18ba2c13168f..57bec4f32dcac 100644 --- a/src/query/storages/fuse/src/io/locations.rs +++ b/src/query/storages/fuse/src/io/locations.rs @@ -346,6 +346,10 @@ impl TableMetaLocationGenerator { } pub fn gen_bloom_index_location_from_block_location(loc: &str) -> String { + Self::gen_bloom_index_location_with_version(loc, BlockFilter::VERSION) + } + + pub fn gen_bloom_index_location_with_version(loc: &str, format_version: u64) -> String { let splits = loc.split('/').collect::>(); let len = splits.len(); let prefix = splits[..len - 2].join("/"); @@ -353,10 +357,7 @@ impl TableMetaLocationGenerator { let id: String = block_name.chars().take(32).collect(); format!( "{}/{}/{}_v{}.parquet", - prefix, - FUSE_TBL_XOR_BLOOM_INDEX_PREFIX, - id, - BlockFilter::VERSION, + prefix, FUSE_TBL_XOR_BLOOM_INDEX_PREFIX, id, format_version, ) } diff --git a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs index a2dbe5c192f7d..7caa74166fd1c 100644 --- a/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs +++ b/src/query/storages/fuse/src/io/read/agg_index/agg_index_reader_parquet.rs @@ -28,14 +28,7 @@ impl AggIndexReader { ) -> Result { let columns_chunks = data.columns_chunks()?; let part = FuseBlockPartInfo::from_part(&part)?; - let block = self.reader.deserialize_parquet_chunks( - part.nums_rows, - &part.columns_meta, - columns_chunks, - &part.compression, - &part.location, - None, - )?; + let block = self.reader.deserialize_part(part, columns_chunks, None)?; self.apply_agg_info(block) } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader.rs b/src/query/storages/fuse/src/io/read/block/block_reader.rs index 01dc040fab256..56f7ed6a8342c 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader.rs @@ -241,6 +241,10 @@ impl BlockReader { } impl BlockReadContext { + pub(crate) fn table_context(&self) -> &Arc { + &self.ctx + } + pub fn operator(&self) -> &Operator { &self.operator } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs index 9dfe85a7b817d..e9e04a4f92730 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_deserialize.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::future::Future; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::Result; @@ -27,11 +28,22 @@ use databend_storages_common_table_meta::meta::column_oriented_segment::BlockRea use super::BlockReader; use crate::BlockReadResult; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; +use crate::fuse_part::project_column_groups; use crate::io::read::block::block_reader_merge_io::DataItem; use crate::unsupported_storage_format_error; impl BlockReader { + pub(crate) fn projected_column_groups(&self, meta: &BlockMeta) -> Vec { + let projected_column_ids = self + .project_column_nodes + .iter() + .flat_map(|node| node.leaf_column_ids.iter().copied()) + .collect::>(); + project_column_groups(meta, &projected_column_ids) + } + /// Deserialize column chunks data from parquet format to DataBlock. pub fn deserialize_chunks_with_part_info( &self, @@ -40,15 +52,10 @@ impl BlockReader { storage_format: &FuseStorageFormat, ) -> Result { let part = FuseBlockPartInfo::from_part(&part)?; - - self.deserialize_chunks( - &part.location, - part.nums_rows, - &part.compression, - &part.columns_meta, - chunks, - storage_format, - ) + match storage_format { + FuseStorageFormat::Parquet => self.deserialize_part(part, chunks, None), + FuseStorageFormat::Unsupported => Err(unsupported_storage_format_error()), + } } pub fn deserialize_chunks( @@ -82,11 +89,23 @@ impl BlockReader { storage_format: &FuseStorageFormat, ) -> Result { // Get the merged IO read result. - let merge_io_read_result = self - .read_columns_data_by_merge_io(settings, &meta.location.0, &meta.col_metas, &None) - .await?; - - self.deserialize_chunks_with_meta(&meta.into(), storage_format, merge_io_read_result) + let column_groups = self.projected_column_groups(meta); + // Type erasure breaks the recursive async future formed by virtual-column reads that + // return to `read_by_meta` through the merge-IO path. + let read: std::pin::Pin> + Send + '_>> = + Box::pin(self.read_column_groups_data_by_merge_io(settings, &column_groups, &None)); + let merge_io_read_result = read.await?; + let column_chunks = merge_io_read_result.columns_chunks()?; + match storage_format { + FuseStorageFormat::Parquet => self.deserialize_column_groups( + meta.row_count as usize, + &column_groups, + column_chunks, + &meta.compression, + None, + ), + FuseStorageFormat::Unsupported => Err(unsupported_storage_format_error()), + } } pub fn deserialize_chunks_with_meta( diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs index cdb61afcc436a..10c42e30e733f 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io.rs @@ -15,7 +15,6 @@ use std::collections::HashMap; use std::sync::Arc; -use bytes::Bytes; use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_storages_common_cache::ColumnData; @@ -34,7 +33,7 @@ pub enum DataItem<'a> { } pub struct BlockReadResult { - merge_io_result: MergeIOReadResult, + merge_io_results: Vec, pub(crate) cached_column_data: CachedColumnData, pub(crate) cached_column_array: CachedColumnArray, } @@ -46,22 +45,46 @@ impl BlockReadResult { cached_column_array: CachedColumnArray, ) -> BlockReadResult { BlockReadResult { - merge_io_result, + merge_io_results: vec![merge_io_result], + cached_column_data, + cached_column_array, + } + } + + pub(crate) fn merge(results: Vec) -> BlockReadResult { + let mut merge_io_results = Vec::with_capacity(results.len()); + let mut cached_column_data = vec![]; + let mut cached_column_array = vec![]; + + for result in results { + merge_io_results.extend(result.merge_io_results); + cached_column_data.extend(result.cached_column_data); + cached_column_array.extend(result.cached_column_array); + } + + BlockReadResult { + merge_io_results, cached_column_data, cached_column_array, } } pub fn columns_chunks(&self) -> Result>> { - let mut res = HashMap::with_capacity(self.merge_io_result.columns_chunk_offsets.len()); + let capacity = self + .merge_io_results + .iter() + .map(|result| result.columns_chunk_offsets.len()) + .sum(); + let mut res = HashMap::with_capacity(capacity); // merge column data fetched from object storage - for (column_id, (chunk_idx, range)) in &self.merge_io_result.columns_chunk_offsets { - let chunk = self - .merge_io_result - .owner_memory - .get_chunk(*chunk_idx, &self.merge_io_result.block_path)?; - res.insert(*column_id, DataItem::RawData(chunk.slice(range.clone()))); + for merge_io_result in &self.merge_io_results { + for (column_id, (chunk_idx, range)) in &merge_io_result.columns_chunk_offsets { + let chunk = merge_io_result + .owner_memory + .get_chunk(*chunk_idx, &merge_io_result.block_path)?; + res.insert(*column_id, DataItem::RawData(chunk.slice(range.clone()))); + } } // merge column data from cache @@ -76,24 +99,4 @@ impl BlockReadResult { Ok(res) } - - pub fn column_buffers(&self) -> Result> { - let mut res = HashMap::with_capacity(self.merge_io_result.columns_chunk_offsets.len()); - - // merge column data fetched from object storage - for (column_id, (chunk_idx, range)) in &self.merge_io_result.columns_chunk_offsets { - let chunk = self - .merge_io_result - .owner_memory - .get_chunk(*chunk_idx, &self.merge_io_result.block_path)?; - res.insert(*column_id, chunk.slice(range.clone()).to_bytes()); - } - - // merge column data from cache - for (column_id, data) in &self.cached_column_data { - res.insert(*column_id, data.bytes()); - } - - Ok(res) - } } diff --git a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs index 1d0be8fa3cb0c..cfe3fd3208b89 100644 --- a/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs +++ b/src/query/storages/fuse/src/io/read/block/block_reader_merge_io_async.rs @@ -29,10 +29,28 @@ use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::ColumnMeta; use crate::BlockReadResult; +use crate::FuseColumnGroupPartInfo; use crate::io::BlockReadContext; use crate::io::BlockReader; +fn column_group_read_concurrency(max_threads: usize, max_storage_io_requests: usize) -> usize { + let outer_readers = max_threads.min(max_storage_io_requests).max(1); + max_storage_io_requests.max(1).div_ceil(outer_readers) +} + impl BlockReader { + #[async_backtrace::framed] + pub(crate) async fn read_column_groups_data_by_merge_io( + &self, + settings: &ReadSettings, + column_groups: &[FuseColumnGroupPartInfo], + ignore_column_ids: &Option>, + ) -> Result { + self.read_context() + .read_column_groups_data_by_merge_io(settings, column_groups, ignore_column_ids) + .await + } + #[async_backtrace::framed] pub async fn read_columns_data_by_merge_io( &self, @@ -48,6 +66,33 @@ impl BlockReader { } impl BlockReadContext { + #[async_backtrace::framed] + pub(crate) async fn read_column_groups_data_by_merge_io( + &self, + settings: &ReadSettings, + column_groups: &[FuseColumnGroupPartInfo], + ignore_column_ids: &Option>, + ) -> Result { + let query_settings = self.table_context().get_settings(); + let concurrency = column_group_read_concurrency( + query_settings.get_max_threads()? as usize, + query_settings.get_max_storage_io_requests()? as usize, + ); + let mut results = Vec::with_capacity(column_groups.len()); + for groups in column_groups.chunks(concurrency) { + let reads = groups.iter().map(|group| { + self.read_columns_data_by_merge_io( + settings, + &group.location, + &group.columns_meta, + ignore_column_ids, + ) + }); + results.extend(futures::future::try_join_all(reads).await?); + } + Ok(BlockReadResult::merge(results)) + } + #[async_backtrace::framed] pub async fn read_columns_data_by_merge_io( &self, @@ -172,3 +217,17 @@ impl<'a> ColumnCacheKeyBuilder<'a> { TableDataCacheKey::new(self.block_path, *column_id, offset, len) } } + +#[cfg(test)] +mod tests { + use super::column_group_read_concurrency; + + #[test] + fn test_column_group_read_concurrency() { + assert_eq!(column_group_read_concurrency(8, 64), 8); + assert_eq!(column_group_read_concurrency(8, 10), 2); + assert_eq!(column_group_read_concurrency(64, 8), 1); + assert_eq!(column_group_read_concurrency(0, 0), 1); + assert_eq!(column_group_read_concurrency(0, 8), 8); + } +} diff --git a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs index aaeb22d5291b3..3bcbcc95f3316 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/mod.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/mod.rs @@ -42,6 +42,7 @@ pub use deserialize::column_chunks_to_record_batch; pub use row_selection::RowSelection; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::io::BlockReader; use crate::io::read::block::block_reader_merge_io::DataItem; @@ -52,12 +53,35 @@ impl BlockReader { column_chunks: HashMap, selection: Option<&RowSelection>, ) -> databend_common_exception::Result { - self.deserialize_parquet_chunks( + self.deserialize_column_groups( part.nums_rows, - &part.columns_meta, + &part.column_groups, column_chunks, &part.compression, - &part.location, + selection, + ) + } + + pub(crate) fn deserialize_column_groups( + &self, + num_rows: usize, + column_groups: &[FuseColumnGroupPartInfo], + column_chunks: HashMap, + compression: &Compression, + selection: Option<&RowSelection>, + ) -> databend_common_exception::Result { + self.deserialize_parquet_chunks_with_cache_key( + num_rows, + column_chunks, + compression, + |column_id| { + column_groups.iter().find_map(|group| { + group.columns_meta.get(&column_id).map(|column_meta| { + let (offset, len) = column_meta.offset_length(); + TableDataCacheKey::new(&group.location, column_id, offset, len) + }) + }) + }, selection, ) } @@ -71,6 +95,31 @@ impl BlockReader { block_path: &str, selection: Option<&RowSelection>, ) -> databend_common_exception::Result { + self.deserialize_parquet_chunks_with_cache_key( + num_rows, + column_chunks, + compression, + |column_id| { + column_metas.get(&column_id).map(|column_meta| { + let (offset, len) = column_meta.offset_length(); + TableDataCacheKey::new(block_path, column_id, offset, len) + }) + }, + selection, + ) + } + + fn deserialize_parquet_chunks_with_cache_key( + &self, + num_rows: usize, + column_chunks: HashMap, + compression: &Compression, + cache_key_for_column: F, + selection: Option<&RowSelection>, + ) -> databend_common_exception::Result + where + F: Fn(ColumnId) -> Option, + { let result_rows = selection.map(|s| s.selected_rows).unwrap_or(num_rows); // If projection is empty, return a DataBlock with the appropriate row count but no columns if self.projected_schema.fields.is_empty() { @@ -128,10 +177,7 @@ impl BlockReader { let arrow_array = column_by_name(&record_batch, &name_paths[i]); if !column_node.is_nested { if let Some(cache) = &array_cache { - let meta = column_metas.get(&field.column_id).unwrap(); - let (offset, len) = meta.offset_length(); - let key = - TableDataCacheKey::new(block_path, field.column_id, offset, len); + let key = cache_key_for_column(field.column_id).unwrap(); let array_memory_size = arrow_array.get_array_memory_size(); cache.insert(key.into(), (arrow_array.clone(), array_memory_size)); } diff --git a/src/query/storages/fuse/src/io/write/block_writer.rs b/src/query/storages/fuse/src/io/write/block_writer.rs index 478d7807fb483..03eceb2faa99f 100644 --- a/src/query/storages/fuse/src/io/write/block_writer.rs +++ b/src/query/storages/fuse/src/io/write/block_writer.rs @@ -14,12 +14,13 @@ use std::collections::BTreeMap; use std::collections::HashMap; -use std::collections::hash_map::Entry; +use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; use chrono::Utc; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfo; use databend_common_expression::ColumnId; @@ -47,12 +48,16 @@ use databend_common_metrics::storage::metrics_inc_block_write_nums; use databend_storages_common_blocks::SerializedParquet; use databend_storages_common_blocks::blocks_to_parquet_with_stats; use databend_storages_common_index::NgramArgs; +use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::BlockTopN; use databend_storages_common_table_meta::meta::ClusterStatistics; +use databend_storages_common_table_meta::meta::ColumnGroupBloomMeta; +use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::ExtendedBlockMeta; +use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::encode_column_hll; @@ -63,6 +68,7 @@ use crate::FuseStorageFormat; use crate::io::BlockStatsBuilder; use crate::io::BloomIndexState; use crate::io::TableMetaLocationGenerator; +use crate::io::build_column_hlls; use crate::io::write::InvertedIndexBuilder; use crate::io::write::InvertedIndexState; use crate::io::write::SpatialIndexBuilder; @@ -163,7 +169,104 @@ pub struct BlockBuilder { pub serialize_hll: bool, } +struct ColumnGroupUpdate { + active_column_ids: Vec, + location: Location, + file_size: u64, + uncompressed_size: u64, + column_metas: HashMap, + column_stats: StatisticsOfColumns, + bloom: Option, +} + +fn merge_column_group_metadata( + origin: &BlockMeta, + current_column_ids: &HashSet, + update: ColumnGroupUpdate, +) -> BlockMeta { + let updated_column_ids = update + .active_column_ids + .iter() + .copied() + .collect::>(); + let mut column_groups = origin.physical_column_groups().into_owned(); + if origin.column_groups.is_empty() { + column_groups[0].bloom = + origin + .bloom_filter_index_location + .as_ref() + .map(|location| ColumnGroupBloomMeta { + format_version: location.1, + file_size: origin.bloom_filter_index_size, + }); + } + for group in &mut column_groups { + group.active_column_ids.retain(|column_id| { + current_column_ids.contains(column_id) && !updated_column_ids.contains(column_id) + }); + } + column_groups.retain(|group| !group.active_column_ids.is_empty()); + column_groups.push(ColumnGroupFileMeta { + active_column_ids: update.active_column_ids, + location: update.location.clone(), + file_size: update.file_size, + uncompressed_size: update.uncompressed_size, + leaf_column_metas: update.column_metas.clone(), + bloom: update.bloom, + }); + + let mut block_meta = origin.clone(); + block_meta.location = update.location; + block_meta.file_size = column_groups.iter().map(|group| group.file_size).sum(); + block_meta.block_size = column_groups + .iter() + .map(|group| group.uncompressed_size) + .sum(); + block_meta.column_groups = column_groups; + block_meta + .col_metas + .retain(|column_id, _| current_column_ids.contains(column_id)); + block_meta.col_metas.extend(update.column_metas); + block_meta + .col_stats + .retain(|column_id, _| current_column_ids.contains(column_id)); + block_meta.col_stats.extend(update.column_stats); + block_meta.bloom_filter_index_location = None; + block_meta.bloom_filter_index_size = block_meta + .column_groups + .iter() + .filter_map(|group| group.bloom.as_ref()) + .map(|bloom| bloom.file_size) + .sum(); + block_meta +} + impl BlockBuilder { + fn add_hll_distinct_counts( + column_distinct_count: &mut HashMap, + column_hlls: &Option, + ) { + if let Some(hlls) = column_hlls { + for (column_id, hll) in hlls { + column_distinct_count + .entry(*column_id) + .or_insert_with(|| hll.count()); + } + } + } + + fn finalize_column_hlls(&self, column_hlls: Option) -> Result> { + column_hlls + .map(|hlls| { + if self.serialize_hll { + encode_column_hll(&hlls).map(BlockHLLState::Serialized) + } else { + Ok(BlockHLLState::Deserialized(hlls)) + } + }) + .transpose() + } + pub fn build(&self, data_block: DataBlock, f: F) -> Result where F: Fn(DataBlock, &ClusterStatsGenerator) -> Result<(Option, DataBlock)> { @@ -201,13 +304,7 @@ impl BlockBuilder { } else { (None, None) }; - if let Some(hlls) = &column_hlls { - for (key, val) in hlls { - if let Entry::Vacant(entry) = column_distinct_count.entry(*key) { - entry.insert(val.count()); - } - } - } + Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); let mut inverted_index_states = Vec::with_capacity(self.inverted_index_builders.len()); for inverted_index_builder in &self.inverted_index_builders { @@ -284,6 +381,7 @@ impl BlockBuilder { file_size, col_stats, col_metas, + column_groups: vec![], cluster_stats, location: block_location, bloom_filter_index_location: bloom_index_state.as_ref().map(|v| v.location.clone()), @@ -307,15 +405,7 @@ impl BlockBuilder { create_on: Some(Utc::now()), }; - let column_hlls = column_hlls - .map(|hlls| { - if self.serialize_hll { - encode_column_hll(&hlls).map(BlockHLLState::Serialized) - } else { - Ok(BlockHLLState::Deserialized(hlls)) - } - }) - .transpose()?; + let column_hlls = self.finalize_column_hlls(column_hlls)?; let serialized = BlockSerialization { block_raw_data: buffer, block_meta, @@ -329,6 +419,147 @@ impl BlockBuilder { }; Ok(serialized) } + + /// Serialize only the fields changed by an UPDATE and merge their physical + /// metadata back into the original logical block. + /// + /// # Supported feature scope + /// + /// This method is defined only for the Partial UPDATE feature domain: no partitioning, computed + /// columns, column-oriented segments, derived table indexes (inverted, Ngram, vector, or + /// spatial), or Fuse virtual columns. These exclusions are contract preconditions, + /// intentionally not validated or handled by fallback here. Within that contract, this method + /// maintains data, ordinary Bloom, HLL, and column statistics only; handling metadata for + /// excluded features is out of scope. REPLACE INTO must not consume column-group blocks + /// produced by this method; that operation is outside the same contract. + pub fn build_column_group( + &self, + data_block: DataBlock, + origin: &BlockMeta, + updated_field_indices: &[FieldIndex], + ) -> Result { + if data_block.num_rows() as u64 != origin.row_count { + return Err(ErrorCode::Internal( + "column-group update changed the block row count", + )); + } + + let mut updated_field_indices = updated_field_indices.to_vec(); + updated_field_indices.sort_unstable(); + updated_field_indices.dedup(); + if updated_field_indices.is_empty() { + return Err(ErrorCode::Internal( + "column-group update has no updated fields", + )); + } + if updated_field_indices + .iter() + .any(|index| *index >= self.source_schema.fields().len()) + { + return Err(ErrorCode::Internal( + "column-group update field is outside the table schema", + )); + } + + let updated_schema = Arc::new(self.source_schema.project(&updated_field_indices)); + if data_block.num_columns() != updated_schema.fields().len() { + return Err(ErrorCode::Internal( + "column-group update block does not match updated fields", + )); + } + let updated_block = data_block; + let updated_column_ids = updated_schema.to_leaf_column_ids(); + + let (data_location, block_id) = self + .meta_locations + .gen_block_location(self.table_meta_timestamps); + + let project_updated_fields = |fields: &BTreeMap| { + updated_field_indices + .iter() + .enumerate() + .filter_map(|(updated_index, source_index)| { + fields + .get(source_index) + .cloned() + .map(|field| (updated_index, field)) + }) + .collect::>() + }; + let updated_bloom_columns_map = project_updated_fields(&self.bloom_columns_map); + let rebuild_bloom_index = !updated_bloom_columns_map.is_empty(); + let bloom_index_state = if rebuild_bloom_index { + let location = self.meta_locations.block_bloom_index_location(&block_id); + BloomIndexState::from_data_block( + self.ctx.clone(), + &updated_block, + location, + self.write_settings.bloom_index_type, + updated_bloom_columns_map, + &[], + )? + } else { + None + }; + + let mut column_distinct_count = bloom_index_state + .as_ref() + .map(|index| index.column_distinct_count.clone()) + .unwrap_or_default(); + let updated_ndv_columns_map = project_updated_fields(&self.ndv_columns_map); + let column_hlls = build_column_hlls(&updated_block, &updated_ndv_columns_map)?; + Self::add_hll_distinct_counts(&mut column_distinct_count, &column_hlls); + + let updated_col_stats = gen_columns_statistics( + &updated_block, + Some(column_distinct_count), + &updated_schema, + &self.write_settings.col_stats_truncate_lens, + )?; + let uncompressed_size = + updated_block.estimate_block_size(updated_block.num_columns()) as u64; + let mut write_settings = self.write_settings.clone(); + write_settings.table_compression = origin.compression.into(); + let (updated_col_metas, buffer) = serialize_block_with_column_stats( + &write_settings, + &updated_schema, + Some(&updated_col_stats), + updated_block, + )?; + let file_size = buffer.len() as u64; + + let current_column_ids = self.source_schema.to_leaf_column_id_set(); + let mut block_meta = + merge_column_group_metadata(origin, ¤t_column_ids, ColumnGroupUpdate { + active_column_ids: updated_column_ids, + location: data_location, + file_size, + uncompressed_size, + column_metas: updated_col_metas, + column_stats: updated_col_stats, + bloom: bloom_index_state + .as_ref() + .map(|state| ColumnGroupBloomMeta { + format_version: state.location.1, + file_size: state.size, + }), + }); + block_meta.create_on = Some(Utc::now()); + + let column_hlls = self.finalize_column_hlls(column_hlls)?; + + Ok(BlockSerialization { + block_raw_data: buffer, + block_meta, + bloom_index_state, + inverted_index_states: vec![], + virtual_column_state: None, + vector_index_state: None, + spatial_index_state: None, + column_hlls, + column_top_n: None, + }) + } } pub struct BlockWriter; diff --git a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs index 6a1447a9d0089..36bf112287081 100644 --- a/src/query/storages/fuse/src/io/write/bloom_index_writer.rs +++ b/src/query/storages/fuse/src/io/write/bloom_index_writer.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use databend_common_catalog::plan::Projection; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; @@ -139,6 +140,18 @@ pub struct BloomIndexRebuilder { } impl BloomIndexRebuilder { + pub(crate) fn validate_rebuild_version(bloom_index_location: &Location) -> Result<()> { + if bloom_index_location.1 != BlockFilter::VERSION { + return Err(ErrorCode::DeprecatedIndexFormat(format!( + "cannot rebuild Bloom index {:?} in legacy format {} with current format {}", + bloom_index_location, + bloom_index_location.1, + BlockFilter::VERSION + ))); + } + Ok(()) + } + pub async fn bloom_index_state_from_block_meta( &self, bloom_index_location: &Location, @@ -158,7 +171,6 @@ impl BloomIndexRebuilder { )?; let settings = ReadSettings::from_ctx(&self.table_ctx)?; - let merge_io_read_result = block_reader .read_columns_data_by_merge_io( &settings, @@ -173,7 +185,7 @@ impl BloomIndexRebuilder { merge_io_read_result, )?; - assert_eq!(bloom_index_location.1, BlockFilter::VERSION); + Self::validate_rebuild_version(bloom_index_location)?; let mut builder = BloomIndexBuilder::create( self.table_ctx.get_function_context()?, self.bloom_index_type, @@ -192,3 +204,22 @@ impl BloomIndexRebuilder { } } } + +#[cfg(test)] +mod tests { + use databend_common_exception::ErrorCode; + use databend_storages_common_index::filters::BlockFilter; + use databend_storages_common_table_meta::meta::Versioned; + + use super::BloomIndexRebuilder; + + #[test] + fn test_validate_rebuild_version_rejects_legacy_format() { + let legacy_location = ("legacy-bloom".to_string(), BlockFilter::VERSION - 1); + let err = BloomIndexRebuilder::validate_rebuild_version(&legacy_location).unwrap_err(); + assert_eq!(err.code(), ErrorCode::DEPRECATED_INDEX_FORMAT); + + let current_location = ("current-bloom".to_string(), BlockFilter::VERSION); + BloomIndexRebuilder::validate_rebuild_version(¤t_location).unwrap(); + } +} diff --git a/src/query/storages/fuse/src/io/write/stream/block_builder.rs b/src/query/storages/fuse/src/io/write/stream/block_builder.rs index 91b47288a6d7b..b9cb3fafa321e 100644 --- a/src/query/storages/fuse/src/io/write/stream/block_builder.rs +++ b/src/query/storages/fuse/src/io/write/stream/block_builder.rs @@ -403,6 +403,7 @@ impl StreamBlockBuilder { file_size: file_size as u64, col_stats, col_metas, + column_groups: vec![], // Stream block writing is only enabled for tables without a cluster key, so cluster // statistics cannot be produced on this path. cluster_stats: None, diff --git a/src/query/storages/fuse/src/lib.rs b/src/query/storages/fuse/src/lib.rs index 9bebd92396acf..edb35dd810ac0 100644 --- a/src/query/storages/fuse/src/lib.rs +++ b/src/query/storages/fuse/src/lib.rs @@ -72,7 +72,9 @@ use databend_common_catalog::table::TableStatistics; pub use databend_common_catalog::table_context::TableContext; pub use fuse_column::FuseTableColumnStatisticsProvider; pub use fuse_part::FuseBlockPartInfo; +pub use fuse_part::FuseColumnGroupPartInfo; pub use fuse_part::FuseLazyPartInfo; +pub use fuse_part::block_bloom_index_locations; pub use fuse_table::FuseTable; pub use fuse_table::RetentionPolicy; pub use fuse_type::FuseSegmentFormat; diff --git a/src/query/storages/fuse/src/operations/commit.rs b/src/query/storages/fuse/src/operations/commit.rs index 96908da07ddfa..7d71484329ffb 100644 --- a/src/query/storages/fuse/src/operations/commit.rs +++ b/src/query/storages/fuse/src/operations/commit.rs @@ -94,7 +94,7 @@ impl FuseTable { ) })?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( self, ctx.clone(), @@ -105,7 +105,7 @@ impl FuseTable { MutationKind::Insert, table_meta_timestamps, ) - }); + })?; let snapshot_gen = AppendGenerator::new(ctx.clone(), overwrite); pipeline.add_sink(|input| { diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs index 3cf8f498a9e2f..9fa5e6393a41c 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_mutation_aggregator.rs @@ -14,6 +14,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::hash_map::Entry; use std::sync::Arc; use std::time::Instant; @@ -26,6 +27,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::BlockThresholds; +use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::TableSchemaRef; @@ -35,6 +37,7 @@ use databend_common_pipeline_transforms::processors::AsyncAccumulatingTransform; use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_sql::parse_cluster_keys; use databend_storages_common_cache::SegmentStatistics; +use databend_storages_common_index::RangeIndex; use databend_storages_common_table_meta::meta::AdditionalStatsMeta; use databend_storages_common_table_meta::meta::BlockHLL; use databend_storages_common_table_meta::meta::BlockHLLState; @@ -49,6 +52,8 @@ use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::VirtualBlockMeta; +use databend_storages_common_table_meta::meta::decode_column_hll; +use databend_storages_common_table_meta::meta::encode_column_hll; use databend_storages_common_table_meta::meta::merge_column_hll_mut; use databend_storages_common_table_meta::meta::merge_column_top_n_mut; use itertools::Itertools; @@ -98,6 +103,7 @@ pub struct TableMutationAggregator { top_n: BlockTopN, logical_updated_rows: u64, logical_deleted_rows: u64, + current_hll_column_ids: Arc>, write_segment_ctx: WriteSegmentCtx, processed_log_entries: usize, @@ -196,9 +202,23 @@ impl TableMutationAggregator { removed_statistics: Statistics, kind: MutationKind, table_meta_timestamps: TableMetaTimestamps, - ) -> Self { + ) -> Result { let fill_missing_cluster_stats = table.resolve_physical_cluster_keys().is_some(); + let hll_schema = if matches!(kind, MutationKind::Insert | MutationKind::Replace) { + table.schema() + } else { + table.schema_with_stream() + }; + let current_hll_column_ids = Arc::new( + table + .approx_distinct_cols() + .distinct_column_fields(hll_schema, RangeIndex::supported_table_type)? + .into_values() + .map(|field| field.column_id()) + .collect(), + ); + let virtual_schema = table.table_info.meta.virtual_schema.clone(); let cluster_key_exprs = if fill_missing_cluster_stats { table @@ -224,7 +244,7 @@ impl TableMutationAggregator { table_meta_timestamps, fill_missing_cluster_stats, }; - TableMutationAggregator { + Ok(TableMutationAggregator { ctx, mutations: HashMap::new(), extended_mutations: HashMap::new(), @@ -240,10 +260,11 @@ impl TableMutationAggregator { top_n: HashMap::new(), logical_updated_rows: 0, logical_deleted_rows: 0, + current_hll_column_ids, write_segment_ctx, processed_log_entries: 0, table_id: table.get_id(), - } + }) } fn accumulate_top_n(&mut self, top_n: Option) -> Result<()> { @@ -544,6 +565,7 @@ impl TableMutationAggregator { let segment_mutation = self.mutations.remove(&index).unwrap(); let location = self.base_segments.get(index).cloned(); let write_segment_ctx = self.write_segment_ctx.clone(); + let current_hll_column_ids = self.current_hll_column_ids.clone(); tasks.push(async move { let mut force_all_blocks_perfect = false; @@ -570,16 +592,29 @@ impl TableMutationAggregator { .into_iter() .enumerate() .map(|(block_idx, block_meta)| { - let hll = stats - .as_ref() - .and_then(|v| v.block_hlls.get(block_idx)) - .cloned(); - (block_idx, (block_meta, hll)) + let hll = decode_current_block_hll( + stats.as_ref().and_then(|v| v.block_hlls.get(block_idx)), + ¤t_hll_column_ids, + )?; + Ok((block_idx, (block_meta, hll))) }) - .collect::>(); - - for (idx, new_meta) in segment_mutation.replaced_blocks { - block_editor.insert(idx, new_meta); + .collect::>>()?; + + for (idx, (new_meta, new_hll)) in segment_mutation.replaced_blocks { + let previous_hll = block_editor.remove(&idx).and_then(|(_, hll)| hll); + let new_hll = if let Some(updated_column_ids) = new_meta + .column_groups + .last() + .map(|group| group.active_column_ids.as_slice()) + { + replace_partial_block_hll(previous_hll, new_hll, updated_column_ids)? + } else { + new_hll + .map(|hll| decode_column_hll(&hll)) + .transpose()? + .flatten() + }; + block_editor.insert(idx, (new_meta, new_hll)); } for idx in segment_mutation.deleted_blocks { block_editor.remove(&idx); @@ -594,7 +629,17 @@ impl TableMutationAggregator { } // assign back the mutated blocks to segment - let (new_blocks, new_hlls) = block_editor.into_values().unzip(); + let (new_blocks, new_hlls) = block_editor + .into_values() + .map(|(block_meta, hll)| { + Ok(( + block_meta, + encode_current_block_hll(hll, ¤t_hll_column_ids)?, + )) + }) + .collect::>>()? + .into_iter() + .unzip(); let stats = generate_segment_stats(new_hlls)?; (new_blocks, stats, Some(segment_info.summary)) } else { @@ -616,7 +661,14 @@ impl TableMutationAggregator { .replaced_blocks .into_iter() .sorted_by(|a, b| a.0.cmp(&b.0)) - .map(|(_, meta)| meta) + .map(|(_, (block_meta, hll))| { + Ok(( + block_meta, + normalize_current_block_hll(hll, ¤t_hll_column_ids)?, + )) + }) + .collect::>>()? + .into_iter() .unzip(); let stats = generate_segment_stats(new_hlls)?; (new_blocks, stats, None) @@ -974,3 +1026,60 @@ fn generate_segment_stats(hlls: Vec>) -> Result, + replacement: Option, + updated_column_ids: &[ColumnId], +) -> Result> { + let mut merged = previous.unwrap_or_default(); + for column_id in updated_column_ids { + merged.remove(column_id); + } + if let Some(replacement) = replacement { + if let Some(replacement) = decode_column_hll(&replacement)? { + merged.extend(replacement); + } + } + + if merged.is_empty() { + Ok(None) + } else { + Ok(Some(merged)) + } +} + +fn retain_hll_columns( + mut hll: BlockHLL, + current_hll_column_ids: &HashSet, +) -> Option { + hll.retain(|column_id, _| current_hll_column_ids.contains(column_id)); + (!hll.is_empty()).then_some(hll) +} + +fn decode_current_block_hll( + hll: Option<&RawBlockHLL>, + current_hll_column_ids: &HashSet, +) -> Result> { + let Some(hll) = hll.map(decode_column_hll).transpose()?.flatten() else { + return Ok(None); + }; + Ok(retain_hll_columns(hll, current_hll_column_ids)) +} + +fn encode_current_block_hll( + hll: Option, + current_hll_column_ids: &HashSet, +) -> Result> { + hll.and_then(|hll| retain_hll_columns(hll, current_hll_column_ids)) + .map(|hll| encode_column_hll(&hll)) + .transpose() +} + +fn normalize_current_block_hll( + hll: Option, + current_hll_column_ids: &HashSet, +) -> Result> { + let hll = hll.as_ref().map(decode_column_hll).transpose()?.flatten(); + encode_current_block_hll(hll, current_hll_column_ids) +} diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_partition_by.rs b/src/query/storages/fuse/src/operations/common/processors/transform_partition_by.rs index 29ef8a9a60f03..079aadb96ba0f 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_partition_by.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_partition_by.rs @@ -247,6 +247,7 @@ mod tests { ClusterStatsGenType::Generally, 2, 0, + None, )); let block = DataBlock::new_from_columns(vec![ Int32Type::from_data(vec![0, 0, 1, 1, 2]), @@ -283,6 +284,7 @@ mod tests { ClusterStatsGenType::Generally, 1, 0, + None, )); let block = DataBlock::new_from_columns(vec![Int32Type::from_data(vec![1])]) .add_meta(Some(Box::new(meta)))?; diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs index 869c13339a849..74607d629d5dc 100644 --- a/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs +++ b/src/query/storages/fuse/src/operations/common/processors/transform_serialize_block.rs @@ -23,6 +23,7 @@ use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::ComputedExpr; use databend_common_expression::DataBlock; +use databend_common_expression::FieldIndex; use databend_common_expression::TableSchema; use databend_common_metrics::storage::metrics_inc_recluster_write_block_nums; use databend_common_pipeline::core::Event; @@ -35,6 +36,7 @@ use databend_common_sql::executor::physical_plans::MutationKind; use databend_common_storage::MutationStatus; use databend_storages_common_index::BloomIndex; use databend_storages_common_index::RangeIndex; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use opendal::Operator; @@ -53,17 +55,23 @@ use crate::operations::mutation::ClusterStatsGenType; use crate::operations::mutation::SerializeDataMeta; use crate::statistics::ClusterStatsGenerator; +struct PendingSerialization { + block: DataBlock, + stats_type: ClusterStatsGenType, + index: Option, +} + #[allow(clippy::large_enum_variant)] enum State { Consume, NeedSerialize { - block: DataBlock, - stats_type: ClusterStatsGenType, - index: Option, + pending: PendingSerialization, + origin_block_meta: Option>, }, Serialized { serialized: BlockSerialization, index: Option, + write_progress: ProgressValues, }, } @@ -77,6 +85,7 @@ pub struct TransformSerializeBlock { dal: Operator, table_id: Option, // Only used in multi table insert kind: MutationKind, + updated_field_indices: Option>, pending_merge_hll: bool, pending_logical_change: (u64, u64), } @@ -99,6 +108,7 @@ impl TransformSerializeBlock { cluster_stats_gen, kind, false, + None, table_meta_timestamps, ) } @@ -120,6 +130,30 @@ impl TransformSerializeBlock { cluster_stats_gen, kind, true, + None, + table_meta_timestamps, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn try_create_for_update( + ctx: Arc, + input: Arc, + output: Arc, + table: &FuseTable, + cluster_stats_gen: ClusterStatsGenerator, + updated_field_indices: Vec, + table_meta_timestamps: TableMetaTimestamps, + ) -> Result { + Self::do_create( + ctx, + input, + output, + table, + cluster_stats_gen, + MutationKind::Update, + false, + Some(updated_field_indices), table_meta_timestamps, ) } @@ -132,6 +166,7 @@ impl TransformSerializeBlock { cluster_stats_gen: ClusterStatsGenerator, kind: MutationKind, with_tid: bool, + updated_field_indices: Option>, table_meta_timestamps: TableMetaTimestamps, ) -> Result { let schema = table.schema(); @@ -223,6 +258,7 @@ impl TransformSerializeBlock { dal: table.get_operator(), table_id: if with_tid { Some(table.get_id()) } else { None }, kind, + updated_field_indices, pending_merge_hll: false, pending_logical_change: (0, 0), }) @@ -335,9 +371,12 @@ impl Processor for TransformSerializeBlock { serialize_block.logical_deleted_rows, ); self.state = State::NeedSerialize { - block: input_data, - stats_type: serialize_block.stats_type, - index: Some(serialize_block.index), + pending: PendingSerialization { + block: input_data, + stats_type: serialize_block.stats_type, + index: Some(serialize_block.index), + }, + origin_block_meta: serialize_block.origin_block_meta, }; Ok(Event::Sync) } @@ -345,9 +384,12 @@ impl Processor for TransformSerializeBlock { SerializeDataMeta::SerializeAppend => { self.pending_merge_hll = true; self.state = State::NeedSerialize { - block: input_data, - stats_type: ClusterStatsGenType::Generally, - index: None, + pending: PendingSerialization { + block: input_data, + stats_type: ClusterStatsGenType::Generally, + index: None, + }, + origin_block_meta: None, }; Ok(Event::Sync) } @@ -371,9 +413,12 @@ impl Processor for TransformSerializeBlock { Ok(Event::NeedConsume) } else { self.state = State::NeedSerialize { - block: input_data, - stats_type: ClusterStatsGenType::Generally, - index: None, + pending: PendingSerialization { + block: input_data, + stats_type: ClusterStatsGenType::Generally, + index: None, + }, + origin_block_meta: None, }; Ok(Event::Sync) } @@ -382,25 +427,62 @@ impl Processor for TransformSerializeBlock { fn process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { State::NeedSerialize { - block, - stats_type, - index, + pending: + PendingSerialization { + block, + stats_type, + index, + }, + origin_block_meta, } => { // Check if the datablock is valid, this is needed to ensure data is correct block.check_valid()?; - let serialized = - self.block_builder - .build(block, |block, generator| match &stats_type { - ClusterStatsGenType::Generally => generator.gen_stats_for_append(block), - ClusterStatsGenType::WithOrigin(origin_stats) => { - let cluster_stats = generator - .gen_with_origin_stats(&block, origin_stats.clone())?; - Ok((cluster_stats, block)) - } - })?; - - self.state = State::Serialized { serialized, index }; + let (serialized, write_progress_bytes) = + if let (Some(updated_field_indices), Some(origin_block_meta)) = + (&self.updated_field_indices, origin_block_meta) + { + let write_progress_bytes = block.estimate_block_size(block.num_columns()); + let serialized = self.block_builder.build_column_group( + block, + &origin_block_meta, + updated_field_indices, + )?; + (serialized, write_progress_bytes) + } else { + let serialized = + self.block_builder.build( + block, + |block, generator| match &stats_type { + ClusterStatsGenType::Generally => { + generator.gen_stats_for_append(block) + } + ClusterStatsGenType::WithOrigin(origin_stats) => { + let cluster_stats = generator + .gen_with_origin_stats(&block, origin_stats.clone())?; + Ok((cluster_stats, block)) + } + }, + )?; + let virtual_column_size = serialized + .virtual_column_state + .as_ref() + .map(|state| state.draft_virtual_block_meta.virtual_column_size) + .unwrap_or_default(); + let write_progress_bytes = + (serialized.block_meta.block_size + virtual_column_size) as usize; + (serialized, write_progress_bytes) + }; + let write_progress = ProgressValues { + rows: serialized.block_meta.row_count as usize, + bytes: write_progress_bytes, + }; + + self.state = State::Serialized { + serialized, + index, + write_progress, + }; } _ => return Err(ErrorCode::Internal("It's a bug.")), } @@ -410,28 +492,20 @@ impl Processor for TransformSerializeBlock { #[async_backtrace::framed] async fn async_process(&mut self) -> Result<()> { match std::mem::replace(&mut self.state, State::Consume) { - State::Serialized { serialized, index } => { + State::Serialized { + serialized, + index, + write_progress, + } => { let merge_hll = std::mem::take(&mut self.pending_merge_hll); let (logical_updated_rows, logical_deleted_rows) = std::mem::take(&mut self.pending_logical_change); let extended_block_meta = BlockWriter::write_down(&self.dal, serialized).await?; - let bytes = if let Some(draft_virtual_block_meta) = - &extended_block_meta.draft_virtual_block_meta - { - (extended_block_meta.block_meta.block_size - + draft_virtual_block_meta.virtual_column_size) as usize - } else { - extended_block_meta.block_meta.block_size as usize - }; - let progress_values = ProgressValues { - rows: extended_block_meta.block_meta.row_count as usize, - bytes, - }; self.block_builder .ctx .get_write_progress() - .incr(&progress_values); + .incr(&write_progress); let mutation_log_data_block = if let Some(index) = index { // we are replacing the block represented by the `index` diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs index 3629d9d297367..5a19c794cf9c4 100644 --- a/src/query/storages/fuse/src/operations/gc.rs +++ b/src/query/storages/fuse/src/operations/gc.rs @@ -49,6 +49,7 @@ use log::warn; use crate::FUSE_TBL_SNAPSHOT_PREFIX; use crate::FuseTable; +use crate::block_bloom_index_locations; use crate::index::InvertedIndexFile; use crate::io::InvertedIndexReader; use crate::io::SegmentsIO; @@ -803,10 +804,13 @@ impl TryFrom> for LocationTuple { let mut hll_location = HashSet::new(); let block_metas = value.block_metas()?; for block_meta in block_metas.into_iter() { - block_location.insert(block_meta.location.0.clone()); - if let Some(bloom_loc) = &block_meta.bloom_filter_index_location { - bloom_location.insert(bloom_loc.0.clone()); - } + block_location.extend( + block_meta + .data_file_locations() + .map(|location| location.0.clone()), + ); + bloom_location + .extend(block_bloom_index_locations(&block_meta).map(|location| location.0)); } if let Some(loc) = value.as_ref().summary.additional_stats_loc() { hll_location.insert(loc.0); diff --git a/src/query/storages/fuse/src/operations/mod.rs b/src/query/storages/fuse/src/operations/mod.rs index 767991f34ef8f..2edba084ee5c9 100644 --- a/src/query/storages/fuse/src/operations/mod.rs +++ b/src/query/storages/fuse/src/operations/mod.rs @@ -61,6 +61,8 @@ pub use snapshot_hint::*; pub use table_index::do_refresh_table_index; pub use util::*; pub use vacuum::ASSUMPTION_MAX_TXN_DURATION; +pub use vacuum::VacuumObjectKeyPolicy; +pub use vacuum::is_vacuum_object_gc_candidate; pub use vacuum::vacuum_tables_from_info; pub use virtual_column::VirtualColumnVacuumResult; pub use virtual_column::cleanup_vacuum_virtual_column_files; diff --git a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs index fb2b7b11ab140..f5aaaf90419fa 100644 --- a/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs +++ b/src/query/storages/fuse/src/operations/mutation/meta/mutation_meta.rs @@ -21,6 +21,7 @@ use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::ClusterStatistics; use crate::BlockReadResult; +use crate::FuseColumnGroupPartInfo; use crate::operations::common::BlockMetaIndex; use crate::operations::mutation::CompactExtraInfo; use crate::operations::mutation::DeletedSegmentInfo; @@ -59,6 +60,8 @@ pub struct SerializeBlock { pub logical_updated_rows: u64, #[serde(default)] pub logical_deleted_rows: u64, + #[serde(default)] + pub origin_block_meta: Option>, } impl SerializeBlock { @@ -67,20 +70,25 @@ impl SerializeBlock { stats_type: ClusterStatsGenType, logical_updated_rows: u64, logical_deleted_rows: u64, + origin_block_meta: Option>, ) -> Self { SerializeBlock { index, stats_type, logical_updated_rows, logical_deleted_rows, + origin_block_meta, } } } pub enum CompactSourceMeta { Concat { - read_res: Vec, - metas: Vec>, + blocks: Vec<( + BlockReadResult, + Arc, + Vec, + )>, index: BlockMetaIndex, }, Extras(CompactExtraInfo), diff --git a/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs b/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs index 5a8d2564c9052..7cb314339fea3 100644 --- a/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs +++ b/src/query/storages/fuse/src/operations/mutation/meta/mutation_part.rs @@ -16,13 +16,14 @@ use std::any::Any; use std::collections::hash_map::DefaultHasher; use std::hash::Hash; use std::hash::Hasher; +use std::sync::Arc; use databend_common_catalog::plan::PartInfo; use databend_common_catalog::plan::PartInfoPtr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_storages_common_pruner::BlockMetaIndex; -use databend_storages_common_table_meta::meta::ClusterStatistics; +use databend_storages_common_table_meta::meta::BlockMeta; use databend_storages_common_table_meta::meta::Statistics; use crate::operations::mutation::SegmentIndex; @@ -82,7 +83,7 @@ impl DeletedSegmentInfo { #[derive(serde::Serialize, serde::Deserialize, PartialEq)] pub struct MutationPartInfo { pub index: BlockMetaIndex, - pub cluster_stats: Option, + pub block_meta: Arc, pub inner_part: PartInfoPtr, pub whole_block_mutation: bool, } diff --git a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs index bf0868f931d95..0bf81ec616230 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/compact_source.rs @@ -19,6 +19,7 @@ use databend_common_base::base::Progress; use databend_common_base::base::ProgressValues; use databend_common_catalog::plan::gen_mutation_stream_meta; use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_metrics::storage::*; @@ -89,14 +90,15 @@ impl PrefetchAsyncSource for CompactSource { metrics_inc_compact_block_read_bytes(block.block_size); } - block_reader - .read_columns_data_by_merge_io( + let column_groups = block_reader.projected_column_groups(&block); + let read_res = block_reader + .read_column_groups_data_by_merge_io( &settings, - &block.location.0, - &block.col_metas, + &column_groups, &None, ) - .await + .await?; + Ok::<_, ErrorCode>((read_res, block, column_groups)) }) .await .unwrap() @@ -105,14 +107,13 @@ impl PrefetchAsyncSource for CompactSource { let start = Instant::now(); - let read_res = futures::future::try_join_all(task_futures).await?; + let blocks = futures::future::try_join_all(task_futures).await?; // Perf. { metrics_inc_compact_block_read_milliseconds(start.elapsed().as_millis() as u64); } Box::new(CompactSourceMeta::Concat { - read_res, - metas: task.blocks.clone(), + blocks, index: task.index.clone(), }) } @@ -154,20 +155,25 @@ impl BlockMetaTransform for CompactTransform { fn transform(&mut self, meta: CompactSourceMeta) -> Result> { match meta { - CompactSourceMeta::Concat { - read_res, - metas, - index, - } => { - let blocks = read_res + CompactSourceMeta::Concat { blocks, index } => { + let blocks = blocks .into_iter() - .zip(metas.into_iter()) - .map(|(data, meta)| { - let mut block = self.block_reader.deserialize_chunks_with_meta( - &meta.as_ref().into(), - &self.storage_format, - data, - )?; + .map(|(data, meta, column_groups)| { + let chunks = data.columns_chunks()?; + let mut block = match self.storage_format { + FuseStorageFormat::Parquet => { + self.block_reader.deserialize_column_groups( + meta.row_count as usize, + &column_groups, + chunks, + &meta.compression, + None, + )? + } + FuseStorageFormat::Unsupported => { + return Err(crate::unsupported_storage_format_error()); + } + }; self.scan_progress.incr(&ProgressValues { rows: block.num_rows(), @@ -189,6 +195,7 @@ impl BlockMetaTransform for CompactTransform { ClusterStatsGenType::Generally, 0, 0, + None, ))); let new_block = block.add_meta(Some(meta))?; Ok(vec![new_block]) diff --git a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs index 6c585915dce5c..b19bf08523071 100644 --- a/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation/processors/mutation_source.rs @@ -38,6 +38,7 @@ use databend_common_pipeline::core::ProcessorPtr; use databend_common_sql::evaluator::BlockOperator; use databend_common_storage::MutationStatus; use databend_storages_common_io::ReadSettings; +use databend_storages_common_table_meta::meta::BlockMeta; use crate::BlockReadResult; use crate::FuseStorageFormat; @@ -90,6 +91,7 @@ pub struct MutationSource { index: BlockMetaIndex, stats_type: ClusterStatsGenType, update_rows: u64, + origin_block_meta: Option>, deleted_rows: u64, } @@ -120,6 +122,7 @@ impl MutationSource { index: BlockMetaIndex::default(), stats_type: ClusterStatsGenType::Generally, update_rows: 0, + origin_block_meta: None, deleted_rows: 0, }))) } @@ -226,6 +229,7 @@ impl Processor for MutationSource { self.stats_type.clone(), 0, affect_rows as u64, + None, ), )); self.state = State::Output( @@ -330,6 +334,7 @@ impl Processor for MutationSource { self.stats_type.clone(), update_rows, deleted_rows, + self.origin_block_meta.take(), ))); let meta: BlockMetaInfoPtr = if self.update_stream_columns { Box::new(gen_mutation_stream_meta(Some(inner_meta), &path)?) @@ -364,8 +369,11 @@ impl Processor for MutationSource { block_idx: part.index.block_idx, }; if matches!(self.action, MutationAction::Deletion) { - self.stats_type = - ClusterStatsGenType::WithOrigin(part.cluster_stats.clone()); + self.stats_type = ClusterStatsGenType::WithOrigin( + part.block_meta.cluster_stats.clone(), + ); + } else { + self.origin_block_meta = Some(part.block_meta.clone()); } let inner_part = part.inner_part.clone(); @@ -382,6 +390,7 @@ impl Processor for MutationSource { self.stats_type.clone(), 0, fuse_part.nums_rows as u64, + None, ), )); self.state = State::Output( @@ -391,10 +400,9 @@ impl Processor for MutationSource { } else { let read_res = self .block_reader - .read_columns_data_by_merge_io( + .read_column_groups_data_by_merge_io( &settings, - &fuse_part.location, - &fuse_part.columns_meta, + &fuse_part.column_groups, &None, ) .await?; @@ -413,10 +421,9 @@ impl Processor for MutationSource { let settings = ReadSettings::from_ctx(&self.ctx)?; let read_res = remain_reader - .read_columns_data_by_merge_io( + .read_column_groups_data_by_merge_io( &settings, - &fuse_part.location, - &fuse_part.columns_meta, + &fuse_part.column_groups, &None, ) .await?; diff --git a/src/query/storages/fuse/src/operations/mutation_source.rs b/src/query/storages/fuse/src/operations/mutation_source.rs index 2be438e301461..56e06668825ca 100644 --- a/src/query/storages/fuse/src/operations/mutation_source.rs +++ b/src/query/storages/fuse/src/operations/mutation_source.rs @@ -56,14 +56,17 @@ impl FuseTable { col_indices: Vec, pipeline: &mut Pipeline, mutation_action: MutationAction, + partial_update: bool, ) -> Result<()> { let all_column_indices = self.all_column_indices(); - let col_indices = + let mut col_indices = if matches!(mutation_action, MutationAction::Deletion) || !col_indices.is_empty() { col_indices } else { all_column_indices.clone() }; + col_indices.sort_unstable(); + col_indices.dedup(); let projection = Projection::Columns(col_indices.clone()); let update_stream_columns = self.change_tracking_enabled(); let block_reader = self.create_block_reader(ctx.clone(), projection, false)?; @@ -76,10 +79,14 @@ impl FuseTable { })); let num_column_indices = self.schema_with_stream().fields().len(); - let remain_column_indices: Vec = all_column_indices - .into_iter() - .filter(|index| !col_indices.contains(index)) - .collect(); + let remain_column_indices: Vec = if partial_update { + vec![] + } else { + all_column_indices + .into_iter() + .filter(|index| !col_indices.contains(index)) + .collect() + }; let mut source_col_indices = col_indices; if matches!(mutation_action, MutationAction::Deletion) && update_stream_columns || matches!(mutation_action, MutationAction::Update) && filter_expr.is_some() @@ -261,13 +268,12 @@ impl FuseTable { .into_iter() .zip(inner_parts.partitions.into_iter()) .map(|((index, block_meta), inner_part)| { - let cluster_stats = block_meta.cluster_stats.clone(); let key = (index.segment_idx, index.block_idx); let whole_block_mutation = whole_block_deletions.contains(&key); let part_info_ptr: PartInfoPtr = Arc::new(Box::new(Mutation::MutationPartInfo(MutationPartInfo { index, - cluster_stats, + block_meta, inner_part, whole_block_mutation, }))); diff --git a/src/query/storages/fuse/src/operations/read/block_format/parquet.rs b/src/query/storages/fuse/src/operations/read/block_format/parquet.rs index 3f78bf4882858..9e6990851b577 100644 --- a/src/query/storages/fuse/src/operations/read/block_format/parquet.rs +++ b/src/query/storages/fuse/src/operations/read/block_format/parquet.rs @@ -12,19 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; -use std::collections::HashSet; - -use databend_common_exception::Result; -use databend_common_expression::ColumnId; use databend_common_storage::read_metadata_async; -use databend_storages_common_io::ReadSettings; -use databend_storages_common_table_meta::meta::ColumnMeta; use opendal::Operator; use super::ReadBlockMeta; -use crate::io::BlockReadContext; -use crate::io::BlockReadResult; use crate::io::build_columns_meta; pub struct FuseParquetBlockFormat; @@ -34,20 +25,6 @@ impl FuseParquetBlockFormat { Self } - /// Reads raw column data from the given block location. - pub async fn read_data_by_merge_io( - &self, - read_ctx: &BlockReadContext, - settings: &ReadSettings, - location: &str, - columns_meta: &HashMap, - ignore_column_ids: &Option>, - ) -> Result { - read_ctx - .read_columns_data_by_merge_io(settings, location, columns_meta, ignore_column_ids) - .await - } - /// Reads the metadata needed to fetch an arbitrary block location. pub async fn read_block_meta( &self, diff --git a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs index 375acb536a130..7ca09a46796c8 100644 --- a/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs +++ b/src/query/storages/fuse/src/operations/read/parquet_rows_fetcher.rs @@ -27,7 +27,6 @@ use databend_common_catalog::plan::split_row_id; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::ColumnId; use databend_common_expression::DataBlock; use databend_common_expression::TableSchemaRef; use databend_common_storage::ColumnNodes; @@ -37,7 +36,6 @@ use databend_storages_common_cache::InMemoryLruCache; use databend_storages_common_cache::LoadParams; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::BlockMeta; -use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::Compression; use databend_storages_common_table_meta::meta::TableSnapshot; use futures_util::stream::FuturesUnordered; @@ -50,6 +48,7 @@ use super::fuse_rows_fetcher::RowsFetchMetadata; use super::fuse_rows_fetcher::RowsFetcher; use crate::BlockReadResult; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::FuseTable; use crate::io::BlockReader; use crate::io::CompactSegmentInfoReader; @@ -93,10 +92,9 @@ pub struct RowsFetchMetadataImpl { // block_bytes after projection pub block_bytes: usize, - pub location: String, pub nums_rows: usize, pub compression: Compression, - pub columns_meta: HashMap, + pub column_groups: Vec, } impl RowsFetchMetadata for RowsFetchMetadataImpl { @@ -328,12 +326,7 @@ impl ParquetRowsFetcher { .await .expect("row-fetch io semaphore never closed"); let chunk = reader - .read_columns_data_by_merge_io( - &settings, - &metadata.location, - &metadata.columns_meta, - &None, - ) + .read_column_groups_data_by_merge_io(&settings, &metadata.column_groups, &None) .await?; Ok(( @@ -363,20 +356,24 @@ impl ParquetRowsFetcher { let compression_ratio = block_meta.block_size as f64 / block_meta.file_size as f64; let mut block_bytes = 0; let mut average_bytes = 0; - for (column_id, column_meta) in &fuse_part.columns_meta { - if let Some(columns_stat) = &fuse_part.columns_stat { - if let Some(column_stat) = columns_stat.get(column_id) { - average_bytes += column_stat.in_memory_size as usize / fuse_part.nums_rows; - block_bytes += column_stat.in_memory_size as usize; - - continue; + for group in &fuse_part.column_groups { + for (column_id, column_meta) in &group.columns_meta { + if let Some(columns_stat) = &fuse_part.columns_stat { + if let Some(column_stat) = columns_stat.get(column_id) { + average_bytes += + column_stat.in_memory_size as usize / fuse_part.nums_rows; + block_bytes += column_stat.in_memory_size as usize; + + continue; + } } - } - let compressed_size = column_meta.read_bytes(&None); - let estimate_memory_size = (compressed_size as f64 * compression_ratio) as usize; - block_bytes += estimate_memory_size; - average_bytes += estimate_memory_size / fuse_part.nums_rows; + let compressed_size = column_meta.read_bytes(&None); + let estimate_memory_size = + (compressed_size as f64 * compression_ratio) as usize; + block_bytes += estimate_memory_size; + average_bytes += estimate_memory_size / fuse_part.nums_rows; + } } metadata.push(RowsFetchMetadataImpl { @@ -384,8 +381,7 @@ impl ParquetRowsFetcher { block_bytes, nums_rows: fuse_part.nums_rows, compression: fuse_part.compression, - location: fuse_part.location.clone(), - columns_meta: fuse_part.columns_meta.clone(), + column_groups: fuse_part.column_groups.clone(), }); } @@ -398,12 +394,11 @@ impl ParquetRowsFetcher { chunk: BlockReadResult, ) -> Result { let columns_chunks = chunk.columns_chunks()?; - reader.deserialize_parquet_chunks( + reader.deserialize_column_groups( metadata.nums_rows, - &metadata.columns_meta, + &metadata.column_groups, columns_chunks, &metadata.compression, - &metadata.location, None, ) } diff --git a/src/query/storages/fuse/src/operations/read/read_block_context.rs b/src/query/storages/fuse/src/operations/read/read_block_context.rs index a0121ee638a9e..b6e912a30253b 100644 --- a/src/query/storages/fuse/src/operations/read/read_block_context.rs +++ b/src/query/storages/fuse/src/operations/read/read_block_context.rs @@ -23,6 +23,7 @@ use log::debug; use super::block_format::FuseParquetBlockFormat; use super::parquet_data_source::ParquetDataSource; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; use crate::FuseStorageFormat; use crate::io::AggIndexReader; use crate::io::BlockReadContext; @@ -77,12 +78,10 @@ impl ReadBlockContext { .and_then(|source| source.ignore_column_ids.clone()); let data = self - .block_format - .read_data_by_merge_io( - &self.block_read_ctx, + .block_read_ctx + .read_column_groups_data_by_merge_io( &self.read_settings, - &fuse_part.location, - &fuse_part.columns_meta, + &fuse_part.column_groups, &ignore_column_ids, ) .await?; @@ -112,15 +111,13 @@ impl ReadBlockContext { return Ok(None); }; - let data = match self - .block_format - .read_data_by_merge_io( - &index_block_read_ctx, - &self.read_settings, - &location, - &block_meta.columns_meta, - &None, - ) + let num_rows = block_meta.num_rows; + let column_groups = vec![FuseColumnGroupPartInfo { + location: location.clone(), + columns_meta: block_meta.columns_meta, + }]; + let data = match index_block_read_ctx + .read_column_groups_data_by_merge_io(&self.read_settings, &column_groups, &None) .await { Ok(data) => data, @@ -134,8 +131,9 @@ impl ReadBlockContext { location, None, 0, - block_meta.num_rows, - block_meta.columns_meta, + vec![], + num_rows, + column_groups, None, index_reader.compression().into(), None, diff --git a/src/query/storages/fuse/src/operations/read_partitions.rs b/src/query/storages/fuse/src/operations/read_partitions.rs index e29fbfc6ee72b..dff999961af3e 100644 --- a/src/query/storages/fuse/src/operations/read_partitions.rs +++ b/src/query/storages/fuse/src/operations/read_partitions.rs @@ -87,6 +87,10 @@ use crate::FuseLazyPartInfo; use crate::FuseSegmentFormat; use crate::FuseTable; use crate::fuse_part::FuseBlockPartInfo; +use crate::fuse_part::block_bloom_index_size; +use crate::fuse_part::column_group_bloom_files; +use crate::fuse_part::legacy_bloom_index_location; +use crate::fuse_part::project_column_groups; use crate::io::BloomIndexRebuilder; use crate::pruning::BlockPruner; use crate::pruning::FusePruner; @@ -1364,23 +1368,22 @@ impl FuseTable { top_k: &Option<(TopK, Scalar)>, meta: &BlockMeta, ) -> PartInfoPtr { - let mut columns_meta = HashMap::with_capacity(meta.col_metas.len()); let mut columns_stats = HashMap::with_capacity(meta.col_stats.len()); let mut spatial_stats = HashMap::new(); - for column_id in meta.col_metas.keys() { - // ignore all deleted field - if let Some(schema) = schema { - if schema.is_column_deleted(*column_id) { - continue; - } - } - - // ignore column this block dose not exist - if let Some(meta) = meta.col_metas.get(column_id) { - columns_meta.insert(*column_id, meta.clone()); - } + let mut projected_column_ids = if meta.column_groups.is_empty() { + meta.col_metas.keys().copied().collect::>() + } else { + meta.physical_column_groups() + .iter() + .flat_map(|group| group.active_column_ids.iter().copied()) + .collect::>() + }; + if let Some(schema) = schema { + projected_column_ids.retain(|column_id| !schema.is_column_deleted(*column_id)); + } + for column_id in &projected_column_ids { if let Some(stat) = meta.col_stats.get(column_id) { columns_stats.insert(*column_id, stat.clone()); } @@ -1392,6 +1395,7 @@ impl FuseTable { spatial_stats.insert(*column_id, stats.clone()); } } + let column_groups = project_column_groups(meta, &projected_column_ids); let rows_count = meta.row_count; let location = meta.location.0.clone(); @@ -1406,10 +1410,11 @@ impl FuseTable { FuseBlockPartInfo::create( location, - meta.bloom_filter_index_location.clone(), - meta.bloom_filter_index_size, + legacy_bloom_index_location(meta).cloned(), + block_bloom_index_size(meta), + column_group_bloom_files(meta), rows_count, - columns_meta, + column_groups, Some(columns_stats), meta.compression(), sort_min_max, @@ -1425,17 +1430,14 @@ impl FuseTable { top_k: Option<(TopK, Scalar)>, projection: &Projection, ) -> PartInfoPtr { - let mut columns_meta = HashMap::with_capacity(projection.len()); let mut columns_stat = HashMap::with_capacity(projection.len()); let mut spatial_stats = HashMap::new(); + let mut projected_column_ids = HashSet::with_capacity(projection.len()); let columns = projection.project_column_nodes(column_nodes).unwrap(); for column in &columns { for column_id in &column.leaf_column_ids { - // ignore column this block dose not exist - if let Some(column_meta) = meta.col_metas.get(column_id) { - columns_meta.insert(*column_id, column_meta.clone()); - } + projected_column_ids.insert(*column_id); if let Some(column_stat) = meta.col_stats.get(column_id) { columns_stat.insert(*column_id, column_stat.clone()); } @@ -1448,6 +1450,7 @@ impl FuseTable { } } } + let column_groups = project_column_groups(meta, &projected_column_ids); let rows_count = meta.row_count; let location = meta.location.0.clone(); @@ -1464,10 +1467,11 @@ impl FuseTable { // not the count the rows in this partition FuseBlockPartInfo::create( location, - meta.bloom_filter_index_location.clone(), - meta.bloom_filter_index_size, + legacy_bloom_index_location(meta).cloned(), + block_bloom_index_size(meta), + column_group_bloom_files(meta), rows_count, - columns_meta, + column_groups, Some(columns_stat), meta.compression(), sort_min_max, @@ -1479,6 +1483,11 @@ impl FuseTable { #[cfg(test)] mod tests { + use databend_storages_common_table_meta::meta::ColumnGroupFileMeta; + use databend_storages_common_table_meta::meta::ColumnMeta; + use databend_storages_common_table_meta::meta::Compression; + use databend_storages_common_table_meta::meta::SingleColumnMeta; + use super::*; fn segment_location(segment_idx: usize, location: &str, snapshot_loc: &str) -> SegmentLocation { @@ -1506,4 +1515,79 @@ mod tests { changed_segment[1].location = ("segment-c".to_string(), 1); assert!(!same_segment_locations(&original, &changed_segment)); } + + #[test] + fn test_projected_column_groups() { + let column_1_meta = ColumnMeta::Parquet(SingleColumnMeta::new(0, 10, 3)); + let stale_column_2_meta = ColumnMeta::Parquet(SingleColumnMeta::new(10, 10, 3)); + let active_column_2_meta = ColumnMeta::Parquet(SingleColumnMeta::new(0, 12, 3)); + let mut block_meta = BlockMeta::new( + 3, + 30, + 22, + HashMap::new(), + HashMap::from([ + (1, column_1_meta.clone()), + (2, active_column_2_meta.clone()), + ]), + None, + ("group-2.parquet".to_string(), 2), + None, + 0, + None, + None, + None, + None, + None, + None, + None, + None, + Compression::Zstd, + None, + ); + + let legacy_groups = project_column_groups(&block_meta, &HashSet::from([2])); + assert_eq!(legacy_groups.len(), 1); + assert_eq!(legacy_groups[0].location, "group-2.parquet"); + assert_eq!( + legacy_groups[0].columns_meta, + HashMap::from([(2, active_column_2_meta.clone())]) + ); + + block_meta.bloom_filter_index_location = Some(("stale-bloom.parquet".to_string(), 4)); + block_meta.bloom_filter_index_size = 10; + block_meta.column_groups = vec![ + ColumnGroupFileMeta { + active_column_ids: vec![1], + location: ("group-1.parquet".to_string(), 2), + file_size: 10, + uncompressed_size: 10, + leaf_column_metas: HashMap::from([(1, column_1_meta), (2, stale_column_2_meta)]), + bloom: None, + }, + ColumnGroupFileMeta { + active_column_ids: vec![2], + location: block_meta.location.clone(), + file_size: 12, + uncompressed_size: 12, + leaf_column_metas: HashMap::from([(2, active_column_2_meta.clone())]), + bloom: None, + }, + ]; + + let column_groups = project_column_groups(&block_meta, &HashSet::from([2])); + + assert_eq!(column_groups.len(), 1); + assert_eq!(column_groups[0].location, "group-2.parquet"); + assert_eq!( + column_groups[0].columns_meta, + HashMap::from([(2, active_column_2_meta)]) + ); + + let part = FuseTable::all_columns_part(None, &None, &None, &block_meta); + let part = FuseBlockPartInfo::from_part(&part).unwrap(); + assert!(part.bloom_filter_index_location.is_none()); + assert_eq!(part.bloom_filter_index_size, 0); + assert!(part.bloom_index_layout().is_none()); + } } diff --git a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs index 831d87b7a9b88..b626dfb90279b 100644 --- a/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs +++ b/src/query/storages/fuse/src/operations/replace_into/mutator/replace_into_operation_agg.rs @@ -637,6 +637,9 @@ impl AggregationContext { } async fn read_block(&self, reader: &BlockReader, block_meta: &BlockMeta) -> Result { + // REPLACE INTO is outside the Partial UPDATE feature contract. This method is defined only + // for legacy single-file blocks and intentionally does not read `block_meta.column_groups`; + // behavior for column-group blocks is outside its contract. let merged_io_read_result = reader .read_columns_data_by_merge_io( &self.read_settings, diff --git a/src/query/storages/fuse/src/operations/table_index.rs b/src/query/storages/fuse/src/operations/table_index.rs index 8ce35fd216b3b..1bcabc2d9b69d 100644 --- a/src/query/storages/fuse/src/operations/table_index.rs +++ b/src/query/storages/fuse/src/operations/table_index.rs @@ -258,7 +258,7 @@ pub async fn do_refresh_table_index( pipeline.try_resize(1)?; let table_meta_timestamps = ctx.get_table_meta_timestamps(fuse_table, Some(snapshot.clone()))?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( fuse_table, ctx.clone(), @@ -269,7 +269,7 @@ pub async fn do_refresh_table_index( MutationKind::Refresh, table_meta_timestamps, ) - }); + })?; let prev_snapshot_id = snapshot.snapshot_id; let snapshot_gen = MutationGenerator::new(Some(snapshot), MutationKind::Refresh); diff --git a/src/query/storages/fuse/src/operations/util.rs b/src/query/storages/fuse/src/operations/util.rs index c2a65329b937d..8f893ad5eca4c 100644 --- a/src/query/storages/fuse/src/operations/util.rs +++ b/src/query/storages/fuse/src/operations/util.rs @@ -119,13 +119,9 @@ pub async fn read_block( block_meta: &BlockMeta, read_settings: &ReadSettings, ) -> Result { + let column_groups = reader.projected_column_groups(block_meta); let merged_io_read_result = reader - .read_columns_data_by_merge_io( - read_settings, - &block_meta.location.0, - &block_meta.col_metas, - &None, - ) + .read_column_groups_data_by_merge_io(read_settings, &column_groups, &None) .await?; // deserialize block data @@ -136,14 +132,16 @@ pub async fn read_block( GlobalIORuntime::instance() .spawn(async move { let column_chunks = merged_io_read_result.columns_chunks()?; - reader.deserialize_chunks( - block_meta_ptr.location.0.as_str(), - block_meta_ptr.row_count as usize, - &block_meta_ptr.compression, - &block_meta_ptr.col_metas, - column_chunks, - &storage_format, - ) + match storage_format { + FuseStorageFormat::Parquet => reader.deserialize_column_groups( + block_meta_ptr.row_count as usize, + &column_groups, + column_chunks, + &block_meta_ptr.compression, + None, + ), + FuseStorageFormat::Unsupported => Err(crate::unsupported_storage_format_error()), + } }) .await .map_err(|e| { diff --git a/src/query/storages/fuse/src/operations/vacuum.rs b/src/query/storages/fuse/src/operations/vacuum.rs index ab45f91e4205f..ef40c120307be 100644 --- a/src/query/storages/fuse/src/operations/vacuum.rs +++ b/src/query/storages/fuse/src/operations/vacuum.rs @@ -36,6 +36,7 @@ use databend_storages_common_cache::TableSnapshot; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::VACUUM2_OBJECT_KEY_PREFIX; use databend_storages_common_table_meta::meta::is_uuid_v7; +use databend_storages_common_table_meta::meta::try_extract_uuid_str_from_path; use databend_storages_common_table_meta::meta::uuid_from_date_time; use futures_util::TryStreamExt; use log::info; @@ -44,6 +45,7 @@ use opendal::Entry; use opendal::ErrorKind; use opendal::Operator; use opendal::Scheme; +use uuid::Uuid; use crate::FuseTable; use crate::RetentionPolicy; @@ -84,6 +86,61 @@ use crate::io::TableMetaLocationGenerator; /// the above risks will not exist. pub const ASSUMPTION_MAX_TXN_DURATION: Duration = Duration::days(3); +/// Object-key policy used when deciding whether an unreferenced Vacuum2 object predates the GC +/// root. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VacuumObjectKeyPolicy { + /// The listing has already applied the lexicographic cutoff for `h`-prefixed Vacuum2 keys. + Vacuum2PrefixedBeforeRoot, + /// Current objects use a canonical, prefixless UUID V7 as the first file-name component. + PrefixlessUuidV7 { gc_root_timestamp: DateTime }, +} + +impl VacuumObjectKeyPolicy { + /// Return a decision when the object key itself is trustworthy. `None` selects the legacy + /// transaction-window fallback. + fn trusted_gc_candidate(self, path: &str) -> Option { + match self { + Self::Vacuum2PrefixedBeforeRoot => path + .rsplit('/') + .next() + .is_some_and(|name| name.starts_with(VACUUM2_OBJECT_KEY_PREFIX)) + .then_some(true), + Self::PrefixlessUuidV7 { gc_root_timestamp } => { + let uuid_str = try_extract_uuid_str_from_path(path).ok()?; + let file_name = path.rsplit('/').next()?; + if !file_name.starts_with(uuid_str) { + return None; + } + + let uuid = Uuid::parse_str(uuid_str).ok()?; + if !is_uuid_v7(&uuid) || uuid.as_simple().to_string() != uuid_str { + return None; + } + + let (seconds, nanos) = uuid.get_timestamp()?.to_unix(); + let created_at = DateTime::::from_timestamp(seconds.try_into().ok()?, nanos)?; + Some(created_at.timestamp_millis() < gc_root_timestamp.timestamp_millis()) + } + } + } +} + +/// Decide whether an unreferenced object is eligible for Vacuum2 collection. +/// +/// Trusted current keys use their embedded creation time and a strict GC-root cutoff. Legacy or +/// malformed keys use the compatibility window based on object modification time. +pub fn is_vacuum_object_gc_candidate( + path: &str, + modified: DateTime, + gc_root_meta_ts: DateTime, + key_policy: VacuumObjectKeyPolicy, +) -> bool { + key_policy + .trusted_gc_candidate(path) + .unwrap_or(modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts) +} + pub struct SnapshotGcSelection { pub gc_root: Arc, pub snapshots_to_gc: Vec, @@ -171,9 +228,9 @@ async fn is_gc_candidate_segment_block( gc_root_meta_ts: DateTime, ) -> Result { let path = entry.path(); - let last_part = path.rsplit('/').next().unwrap(); - if last_part.starts_with(VACUUM2_OBJECT_KEY_PREFIX) { - return Ok(true); + let key_policy = VacuumObjectKeyPolicy::Vacuum2PrefixedBeforeRoot; + if let Some(candidate) = key_policy.trusted_gc_candidate(path) { + return Ok(candidate); } let last_modified = if let Some(v) = entry.metadata().last_modified() { v @@ -188,7 +245,12 @@ async fn is_gc_candidate_segment_block( })? }; - Ok(last_modified + ASSUMPTION_MAX_TXN_DURATION < gc_root_meta_ts) + Ok(is_vacuum_object_gc_candidate( + path, + last_modified, + gc_root_meta_ts, + key_policy, + )) } impl FuseTable { @@ -721,3 +783,106 @@ pub fn slice_summary(s: &[T]) -> String { format!("{:?}", s) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn bloom_path(uuid: Uuid) -> String { + format!("1/2/_i_b_v2/{}_v4.parquet", uuid.as_simple()) + } + + #[test] + fn test_prefixless_uuid_v7_gc_classification() { + let gc_root_timestamp = + DateTime::::from_timestamp(1_700_000_000, 500_000_000).unwrap(); + let gc_root_meta_ts = gc_root_timestamp + Duration::days(10); + let recent = gc_root_meta_ts - Duration::hours(1); + let old = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION - Duration::hours(1); + let policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; + + let before_root = bloom_path(uuid_from_date_time( + gc_root_timestamp - Duration::milliseconds(1), + )); + let at_root = bloom_path(uuid_from_date_time(gc_root_timestamp)); + let after_root = bloom_path(uuid_from_date_time( + gc_root_timestamp + Duration::milliseconds(1), + )); + + assert!(is_vacuum_object_gc_candidate( + &before_root, + recent, + gc_root_meta_ts, + policy, + )); + assert!(!is_vacuum_object_gc_candidate( + &at_root, + old, + gc_root_meta_ts, + policy, + )); + assert!(!is_vacuum_object_gc_candidate( + &after_root, + old, + gc_root_meta_ts, + policy, + )); + } + + #[test] + fn test_legacy_or_malformed_gc_classification_uses_transaction_window() { + let gc_root_timestamp = DateTime::::from_timestamp(1_700_000_000, 0).unwrap(); + let gc_root_meta_ts = gc_root_timestamp + Duration::days(10); + let policy = VacuumObjectKeyPolicy::PrefixlessUuidV7 { gc_root_timestamp }; + let valid_v7 = bloom_path(uuid_from_date_time( + gc_root_timestamp - Duration::milliseconds(1), + )); + let uppercase_v7 = valid_v7.to_ascii_uppercase(); + let v4 = bloom_path(Uuid::new_v4()); + let malformed = "1/2/_i_b_v2/not-a-uuid_v4.parquet"; + let at_window = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION; + let before_window = at_window - Duration::milliseconds(1); + + for path in [&uppercase_v7, &v4, malformed] { + assert!(!is_vacuum_object_gc_candidate( + path, + at_window, + gc_root_meta_ts, + policy, + )); + assert!(is_vacuum_object_gc_candidate( + path, + before_window, + gc_root_meta_ts, + policy, + )); + } + } + + #[test] + fn test_prefixed_policy_preserves_existing_listing_classification() { + let gc_root_meta_ts = Utc::now(); + let recent = gc_root_meta_ts - Duration::hours(1); + let old = gc_root_meta_ts - ASSUMPTION_MAX_TXN_DURATION - Duration::hours(1); + let policy = VacuumObjectKeyPolicy::Vacuum2PrefixedBeforeRoot; + + assert!(is_vacuum_object_gc_candidate( + "1/2/_b/h-malformed.parquet", + recent, + gc_root_meta_ts, + policy, + )); + assert!(!is_vacuum_object_gc_candidate( + "1/2/_b/legacy.parquet", + recent, + gc_root_meta_ts, + policy, + )); + assert!(is_vacuum_object_gc_candidate( + "1/2/_b/legacy.parquet", + old, + gc_root_meta_ts, + policy, + )); + } +} diff --git a/src/query/storages/fuse/src/operations/virtual_column.rs b/src/query/storages/fuse/src/operations/virtual_column.rs index a4dd9848dd48c..b94537a7fb5f9 100644 --- a/src/query/storages/fuse/src/operations/virtual_column.rs +++ b/src/query/storages/fuse/src/operations/virtual_column.rs @@ -367,7 +367,7 @@ pub async fn commit_refresh_virtual_column( let table_meta_timestamps = ctx.get_table_meta_timestamps(fuse_table, Some(latest_snapshot.clone()))?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( fuse_table, ctx.clone(), @@ -378,7 +378,7 @@ pub async fn commit_refresh_virtual_column( MutationKind::Refresh, table_meta_timestamps, ) - }); + })?; let snapshot_gen = MutationGenerator::new(Some(latest_snapshot), MutationKind::Refresh); pipeline.add_sink(|input| { @@ -444,7 +444,7 @@ pub async fn do_vacuum_virtual_column( let table_meta_timestamps = ctx.get_table_meta_timestamps(fuse_table, Some(latest_snapshot.clone()))?; - pipeline.add_async_accumulating_transformer(|| { + pipeline.try_add_async_accumulating_transformer(|| { TableMutationAggregator::create( fuse_table, ctx.clone(), @@ -455,7 +455,7 @@ pub async fn do_vacuum_virtual_column( MutationKind::Refresh, table_meta_timestamps, ) - }); + })?; let prev_snapshot_id = latest_snapshot.snapshot_id; let snapshot_gen = MutationGenerator::new(Some(latest_snapshot), MutationKind::Refresh); diff --git a/src/query/storages/fuse/src/pruning/block_pruner.rs b/src/query/storages/fuse/src/pruning/block_pruner.rs index 3142b7893f637..e16036014631a 100644 --- a/src/query/storages/fuse/src/pruning/block_pruner.rs +++ b/src/query/storages/fuse/src/pruning/block_pruner.rs @@ -33,6 +33,7 @@ use log::debug; use tokio::sync::OwnedSemaphorePermit; use super::SegmentLocation; +use crate::fuse_part::bloom_index_layout; use crate::pruning::PruningContext; use crate::pruning::PruningCostKind; use crate::pruning::RuntimeStatsPruner; @@ -339,8 +340,7 @@ impl BlockPruner { .measure_async( PruningCostKind::BlocksBloom, bloom_pruner.should_keep( - &block_meta.bloom_filter_index_location, - block_meta.bloom_filter_index_size, + bloom_index_layout(&block_meta), &block_meta.col_stats, column_ids, &block_meta.as_ref().into(), diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs index 5d7fd27389c27..8608aec86e5db 100644 --- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs +++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs @@ -23,6 +23,7 @@ use databend_common_expression::ColumnId; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; +use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRef; @@ -35,12 +36,15 @@ use databend_storages_common_index::filters::BlockFilter; use databend_storages_common_io::ReadSettings; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::StatisticsOfColumns; +use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::meta::column_oriented_segment::BlockReadInfo; use log::info; use log::warn; use opendal::Operator; use crate::FuseBlockPartInfo; +use crate::fuse_part::BloomIndexLayout; +use crate::fuse_part::FuseBloomIndexFileInfo; use crate::io::BlockWriter; use crate::io::BloomBlockFilterReader; use crate::io::BloomIndexRebuilder; @@ -52,8 +56,7 @@ pub trait BloomPruner { // returns true, if target should NOT be pruned (false positive allowed) async fn should_keep( &self, - index_location: &Option, - index_length: u64, + index_layout: Option>, column_stats: &StatisticsOfColumns, column_ids: Vec, block_meta: &BlockReadInfo, @@ -68,10 +71,6 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( expr: &Expr, part: &FuseBlockPartInfo, ) -> Result { - let Some(index_location) = part.bloom_filter_index_location.as_ref() else { - return Ok(false); - }; - if part.bloom_filter_index_size == 0 { return Ok(false); } @@ -95,26 +94,41 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( } } - let index_columns = result - .bloom_fields - .iter() - .map(|field| BloomIndex::build_filter_bloom_name(index_location.1, field)) - .collect::>>()?; - let filter = index_location - .read_block_filter( - dal.clone(), - settings, - &index_columns, - part.bloom_filter_index_size, - ) - .await?; - - let bloom_index = BloomIndex::from_filter_block( - func_ctx.clone(), - filter.filter_schema, - filter.filters, - index_location.1, - )?; + let bloom_index = match part.bloom_index_layout() { + None => return Ok(false), + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { + let index_columns = result + .bloom_fields + .iter() + .map(|field| BloomIndex::build_filter_bloom_name(location.1, field)) + .collect::>>()?; + let filter = location + .read_block_filter(dal.clone(), settings, &index_columns, file_size) + .await?; + BloomIndex::from_filter_block( + func_ctx.clone(), + filter.filter_schema, + filter.filters, + location.1, + )? + } + Some(BloomIndexLayout::ColumnGroups { files }) => { + let Some(filter) = + read_multi_file_block_filter(dal, settings, &result.bloom_fields, &files).await? + else { + return Ok(false); + }; + BloomIndex::from_filter_block( + func_ctx.clone(), + filter.block_filter.filter_schema, + filter.block_filter.filters, + filter.format_version, + )? + } + }; let like_scalar_map = HashMap::new(); let empty_stats = StatisticsOfColumns::new(); @@ -129,6 +143,97 @@ pub(crate) async fn should_prune_runtime_inlist_by_bloom_index( )? == FilterEvalResult::MustFalse) } +struct MultiFileBlockFilter { + block_filter: BlockFilter, + format_version: u64, +} + +fn merged_filter_version(versions: impl IntoIterator) -> Option { + let mut has_v2 = false; + let mut has_digest_encoding = false; + for version in versions { + if version == 2 { + has_v2 = true; + } else { + has_digest_encoding = true; + } + } + match (has_v2, has_digest_encoding) { + (true, true) => None, + (true, false) => Some(2), + _ => Some(BlockFilter::VERSION), + } +} + +async fn read_multi_file_block_filter( + dal: &Operator, + settings: &ReadSettings, + index_fields: &[TableField], + index_files: &[FuseBloomIndexFileInfo], +) -> Result> { + // The owning data group may contain active columns for which its Bloom file has no filter. + // Decide encoding compatibility from the filters actually loaded, not merely from the data + // group's active ids. + let mut loaded_filters = Vec::new(); + + for file in index_files { + let mut fields_by_filter_name = HashMap::new(); + let mut columns = Vec::new(); + for field in index_fields { + if !file.active_column_ids.contains(&field.column_id()) { + continue; + } + let physical = BloomIndex::build_filter_bloom_name(file.location.1, field)?; + fields_by_filter_name.insert(physical.clone(), field); + columns.push(physical); + } + if columns.is_empty() { + continue; + } + + let filter = file + .location + .read_block_filter(dal.clone(), settings, &columns, file.file_size) + .await?; + for (field, value) in filter + .filter_schema + .fields() + .iter() + .zip(filter.filters.into_iter()) + { + if let Some(index_field) = fields_by_filter_name.get(field.name()) { + loaded_filters.push((*index_field, file.location.1, value)); + } + } + } + + if loaded_filters.is_empty() { + return Ok(None); + } + let Some(format_version) = + merged_filter_version(loaded_filters.iter().map(|(_, version, _)| *version)) + else { + // V2 filters use legacy scalar encoding and cannot be evaluated together with the digest + // encoding of newer files. Keeping the block is the safe compatibility behavior. + return Ok(None); + }; + let mut merged_fields = Vec::with_capacity(loaded_filters.len()); + let mut merged_filters = Vec::with_capacity(loaded_filters.len()); + for (field, _, filter) in loaded_filters { + let name = BloomIndex::build_filter_bloom_name(format_version, field)?; + merged_fields.push(TableField::new(&name, TableDataType::Binary)); + merged_filters.push(filter); + } + + Ok(Some(MultiFileBlockFilter { + block_filter: BlockFilter { + filter_schema: Arc::new(TableSchema::new(merged_fields)), + filters: merged_filters, + }, + format_version, + })) +} + pub struct BloomPrunerCreator { func_ctx: FunctionContext, @@ -329,6 +434,37 @@ impl BloomPrunerCreator { } } + async fn apply_files( + &self, + index_files: &[FuseBloomIndexFileInfo], + column_stats: &StatisticsOfColumns, + ) -> Result { + let maybe_filter = read_multi_file_block_filter( + &self.dal, + &self.settings, + &self.index_fields, + index_files, + ) + .await; + let Some(filter) = maybe_filter? else { + return Ok(true); + }; + Ok(BloomIndex::from_filter_block( + self.func_ctx.clone(), + filter.block_filter.filter_schema, + filter.block_filter.filters, + filter.format_version, + )? + .apply( + self.filter_expression.clone(), + &self.eq_scalar_map, + &self.like_scalar_map, + &self.ngram_args, + column_stats, + self.data_schema.clone(), + )? != FilterEvalResult::MustFalse) + } + async fn try_rebuild_missing_bloom_index( &self, bloom_index_location: &Location, @@ -336,6 +472,14 @@ impl BloomPrunerCreator { index_columns: &[String], block_read_info: &BlockReadInfo, ) -> Result> { + if let Err(e) = BloomIndexRebuilder::validate_rebuild_version(bloom_index_location) { + info!( + "cannot rebuild missing legacy Bloom index {:?}: {}", + bloom_index_location, e + ); + return Ok(None); + } + if self.dal.exists(bloom_index_location.0.as_str()).await? { info!("bloom index exists, ignore"); return Ok(None); @@ -385,27 +529,58 @@ impl BloomPruner for BloomPrunerCreator { #[async_backtrace::framed] async fn should_keep( &self, - index_location: &Option, - index_length: u64, + index_layout: Option>, column_stats: &StatisticsOfColumns, column_ids: Vec, block_meta: &BlockReadInfo, ) -> bool { - if let Some(loc) = index_location { - // load filter, and try pruning according to filter expression - match self - .apply(loc, index_length, column_stats, column_ids, block_meta) - .await - { - Ok(v) => v, - Err(e) => { - // swallow exceptions intentionally, corrupted index should not prevent execution - warn!("failed to apply bloom pruner, returning true. {}", e); - true + match index_layout { + Some(BloomIndexLayout::ColumnGroups { files }) => { + match self.apply_files(&files, column_stats).await { + Ok(value) => value, + Err(e) => { + warn!( + "failed to apply multi-file bloom pruner, returning true. {}", + e + ); + true + } } } - } else { - true + Some(BloomIndexLayout::Legacy { + location, + file_size, + }) => { + // Load the filter and try pruning according to its expression. + match self + .apply(location, file_size, column_stats, column_ids, block_meta) + .await + { + Ok(value) => value, + Err(e) => { + // Swallow exceptions intentionally: a corrupt index must not fail a query. + warn!("failed to apply bloom pruner, returning true. {}", e); + true + } + } + } + None => true, } } } + +#[cfg(test)] +mod tests { + use databend_storages_common_index::filters::BlockFilter; + use databend_storages_common_table_meta::meta::Versioned; + + use super::merged_filter_version; + + #[test] + fn test_merged_filter_version() { + assert_eq!(merged_filter_version([2]), Some(2)); + assert_eq!(merged_filter_version([2, 4]), None); + assert_eq!(merged_filter_version([3, 4]), Some(BlockFilter::VERSION)); + assert_eq!(merged_filter_version([]), Some(BlockFilter::VERSION)); + } +} diff --git a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs index 85db98015890f..f4b65db5a0f48 100644 --- a/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs +++ b/src/query/storages/fuse/src/pruning/expr_runtime_pruner.rs @@ -537,8 +537,9 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, + vec![], 4, - HashMap::new(), + vec![], Some(column_stats), Compression::Lz4Raw, None, @@ -727,8 +728,9 @@ mod tests { "memory:///block".to_string(), bloom_filter_index_location, bloom_filter_index_size, + vec![], 1000, - HashMap::new(), + vec![], Some(column_stats), Compression::Lz4Raw, None, diff --git a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs index 01924a907b424..3f793fe1175e5 100644 --- a/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs +++ b/src/query/storages/fuse/src/pruning_pipeline/column_oriented_block_prune.rs @@ -42,6 +42,8 @@ use tokio::sync::OwnedSemaphorePermit; use super::PrunedColumnOrientedSegmentMeta; use crate::FuseBlockPartInfo; +use crate::FuseColumnGroupPartInfo; +use crate::fuse_part::BloomIndexLayout; use crate::pruning::BlockPruner; use crate::pruning_pipeline::RuntimeFilterPruneContext; @@ -220,8 +222,12 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { if !bloom_pruner .should_keep( - &bloom_filter_index_location, - bloom_filter_index_size, + bloom_filter_index_location.as_ref().map(|location| { + BloomIndexLayout::Legacy { + location, + file_size: bloom_filter_index_size, + } + }), &columns_stat, column_ids.clone(), &block_read_info, @@ -276,12 +282,17 @@ impl AsyncSink for ColumnOrientedBlockPruneSink { } } + let column_groups = vec![FuseColumnGroupPartInfo { + location: location_path.clone(), + columns_meta, + }]; let part_info = FuseBlockPartInfo::create( location_path, bloom_filter_index_location, bloom_filter_index_size, + vec![], row_count, - columns_meta, + column_groups, Some(columns_stat), compression, None, // TODO(Sky): sort_min_max diff --git a/src/query/storages/fuse/src/table_functions/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_block.rs index f75d4094fda72..4d6a2eb4671d0 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_block.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use databend_common_catalog::table::Table; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockEntry; use databend_common_expression::Column; @@ -28,11 +29,15 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; +use databend_common_expression::types::VariantType; use databend_common_expression::types::string::StringColumnBuilder; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; +use serde::Serialize; use crate::FuseTable; +use crate::fuse_part::block_bloom_index_size; +use crate::fuse_part::legacy_bloom_index_location; use crate::io::SegmentsIO; use crate::sessions::TableContext; use crate::table_functions::TableMetaFuncTemplate; @@ -41,6 +46,13 @@ use crate::table_functions::function_template::TableMetaFunc; pub struct FuseBlock; pub type FuseBlockFunc = TableMetaFuncTemplate; +fn serialize_variant(value: &impl Serialize) -> Result> { + let json = serde_json::to_vec(value)?; + Ok(jsonb::parse_value(&json) + .map_err(|error| ErrorCode::Internal(error.to_string()))? + .to_vec()) +} + #[async_trait::async_trait] impl TableMetaFunc for FuseBlock { fn schema() -> Arc { @@ -79,6 +91,7 @@ impl TableMetaFunc for FuseBlock { "virtual_column_size", TableDataType::Nullable(Box::new(TableDataType::Number(NumberDataType::UInt64))), ), + TableField::new("column_groups", TableDataType::Variant), ]) } @@ -104,6 +117,7 @@ impl TableMetaFunc for FuseBlock { let mut vector_index_size = Vec::with_capacity(len); let mut spatial_index_size = Vec::with_capacity(len); let mut virtual_column_size = Vec::with_capacity(len); + let mut column_groups = Vec::with_capacity(len); let segments_io = SegmentsIO::create(ctx.clone(), tbl.operator.clone(), tbl.schema()); @@ -124,12 +138,9 @@ impl TableMetaFunc for FuseBlock { file_size.push(block.file_size); row_count.push(block.row_count); bloom_filter_location.push( - block - .bloom_filter_index_location - .as_ref() - .map(|s| s.0.clone()), + legacy_bloom_index_location(block).map(|location| location.0.clone()), ); - bloom_filter_size.push(block.bloom_filter_index_size); + bloom_filter_size.push(block_bloom_index_size(block)); inverted_index_size.push(block.inverted_index_size); ngram_index_size.push(block.ngram_filter_index_size); vector_index_size.push(block.vector_index_size); @@ -140,7 +151,22 @@ impl TableMetaFunc for FuseBlock { .as_ref() .map(|m| m.virtual_column_size), ); - + column_groups.push(serialize_variant( + &block + .column_groups + .iter() + .map(|group| { + serde_json::json!({ + "active_column_ids": group.active_column_ids, + "location": group.location.0, + "format_version": group.location.1, + "file_size": group.file_size, + "uncompressed_size": group.uncompressed_size, + "bloom": group.bloom, + }) + }) + .collect::>(), + )?); num_rows += 1; if num_rows >= limit { break 'FOR; @@ -164,6 +190,7 @@ impl TableMetaFunc for FuseBlock { UInt64Type::from_opt_data(vector_index_size).into(), UInt64Type::from_opt_data(spatial_index_size).into(), UInt64Type::from_opt_data(virtual_column_size).into(), + VariantType::from_data(column_groups).into(), ], num_rows, )) diff --git a/src/query/storages/fuse/src/table_functions/fuse_column.rs b/src/query/storages/fuse/src/table_functions/fuse_column.rs index 7f098b858404d..a9078ab097038 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_column.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_column.rs @@ -130,27 +130,29 @@ impl FuseColumn { for segment in segments { let segment = segment?; for block in segment.block_metas()? { - for (id, column) in block.col_metas(&col_ids).iter() { - if let Some(f) = leaf_fields.iter().find(|f| f.column_id == *id) { - block_location.put_and_commit(block.location_path()); - block_size.push(block.block_size()); - file_size.push(block.file_size()); - row_count.push(column.total_rows() as u64); + for (location, col_metas) in block.col_metas_by_location(&col_ids) { + for (id, column) in &col_metas { + if let Some(f) = leaf_fields.iter().find(|f| f.column_id == *id) { + block_location.put_and_commit(&location); + block_size.push(block.block_size()); + file_size.push(block.file_size()); + row_count.push(column.total_rows() as u64); - column_name.put_and_commit(&f.name); + column_name.put_and_commit(&f.name); - column_type.put_and_commit(f.data_type.to_string()); + column_type.put_and_commit(f.data_type.to_string()); - column_id.push(*id); + column_id.push(*id); - let (offset, length) = column.offset_length(); - block_offset.push(offset); - bytes_compressed.push(length); + let (offset, length) = column.offset_length(); + block_offset.push(offset); + bytes_compressed.push(length); - num_rows += 1; + num_rows += 1; - if num_rows >= limit { - break 'FOR; + if num_rows >= limit { + break 'FOR; + } } } } diff --git a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs index b44d0257d2746..3c208ee934e99 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_encoding.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_encoding.rs @@ -48,6 +48,7 @@ use databend_common_expression::types::UInt64Type; use databend_common_expression::types::nullable::NullableColumnBuilder; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_storages_common_table_meta::meta::ColumnMeta; use databend_storages_common_table_meta::meta::SegmentInfo; use databend_storages_common_table_meta::meta::TableSnapshot; use futures::stream; @@ -306,7 +307,7 @@ impl<'a> FuseEncodingImpl<'a> { storage_format: FuseStorageFormat, table_name: Arc, fields: Arc>, - column_id_to_index: Arc>, + column_metas: Arc>, column_name_filter: Arc>, ) -> Result> { let file_meta = read_thrift_file_metadata(operator, &location, Some(file_size)).await?; @@ -319,6 +320,14 @@ impl<'a> FuseEncodingImpl<'a> { } let row_group = &file_meta.row_groups()[0]; let columns = row_group.columns(); + let mut columns_by_range = HashMap::with_capacity(columns.len()); + for column in columns { + // Keep the first match to preserve the previous `iter().find()` behavior for + // malformed metadata containing duplicate byte ranges. + columns_by_range + .entry(column.byte_range()) + .or_insert(column); + } let mut block_rows = Vec::new(); for field in fields.iter() { @@ -329,11 +338,11 @@ impl<'a> FuseEncodingImpl<'a> { continue; } let column_id = field.column_id; - let Some(column_idx) = column_id_to_index.get(&column_id) else { + let Some(column_meta) = column_metas.get(&column_id) else { continue; }; - let Some(column_chunk) = columns.get(*column_idx) else { - // Missing column caused by schema evolutions + let column_range = column_meta.offset_length(); + let Some(column_chunk) = columns_by_range.get(&column_range).copied() else { continue; }; let compressed_size = u64::try_from(column_chunk.compressed_size()).map_err(|_| { @@ -381,14 +390,6 @@ impl<'a> FuseEncodingImpl<'a> { ) -> Result<()> { let schema = table.schema(); let fields = Arc::new(schema.fields().clone()); - let column_id_to_index: Arc> = Arc::new( - schema - .to_leaf_column_ids() - .into_iter() - .enumerate() - .map(|(idx, id)| (id, idx)) - .collect(), - ); let column_name_filter = Arc::new(self.column_name_filter.clone()); let table_name = Arc::new(table.name().to_string()); let storage_format = table.storage_format; @@ -401,28 +402,36 @@ impl<'a> FuseEncodingImpl<'a> { let segments = segments_io .read_segments::(chunk, false) .await?; - let mut block_locations = Vec::new(); + let mut block_groups = Vec::new(); for segment in segments { let segment = segment?; for block in segment.blocks.iter() { - block_locations.push((block.location.0.clone(), block.file_size)); + for group in block.physical_column_groups().iter() { + let column_metas = group + .active_leaf_column_metas() + .map(|(column_id, column_meta)| (column_id, column_meta.clone())) + .collect(); + block_groups.push(( + group.location.0.clone(), + group.file_size, + Arc::new(column_metas), + )); + } } } - if block_locations.is_empty() { + if block_groups.is_empty() { continue; } let table_operator = table.operator.clone(); let fields_arc = fields.clone(); - let column_id_to_index_arc = column_id_to_index.clone(); let table_name_arc = table_name.clone(); let column_name_filter_arc = column_name_filter.clone(); - let mut block_stream = stream::iter(block_locations.into_iter().map( - move |(location, file_size)| { + let mut block_stream = stream::iter(block_groups.into_iter().map( + move |(location, file_size, column_metas)| { let operator = table_operator.clone(); let fields = fields_arc.clone(); - let column_id_to_index = column_id_to_index_arc.clone(); let table_name = table_name_arc.clone(); let column_name_filter = column_name_filter_arc.clone(); async move { @@ -433,7 +442,7 @@ impl<'a> FuseEncodingImpl<'a> { storage_format, table_name, fields, - column_id_to_index, + column_metas, column_name_filter, ) .await diff --git a/src/query/storages/fuse/src/table_functions/fuse_page.rs b/src/query/storages/fuse/src/table_functions/fuse_page.rs index f98117460e3dd..a1f721c9eb514 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_page.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_page.rs @@ -144,48 +144,52 @@ impl TableMetaFunc for FusePage { let segment = segment?; for block in segment.blocks.iter() { let block = block.as_ref(); - for field in schema.fields().iter() { - if field.is_nested() { - continue; - } - let column_id = field.column_id; - let Some(column_meta) = block.col_metas.get(&column_id) else { - continue; - }; - let Some(parquet_meta) = column_meta.as_parquet() else { - continue; - }; + for group in block.physical_column_groups().iter() { + for field in schema.fields().iter() { + if field.is_nested() + || !group.active_column_ids.contains(&field.column_id) + { + continue; + } + let Some(column_meta) = group.leaf_column_metas.get(&field.column_id) + else { + continue; + }; + let Some(parquet_meta) = column_meta.as_parquet() else { + continue; + }; - let column_bytes = read_parquet_column_chunk( - &tbl.operator, - &block.location.0, - parquet_meta.offset, - parquet_meta.len, - ) - .await?; - for (page_ordinal_in_column, page) in parse_page_headers( - field.name(), - parquet_meta.offset, - column_bytes.as_slice(), - )? - .into_iter() - .enumerate() - { - block_location.put_and_commit(&block.location.0); - column_name.put_and_commit(page.column_name); - page_type.put_and_commit(page.page_type); - encoding.put_and_commit(page.encoding); - page_ordinal.push(page_ordinal_in_column as u64); - num_values.push(page.num_values); - num_rows.push(page.num_rows); - compressed_size.push(page.compressed_size); - uncompressed_size.push(page.uncompressed_size); - header_size.push(page.header_size); - page_offset.push(page.page_offset); + let column_bytes = read_parquet_column_chunk( + &tbl.operator, + &group.location.0, + parquet_meta.offset, + parquet_meta.len, + ) + .await?; + for (page_ordinal_in_column, page) in parse_page_headers( + field.name(), + parquet_meta.offset, + column_bytes.as_slice(), + )? + .into_iter() + .enumerate() + { + block_location.put_and_commit(&group.location.0); + column_name.put_and_commit(page.column_name); + page_type.put_and_commit(page.page_type); + encoding.put_and_commit(page.encoding); + page_ordinal.push(page_ordinal_in_column as u64); + num_values.push(page.num_values); + num_rows.push(page.num_rows); + compressed_size.push(page.compressed_size); + uncompressed_size.push(page.uncompressed_size); + header_size.push(page.header_size); + page_offset.push(page.page_offset); - rows += 1; - if rows >= limit { - break 'FOR; + rows += 1; + if rows >= limit { + break 'FOR; + } } } } diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test new file mode 100644 index 0000000000000..2b23cac2a8512 --- /dev/null +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0054_partial_update.test @@ -0,0 +1,77 @@ +statement ok +drop table if exists t_partial_update + +statement ok +set enable_partial_update = 0 + +statement ok +create table t_partial_update(id int, value int) + bloom_index_columns = 'id,value' + enable_partial_update = true + +statement ok +insert into t_partial_update values (1, 10), (2, 20) + +# The table option alone is not enough to enable partial updates. +statement ok +update t_partial_update set value = value + 1 where id = 1 + +query B +select column_groups = parse_json('[]') from fuse_block('default', 't_partial_update') +---- +1 + +statement ok +set enable_partial_update = 1 + +statement ok +update t_partial_update set value = value + 1 where id = 2 + +query II +select * from t_partial_update order by id +---- +1 11 +2 21 + +query B +select column_groups != parse_json('[]') +from fuse_block('default', 't_partial_update') +---- +1 + +query I +select count(distinct column_name) +from fuse_page('default', 't_partial_update') +where column_name in ('id', 'value') +---- +2 + +query I +select count(distinct column_name) +from fuse_encoding('default', 't_partial_update') +where column_name in ('id', 'value') +---- +2 + +query I +select count(distinct block_location) +from fuse_column('default', 't_partial_update') +where column_name in ('id', 'value') +---- +2 + +# Unsetting the table option forces the next rewrite back to the single-file layout. +statement ok +alter table t_partial_update unset options(enable_partial_update) + +statement ok +update t_partial_update set value = value + 1 where id = 1 + +query B +select column_groups = parse_json('[]') +from fuse_block('default', 't_partial_update') +---- +1 + +statement ok +drop table t_partial_update