diff --git a/Cargo.lock b/Cargo.lock index 4bd77dd53fbf2..2ad2394a3df1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5036,6 +5036,7 @@ dependencies = [ "async-channel", "async-trait", "backoff", + "bumpalo", "bytes", "chrono", "databend-common-ast", diff --git a/src/query/ast/src/ast/statements/materialized_view.rs b/src/query/ast/src/ast/statements/materialized_view.rs new file mode 100644 index 0000000000000..3876139570166 --- /dev/null +++ b/src/query/ast/src/ast/statements/materialized_view.rs @@ -0,0 +1,149 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Display; +use std::fmt::Formatter; + +use databend_common_ast_visit_derive::Walk; +use databend_common_ast_visit_derive::WalkMut; +use derive_visitor::Drive; +use derive_visitor::DriveMut; + +use crate::ast::CreateOption; +use crate::ast::Identifier; +use crate::ast::Query; +use crate::ast::ShowLimit; +use crate::ast::write_comma_separated_list; +use crate::ast::write_dot_separated_list; + +#[derive(Debug, Clone, PartialEq, Drive, DriveMut, Walk, WalkMut)] +pub struct CreateMaterializedViewStmt { + pub create_option: CreateOption, + pub catalog: Option, + pub database: Option, + pub view: Identifier, + pub columns: Vec, + pub query: Box, +} + +impl Display for CreateMaterializedViewStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "CREATE ")?; + if let CreateOption::CreateOrReplace = self.create_option { + write!(f, "OR REPLACE ")?; + } + write!(f, "MATERIALIZED VIEW ")?; + if let CreateOption::CreateIfNotExists = self.create_option { + write!(f, "IF NOT EXISTS ")?; + } + write_dot_separated_list( + f, + self.catalog + .iter() + .chain(&self.database) + .chain(Some(&self.view)), + )?; + if !self.columns.is_empty() { + write!(f, " (")?; + write_comma_separated_list(f, &self.columns)?; + write!(f, ")")?; + } + write!(f, " AS {}", self.query) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut, Walk, WalkMut)] +pub struct DropMaterializedViewStmt { + pub if_exists: bool, + pub catalog: Option, + pub database: Option, + pub view: Identifier, +} + +impl Display for DropMaterializedViewStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "DROP MATERIALIZED VIEW ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write_dot_separated_list( + f, + self.catalog + .iter() + .chain(&self.database) + .chain(Some(&self.view)), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut, Walk, WalkMut)] +pub struct RefreshMaterializedViewStmt { + pub catalog: Option, + pub database: Option, + pub view: Identifier, +} + +impl Display for RefreshMaterializedViewStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "REFRESH MATERIALIZED VIEW ")?; + write_dot_separated_list( + f, + self.catalog + .iter() + .chain(&self.database) + .chain(Some(&self.view)), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut, Walk, WalkMut)] +pub struct DescribeMaterializedViewStmt { + pub catalog: Option, + pub database: Option, + pub view: Identifier, +} + +impl Display for DescribeMaterializedViewStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "DESCRIBE MATERIALIZED VIEW ")?; + write_dot_separated_list( + f, + self.catalog + .iter() + .chain(&self.database) + .chain(Some(&self.view)), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Drive, DriveMut, Walk, WalkMut)] +pub struct ShowMaterializedViewsStmt { + pub catalog: Option, + pub database: Option, + pub limit: Option, +} + +impl Display for ShowMaterializedViewsStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "SHOW MATERIALIZED VIEWS")?; + if self.catalog.is_some() || self.database.is_some() { + write!(f, " FROM ")?; + write_dot_separated_list(f, self.catalog.iter().chain(&self.database))?; + } + if let Some(limit) = &self.limit { + write!(f, " {limit}")?; + } + Ok(()) + } +} diff --git a/src/query/ast/src/ast/statements/mod.rs b/src/query/ast/src/ast/statements/mod.rs index 1fa8c7b5aae75..960831543337c 100644 --- a/src/query/ast/src/ast/statements/mod.rs +++ b/src/query/ast/src/ast/statements/mod.rs @@ -30,6 +30,7 @@ mod insert; mod insert_multi_table; mod kill; mod lock; +mod materialized_view; mod merge_into; mod network_policy; mod notification; @@ -81,6 +82,7 @@ pub use insert::*; pub use insert_multi_table::*; pub use kill::*; pub use lock::*; +pub use materialized_view::*; pub use merge_into::*; pub use network_policy::*; pub use notification::*; diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 4d935b5d66c63..864697469c080 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -219,6 +219,13 @@ pub enum Statement { ShowViews(ShowViewsStmt), DescribeView(DescribeViewStmt), + // Materialized Views + CreateMaterializedView(CreateMaterializedViewStmt), + DropMaterializedView(DropMaterializedViewStmt), + RefreshMaterializedView(RefreshMaterializedViewStmt), + DescribeMaterializedView(DescribeMaterializedViewStmt), + ShowMaterializedViews(ShowMaterializedViewsStmt), + // Streams CreateStream(CreateStreamStmt), DropStream(DropStreamStmt), @@ -554,6 +561,8 @@ impl Statement { | Statement::ShowColumns(..) | Statement::ShowViews(..) | Statement::DescribeView(..) + | Statement::ShowMaterializedViews(..) + | Statement::DescribeMaterializedView(..) | Statement::ShowStreams(..) | Statement::DescribeStream(..) | Statement::RefreshIndex(..) @@ -619,6 +628,9 @@ impl Statement { | Statement::DropDatabase(..) | Statement::DropTable(..) | Statement::DropView(..) + | Statement::DropMaterializedView(..) + | Statement::CreateMaterializedView(..) + | Statement::RefreshMaterializedView(..) | Statement::DropIndex(..) | Statement::DropSequence(..) | Statement::DropDictionary(..) @@ -932,6 +944,11 @@ impl Display for Statement { Statement::DropView(stmt) => write!(f, "{stmt}")?, Statement::ShowViews(stmt) => write!(f, "{stmt}")?, Statement::DescribeView(stmt) => write!(f, "{stmt}")?, + Statement::CreateMaterializedView(stmt) => write!(f, "{stmt}")?, + Statement::DropMaterializedView(stmt) => write!(f, "{stmt}")?, + Statement::RefreshMaterializedView(stmt) => write!(f, "{stmt}")?, + Statement::DescribeMaterializedView(stmt) => write!(f, "{stmt}")?, + Statement::ShowMaterializedViews(stmt) => write!(f, "{stmt}")?, Statement::CreateStream(stmt) => write!(f, "{stmt}")?, Statement::DropStream(stmt) => write!(f, "{stmt}")?, Statement::ShowStreams(stmt) => write!(f, "{stmt}")?, diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index 70980092ee1bd..41a0db740b239 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -879,6 +879,7 @@ pub enum Engine { Null, Memory, Fuse, + MaterializedView, View, Random, Iceberg, @@ -893,6 +894,7 @@ impl Display for Engine { Engine::Null => write!(f, "NULL"), Engine::Memory => write!(f, "MEMORY"), Engine::Fuse => write!(f, "FUSE"), + Engine::MaterializedView => write!(f, "MATERIALIZED_VIEW"), Engine::View => write!(f, "VIEW"), Engine::Random => write!(f, "RANDOM"), Engine::Iceberg => write!(f, "ICEBERG"), @@ -909,6 +911,7 @@ impl From<&str> for Engine { "null" => Engine::Null, "memory" => Engine::Memory, "fuse" => Engine::Fuse, + "materialized_view" => Engine::MaterializedView, "view" => Engine::View, "random" => Engine::Random, "iceberg" => Engine::Iceberg, diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 0a4e19858cfc1..43e397ac0f8ea 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -1613,6 +1613,95 @@ pub fn statement_body(i: Input) -> IResult { }, ); + let create_materialized_view = map_res( + rule! { + CREATE ~ ( OR ~ ^REPLACE )? ~ MATERIALIZED ~ ^VIEW ~ ( IF ~ ^NOT ~ ^EXISTS )? + ~ #dot_separated_idents_1_to_3 + ~ ( "(" ~ #comma_separated_list1(ident) ~ ")" )? + ~ AS ~ #query + }, + |( + _, + opt_or_replace, + _, + _, + opt_if_not_exists, + (catalog, database, view), + opt_columns, + _, + query, + )| { + let create_option = + parse_create_option(opt_or_replace.is_some(), opt_if_not_exists.is_some())?; + Ok(Statement::CreateMaterializedView( + CreateMaterializedViewStmt { + create_option, + catalog, + database, + view, + columns: opt_columns + .map(|(_, columns, _)| columns) + .unwrap_or_default(), + query: Box::new(query), + }, + )) + }, + ); + let drop_materialized_view = map( + rule! { + DROP ~ MATERIALIZED ~ ^VIEW ~ ( IF ~ ^EXISTS )? ~ #dot_separated_idents_1_to_3 + }, + |(_, _, _, opt_if_exists, (catalog, database, view))| { + Statement::DropMaterializedView(DropMaterializedViewStmt { + if_exists: opt_if_exists.is_some(), + catalog, + database, + view, + }) + }, + ); + let refresh_materialized_view = map( + rule! { + REFRESH ~ MATERIALIZED ~ ^VIEW ~ #dot_separated_idents_1_to_3 + }, + |(_, _, _, (catalog, database, view))| { + Statement::RefreshMaterializedView(RefreshMaterializedViewStmt { + catalog, + database, + view, + }) + }, + ); + let describe_materialized_view = map( + rule! { + ( DESC | DESCRIBE ) ~ MATERIALIZED ~ ^VIEW ~ #dot_separated_idents_1_to_3 + }, + |(_, _, _, (catalog, database, view))| { + Statement::DescribeMaterializedView(DescribeMaterializedViewStmt { + catalog, + database, + view, + }) + }, + ); + let show_materialized_views = map( + rule! { + SHOW ~ MATERIALIZED ~ ^VIEWS ~ ( ( FROM | IN ) ~ #dot_separated_idents_1_to_2 )? ~ #show_limit? + }, + |(_, _, _, ctl_db, limit)| { + let (catalog, database) = match ctl_db { + Some((_, (Some(c), d))) => (Some(c), Some(d)), + Some((_, (None, d))) => (None, Some(d)), + _ => (None, None), + }; + Statement::ShowMaterializedViews(ShowMaterializedViewsStmt { + catalog, + database, + limit, + }) + }, + ); + let create_index = map_res( rule! { CREATE @@ -2913,6 +3002,7 @@ pub fn statement_body(i: Input) -> IResult { | #show_tables_status : "`SHOW TABLES STATUS [FROM ] []`" | #show_drop_tables_status : "`SHOW DROP TABLES [FROM ]`" | #show_views : "`SHOW [FULL] VIEWS [FROM ] []`" + | #show_materialized_views : "`SHOW MATERIALIZED VIEWS [FROM [.]] []`" | #show_virtual_columns : "`SHOW VIRTUAL COLUMNS FROM [FROM|IN .] []`" ) | ( @@ -2987,7 +3077,8 @@ pub fn statement_body(i: Input) -> IResult { ATTACH => rule!(#attach_table : "`ATTACH TABLE [.]
`" ).parse(i), REFRESH => rule!( - #refresh_index: "`REFRESH INDEX [LIMIT ]`" + #refresh_materialized_view: "`REFRESH MATERIALIZED VIEW [.]`" + | #refresh_index: "`REFRESH INDEX [LIMIT ]`" | #refresh_table_index: "`REFRESH INDEX ON [.]
[LIMIT ]`" | #refresh_virtual_column: "`REFRESH VIRTUAL COLUMN FOR [.]
`" ).parse(i), @@ -3015,6 +3106,7 @@ pub fn statement_body(i: Input) -> IResult { DESC | DESCRIBE => rule!( #desc_task : "`DESC | DESCRIBE TASK `" | #describe_view : "`DESCRIBE VIEW [.]`" + | #describe_materialized_view : "`DESCRIBE MATERIALIZED VIEW [.]`" | #describe_user: "`DESCRIBE USER `" | #describe_row_access : "`DESC[RIBE] ROW ACCESS POLICY `" | #desc_stage: "`DESC STAGE `" @@ -3049,6 +3141,7 @@ AS | #create_table : "`CREATE [OR REPLACE] TABLE [IF NOT EXISTS] [.]
[] []`" | #create_dictionary : "`CREATE [OR REPLACE] DICTIONARY [IF NOT EXISTS] [(, ...)] PRIMARY KEY [, ...] SOURCE ( ([])) [COMMENT ] `" | #create_view : "`CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [.] [(, ...)] AS SELECT ...`" + | #create_materialized_view : "`CREATE [OR REPLACE] MATERIALIZED VIEW [IF NOT EXISTS] [.] [(, ...)] AS SELECT ...`" | #create_index: "`CREATE [OR REPLACE] AGGREGATING INDEX [IF NOT EXISTS] AS SELECT ...`" | #create_table_index: "`CREATE [OR REPLACE] INDEX [IF NOT EXISTS] ON [.]
(, ...)`" ) @@ -3097,6 +3190,7 @@ AS | #drop_table : "`DROP TABLE [IF EXISTS] [.]
`" | #drop_dictionary : "`DROP DICTIONARY [IF EXISTS] `" | #drop_view : "`DROP VIEW [IF EXISTS] [.]`" + | #drop_materialized_view : "`DROP MATERIALIZED VIEW [IF EXISTS] [.]`" | #drop_index: "`DROP INDEX [IF EXISTS] `" | #drop_table_index: "`DROP INDEX [IF EXISTS] ON [.]
`" ) diff --git a/src/query/ast/src/visit/statement.rs b/src/query/ast/src/visit/statement.rs index 248b3dad4e053..2485eee940493 100644 --- a/src/query/ast/src/visit/statement.rs +++ b/src/query/ast/src/visit/statement.rs @@ -169,6 +169,11 @@ impl Walk for Statement { Statement::DropView(stmt) => try_walk!(stmt.walk(visitor)), Statement::DescribeView(stmt) => try_walk!(stmt.walk(visitor)), Statement::ShowViews(stmt) => try_walk!(stmt.walk(visitor)), + Statement::CreateMaterializedView(stmt) => try_walk!(stmt.walk(visitor)), + Statement::DropMaterializedView(stmt) => try_walk!(stmt.walk(visitor)), + Statement::RefreshMaterializedView(stmt) => try_walk!(stmt.walk(visitor)), + Statement::DescribeMaterializedView(stmt) => try_walk!(stmt.walk(visitor)), + Statement::ShowMaterializedViews(stmt) => try_walk!(stmt.walk(visitor)), Statement::CreateStream(stmt) => try_walk!(stmt.walk(visitor)), Statement::DropStream(stmt) => try_walk!(stmt.walk(visitor)), Statement::DescribeStream(stmt) => try_walk!(stmt.walk(visitor)), @@ -384,6 +389,11 @@ impl WalkMut for Statement { Statement::DropView(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::DescribeView(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::ShowViews(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::CreateMaterializedView(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::DropMaterializedView(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::RefreshMaterializedView(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::DescribeMaterializedView(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::ShowMaterializedViews(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::CreateStream(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::DropStream(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::DescribeStream(stmt) => try_walk!(stmt.walk_mut(visitor)), diff --git a/src/query/catalog/src/catalog/interface.rs b/src/query/catalog/src/catalog/interface.rs index 33b9933e50172..f0b324477e0d3 100644 --- a/src/query/catalog/src/catalog/interface.rs +++ b/src/query/catalog/src/catalog/interface.rs @@ -81,6 +81,8 @@ use databend_common_meta_app::schema::ListTableCopiedFileReply; use databend_common_meta_app::schema::ListTableTagsReq; use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; +use databend_common_meta_app::schema::MVDefinition; +use databend_common_meta_app::schema::MVInfo; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::schema::RenameDictionaryReq; @@ -265,6 +267,43 @@ pub trait Catalog: DynClone + Send + Sync + Debug { /// Get the table meta by table id. async fn get_table_meta_by_id(&self, table_id: u64) -> Result>>; + /// Get a materialized-view definition by its table ID. + async fn get_mv_definition( + &self, + _tenant: &Tenant, + _mv_table_id: u64, + ) -> Result>> { + Err(ErrorCode::Unimplemented(format!( + "'get_mv_definition' not implemented for catalog {}", + self.name() + ))) + } + + /// List all materialized views that depend on a source table. + async fn list_mvs_by_source_table_id( + &self, + _tenant: &Tenant, + _source_table_id: u64, + ) -> Result> { + Err(ErrorCode::Unimplemented(format!( + "'list_mvs_by_source_table_id' not implemented for catalog {}", + self.name() + ))) + } + + /// List materialized views valid for the caller's source binding generation. + async fn list_valid_mvs_by_source_table_id( + &self, + _tenant: &Tenant, + _source_table_id: u64, + _expected_source_generation: u64, + ) -> Result> { + Err(ErrorCode::Unimplemented(format!( + "'list_valid_mvs_by_source_table_id' not implemented for catalog {}", + self.name() + ))) + } + /// List the tables name by meta ids. This function should not be used to list temporary tables. async fn mget_table_names_by_ids( &self, diff --git a/src/query/service/src/catalogs/default/database_catalog.rs b/src/query/service/src/catalogs/default/database_catalog.rs index 1d08d72862127..aa73665a23531 100644 --- a/src/query/service/src/catalogs/default/database_catalog.rs +++ b/src/query/service/src/catalogs/default/database_catalog.rs @@ -88,6 +88,8 @@ use databend_common_meta_app::schema::ListTableCopiedFileReply; use databend_common_meta_app::schema::ListTableTagsReq; use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; +use databend_common_meta_app::schema::MVDefinition; +use databend_common_meta_app::schema::MVInfo; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::schema::RenameDictionaryReq; @@ -310,6 +312,37 @@ impl Catalog for DatabaseCatalog { } } + async fn get_mv_definition( + &self, + tenant: &Tenant, + mv_table_id: u64, + ) -> Result>> { + self.mutable_catalog + .get_mv_definition(tenant, mv_table_id) + .await + } + + async fn list_mvs_by_source_table_id( + &self, + tenant: &Tenant, + source_table_id: u64, + ) -> Result> { + self.mutable_catalog + .list_mvs_by_source_table_id(tenant, source_table_id) + .await + } + + async fn list_valid_mvs_by_source_table_id( + &self, + tenant: &Tenant, + source_table_id: u64, + expected_source_generation: u64, + ) -> Result> { + self.mutable_catalog + .list_valid_mvs_by_source_table_id(tenant, source_table_id, expected_source_generation) + .await + } + #[async_backtrace::framed] async fn mget_table_names_by_ids( &self, diff --git a/src/query/service/src/catalogs/default/mutable_catalog.rs b/src/query/service/src/catalogs/default/mutable_catalog.rs index 42a7b28355c4b..a30db95a9ea7e 100644 --- a/src/query/service/src/catalogs/default/mutable_catalog.rs +++ b/src/query/service/src/catalogs/default/mutable_catalog.rs @@ -31,6 +31,7 @@ use databend_common_meta_api::DictionaryApi; use databend_common_meta_api::GarbageCollectionApi; use databend_common_meta_api::IndexApi; use databend_common_meta_api::LockApi2; +use databend_common_meta_api::MaterializedViewApi; use databend_common_meta_api::RefApi; use databend_common_meta_api::SecurityApi; use databend_common_meta_api::SequenceApi; @@ -105,6 +106,8 @@ use databend_common_meta_app::schema::ListTableCopiedFileReply; use databend_common_meta_app::schema::ListTableTagsReq; use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; +use databend_common_meta_app::schema::MVDefinition; +use databend_common_meta_app::schema::MVInfo; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::schema::RenameDictionaryReq; @@ -557,6 +560,43 @@ impl Catalog for MutableCatalog { Ok(res) } + async fn get_mv_definition( + &self, + tenant: &Tenant, + mv_table_id: u64, + ) -> Result>> { + self.ctx + .meta + .get_mv_definition(tenant, mv_table_id) + .await + .map_err(meta_service_error) + } + + async fn list_mvs_by_source_table_id( + &self, + tenant: &Tenant, + source_table_id: u64, + ) -> Result> { + self.ctx + .meta + .list_mvs_by_source_table_id(tenant, source_table_id) + .await + .map_err(meta_service_error) + } + + async fn list_valid_mvs_by_source_table_id( + &self, + tenant: &Tenant, + source_table_id: u64, + expected_source_generation: u64, + ) -> Result> { + self.ctx + .meta + .list_valid_mvs_by_source_table_id(tenant, source_table_id, expected_source_generation) + .await + .map_err(meta_service_error) + } + async fn mget_table_names_by_ids( &self, _tenant: &Tenant, diff --git a/src/query/service/src/catalogs/default/session_catalog.rs b/src/query/service/src/catalogs/default/session_catalog.rs index 717254c984ba2..c693a9cd1ddd4 100644 --- a/src/query/service/src/catalogs/default/session_catalog.rs +++ b/src/query/service/src/catalogs/default/session_catalog.rs @@ -83,6 +83,8 @@ use databend_common_meta_app::schema::ListTableCopiedFileReply; use databend_common_meta_app::schema::ListTableTagsReq; use databend_common_meta_app::schema::LockInfo; use databend_common_meta_app::schema::LockMeta; +use databend_common_meta_app::schema::MVDefinition; +use databend_common_meta_app::schema::MVInfo; use databend_common_meta_app::schema::RenameDatabaseReply; use databend_common_meta_app::schema::RenameDatabaseReq; use databend_common_meta_app::schema::RenameDictionaryReq; @@ -321,6 +323,35 @@ impl Catalog for SessionCatalog { } } + async fn get_mv_definition( + &self, + tenant: &Tenant, + mv_table_id: u64, + ) -> Result>> { + self.inner.get_mv_definition(tenant, mv_table_id).await + } + + async fn list_mvs_by_source_table_id( + &self, + tenant: &Tenant, + source_table_id: u64, + ) -> Result> { + self.inner + .list_mvs_by_source_table_id(tenant, source_table_id) + .await + } + + async fn list_valid_mvs_by_source_table_id( + &self, + tenant: &Tenant, + source_table_id: u64, + expected_source_generation: u64, + ) -> Result> { + self.inner + .list_valid_mvs_by_source_table_id(tenant, source_table_id, expected_source_generation) + .await + } + async fn mget_table_names_by_ids( &self, tenant: &Tenant, diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index 84f8745b93c1c..97c11251db1ca 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -2368,6 +2368,11 @@ impl AccessChecker for PrivilegeAccess { ) .await?; } + // TODO + Plan::CreateMaterializedView(_) => {}, + Plan::DropMaterializedView(_) => {}, + Plan::DescribeMaterializedView(_) => {}, + Plan::RefreshMaterializedView(_) => todo!(), } Ok(()) diff --git a/src/query/service/src/interpreters/common/table_option_validation.rs b/src/query/service/src/interpreters/common/table_option_validation.rs index cf8e0b3386ff5..047a3319bb381 100644 --- a/src/query/service/src/interpreters/common/table_option_validation.rs +++ b/src/query/service/src/interpreters/common/table_option_validation.rs @@ -59,6 +59,7 @@ use databend_storages_common_table_meta::table::OPT_KEY_CHANGE_TRACKING; use databend_storages_common_table_meta::table::OPT_KEY_COMMENT; use databend_storages_common_table_meta::table::OPT_KEY_CONNECTION_NAME; use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; +use databend_storages_common_table_meta::table::OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID; use databend_storages_common_table_meta::table::OPT_KEY_ENABLE_COPY_DEDUP_FULL_PATH; use databend_storages_common_table_meta::table::OPT_KEY_ENABLE_SCHEMA_EVOLUTION; use databend_storages_common_table_meta::table::OPT_KEY_ENGINE; @@ -122,6 +123,15 @@ pub static CREATE_FUSE_OPTIONS: LazyLock> = LazyLock::new( r }); +/// Table option keys that can occur in 'create materialized view statement'. +pub static CREATE_MATERIALIZED_VIEW_OPTIONS: LazyLock> = + LazyLock::new(|| { + let mut r = HashSet::new(); + r.insert(OPT_KEY_DATABASE_ID); + r.insert(OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID); + r +}); + pub static CREATE_LAKE_OPTIONS: LazyLock> = LazyLock::new(|| { let mut r = HashSet::new(); r.insert(OPT_KEY_ENGINE); @@ -195,6 +205,7 @@ pub fn is_valid_create_opt>(opt_key: S, engine: &Engine) -> bool { Engine::Memory => CREATE_MEMORY_OPTIONS.contains(&opt_key), Engine::Proxy => CREATE_PROXY_OPTIONS.contains(&opt_key), Engine::Null | Engine::View => opt_key == OPT_KEY_ENGINE, + Engine::MaterializedView => CREATE_MATERIALIZED_VIEW_OPTIONS.contains(opt_key), } } diff --git a/src/query/service/src/interpreters/interpreter_copy_into_table.rs b/src/query/service/src/interpreters/interpreter_copy_into_table.rs index 5efd01cbfdecc..e8e1a03b89882 100644 --- a/src/query/service/src/interpreters/interpreter_copy_into_table.rs +++ b/src/query/service/src/interpreters/interpreter_copy_into_table.rs @@ -289,6 +289,7 @@ impl CopyIntoTableInterpreter { stat_info: None, table_index: None, internal_column: None, + materialized_view: None, source: Box::new(data_source_plan), meta: PhysicalPlanMeta::new("TableScan"), })), diff --git a/src/query/service/src/interpreters/interpreter_factory.rs b/src/query/service/src/interpreters/interpreter_factory.rs index 427784b0e7c3d..48f083b402aa6 100644 --- a/src/query/service/src/interpreters/interpreter_factory.rs +++ b/src/query/service/src/interpreters/interpreter_factory.rs @@ -517,6 +517,16 @@ impl InterpreterFactory { *describe_view.clone(), )?)), + // Materialized Views + Plan::CreateMaterializedView(create_view) => Ok(Arc::new( + CreateMaterializedViewInterpreter::try_create(ctx, *create_view.clone())?, + )), + Plan::DropMaterializedView(drop_view) => Ok(Arc::new( + DropMaterializedViewInterpreter::try_create(ctx, *drop_view.clone())?, + )), + Plan::DescribeMaterializedView(_) => todo!(), + Plan::RefreshMaterializedView(_) => todo!(), + // Streams Plan::CreateStream(create_stream) => Ok(Arc::new(CreateStreamInterpreter::try_create( ctx, diff --git a/src/query/service/src/interpreters/interpreter_insert.rs b/src/query/service/src/interpreters/interpreter_insert.rs index df8cea1bcf509..a7afadad6a3f7 100644 --- a/src/query/service/src/interpreters/interpreter_insert.rs +++ b/src/query/service/src/interpreters/interpreter_insert.rs @@ -78,6 +78,8 @@ use crate::pipelines::PipelineBuildResult; use crate::pipelines::PipelineBuilder; use crate::pipelines::RawValueSource; use crate::pipelines::ValueSource; +use crate::pipelines::builders::MultiTableWriteTarget; +use crate::pipelines::builders::load_materialized_views; use crate::schedulers::build_query_pipeline_without_render_result_set; use crate::sessions::QueryContext; use crate::sessions::TableContext; @@ -338,6 +340,18 @@ impl Interpreter for InsertInterpreter { // just passes a placeholder value here TableMetaTimestamps::new(None, Duration::hours(1)) }; + let materialized_view_write = if self.plan.table_info.is_some() + || self.plan.branch.is_some() + || matches!(&self.plan.source, InsertInputSource::SelectPlan(_)) + { + // Fixed-table-info inserts (currently used by CTAS) must not update the MV set + // of the table resolved by the current name. Branch inserts must not update the + // main table's MV set either. MV maintenance for INSERT ... SELECT is deferred + // until MV targets are integrated into its distributed physical plan. + None + } else { + load_materialized_views(self.ctx.clone(), &self.plan.catalog, &table).await? + }; let mut build_res = PipelineBuildResult::create(); @@ -452,6 +466,11 @@ impl Interpreter for InsertInterpreter { let update_stream_meta = dml_build_update_stream_req(self.ctx.clone()).await?; + // TODO SelectPlan support Materialized View + // MV maintenance for INSERT ... SELECT is intentionally deferred. Unlike local + // VALUES/streaming inputs, SELECT may use a distributed insert physical plan; + // supporting it requires carrying the MV targets through that plan so row + // preparation, fan-out, and the source/MV commit remain consistent and atomic. let mut insert_select_plan = build_insert_select_physical_plan( select_plan, plan.schema(), @@ -537,17 +556,40 @@ impl Interpreter for InsertInterpreter { })?; } - PipelineBuilder::build_append2table_with_commit_pipeline( - self.ctx.clone(), - &mut build_res.main_pipeline, - table.clone(), - self.plan.dest_schema(), - None, - vec![], - self.plan.overwrite, - unsafe { self.ctx.get_settings().get_deduplicate_label()? }, - table_meta_timestamps, - )?; + if let Some(mv_write) = materialized_view_write { + PipelineBuilder::fill_and_reorder_columns( + self.ctx.clone(), + &mut build_res.main_pipeline, + table.clone(), + self.plan.dest_schema(), + )?; + PipelineBuilder::build_materialized_view_multi_table_write_pipeline( + self.ctx.clone(), + &mut build_res.main_pipeline, + MultiTableWriteTarget { + table: table.clone(), + table_meta_timestamps, + materialized_view_source_table_id: None, + }, + mv_write.materialized_views, + mv_write.catalog, + self.plan.overwrite, + vec![], + unsafe { self.ctx.get_settings().get_deduplicate_label()? }, + )?; + } else { + PipelineBuilder::build_append2table_with_commit_pipeline( + self.ctx.clone(), + &mut build_res.main_pipeline, + table.clone(), + self.plan.dest_schema(), + None, + vec![], + self.plan.overwrite, + unsafe { self.ctx.get_settings().get_deduplicate_label()? }, + table_meta_timestamps, + )?; + } // Execute the hook operator. if self.plan.branch.is_none() { diff --git a/src/query/service/src/interpreters/interpreter_materialized_view_create.rs b/src/query/service/src/interpreters/interpreter_materialized_view_create.rs new file mode 100644 index 0000000000000..6486292a0efbe --- /dev/null +++ b/src/query/service/src/interpreters/interpreter_materialized_view_create.rs @@ -0,0 +1,87 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_catalog::table_context::TableContextTableAccess; +use databend_common_exception::Result; +use databend_common_meta_app::schema::CreateMaterializedViewMeta; +use databend_common_meta_app::schema::CreateTableReq; +use databend_common_meta_app::schema::MATERIALIZED_VIEW_ENGINE; +use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::schema::TableNameIdent; +use databend_common_sql::plans::CreateMaterializedViewPlan; + +use crate::interpreters::Interpreter; +use crate::pipelines::PipelineBuildResult; +use crate::sessions::QueryContext; + +pub struct CreateMaterializedViewInterpreter { + ctx: Arc, + plan: CreateMaterializedViewPlan, +} + +impl CreateMaterializedViewInterpreter { + pub fn try_create(ctx: Arc, plan: CreateMaterializedViewPlan) -> Result { + Ok(CreateMaterializedViewInterpreter { ctx, plan }) + } +} + +#[async_trait::async_trait] +impl Interpreter for CreateMaterializedViewInterpreter { + fn name(&self) -> &str { + "CreateMaterializedViewInterpreter" + } + + fn is_ddl(&self) -> bool { + true + } + + #[async_backtrace::framed] + async fn execute2(&self) -> Result { + let catalog = self.ctx.get_catalog(&self.plan.catalog).await?; + + let materialized_view = CreateMaterializedViewMeta { + definition: self.plan.mv_definition.clone(), + expected_source_generation: 0, + }; + + let plan = CreateTableReq { + create_option: self.plan.create_option, + catalog_name: if self.plan.create_option.is_overriding() { + Some(self.plan.catalog.to_string()) + } else { + None + }, + name_ident: TableNameIdent { + tenant: self.plan.tenant.clone(), + db_name: self.plan.database.clone(), + table_name: self.plan.view_name.clone(), + }, + table_meta: TableMeta { + schema: self.plan.schema.clone(), + engine: MATERIALIZED_VIEW_ENGINE.to_string(), + options: self.plan.options.clone(), + ..Default::default() + }, + as_dropped: false, + materialized_view: Some(materialized_view), + table_properties: None, + table_partition: None, + }; + catalog.create_table(plan).await?; + + Ok(PipelineBuildResult::create()) + } +} diff --git a/src/query/service/src/interpreters/interpreter_materialized_view_drop.rs b/src/query/service/src/interpreters/interpreter_materialized_view_drop.rs new file mode 100644 index 0000000000000..a3658e4380d1a --- /dev/null +++ b/src/query/service/src/interpreters/interpreter_materialized_view_drop.rs @@ -0,0 +1,113 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_meta_app::schema::DropTableByIdReq; +use databend_common_meta_app::schema::MATERIALIZED_VIEW_ENGINE; +use databend_common_sql::plans::DropMaterializedViewPlan; +use databend_common_storages_basic::view_table::VIEW_ENGINE; +use databend_common_storages_stream::stream_table::STREAM_ENGINE; +use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; + +use crate::interpreters::Interpreter; +use crate::pipelines::PipelineBuildResult; +use crate::sessions::QueryContext; +use crate::sessions::TableContextTableAccess; + +pub struct DropMaterializedViewInterpreter { + ctx: Arc, + plan: DropMaterializedViewPlan, +} + +impl DropMaterializedViewInterpreter { + pub fn try_create(ctx: Arc, plan: DropMaterializedViewPlan) -> Result { + Ok(DropMaterializedViewInterpreter { ctx, plan }) + } +} + +#[async_trait::async_trait] +impl Interpreter for DropMaterializedViewInterpreter { + fn name(&self) -> &str { + "DropMaterializedViewInterpreter" + } + + fn is_ddl(&self) -> bool { + true + } + + #[async_backtrace::framed] + async fn execute2(&self) -> Result { + let catalog_name = self.plan.catalog.clone(); + let db_name = self.plan.database.clone(); + let view_name = self.plan.view_name.clone(); + let tbl = self + .ctx + .get_table(&catalog_name, &db_name, &view_name) + .await + .ok(); + + if tbl.is_none() && !self.plan.if_exists { + return Err(ErrorCode::UnknownTable(format!( + "unknown materialized view `{}`.`{}` in catalog '{}'", + db_name, view_name, &catalog_name + ))); + } + + if let Some(table) = &tbl { + let engine = table.get_table_info().engine(); + if engine != MATERIALIZED_VIEW_ENGINE { + return Err(ErrorCode::TableEngineNotSupported(format!( + "{}.{} is not MATERIALIZED VIEW, please use `DROP {} {}.{}`", + &self.plan.database, + &self.plan.view_name, + if engine == STREAM_ENGINE { + "STREAM" + } else if engine == VIEW_ENGINE { + "VIEW" + } else { + "TABLE" + }, + &self.plan.database, + &self.plan.view_name + ))); + } + + let catalog = self.ctx.get_catalog(&self.plan.catalog).await?; + let db = catalog + .get_database(&self.plan.tenant, &self.plan.database) + .await?; + catalog + .drop_table_by_id(DropTableByIdReq { + if_exists: self.plan.if_exists, + tenant: self.plan.tenant.clone(), + table_name: self.plan.view_name.clone(), + tb_id: table.get_id(), + db_id: db.get_db_info().database_id.db_id, + db_name: db.name().to_string(), + engine: table.engine().to_string(), + temp_prefix: table + .options() + .get(OPT_KEY_TEMP_PREFIX) + .cloned() + .unwrap_or_default(), + }) + .await?; + }; + + Ok(PipelineBuildResult::create()) + } +} diff --git a/src/query/service/src/interpreters/mod.rs b/src/query/service/src/interpreters/mod.rs index 4b491ff3bed97..f807e29ea4c66 100644 --- a/src/query/service/src/interpreters/mod.rs +++ b/src/query/service/src/interpreters/mod.rs @@ -66,6 +66,8 @@ mod interpreter_insert; mod interpreter_insert_multi_table; mod interpreter_inspect_warehouse; mod interpreter_kill; +mod interpreter_materialized_view_create; +mod interpreter_materialized_view_drop; mod interpreter_metrics; mod interpreter_mutation; mod interpreter_network_policies_show; @@ -253,6 +255,8 @@ pub use interpreter_insert::InsertInterpreter; pub use interpreter_insert::build_insert_select_physical_plan; pub use interpreter_insert_multi_table::InsertMultiTableInterpreter; pub use interpreter_kill::KillInterpreter; +pub use interpreter_materialized_view_create::CreateMaterializedViewInterpreter; +pub use interpreter_materialized_view_drop::DropMaterializedViewInterpreter; pub use interpreter_metrics::InterpreterMetrics; pub use interpreter_mutation::MutationInterpreter; pub use interpreter_network_policies_show::ShowNetworkPoliciesInterpreter; diff --git a/src/query/service/src/physical_plans/physical_multi_table_insert.rs b/src/query/service/src/physical_plans/physical_multi_table_insert.rs index a8da67f856f56..cdc16e34f2c69 100644 --- a/src/query/service/src/physical_plans/physical_multi_table_insert.rs +++ b/src/query/service/src/physical_plans/physical_multi_table_insert.rs @@ -22,28 +22,17 @@ use databend_common_catalog::catalog::CatalogManager; use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; -use databend_common_expression::LimitType; use databend_common_expression::RemoteExpr; -use databend_common_expression::SortColumnDescription; use databend_common_meta_app::schema::CatalogInfo; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_common_pipeline::core::DynTransformBuilder; use databend_common_pipeline::core::ProcessorPtr; -use databend_common_pipeline::sinks::AsyncSinker; -use databend_common_pipeline_transforms::AccumulatingTransformer; use databend_common_pipeline_transforms::AsyncTransformer; use databend_common_pipeline_transforms::Transformer; -use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::columns::TransformAddComputedColumns; -use databend_common_pipeline_transforms::sorts::TransformSortPartial; use databend_common_sql::DefaultExprBinder; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::operations::CommitMultiTableInsert; -use databend_common_storages_fuse::operations::TransformPartitionBy; -use databend_common_storages_fuse::operations::TransformVectorCluster; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use crate::physical_plans::format::ChunkAppendDataFormatter; @@ -59,6 +48,7 @@ use crate::physical_plans::physical_plan::IPhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlan; use crate::physical_plans::physical_plan::PhysicalPlanMeta; use crate::pipelines::PipelineBuilder; +use crate::pipelines::builders::MultiTableWriteTarget; use crate::pipelines::processors::transforms::TransformAsyncFunction; use crate::sessions::TableContextSession; @@ -671,153 +661,20 @@ impl IPhysicalPlan for ChunkAppendData { fn build_pipeline2(&self, builder: &mut PipelineBuilder) -> Result<()> { self.input.build_pipeline(builder)?; - let mut compact_task_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut compact_transform_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut serialize_block_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut eval_cluster_key_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut eval_cluster_key_num = 0; - let mut vector_cluster_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut vector_cluster_num = 0; - let mut sort_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut sort_num = 0; - let mut partition_builders: Vec = - Vec::with_capacity(self.target_tables.len()); - let mut partition_num = 0; - - for append_data in self.target_tables.iter() { - let table = builder - .ctx - .build_table_by_table_info(&append_data.target_table_info, None)?; - let block_thresholds = table.get_block_thresholds(); - compact_task_builders.push(Box::new( - builder.block_compact_task_builder(block_thresholds)?, - )); - compact_transform_builders.push(Box::new(builder.block_compact_transform_builder()?)); - let schema: Arc = DataSchema::from(table.schema()).into(); - let num_input_columns = schema.num_fields(); - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let cluster_stats_gen = fuse_table.get_cluster_stats_gen( - builder.ctx.clone(), - 0, - block_thresholds, - schema, - )?; - let operators = cluster_stats_gen.operators.clone(); - if !operators.is_empty() { - let func_ctx2 = cluster_stats_gen.func_ctx.clone(); - - eval_cluster_key_builders.push(Box::new(move |input, output| { - Ok(ProcessorPtr::create(CompoundBlockOperator::create( - input, - output, - num_input_columns, - func_ctx2.clone(), - operators.clone(), - ))) - })); - eval_cluster_key_num += 1; - } else { - eval_cluster_key_builders.push(Box::new(builder.dummy_transform_builder())); - } - if let Some(vector_operator) = cluster_stats_gen.vector_operator.clone() { - let rows_per_block = block_thresholds.max_rows_per_block; - vector_cluster_builders.push(Box::new(move |input, output| { - Ok(ProcessorPtr::create(AccumulatingTransformer::create( - input, - output, - TransformVectorCluster::new( - vector_operator.vector_column_input_offset, - vector_operator.info.dimension, - vector_operator.info.distance_type, - rows_per_block, - ), - ))) - })); - vector_cluster_num += 1; - } else { - vector_cluster_builders.push(Box::new(builder.dummy_transform_builder())); - } - - let sort_desc: Vec = cluster_stats_gen.sort_descs(); - if !sort_desc.is_empty() { - let sort_desc: Arc<[_]> = sort_desc.into(); - sort_builders.push(Box::new( - move |transform_input_port, transform_output_port| { - Ok(ProcessorPtr::create(TransformSortPartial::try_create( - transform_input_port, - transform_output_port, - LimitType::None, - sort_desc.clone(), - )?)) - }, - )); - sort_num += 1; - } else { - sort_builders.push(Box::new(builder.dummy_transform_builder())); - } - let partition_key_indices: Arc<[_]> = - cluster_stats_gen.cluster_key_index[..cluster_stats_gen.partition_key_count].into(); - if !partition_key_indices.is_empty() { - partition_builders.push(Box::new(move |input, output| { - Ok(ProcessorPtr::create(AccumulatingTransformer::create( - input, - output, - TransformPartitionBy::new(partition_key_indices.clone()), - ))) - })); - partition_num += 1; - } else { - partition_builders.push(Box::new(builder.dummy_transform_builder())); - } - serialize_block_builders.push(Box::new( - builder.with_tid_serialize_block_transform_builder( - table, - cluster_stats_gen, - append_data.table_meta_timestamps, - )?, - )); - } - builder - .main_pipeline - .add_transforms_by_chunk(compact_task_builders)?; - - builder - .main_pipeline - .add_transforms_by_chunk(compact_transform_builders)?; - - if eval_cluster_key_num > 0 { - builder - .main_pipeline - .add_transforms_by_chunk(eval_cluster_key_builders)?; - } - - if vector_cluster_num > 0 { - builder - .main_pipeline - .add_transforms_by_chunk(vector_cluster_builders)?; - } - - if sort_num > 0 { - builder - .main_pipeline - .add_transforms_by_chunk(sort_builders)?; - } - - if partition_num > 0 { - builder - .main_pipeline - .add_transforms_by_chunk(partition_builders)?; - } - - builder - .main_pipeline - .add_transforms_by_chunk(serialize_block_builders) + let targets = self + .target_tables + .iter() + .map(|target| { + Ok(MultiTableWriteTarget { + table: builder + .ctx + .build_table_by_table_info(&target.target_table_info, None)?, + table_meta_timestamps: target.table_meta_timestamps, + materialized_view_source_table_id: None, + }) + }) + .collect::>>()?; + builder.build_multi_table_append_data(&targets) } } @@ -1008,27 +865,32 @@ impl IPhysicalPlan for ChunkCommitInsert { } else { add_commit_meta_transforms(builder, &self.targets)? }; - builder.main_pipeline.try_resize(1)?; let catalog = CatalogManager::instance().build_catalog( self.targets[0].target_catalog_info.clone(), builder.ctx.session_state()?, )?; - - builder.main_pipeline.add_sink(|input| { - Ok(ProcessorPtr::create(AsyncSinker::create( - input, - CommitMultiTableInsert::create( - tables.clone(), - builder.ctx.clone(), - self.overwrite, - self.update_stream_meta.clone(), - self.deduplicated_label.clone(), - catalog.clone(), - table_meta_timestampss.clone(), - ), - ))) - }) + let write_targets = self + .targets + .iter() + .map(|target| MultiTableWriteTarget { + table: tables + .get(&target.target_table_info.ident.table_id) + .unwrap() + .clone(), + table_meta_timestamps: *table_meta_timestampss + .get(&target.target_table_info.ident.table_id) + .unwrap(), + materialized_view_source_table_id: None, + }) + .collect::>(); + builder.build_multi_table_commit_sink( + &write_targets, + catalog, + self.overwrite, + self.update_stream_meta.clone(), + self.deduplicated_label.clone(), + ) } } @@ -1056,33 +918,18 @@ fn add_commit_meta_transforms( ) -> Result { let (tables, table_meta_timestampss) = build_target_tables(builder, targets)?; - let mut serialize_segment_builders: Vec = - Vec::with_capacity(targets.len()); - let mut mutation_aggregator_builders: Vec = - Vec::with_capacity(targets.len()); - - for target in targets { - let table = tables - .get(&target.target_table_info.ident.table_id) - .unwrap() - .clone(); - let block_thresholds = table.get_block_thresholds(); - serialize_segment_builders.push(Box::new(builder.serialize_segment_transform_builder( - table.clone(), - block_thresholds, - target.table_meta_timestamps, - )?)); - mutation_aggregator_builders.push(Box::new( - builder.mutation_aggregator_transform_builder(table, target.table_meta_timestamps)?, - )); - } - - builder - .main_pipeline - .add_transforms_by_chunk(serialize_segment_builders)?; - builder - .main_pipeline - .add_transforms_by_chunk(mutation_aggregator_builders)?; + let write_targets = targets + .iter() + .map(|target| MultiTableWriteTarget { + table: tables + .get(&target.target_table_info.ident.table_id) + .unwrap() + .clone(), + table_meta_timestamps: target.table_meta_timestamps, + materialized_view_source_table_id: None, + }) + .collect::>(); + builder.build_multi_table_commit_meta(&write_targets)?; Ok((tables, table_meta_timestampss)) } diff --git a/src/query/service/src/physical_plans/physical_table_scan.rs b/src/query/service/src/physical_plans/physical_table_scan.rs index 15b33b565ee86..699c6bc39eee5 100644 --- a/src/query/service/src/physical_plans/physical_table_scan.rs +++ b/src/query/service/src/physical_plans/physical_table_scan.rs @@ -46,6 +46,7 @@ use databend_common_expression::TableSchemaRef; use databend_common_expression::type_check::check_function; use databend_common_expression::types::DataType; use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionFactory; use databend_common_pipeline_transforms::TransformPipelineHelper; use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; use databend_common_pipeline_transforms::columns::TransformAddInternalColumns; @@ -55,6 +56,7 @@ use databend_common_sql::ColumnSet; use databend_common_sql::DUMMY_TABLE_INDEX; use databend_common_sql::DerivedColumn; use databend_common_sql::IndexType; +use databend_common_sql::MaterializedViewScanInfo; use databend_common_sql::Metadata; use databend_common_sql::ScalarExpr; use databend_common_sql::Symbol; @@ -67,6 +69,8 @@ use databend_common_sql::executor::cast_expr_to_non_null_boolean; use databend_common_sql::executor::table_read_plan::ToReadDataSourcePlan; use databend_common_sql::plans::FunctionCall; use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::MaterializedViewStateMergePlan; +use databend_common_storages_fuse::operations::TransformMaterializedViewStateMerge; use databend_common_storages_fuse::operations::need_reserve_block_info; use rand::distributions::Bernoulli; use rand::distributions::Distribution; @@ -94,6 +98,7 @@ pub struct TableScan { pub name_mapping: BTreeMap, pub source: Box, pub internal_column: Option>, + pub materialized_view: Option, pub table_index: Option, pub stat_info: Option, @@ -114,7 +119,16 @@ impl IPhysicalPlan for TableScan { #[recursive::recursive] fn output_schema(&self) -> Result { - Self::output_fields(self.source.schema(), &self.name_mapping).map(DataSchema::new_ref) + match &self.materialized_view { + Some(info) => Self::materialized_view_output_fields( + self.source.schema(), + &self.name_mapping, + info, + ) + .map(DataSchema::new_ref), + None => Self::output_fields(self.source.schema(), &self.name_mapping) + .map(DataSchema::new_ref), + } } fn formatter(&self) -> Result> { @@ -179,6 +193,7 @@ impl IPhysicalPlan for TableScan { name_mapping: self.name_mapping.clone(), source: self.source.clone(), internal_column: self.internal_column.clone(), + materialized_view: self.materialized_view.clone(), table_index: self.table_index, stat_info: self.stat_info.clone(), }) @@ -206,6 +221,31 @@ impl IPhysicalPlan for TableScan { true, )?; + if let Some(info) = &self.materialized_view { + let aggregate_functions = info + .aggregate_functions + .iter() + .map(|aggregate| { + AggregateFunctionFactory::instance().get( + &aggregate.name, + aggregate.params.clone(), + aggregate.argument_types.clone(), + vec![], + ) + }) + .collect::>>()?; + let merge_plan = MaterializedViewStateMergePlan { + aggregate_functions, + group_data_types: info.group_data_types.clone(), + physical_data_types: info.physical_data_types.clone(), + final_data_types: info.final_data_types.clone(), + }; + builder.main_pipeline.try_resize(1)?; + builder.main_pipeline.add_accumulating_transformer(|| { + TransformMaterializedViewStateMerge::create(merge_plan.clone()) + }); + } + // Fill internal columns if needed. if let Some(internal_columns) = &self.internal_column { builder @@ -213,18 +253,33 @@ impl IPhysicalPlan for TableScan { .add_transformer(|| TransformAddInternalColumns::new(internal_columns.clone())); } - let schema = self.source.schema(); - let mut projection = self - .name_mapping - .keys() - .map(|name| schema.index_of(name.as_str())) - .collect::>>()?; - projection.sort(); + let projection = match &self.materialized_view { + Some(_) => { + let schema = self.source.schema(); + let mut projection = self + .name_mapping + .keys() + .map(|name| schema.index_of(name)) + .collect::>>()?; + projection.sort_unstable(); + projection + } + None => { + let schema = self.source.schema(); + let mut projection = self + .name_mapping + .keys() + .map(|name| schema.index_of(name.as_str())) + .collect::>>()?; + projection.sort(); + projection + } + }; // if projection is sequential, no need to add projection - if projection != (0..schema.fields().len()).collect::>() { + let num_input_columns = self.source.schema().num_fields(); + if projection != (0..num_input_columns).collect::>() { let ops = vec![BlockOperator::Project { projection }]; - let num_input_columns = schema.num_fields(); builder.main_pipeline.add_transformer(|| { CompoundBlockOperator::new(ops.clone(), builder.func_ctx.clone(), num_input_columns) }); @@ -242,6 +297,7 @@ impl TableScan { table_index: Option, stat_info: Option, internal_column: Option>, + materialized_view: Option, ) -> PhysicalPlan { let name = match &source.source_info { DataSourceInfo::TableSource(_) => "TableScan".to_string(), @@ -259,9 +315,36 @@ impl TableScan { table_index, stat_info, internal_column, + materialized_view, }) } + fn materialized_view_output_fields( + schema: TableSchemaRef, + name_mapping: &BTreeMap, + info: &MaterializedViewScanInfo, + ) -> Result> { + if info.final_data_types.len() != schema.num_fields() { + return Err(ErrorCode::Internal(format!( + "materialized view has {} finalized storage types for a {}-column physical schema", + info.final_data_types.len(), + schema.num_fields() + ))); + } + let mut fields = name_mapping + .iter() + .map(|(name, id)| { + let offset = schema.index_of(name)?; + Ok(( + offset, + DataField::new(id, info.final_data_types[offset].clone()), + )) + }) + .collect::>>()?; + fields.sort_by_key(|(offset, _)| *offset); + Ok(fields.into_iter().map(|(_, field)| field).collect()) + } + pub fn output_fields( schema: TableSchemaRef, name_mapping: &BTreeMap, @@ -411,6 +494,7 @@ impl PhysicalPlanBuilder { let table_entry = metadata.table(scan.table_index); let table = table_entry.table(); + let materialized_view = metadata.materialized_view_scan(scan.table_index).cloned(); if !table.result_can_be_cached() { self.ctx.result_cache_state().set_cacheable(false); @@ -429,12 +513,23 @@ impl PhysicalPlanBuilder { table_schema = Arc::new(schema); } - let push_downs = self.push_downs( + let mut push_downs = self.push_downs( &scan, &table_schema, project_virtual_columns, has_inner_column, )?; + if materialized_view.is_some() { + push_downs.projection = Some(Projection::Columns( + (0..table_schema.num_fields()).collect(), + )); + push_downs.output_columns = None; + push_downs.filters = None; + push_downs.prewhere = None; + push_downs.limit = None; + push_downs.order_by.clear(); + push_downs.lazy_materialization = false; + } // Generate secure cache key extra for Row Access Policy predicates. // Constant-fold so session-dependent functions (e.g. GETVARIABLE) @@ -536,6 +631,7 @@ impl PhysicalPlanBuilder { Some(scan.table_index), Some(stat_info.clone()), internal_column, + materialized_view, ); // Update stream columns if needed. @@ -673,6 +769,7 @@ impl PhysicalPlanBuilder { estimated_rows: 1.0, }), None, + None, )) } diff --git a/src/query/service/src/pipelines/builders/builder_materialized_view.rs b/src/query/service/src/pipelines/builders/builder_materialized_view.rs new file mode 100644 index 0000000000000..7754ec9a80839 --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_materialized_view.rs @@ -0,0 +1,686 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +use databend_common_catalog::catalog::Catalog; +use databend_common_catalog::table::Table; +use databend_common_catalog::table_context::TableContextSettings; +use databend_common_catalog::table_context::TableContextTableAccess; +use databend_common_catalog::table_context::TableContextTableManagement; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::DataSchema; +use databend_common_expression::Expr; +use databend_common_expression::SymbolOrOffset; +use databend_common_functions::aggregates::AggregateFunctionFactory; +use databend_common_meta_app::schema::DatabaseType; +use databend_common_meta_app::schema::MVInfo; +use databend_common_meta_app::schema::TableIdent; +use databend_common_meta_app::schema::TableInfo; +use databend_common_meta_app::schema::UpdateStreamMetaReq; +use databend_common_pipeline::basic::create_resize_item; +use databend_common_pipeline::core::InputPort; +use databend_common_pipeline::core::OutputPort; +use databend_common_pipeline::core::Pipe; +use databend_common_pipeline::core::PipeItem; +use databend_common_pipeline::core::Pipeline; +use databend_common_pipeline::core::ProcessorPtr; +use databend_common_pipeline_transforms::AccumulatingTransformer; +use databend_common_pipeline_transforms::create_dummy_item; +use databend_common_sql::Metadata; +use databend_common_sql::MetadataRef; +use databend_common_sql::Planner; +use databend_common_sql::Symbol; +use databend_common_sql::TypeCheck; +use databend_common_sql::executor::cast_expr_to_non_null_boolean; +use databend_common_sql::optimizer::ir::SExpr; +use databend_common_sql::plans::AggregateMode; +use databend_common_sql::plans::EvalScalar; +use databend_common_sql::plans::RelOperator; +use databend_common_sql::plans::ScalarExpr; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::operations::MaterializedViewTransformPlan; +use databend_common_storages_fuse::operations::TransformMaterializedView; +use databend_storages_common_table_meta::meta::TableMetaTimestamps; +use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; +use databend_storages_common_table_meta::table::OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION; +use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; + +use super::MultiTableWriteTarget; +use crate::pipelines::PipelineBuilder; +use crate::sessions::QueryContext; + +#[derive(Clone)] +pub struct MaterializedViewWrite { + pub catalog: Arc, + pub materialized_views: Vec, +} + +#[derive(Clone)] +pub struct MaterializedViewWriteTarget { + pub target: MultiTableWriteTarget, + pub transform_plan: MaterializedViewTransformPlan, +} + +struct MaterializedViewScalarLowerer<'a> { + metadata: &'a Metadata, + input_schema: &'a DataSchema, + definitions: HashMap, +} + +impl<'a> MaterializedViewScalarLowerer<'a> { + fn try_create( + metadata: &'a Metadata, + input_schema: &'a DataSchema, + evals: &[&EvalScalar], + ) -> Result { + let mut definitions = HashMap::new(); + for eval in evals { + for item in &eval.items { + // Binder may emit passthrough items such as `0 := column(0)`. They are not + // derived definitions and registering them would create a false self-cycle. + if matches!( + &item.scalar, + ScalarExpr::BoundColumnRef(column) if column.column.index == item.index + ) { + continue; + } + if let Some(previous) = definitions.insert(item.index, item.scalar.clone()) + && previous != item.scalar + { + return Err(ErrorCode::Internal(format!( + "materialized view column {} has conflicting scalar definitions", + item.index + ))); + } + } + } + Ok(Self { + metadata, + input_schema, + definitions, + }) + } + + fn expand(&self, scalar: &ScalarExpr, visiting: &mut HashSet) -> Result { + let mut scalar = scalar.clone(); + for used in scalar.used_columns() { + let Some(replacement) = self.definitions.get(&used) else { + continue; + }; + if !visiting.insert(used) { + return Err(ErrorCode::Internal(format!( + "cyclic materialized view scalar dependency at column {used}" + ))); + } + let replacement = self.expand(replacement, visiting)?; + visiting.remove(&used); + scalar.replace_column_with_scalar(used, &replacement)?; + } + Ok(scalar) + } + + fn lower(&self, scalar: &ScalarExpr) -> Result { + let scalar = self.expand(scalar, &mut HashSet::new())?; + scalar + .type_check(self.metadata)? + .project_column_ref(|symbol| { + materialized_view_source_offset(self.metadata, *symbol, self.input_schema) + }) + } +} + +async fn materialized_view_table_info( + catalog: &Arc, + mv: &MVInfo, +) -> Result { + let table_name = catalog + .get_table_name_by_id(mv.mv_id) + .await? + .ok_or_else(|| { + ErrorCode::UnknownTable(format!("materialized view table id {}", mv.mv_id)) + })?; + let database_id = mv + .table_meta + .data + .options + .get(OPT_KEY_DATABASE_ID) + .ok_or_else(|| { + ErrorCode::Internal(format!( + "materialized view {} has no database id option", + mv.mv_id + )) + })? + .parse::() + .map_err(|error| { + ErrorCode::Internal(format!( + "materialized view {} has invalid database id: {error}", + mv.mv_id + )) + })?; + let database_name = catalog.get_db_name_by_id(database_id).await?; + // Resolve the current name through the catalog instead of inventing one. The ident must use + // TableMeta's sequence, not the MV-definition sequence. + Ok(TableInfo::new_full( + &database_name, + &table_name, + TableIdent::new(mv.mv_id, mv.table_meta.seq), + mv.table_meta.data.clone(), + catalog.info(), + DatabaseType::NormalDB, + )) +} + +pub async fn load_materialized_views( + ctx: Arc, + catalog_name: &str, + base_table: &Arc, +) -> Result> { + if base_table.engine() != "FUSE" { + return Ok(None); + } + let base_snapshot_location = base_table.options().get(OPT_KEY_SNAPSHOT_LOCATION); + let catalog = ctx.get_catalog(catalog_name).await?; + let tenant = ctx.get_tenant(); + let source_mvs = catalog + .list_mvs_by_source_table_id(&tenant, base_table.get_id()) + .await?; + if source_mvs.is_empty() { + return Ok(None); + } + + let mut materialized_views = Vec::with_capacity(source_mvs.len()); + for mv in source_mvs { + let mv_source_snapshot = mv + .table_meta + .data + .options + .get(OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION); + if mv_source_snapshot != base_snapshot_location { + continue; + } + + let table_info = materialized_view_table_info(&catalog, &mv).await?; + let mv_table = catalog.get_table_by_info(&table_info)?; + let fuse = FuseTable::try_from_table(mv_table.as_ref())?; + let snapshot = fuse.read_table_snapshot().await?; + let table_meta_timestamps = ctx.get_table_meta_timestamps(mv_table.as_ref(), snapshot)?; + + let mut planner = Planner::new(ctx.clone()); + let (definition_plan, _) = planner.bind_sql(&mv.definition.data.query).await?; + let databend_common_sql::plans::Plan::Query { + s_expr, metadata, .. + } = definition_plan + else { + return Err(ErrorCode::Internal(format!( + "materialized view {} definition is not a query", + mv.mv_id + ))); + }; + let materialized_view_target = create_materialized_view_target( + base_table.as_ref(), + mv_table, + table_meta_timestamps, + &s_expr, + metadata, + )?; + materialized_views.push(materialized_view_target); + } + + if materialized_views.is_empty() { + return Ok(None); + } + Ok(Some(MaterializedViewWrite { + catalog, + materialized_views, + })) +} + +fn create_materialized_view_target( + base_table: &dyn Table, + mv_table: Arc, + table_meta_timestamps: TableMetaTimestamps, + s_expr: &SExpr, + metadata: MetadataRef, +) -> Result { + // extract materialized view plans + let mut filters = Vec::new(); + let mut evals = Vec::new(); + let mut aggregate = None; + let mut scan = None; + let mut current = s_expr; + loop { + match current.plan() { + RelOperator::Filter(filter) => filters.push(filter), + RelOperator::EvalScalar(eval) => evals.push(eval), + RelOperator::Aggregate(node) => { + if node.mode != AggregateMode::Initial { + return Err(ErrorCode::Internal(format!( + "bind-only materialized view plan contains unexpected aggregate mode {:?}", + node.mode + ))); + } + if aggregate.replace(node).is_some() { + return Err(ErrorCode::Unimplemented( + "materialized view does not support nested aggregates", + )); + } + } + RelOperator::Scan(node) => scan = Some(node), + _ => {} + } + if current.arity() == 0 { + break; + } + if current.arity() != 1 { + return Err(ErrorCode::Unimplemented( + "materialized view does not support joins or multi-input plans", + )); + } + current = current.child(0)?; + } + let scan = scan.ok_or_else(|| { + ErrorCode::Internal("materialized view definition has no base-table scan") + })?; + + let input_schema = DataSchema::from(base_table.schema()); + let metadata = metadata.read(); + if metadata.table(scan.table_index).table().get_id() != base_table.get_id() { + return Err(ErrorCode::Internal(format!( + "materialized view definition scans table {}, expected source table {}", + metadata.table(scan.table_index).table().get_id(), + base_table.get_id() + ))); + } + + let scalar_lowerer = + MaterializedViewScalarLowerer::try_create(&metadata, &input_schema, &evals)?; + let lower_scalar = |scalar: &ScalarExpr| scalar_lowerer.lower(scalar); + + let mut filter_expr = None; + for predicate in filters + .into_iter() + .flat_map(|filter| filter.predicates.iter()) + // Row access policies are attached to Scan during binding, unlike ordinary pushdown and + // prewhere predicates, which only appear after optimization. + .chain(scan.secure_predicates.iter().flatten()) + { + let predicate = lower_scalar(predicate)?; + filter_expr = combine_materialized_view_filters(filter_expr, Some(predicate))?; + } + + let (projection, group_columns, group_data_types, aggregate_functions, aggregate_arguments) = + if let Some(aggregate) = aggregate { + let mut projection = Vec::new(); + let mut projection_by_symbol = HashMap::new(); + + let group_columns = aggregate + .group_items + .iter() + .map(|item| { + add_materialized_view_projection( + item.index, + &item.scalar, + &mut projection, + &mut projection_by_symbol, + &lower_scalar, + ) + }) + .collect::>>()?; + let mut aggregate_arguments: Vec> = + Vec::with_capacity(aggregate.aggregate_functions.len()); + let mut aggregate_sort_offsets: Vec> = + Vec::with_capacity(aggregate.aggregate_functions.len()); + for item in &aggregate.aggregate_functions { + let ScalarExpr::AggregateFunction(function) = &item.scalar else { + return Err(ErrorCode::Unimplemented( + "materialized view only supports built-in aggregate functions", + )); + }; + aggregate_arguments.push( + function + .args + .iter() + .map(|argument| match argument { + ScalarExpr::BoundColumnRef(column) => add_materialized_view_projection( + column.column.index, + argument, + &mut projection, + &mut projection_by_symbol, + &lower_scalar, + ), + _ => Err(ErrorCode::Internal( + "materialized view aggregate argument must be a bound column", + )), + }) + .collect::>>()?, + ); + aggregate_sort_offsets.push( + function + .sort_descs + .iter() + .map(|desc| { + let ScalarExpr::BoundColumnRef(column) = &desc.expr else { + return Err(ErrorCode::Internal( + "materialized view aggregate sort expression must be a bound column", + )); + }; + add_materialized_view_projection( + column.column.index, + &desc.expr, + &mut projection, + &mut projection_by_symbol, + &lower_scalar, + ) + }) + .collect::>>()?, + ); + } + + let group_data_types = group_columns + .iter() + .map(|offset| projection[*offset].data_type().clone()) + .collect::>(); + let mut aggregate_functions = Vec::with_capacity(aggregate.aggregate_functions.len()); + for ((item, arguments), sort_offsets) in aggregate + .aggregate_functions + .iter() + .zip(aggregate_arguments.iter()) + .zip(aggregate_sort_offsets.iter()) + { + let ScalarExpr::AggregateFunction(function) = &item.scalar else { + return Err(ErrorCode::Unimplemented( + "materialized view only supports built-in aggregate functions", + )); + }; + let argument_types = arguments + .iter() + .map(|offset| projection[*offset].data_type().clone()) + .collect::>(); + let sort_descs = function + .sort_descs + .iter() + .zip(sort_offsets.iter()) + .map(|(desc, offset)| { + Ok( + databend_common_functions::aggregates::AggregateFunctionSortDesc { + index: SymbolOrOffset::Offset(*offset), + is_reuse_index: desc.is_reuse_index, + data_type: desc.expr.data_type()?, + nulls_first: desc.nulls_first, + asc: desc.asc, + }, + ) + }) + .collect::>>()?; + aggregate_functions.push( + AggregateFunctionFactory::instance().get( + function + .func_name + .strip_suffix("_state") + .unwrap_or(&function.func_name), + function.params.clone(), + argument_types, + sort_descs, + )?, + ); + } + ( + projection, + group_columns, + group_data_types, + aggregate_functions, + aggregate_arguments, + ) + } else { + let output_eval = evals.first().ok_or_else(|| { + ErrorCode::Internal("non-aggregate materialized view has no output projection") + })?; + let projection = output_eval + .items + .iter() + .map(|item| lower_scalar(&item.scalar)) + .collect::>>()?; + (projection, vec![], vec![], vec![], vec![]) + }; + + let filter_expr = filter_expr.map(cast_expr_to_non_null_boolean).transpose()?; + let output_schema = DataSchema::from(mv_table.schema()); + let transform_plan = MaterializedViewTransformPlan { + table_id: mv_table.get_id(), + filter: filter_expr, + projection, + group_columns, + group_data_types, + aggregate_functions, + aggregate_arguments, + output_columns: output_schema.num_fields(), + output_data_types: output_schema + .fields() + .iter() + .map(|field| field.data_type().clone()) + .collect(), + }; + Ok(MaterializedViewWriteTarget { + target: MultiTableWriteTarget { + table: mv_table, + table_meta_timestamps, + materialized_view_source_table_id: Some(base_table.get_id()), + }, + transform_plan, + }) +} + +fn materialized_view_source_offset( + metadata: &databend_common_sql::Metadata, + symbol: databend_common_sql::Symbol, + input_schema: &DataSchema, +) -> Result { + let column = metadata.column(symbol); + let position = match column { + // Metadata column positions are SQL-style and one-based (`$1` is the first column), while + // DataBlock offsets are zero-based. + databend_common_sql::ColumnEntry::BaseTableColumn(column) => column + .column_position + .and_then(|position| position.checked_sub(1)), + _ => None, + }; + position.ok_or_else(|| { + ErrorCode::Internal(format!( + "materialized view column {} is not a base-table column in its source schema", + column.name() + )) + }).and_then(|position| { + if position < input_schema.num_fields() { + Ok(position) + } else { + Err(ErrorCode::Internal(format!( + "materialized view source column position {position} is outside its {}-column input", + input_schema.num_fields() + ))) + } + }) +} + +fn add_materialized_view_projection( + symbol: databend_common_sql::Symbol, + scalar: &ScalarExpr, + projection: &mut Vec, + projection_by_symbol: &mut HashMap, + lower_scalar: &F, +) -> Result +where + F: Fn(&ScalarExpr) -> Result, +{ + if let Some(offset) = projection_by_symbol.get(&symbol) { + return Ok(*offset); + } + let offset = projection.len(); + projection.push(lower_scalar(scalar)?); + projection_by_symbol.insert(symbol, offset); + Ok(offset) +} + +fn combine_materialized_view_filters( + left: Option, + right: Option, +) -> Result> { + match (left, right) { + (Some(left), Some(right)) => Ok(Some( + databend_common_expression::type_check::check_function( + None, + "and", + &[], + &[left, right], + &databend_common_functions::BUILTIN_FUNCTIONS, + )?, + )), + (Some(filter), None) | (None, Some(filter)) => Ok(Some(filter)), + (None, None) => Ok(None), + } +} + +impl PipelineBuilder { + #[allow(clippy::too_many_arguments)] + pub fn build_materialized_view_multi_table_write_pipeline( + ctx: Arc, + main_pipeline: &mut Pipeline, + base: MultiTableWriteTarget, + materialized_views: Vec, + catalog: Arc, + overwrite: bool, + update_stream_meta: Vec, + deduplicated_label: Option, + ) -> Result<()> { + let mut builder = + PipelineBuilder::create(ctx.get_function_context()?, ctx.get_settings(), ctx); + std::mem::swap(&mut builder.main_pipeline, main_pipeline); + + let result = (|| { + let targets = + builder.build_materialized_view_multi_table_write(base, materialized_views)?; + builder.build_multi_table_commit_sink( + &targets, + catalog, + overwrite, + update_stream_meta, + deduplicated_label, + ) + })(); + + std::mem::swap(&mut builder.main_pipeline, main_pipeline); + result + } + + /// Generates base/MV target branches and runs the shared multi-table write pipeline. + /// + /// On return the pipeline emits one `CommitMeta` lane per target. The caller must attach the + /// shared multi-table commit sink; `build_materialized_view_multi_table_write_pipeline` does + /// so after this function returns. + pub fn build_materialized_view_multi_table_write( + &mut self, + base: MultiTableWriteTarget, + materialized_views: Vec, + ) -> Result> { + if materialized_views.is_empty() { + return Err(ErrorCode::Internal( + "materialized view multi-table write requires at least one MV target".to_string(), + )); + } + + let mut table_ids = HashSet::with_capacity(materialized_views.len() + 1); + table_ids.insert(base.table.get_id()); + for materialized_view in &materialized_views { + let table_id = materialized_view.target.table.get_id(); + if !table_ids.insert(table_id) { + return Err(ErrorCode::Internal(format!( + "duplicate materialized view write target {}", + table_id + ))); + } + } + + let branch_width = self.main_pipeline.output_len(); + let branch_count = materialized_views.len() + 1; + self.main_pipeline.duplicate(true, branch_count)?; + + // Convert lane-major duplicate output to target-major layout: + // [base K lanes][mv0 K lanes]...[mvN K lanes]. + let total_width = branch_width * branch_count; + let mut transpose = vec![0; total_width]; + for branch in 0..branch_count { + for lane in 0..branch_width { + transpose[branch + lane * branch_count] = branch * branch_width + lane; + } + } + self.main_pipeline.reorder_inputs(transpose); + + // Preserve all base lanes, but merge every MV branch to one lane so one hash table sees + // every block from this INSERT. + let mut widths = vec![1; branch_width]; + widths.extend(std::iter::repeat_n(branch_width, materialized_views.len())); + self.main_pipeline.resize_partial_one_with_width(widths)?; + + let materialized_width = branch_width + materialized_views.len(); + let mut items = (0..branch_width) + .map(|_| create_dummy_item()) + .collect::>(); + for materialized_view in &materialized_views { + let input = InputPort::create(); + let output = OutputPort::create(); + let transform = TransformMaterializedView::try_create( + materialized_view.transform_plan.clone(), + self.func_ctx.clone(), + materialized_view.target.table.get_block_thresholds(), + )?; + let processor = ProcessorPtr::create(AccumulatingTransformer::create( + input.clone(), + output.clone(), + transform, + )); + items.push(PipeItem::create(processor, vec![input], vec![output])); + } + self.main_pipeline + .add_pipe(Pipe::create(materialized_width, materialized_width, items)); + + // Restore equal branch widths expected by `add_transforms_by_chunk`. Resize distributes + // aggregate-state blocks across K lanes; it does not duplicate them. + let mut items = (0..branch_width) + .map(|_| create_dummy_item()) + .collect::>(); + items.extend((0..materialized_views.len()).map(|_| create_resize_item(1, branch_width))); + self.main_pipeline + .add_pipe(Pipe::create(materialized_width, total_width, items)); + + let mut targets = Vec::with_capacity(branch_count); + targets.push(base); + targets.extend( + materialized_views + .into_iter() + .map(|materialized_view| materialized_view.target), + ); + + self.build_multi_table_append_data(&targets)?; + + // Merge each target's K block-meta lanes before its own segment writer and aggregator. + self.main_pipeline + .resize_partial_one_with_width(vec![branch_width; branch_count])?; + self.build_multi_table_commit_meta(&targets)?; + + Ok(targets) + } +} diff --git a/src/query/service/src/pipelines/builders/builder_multi_table_write.rs b/src/query/service/src/pipelines/builders/builder_multi_table_write.rs new file mode 100644 index 0000000000000..414198002950c --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_multi_table_write.rs @@ -0,0 +1,291 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use databend_common_catalog::catalog::Catalog; +use databend_common_catalog::table::Table; +use databend_common_exception::Result; +use databend_common_expression::DataSchema; +use databend_common_expression::LimitType; +use databend_common_expression::SortColumnDescription; +use databend_common_meta_app::schema::UpdateStreamMetaReq; +use databend_common_pipeline::core::DynTransformBuilder; +use databend_common_pipeline::core::ProcessorPtr; +use databend_common_pipeline::sinks::AsyncSinker; +use databend_common_pipeline_transforms::AccumulatingTransformer; +use databend_common_pipeline_transforms::AsyncAccumulatingTransformer; +use databend_common_pipeline_transforms::blocks::CompoundBlockOperator; +use databend_common_pipeline_transforms::sorts::TransformSortPartial; +use databend_common_sql::executor::physical_plans::MutationKind; +use databend_common_storages_fuse::FuseTable; +use databend_common_storages_fuse::io::StreamBlockProperties; +use databend_common_storages_fuse::operations::CommitMultiTableInsert; +use databend_common_storages_fuse::operations::TransformBlockBuilder; +use databend_common_storages_fuse::operations::TransformBlockWriter; +use databend_common_storages_fuse::operations::TransformVectorCluster; +use databend_storages_common_table_meta::meta::TableMetaTimestamps; + +use crate::pipelines::PipelineBuilder; + +#[derive(Clone)] +pub struct MultiTableWriteTarget { + pub table: Arc, + pub table_meta_timestamps: TableMetaTimestamps, + pub materialized_view_source_table_id: Option, +} + +impl PipelineBuilder { + /// Writes equally-sized, target-major pipeline branches as independent Fuse tables. + /// + /// Input layout must be `[target0 lanes][target1 lanes]...`; all targets must have the same + /// number of lanes. This is shared by SQL multi-table insert and synchronous MV maintenance. + pub fn build_multi_table_append_data( + &mut self, + targets: &[MultiTableWriteTarget], + ) -> Result<()> { + if targets.is_empty() { + return Ok(()); + } + + let mut compact_task_builders: Vec = Vec::with_capacity(targets.len()); + let mut compact_transform_builders: Vec = + Vec::with_capacity(targets.len()); + let mut eval_cluster_key_builders: Vec = + Vec::with_capacity(targets.len()); + let mut vector_cluster_builders: Vec = + Vec::with_capacity(targets.len()); + let mut sort_builders: Vec = Vec::with_capacity(targets.len()); + let mut block_builder_builders: Vec = + Vec::with_capacity(targets.len()); + let mut block_writer_builders: Vec = Vec::with_capacity(targets.len()); + + let mut eval_cluster_key_num = 0; + let mut vector_cluster_num = 0; + let mut sort_num = 0; + let mut stream_write_num = 0; + + for target in targets { + let fuse_table = FuseTable::try_from_table(target.table.as_ref())?; + let block_thresholds = target.table.get_block_thresholds(); + let enable_stream_write = fuse_table.enable_stream_block_write(self.ctx.clone())?; + + if enable_stream_write { + compact_task_builders.push(Box::new(self.dummy_transform_builder())); + compact_transform_builders.push(Box::new(self.dummy_transform_builder())); + eval_cluster_key_builders.push(Box::new(self.dummy_transform_builder())); + vector_cluster_builders.push(Box::new(self.dummy_transform_builder())); + sort_builders.push(Box::new(self.dummy_transform_builder())); + + let properties = StreamBlockProperties::try_create( + self.ctx.clone(), + fuse_table, + MutationKind::Insert, + target.table_meta_timestamps, + )?; + block_builder_builders.push(Box::new(move |input, output| { + TransformBlockBuilder::try_create(input, output, properties.clone()) + })); + + let table = target.table.clone(); + let ctx = self.ctx.clone(); + block_writer_builders.push(Box::new(move |input, output| { + let fuse_table = FuseTable::try_from_table(table.as_ref())?; + Ok(ProcessorPtr::create(AsyncAccumulatingTransformer::create( + input, + output, + TransformBlockWriter::create( + ctx.clone(), + MutationKind::Insert, + fuse_table, + true, + ), + ))) + })); + stream_write_num += 1; + continue; + } + + compact_task_builders + .push(Box::new(self.block_compact_task_builder(block_thresholds)?)); + compact_transform_builders.push(Box::new(self.block_compact_transform_builder()?)); + + let schema: Arc = DataSchema::from(target.table.schema()).into(); + let num_input_columns = schema.num_fields(); + let cluster_stats_gen = + fuse_table.get_cluster_stats_gen(self.ctx.clone(), 0, block_thresholds, schema)?; + let operators = cluster_stats_gen.operators.clone(); + if operators.is_empty() { + eval_cluster_key_builders.push(Box::new(self.dummy_transform_builder())); + } else { + let func_ctx = cluster_stats_gen.func_ctx.clone(); + eval_cluster_key_builders.push(Box::new(move |input, output| { + Ok(ProcessorPtr::create(CompoundBlockOperator::create( + input, + output, + num_input_columns, + func_ctx.clone(), + operators.clone(), + ))) + })); + eval_cluster_key_num += 1; + } + + if let Some(vector_operator) = cluster_stats_gen.vector_operator.clone() { + let rows_per_block = block_thresholds.max_rows_per_block; + vector_cluster_builders.push(Box::new(move |input, output| { + Ok(ProcessorPtr::create(AccumulatingTransformer::create( + input, + output, + TransformVectorCluster::new( + vector_operator.vector_column_input_offset, + vector_operator.info.dimension, + vector_operator.info.distance_type, + rows_per_block, + ), + ))) + })); + vector_cluster_num += 1; + } else { + vector_cluster_builders.push(Box::new(self.dummy_transform_builder())); + } + + let sort_desc: Vec = cluster_stats_gen.sort_descs(); + if sort_desc.is_empty() { + sort_builders.push(Box::new(self.dummy_transform_builder())); + } else { + let sort_desc: Arc<[_]> = sort_desc.into(); + sort_builders.push(Box::new(move |input, output| { + Ok(ProcessorPtr::create(TransformSortPartial::try_create( + input, + output, + LimitType::None, + sort_desc.clone(), + )?)) + })); + sort_num += 1; + } + + block_builder_builders.push(Box::new( + self.with_tid_serialize_block_transform_builder( + target.table.clone(), + cluster_stats_gen, + target.table_meta_timestamps, + )?, + )); + block_writer_builders.push(Box::new(self.dummy_transform_builder())); + } + + self.main_pipeline + .add_transforms_by_chunk(compact_task_builders)?; + self.main_pipeline + .add_transforms_by_chunk(compact_transform_builders)?; + if eval_cluster_key_num > 0 { + self.main_pipeline + .add_transforms_by_chunk(eval_cluster_key_builders)?; + } + if vector_cluster_num > 0 { + self.main_pipeline + .add_transforms_by_chunk(vector_cluster_builders)?; + } + if sort_num > 0 { + self.main_pipeline.add_transforms_by_chunk(sort_builders)?; + } + self.main_pipeline + .add_transforms_by_chunk(block_builder_builders)?; + if stream_write_num > 0 { + self.main_pipeline + .add_transforms_by_chunk(block_writer_builders)?; + } + Ok(()) + } + + /// Converts one lane per target from block metadata into one `CommitMeta` per target. + pub fn build_multi_table_commit_meta( + &mut self, + targets: &[MultiTableWriteTarget], + ) -> Result<()> { + let mut serialize_segment_builders: Vec = + Vec::with_capacity(targets.len()); + let mut mutation_aggregator_builders: Vec = + Vec::with_capacity(targets.len()); + + for target in targets { + serialize_segment_builders.push(Box::new(self.serialize_segment_transform_builder( + target.table.clone(), + target.table.get_block_thresholds(), + target.table_meta_timestamps, + )?)); + mutation_aggregator_builders.push(Box::new( + self.mutation_aggregator_transform_builder( + target.table.clone(), + target.table_meta_timestamps, + )?, + )); + } + + self.main_pipeline + .add_transforms_by_chunk(serialize_segment_builders)?; + self.main_pipeline + .add_transforms_by_chunk(mutation_aggregator_builders)?; + Ok(()) + } + + /// Collects all target `CommitMeta`s and atomically publishes their table snapshots. + #[allow(clippy::too_many_arguments)] + pub fn build_multi_table_commit_sink( + &mut self, + targets: &[MultiTableWriteTarget], + catalog: Arc, + overwrite: bool, + update_stream_meta: Vec, + deduplicated_label: Option, + ) -> Result<()> { + let tables: HashMap<_, _> = targets + .iter() + .map(|target| (target.table.get_id(), target.table.clone())) + .collect(); + let table_meta_timestamps: HashMap<_, _> = targets + .iter() + .map(|target| (target.table.get_id(), target.table_meta_timestamps)) + .collect(); + let materialized_view_sources: HashMap<_, _> = targets + .iter() + .filter_map(|target| { + target + .materialized_view_source_table_id + .map(|source_table_id| (target.table.get_id(), source_table_id)) + }) + .collect(); + + let ctx = self.ctx.clone(); + self.main_pipeline.try_resize(1)?; + self.main_pipeline.add_sink(|input| { + Ok(ProcessorPtr::create(AsyncSinker::create( + input, + CommitMultiTableInsert::create( + tables.clone(), + ctx.clone(), + overwrite, + update_stream_meta.clone(), + deduplicated_label.clone(), + catalog.clone(), + table_meta_timestamps.clone(), + materialized_view_sources.clone(), + ), + ))) + }) + } +} diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index 159e12a0f4fe8..3620f1c8cdcc2 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -20,8 +20,10 @@ mod builder_append_table; mod builder_fill_missing_columns; mod builder_join; +mod builder_materialized_view; mod builder_copy_into_table; +mod builder_multi_table_write; mod builder_mutation; mod builder_on_finished; mod builder_project; @@ -31,6 +33,10 @@ mod builder_udtf; mod merge_into_join_optimizations; mod transform_builder; +pub use builder_materialized_view::MaterializedViewWrite; +pub use builder_materialized_view::MaterializedViewWriteTarget; +pub use builder_materialized_view::load_materialized_views; +pub use builder_multi_table_write::MultiTableWriteTarget; pub use builder_replace_into::RawValueSource; pub use builder_replace_into::ValueSource; pub use builder_sort::SortPipelineBuilder; diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_materialized_view.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_materialized_view.rs new file mode 100644 index 0000000000000..d8619fe197c5c --- /dev/null +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_materialized_view.rs @@ -0,0 +1,413 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_ast::ast::SampleConfig; +use databend_common_ast::ast::Statement; +use databend_common_ast::ast::TableAlias; +use databend_common_ast::parser::parse_expr; +use databend_common_ast::parser::parse_sql; +use databend_common_ast::parser::tokenize_sql; +use databend_common_catalog::table::Table; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::types::DataType; +use databend_common_functions::aggregates::AggregateFunctionFactory; +use databend_common_meta_app::schema::MVDefinition; +use databend_storages_common_table_meta::table::OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION; +use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; +use log::info; + +use crate::BindContext; +use crate::MaterializedViewAggregateDesc; +use crate::MaterializedViewScanInfo; +use crate::Metadata; +use crate::ScalarExpr; +use crate::Visibility; +use crate::binder::Binder; +use crate::binder::ScalarBinder; +use crate::optimizer::ir::SExpr; +use crate::plans::Aggregate; +use crate::plans::EvalScalar; +use crate::plans::RelOperator; +use crate::plans::ScalarItem; + +impl Binder { + fn find_materialized_view_aggregate(s_expr: &SExpr) -> Option<&Aggregate> { + let mut current = s_expr; + loop { + if let RelOperator::Aggregate(aggregate) = current.plan() { + return Some(aggregate); + } + let Ok(child) = current.child(0) else { + return None; + }; + current = child; + } + } + + /// Whether the MV storage is still consistent with its source table snapshot. + /// + /// Uses the same option pair as the synchronous multi-table insert path: + /// `materialized_view_source_snapshot_location` vs source `snapshot_loc`. + fn is_materialized_view_fresh( + mv_table: &dyn Table, + source_snapshot_location: Option<&String>, + ) -> bool { + let mv_source_snapshot = mv_table + .options() + .get(OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION); + mv_source_snapshot == source_snapshot_location + } + + // Stale MV read: bind the persisted logical definition against the live source table. + // TODO: Support hybrid read in storage layer. + fn bind_materialized_view_fallback( + &mut self, + bind_context: &mut BindContext, + mv_definition: &MVDefinition, + database: &str, + table_name: &str, + alias: &Option, + ) -> Result<(SExpr, BindContext)> { + let tokens = tokenize_sql(&mv_definition.original_query)?; + let (stmt, _) = parse_sql(&tokens, self.dialect)?; + let Statement::Query(query) = &stmt else { + return Err(ErrorCode::Internal( + "Invalid materialized view logical query", + )); + }; + + // Bind with the current binder so the source table enters this query's metadata. + let mut definition_context = BindContext::with_parent(bind_context.clone())?; + let (s_expr, mut logical_context) = self.bind_query(&mut definition_context, query)?; + if logical_context.columns.len() != mv_definition.logical_schema.num_fields() { + return Err(ErrorCode::Internal(format!( + "materialized view logical query has {} columns, expected {}", + logical_context.columns.len(), + mv_definition.logical_schema.num_fields() + ))); + } + for (column, field) in logical_context + .columns + .iter_mut() + .zip(mv_definition.logical_schema.fields()) + { + let expected_type = DataType::from(field.data_type()); + if column.data_type.as_ref() != &expected_type { + return Err(ErrorCode::Internal(format!( + "materialized view logical column '{}' expects {}, query produces {}", + field.name(), + expected_type, + column.data_type + ))); + } + column.column_name = field.name().clone(); + } + if let Some(alias) = alias { + logical_context.apply_table_alias(alias, &self.name_resolution_ctx)?; + } else { + for column in logical_context.columns.iter_mut() { + column.database_name = Some(database.to_string()); + column.table_name = Some(table_name.to_string()); + } + } + Ok((s_expr, logical_context)) + } + + fn build_materialized_view_scan_info( + &self, + s_expr: &SExpr, + physical_schema: &databend_common_expression::TableSchema, + ) -> Result> { + let Some(aggregate) = Self::find_materialized_view_aggregate(s_expr) else { + return Ok(None); + }; + if aggregate.aggregate_functions.is_empty() && aggregate.group_items.is_empty() { + return Ok(None); + } + + let aggregate_functions = aggregate + .aggregate_functions + .iter() + .map(|item| { + let ScalarExpr::AggregateFunction(function) = &item.scalar else { + return Err(ErrorCode::Unimplemented( + "materialized view read only supports built-in aggregate functions", + )); + }; + let argument_types = function + .args + .iter() + .map(ScalarExpr::data_type) + .collect::>>()?; + Ok(MaterializedViewAggregateDesc { + name: function.func_name.clone(), + params: function.params.clone(), + argument_types, + }) + }) + .collect::>>()?; + + let num_aggregates = aggregate_functions.len(); + let num_groups = aggregate.group_items.len(); + if physical_schema.num_fields() != num_aggregates + num_groups { + return Err(ErrorCode::Internal(format!( + "materialized view state schema has {} columns, expected {} aggregate states and {} group columns", + physical_schema.num_fields(), + num_aggregates, + num_groups + ))); + } + + let physical_data_types = physical_schema + .fields() + .iter() + .map(|field| DataType::from(field.data_type())) + .collect::>(); + let group_data_types = physical_data_types[num_aggregates..].to_vec(); + + let mut final_data_types = aggregate_functions + .iter() + .enumerate() + .map(|(offset, aggregate)| { + let result_type = AggregateFunctionFactory::instance() + .get( + &aggregate.name, + aggregate.params.clone(), + aggregate.argument_types.clone(), + vec![], + )? + .return_type()?; + if physical_data_types[offset].is_nullable() && !result_type.is_nullable() { + Ok(result_type.wrap_nullable()) + } else { + Ok(result_type) + } + }) + .collect::>>()?; + final_data_types.extend(group_data_types.iter().cloned()); + + Ok(Some(MaterializedViewScanInfo { + aggregate_functions, + physical_data_types, + final_data_types, + group_data_types, + })) + } + + fn build_materialized_view_final_projection( + &mut self, + physical_context: &BindContext, + logical_schema: &databend_common_expression::TableSchema, + child: SExpr, + ) -> Result<(SExpr, BindContext)> { + let layout_version = logical_schema + .metadata + .get("_materialized_view_layout_version") + .ok_or_else(|| { + ErrorCode::Internal( + "materialized view uses a legacy logical layout; drop and recreate it", + ) + })?; + if layout_version != "1" { + return Err(ErrorCode::Internal(format!( + "unsupported materialized view logical layout version {layout_version}" + ))); + } + + let physical_columns = physical_context + .columns + .iter() + .filter(|column| column.visibility == Visibility::Visible) + .cloned() + .collect::>(); + let mut expression_context = BindContext::new(); + expression_context.columns = physical_columns; + + let mut items = Vec::with_capacity(logical_schema.num_fields()); + let mut output_columns = Vec::with_capacity(logical_schema.num_fields()); + for logical_field in logical_schema.fields() { + // For MVDefinition.schema only, default_expr stores the canonical finalized + // projection expression rather than a table-column default value. + let final_expr = logical_field.default_expr().ok_or_else(|| { + ErrorCode::Internal(format!( + "materialized view logical column '{}' has no final expression; drop and recreate it", + logical_field.name() + )) + })?; + let tokens = tokenize_sql(final_expr)?; + let ast = parse_expr(&tokens, self.dialect)?; + let (scalar, scalar_type) = { + let mut scalar_binder = ScalarBinder::new( + &mut expression_context, + self.ctx.clone(), + &self.name_resolution_ctx, + self.metadata.clone(), + &[], + ); + scalar_binder.forbid_udf(); + scalar_binder.bind(&ast)? + }; + let logical_type = DataType::from(logical_field.data_type()); + if scalar_type != logical_type { + return Err(ErrorCode::Internal(format!( + "materialized view logical column '{}' expects {}, final expression produces {}; drop and recreate it", + logical_field.name(), + logical_type, + scalar_type + ))); + } + let output = + self.create_derived_column_binding(logical_field.name().to_string(), logical_type); + items.push(ScalarItem { + scalar, + index: output.index, + }); + output_columns.push(output); + } + + let projection = + SExpr::create_unary(Arc::new(EvalScalar { items }.into()), Arc::new(child)); + let mut output_context = BindContext::new(); + output_context.parent = physical_context.parent.clone(); + output_context + .cte_context + .set_cte_context(physical_context.cte_context.clone()); + output_context.columns = output_columns; + Ok((projection, output_context)) + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn bind_materialized_view( + &mut self, + bind_context: &mut BindContext, + catalog_name: &str, + database: &str, + table_name: &str, + table_name_alias: Option, + table_meta: Arc, + alias: &Option, + sample: &Option, + cte_suffix_name: Option, + ) -> Result<(SExpr, BindContext)> { + let tenant = self.ctx.get_tenant(); + let (mv_definition, source_snapshot_location) = + databend_common_base::runtime::block_on(async { + let catalog = self.ctx.get_catalog(catalog_name).await?; + let mv_definition = catalog + .get_mv_definition(&tenant, table_meta.get_id()) + .await? + .ok_or_else(|| { + ErrorCode::Internal(format!( + "materialized view {} has no definition", + table_meta.name() + )) + })?; + let source_table_id = table_meta + .get_table_info() + .meta + .materialized_view_source_table_id() + .map_err(ErrorCode::from)?; + let source_meta = catalog + .get_table_meta_by_id(source_table_id) + .await? + .ok_or_else(|| { + ErrorCode::UnknownTable(format!( + "materialized view {} source table id {} not found", + table_meta.name(), + source_table_id + )) + })?; + let source_snapshot_location = source_meta + .data + .options + .get(OPT_KEY_SNAPSHOT_LOCATION) + .cloned(); + Ok::<_, ErrorCode>((mv_definition, source_snapshot_location)) + })?; + + if !Self::is_materialized_view_fresh(table_meta.as_ref(), source_snapshot_location.as_ref()) + { + info!( + "materialized view {} is stale (source_snapshot={:?}, base_snapshot={:?}); fallback to live compute", + table_meta.name(), + table_meta + .options() + .get(OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION), + source_snapshot_location + ); + return self.bind_materialized_view_fallback( + bind_context, + &mv_definition.data, + database, + table_name, + alias, + ); + } + + let tokens = tokenize_sql(&mv_definition.data.query)?; + let (stmt, _) = parse_sql(&tokens, self.dialect)?; + let Statement::Query(query) = &stmt else { + return Err(ErrorCode::Internal("Invalid materialized view query")); + }; + let mut definition_binder = Binder::new( + self.ctx.clone(), + self.catalogs.clone(), + self.name_resolution_ctx.clone(), + Metadata::default_ref(), + ) + .with_subquery_executor(self.subquery_executor.clone()); + let mut definition_context = BindContext::new(); + let (definition_expr, _) = definition_binder.bind_query(&mut definition_context, query)?; + let storage_scan_info = definition_binder + .build_materialized_view_scan_info(&definition_expr, table_meta.schema().as_ref())?; + + let table_index = self.metadata.write().add_table( + catalog_name.to_string(), + database.to_string(), + table_meta, + None, + table_name_alias, + !bind_context.binding_views.is_empty(), + bind_context.planning_agg_index, + false, + cte_suffix_name, + ); + if let Some(scan_info) = storage_scan_info { + let mut metadata = self.metadata.write(); + metadata + .set_materialized_view_column_types(table_index, &scan_info.final_data_types)?; + metadata.set_materialized_view_scan(table_index, scan_info); + } + + let (s_expr, physical_context) = + self.bind_base_table(bind_context, database, table_index, None, sample, false)?; + let (s_expr, mut logical_context) = self.build_materialized_view_final_projection( + &physical_context, + &mv_definition.data.logical_schema, + s_expr, + )?; + if let Some(alias) = alias { + logical_context.apply_table_alias(alias, &self.name_resolution_ctx)?; + } else { + for column in logical_context.columns.iter_mut() { + column.database_name = Some(database.to_string()); + column.table_name = Some(table_name.to_string()); + } + } + Ok((s_expr, logical_context)) + } +} diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 438d1f2a0a62f..d0a1737576fd5 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -27,6 +27,7 @@ use databend_common_catalog::table_with_options::get_with_opt_consume; use databend_common_catalog::table_with_options::get_with_opt_max_batch_size; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_app::schema::MATERIALIZED_VIEW_ENGINE; use databend_common_storages_basic::view_table::QUERY; use databend_storages_common_table_meta::table::get_change_type; @@ -37,7 +38,6 @@ use crate::binder::util::TableIdentifier; use crate::binder::util::legacy_table_ref_removed_error; use crate::optimizer::ir::SExpr; impl Binder { - /// Bind a base table. /// A base table is a table that is not a view or CTE. #[allow(clippy::too_many_arguments)] pub(crate) fn bind_table( @@ -314,6 +314,17 @@ impl Binder { ) } } + MATERIALIZED_VIEW_ENGINE => self.bind_materialized_view( + bind_context, + &catalog, + &database, + &self.normalize_identifier(table).name, + table_name_alias, + table_meta, + alias, + sample, + cte_suffix_name, + ), _ => { let table_index = self.metadata.write().add_table( catalog.clone(), diff --git a/src/query/sql/src/planner/binder/bind_table_reference/mod.rs b/src/query/sql/src/planner/binder/bind_table_reference/mod.rs index 38766d581616a..b631af527d2ac 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/mod.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/mod.rs @@ -17,6 +17,7 @@ mod bind_asof_join; mod bind_cte; mod bind_join; mod bind_location; +mod bind_materialized_view; mod bind_obfuscate; mod bind_subquery; mod bind_table; diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index eff33db8a03e0..7057262fe1a56 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -349,6 +349,21 @@ impl Binder { Statement::DropView(stmt) => self.bind_drop_view(stmt).await?, Statement::ShowViews(stmt) => self.bind_show_views(bind_context, stmt).await?, Statement::DescribeView(stmt) => self.bind_describe_view(stmt).await?, + Statement::CreateMaterializedView(stmt) => { + self.bind_create_materialized_view(stmt).await? + } + Statement::DropMaterializedView(stmt) => self.bind_drop_materialized_view(stmt).await?, + Statement::RefreshMaterializedView(stmt) => { + self.bind_refresh_materialized_view(stmt).await? + } + Statement::DescribeMaterializedView(stmt) => { + self.bind_describe_materialized_view(bind_context, stmt) + .await? + } + Statement::ShowMaterializedViews(stmt) => { + self.bind_show_materialized_views(bind_context, stmt) + .await? + } // Indexes Statement::CreateIndex(stmt) => self.bind_create_index(bind_context, stmt).await?, diff --git a/src/query/sql/src/planner/binder/ddl/materialized_view.rs b/src/query/sql/src/planner/binder/ddl/materialized_view.rs new file mode 100644 index 0000000000000..dd811b24835ad --- /dev/null +++ b/src/query/sql/src/planner/binder/ddl/materialized_view.rs @@ -0,0 +1,504 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_ast::ast::CreateMaterializedViewStmt; +use databend_common_ast::ast::CreateOption as AstCreateOption; +use databend_common_ast::ast::DescribeMaterializedViewStmt; +use databend_common_ast::ast::DropMaterializedViewStmt; +use databend_common_ast::ast::RefreshMaterializedViewStmt; +use databend_common_ast::ast::ShowLimit; +use databend_common_ast::ast::ShowMaterializedViewsStmt; +use databend_common_ast::ast::quote::QuotedIdent; +use databend_common_ast::ast::quote::QuotedString; +use databend_common_ast::visit::WalkMut; +use databend_common_catalog::table::Table; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +use databend_common_expression::TableSchema; +use databend_common_expression::TableSchemaRef; +use databend_common_expression::TableSchemaRefExt; +use databend_common_expression::infer_schema_type; +use databend_common_functions::aggregates::AggregateFunctionFactory; +use databend_common_meta_app::schema::MVDefinition; +use databend_common_meta_app::schema::OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID; +use databend_common_meta_app::schema::is_materialized_view_engine; +use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; +use log::debug; + +use crate::BindContext; +use crate::MetadataRef; +use crate::SelectBuilder; +use crate::binder::Binder; +use crate::optimizer::ir::SExpr; +use crate::planner::SUPPORTED_AGGREGATING_INDEX_FUNCTIONS; +use crate::planner::semantic::MaterializedViewChecker; +use crate::planner::semantic::MaterializedViewRewriter; +use crate::planner::semantic::ViewRewriter; +use crate::planner::semantic::normalize_identifier; +use crate::plans::Aggregate; +use crate::plans::CreateMaterializedViewPlan; +use crate::plans::DropMaterializedViewPlan; +use crate::plans::Plan; +use crate::plans::RelOperator; +use crate::plans::RewriteKind; +use crate::plans::ScalarExpr; + +/// Layout version stored in `MVDefinition.schema.metadata`. +/// Bump this when the logical-to-physical projection contract changes. +const MATERIALIZED_VIEW_LAYOUT_VERSION: &str = "1"; +const MATERIALIZED_VIEW_LAYOUT_VERSION_KEY: &str = "_materialized_view_layout_version"; + +/// Walk a unary plan chain and return the first Aggregate node, if any. +pub(crate) fn find_materialized_view_aggregate(s_expr: &SExpr) -> Option<&Aggregate> { + let mut current = s_expr; + loop { + if let RelOperator::Aggregate(aggregate) = current.plan() { + return Some(aggregate); + } + let Ok(child) = current.child(0) else { + return None; + }; + current = child; + } +} + +/// Table schemas reject bare Null columns; normalize them the same way CREATE TABLE does. +fn normalize_null_fields(schema: TableSchemaRef) -> TableSchemaRef { + let mut fields = schema.fields().clone(); + let mut changed = false; + for field in fields.iter_mut() { + if field.data_type == TableDataType::Null { + field.data_type = TableDataType::String.wrap_nullable(); + changed = true; + } + } + if changed { + TableSchemaRefExt::create(fields) + } else { + schema + } +} + +fn with_layout_version(mut schema: TableSchema) -> TableSchemaRef { + schema.metadata.insert( + MATERIALIZED_VIEW_LAYOUT_VERSION_KEY.to_string(), + MATERIALIZED_VIEW_LAYOUT_VERSION.to_string(), + ); + Arc::new(schema) +} + +fn query_s_expr(plan: &Plan) -> Result<&SExpr> { + match plan { + Plan::Query { s_expr, .. } => Ok(s_expr), + _ => Err(ErrorCode::Internal( + "materialized view AS clause must produce a Query plan", + )), + } +} + +impl Binder { + /// Build the physical schema from rewriter names and types inferred by binding. + /// Aggregate outputs are persisted as serialized states, while group keys and + /// non-aggregate outputs use their bound result types. + fn materialized_view_physical_schema( + s_expr: &SExpr, + bind_context: &BindContext, + metadata: MetadataRef, + physical_names: &[String], + ) -> Result { + let data_types = if let Some(aggregate) = find_materialized_view_aggregate(s_expr) { + let mut data_types = Vec::with_capacity( + aggregate.aggregate_functions.len() + aggregate.group_items.len(), + ); + for item in &aggregate.aggregate_functions { + let ScalarExpr::AggregateFunction(function) = &item.scalar else { + return Err(ErrorCode::Unimplemented( + "materialized view only supports built-in aggregate functions", + )); + }; + let agg_name = function + .func_name + .strip_suffix("_state") + .unwrap_or(&function.func_name); + let argument_types = function + .args + .iter() + .map(ScalarExpr::data_type) + .collect::>>()?; + let state_type = AggregateFunctionFactory::instance() + .get(agg_name, function.params.clone(), argument_types, vec![])? + .serialize_data_type(); + data_types.push(infer_schema_type(&state_type)?); + } + for item in &aggregate.group_items { + data_types.push(infer_schema_type(&item.scalar.data_type()?)?); + } + data_types + } else { + bind_context + .output_table_schema(metadata)? + .fields() + .iter() + .map(|field| field.data_type().clone()) + .collect() + }; + + if data_types.len() != physical_names.len() { + return Err(ErrorCode::Internal(format!( + "materialized view rewriter produced {} physical names, bound plan has {} fields", + physical_names.len(), + data_types.len() + ))); + } + Ok(TableSchemaRefExt::create( + physical_names + .iter() + .zip(data_types) + .map(|(name, data_type)| TableField::new(name, data_type)) + .collect(), + )) + } + + /// Build the logical schema from names and expressions recorded by the rewriter. + /// Types come from the bound original query. + fn materialized_view_logical_schema_from_rewriter( + logical_context: &BindContext, + rewriter: &MaterializedViewRewriter, + metadata: MetadataRef, + ) -> Result { + let schema = logical_context.output_table_schema(metadata)?; + let names = rewriter.logical_names(); + let define_exprs = rewriter.logical_define_exprs(); + if schema.num_fields() != names.len() || schema.num_fields() != define_exprs.len() { + return Err(ErrorCode::Internal(format!( + "materialized view has {} logical outputs, rewriter recorded {} names and {} expressions", + schema.num_fields(), + names.len(), + define_exprs.len() + ))); + } + let fields = schema + .fields() + .iter() + .zip(names) + .zip(define_exprs) + .map(|((field, name), define_expr)| { + TableField::new(name, field.data_type().clone()) + .with_default_expr(Some(define_expr.clone())) + }) + .collect(); + Ok(with_layout_version(TableSchema::new(fields))) + } + + async fn check_materialized_view_replace_loop( + &self, + create_option: &AstCreateOption, + target_catalog: &dyn databend_common_catalog::catalog::Catalog, + tenant: &databend_common_meta_app::tenant::Tenant, + database_name: &str, + view_name: &str, + original_query_plan: &Plan, + ) -> Result<()> { + if !matches!(*create_option, AstCreateOption::CreateOrReplace) { + return Ok(()); + } + let existing = match target_catalog + .get_table(tenant, database_name, view_name) + .await + { + Ok(table) => Some(table), + Err(e) if e.code() == ErrorCode::UNKNOWN_TABLE => None, + Err(e) => return Err(e), + }; + if let Some(existing) = existing + && is_materialized_view_engine(existing.engine()) + && let Plan::Query { metadata, .. } = original_query_plan + { + let metadata = metadata.read(); + let existing_table_id = existing.get_id(); + if metadata + .tables() + .iter() + .any(|table| table.table().get_id() == existing_table_id) + { + return Err(ErrorCode::ViewDependencyError(format!( + "View dependency loop detected (view: {}.{})", + database_name, view_name + ))); + } + } + Ok(()) + } + + /// Resolve the single base table referenced by the defining query. + fn materialize_view_source_table( + original_query_plan: &Plan, + ) -> Result<(Arc, String)> { + let Plan::Query { metadata, .. } = original_query_plan else { + return Err(ErrorCode::Internal( + "materialized view AS clause must produce a Query plan", + )); + }; + let metadata = metadata.read(); + if metadata.tables().len() != 1 { + return Err(ErrorCode::SemanticError( + "Materialized view requires exactly one base table source".to_string(), + )); + } + let table = &metadata.tables()[0]; + Ok((table.table(), table.database().to_string())) + } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_create_materialized_view( + &mut self, + stmt: &CreateMaterializedViewStmt, + ) -> Result { + let CreateMaterializedViewStmt { + create_option, + catalog, + database, + view, + columns, + query, + } = stmt; + + // 1. Resolve target catalog/database and bind the user-facing defining query. + // This yields the logical plan (visible column names/types) and the single + // source table the MV depends on. + let tenant = self.ctx.get_tenant(); + let (catalog_name, database_name, view_name) = + self.normalize_object_identifier_triple(catalog, database, view); + let target_catalog = self.ctx.get_catalog(&catalog_name).await?; + let target_database = target_catalog.get_database(&tenant, &database_name).await?; + let target_database_id = target_database.get_db_info().database_id.db_id; + + let original_query_plan = self.as_query_plan(query).await?; + self.check_materialized_view_replace_loop( + create_option, + target_catalog.as_ref(), + &tenant, + &database_name, + &view_name, + &original_query_plan, + ) + .await?; + + let original_bind_context = original_query_plan.bind_context().unwrap(); + let (source_table, source_database) = + Self::materialize_view_source_table(&original_query_plan)?; + let source_table_id = source_table.get_id(); + + // Preserve the logical query for stale reads, qualifying unqualified source tables just as + // regular views do so fallback binding does not depend on the session's current database. + let mut original_query = query.as_ref().clone(); + original_query.walk_mut(&mut ViewRewriter { + current_database: source_database.clone(), + })?; + + // 2. Validate and rewrite the query into storage form, recording logical→physical + // mapping in the same pass. + let checker = MaterializedViewChecker::check_query(query); + if !checker.is_supported() { + return Err(ErrorCode::SemanticError(format!( + "Materialized View only support simple query, like: \ + `SELECT ... FROM ... WHERE ... GROUP BY ...`, \ + and these aggregate funcs: {}, \ + non-deterministic functions are not support like: NOW()", + SUPPORTED_AGGREGATING_INDEX_FUNCTIONS.join(",") + ))); + } + + let mut physical_query = query.as_ref().clone(); + let specified_columns = columns + .iter() + .map(|column| normalize_identifier(column, &self.name_resolution_ctx).name) + .collect(); + let mut physical_rewriter = MaterializedViewRewriter::new( + checker.is_aggregating(), + &source_database, + specified_columns, + ); + physical_rewriter.rewrite_query(&mut physical_query)?; + + // 3. Bind the rewritten query and build physical storage slots. Names are + // then aligned with the rewriter-assigned aliases so default_expr resolves. + let mut physical_binder = Binder::new( + self.ctx.clone(), + self.catalogs.clone(), + self.name_resolution_ctx.clone(), + crate::Metadata::default_ref(), + ) + .with_subquery_executor(self.subquery_executor.clone()); + let physical_query_plan = physical_binder.as_query_plan(&physical_query).await?; + let physical_bind_context = physical_query_plan.bind_context().unwrap(); + let storage_expr = query_s_expr(&physical_query_plan)?; + let physical_schema = normalize_null_fields(Self::materialized_view_physical_schema( + storage_expr, + &physical_bind_context, + physical_binder.metadata.clone(), + physical_rewriter.physical_names(), + )?); + Self::validate_create_table_schema(&physical_schema)?; + + // 4. Logical schema: original bind types + rewriter final_expr as default_expr. + let logical_schema = + normalize_null_fields(Self::materialized_view_logical_schema_from_rewriter( + &original_bind_context, + &physical_rewriter, + self.metadata.clone(), + )?); + Self::validate_create_table_schema(&logical_schema)?; + + // 5. Assemble CreateTablePlan: TableMeta holds the physical schema used by + // Fuse storage; MVDefinition holds the original query, rewritten storage + // query, and logical schema (with final projection expressions). + let mut options = std::collections::BTreeMap::new(); + options.insert( + OPT_KEY_DATABASE_ID.to_owned(), + target_database_id.to_string(), + ); + options.insert( + OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID.to_owned(), + source_table_id.to_string(), + ); + + let mv_definition = MVDefinition { + original_query: original_query.to_string(), + query: physical_query.to_string(), + logical_schema: logical_schema.as_ref().clone(), + sync_creation: true, + }; + + let plan = CreateMaterializedViewPlan { + create_option: create_option.clone().into(), + tenant, + catalog: catalog_name, + database: database_name, + view_name, + schema: physical_schema, + options, + mv_definition, + }; + + Ok(Plan::CreateMaterializedView(Box::new(plan))) + } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_drop_materialized_view( + &mut self, + stmt: &DropMaterializedViewStmt, + ) -> Result { + let DropMaterializedViewStmt { + if_exists, + catalog, + database, + view, + } = stmt; + + let tenant = self.ctx.get_tenant(); + let (catalog, database, view_name) = + self.normalize_object_identifier_triple(catalog, database, view); + let plan = DropMaterializedViewPlan { + if_exists: *if_exists, + tenant, + catalog, + database, + view_name, + }; + Ok(Plan::DropMaterializedView(Box::new(plan))) + } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_refresh_materialized_view( + &mut self, + _stmt: &RefreshMaterializedViewStmt, + ) -> Result { + Err(ErrorCode::Unimplemented( + "REFRESH MATERIALIZED VIEW is not supported yet", + )) + } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_describe_materialized_view( + &mut self, + _bind_context: &mut BindContext, + _stmt: &DescribeMaterializedViewStmt, + ) -> Result { + Err(ErrorCode::Unimplemented( + "DESCRIBE MATERIALIZED VIEW is not supported yet", + )) + } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_show_materialized_views( + &mut self, + bind_context: &mut BindContext, + stmt: &ShowMaterializedViewsStmt, + ) -> Result { + let ShowMaterializedViewsStmt { + catalog, + database, + limit, + } = stmt; + + let catalog_name = match catalog { + None => self.ctx.get_current_catalog(), + Some(ident) => normalize_identifier(ident, &self.name_resolution_ctx).name, + }; + + let database_name = self.check_database_exist(catalog, database).await?; + + let mut select_builder = SelectBuilder::from(&format!( + "{}.system.tables", + QuotedIdent(catalog_name.to_lowercase(), '`') + )); + select_builder + .with_column("name AS Name") + .with_column("database AS Database") + .with_column("engine AS Engine") + .with_column("created_on AS \"Created On\""); + + select_builder.with_filter(format!("database = {}", QuotedString(&database_name, '\''))); + select_builder.with_filter(format!("catalog = {}", QuotedString(&catalog_name, '\''))); + select_builder.with_filter("table_type = 'MATERIALIZED VIEW'".to_string()); + + select_builder + .with_order_by("database") + .with_order_by("name"); + + let query = match limit { + None => select_builder.build(), + Some(ShowLimit::Like { pattern }) => { + select_builder.with_filter(format!("name LIKE {}", QuotedString(pattern, '\''))); + select_builder.build() + } + Some(ShowLimit::Where { selection }) => { + select_builder.with_filter(format!("({selection})")); + select_builder.build() + } + }; + debug!("show materialized views rewrite to: {:?}", query); + self.bind_rewrite_to_query( + bind_context, + query.as_str(), + RewriteKind::ShowTables(catalog_name, database_name), + ) + .await + } +} diff --git a/src/query/sql/src/planner/binder/ddl/mod.rs b/src/query/sql/src/planner/binder/ddl/mod.rs index 39e8156db46ea..4a367f149d4a4 100644 --- a/src/query/sql/src/planner/binder/ddl/mod.rs +++ b/src/query/sql/src/planner/binder/ddl/mod.rs @@ -21,6 +21,7 @@ pub mod database; mod dictionary; mod dynamic_table; pub(crate) mod index; +pub(crate) mod materialized_view; mod network_policy; mod notification; mod password_policy; diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index 1c288e3b19d95..5ff8d00488aff 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -547,7 +547,10 @@ impl Binder { } } - async fn as_query_plan(&mut self, query: &Query) -> Result { + pub(in crate::planner::binder) async fn as_query_plan( + &mut self, + query: &Query, + ) -> Result { let stmt = Statement::Query(Box::new(query.clone())); let mut bind_context = BindContext::new(); self.bind_statement(&mut bind_context, &stmt).await diff --git a/src/query/sql/src/planner/format/display_plan.rs b/src/query/sql/src/planner/format/display_plan.rs index 646373937120c..eb3a125ba717b 100644 --- a/src/query/sql/src/planner/format/display_plan.rs +++ b/src/query/sql/src/planner/format/display_plan.rs @@ -121,6 +121,12 @@ impl Plan { Plan::DropView(_) => Ok("DropView".to_string()), Plan::DescribeView(_) => Ok("DescribeView".to_string()), + // Materialized Views + Plan::CreateMaterializedView(_) => Ok("CreateMaterializedView".to_string()), + Plan::DropMaterializedView(_) => Ok("DropMaterializedView".to_string()), + Plan::RefreshMaterializedView(_) => Ok("RefreshMaterializedView".to_string()), + Plan::DescribeMaterializedView(_) => Ok("DescribeMaterializedViewStmt".to_string()), + // Streams Plan::CreateStream(_) => Ok("CreateStream".to_string()), Plan::DropStream(_) => Ok("DropStream".to_string()), diff --git a/src/query/sql/src/planner/metadata/metadata.rs b/src/query/sql/src/planner/metadata/metadata.rs index 2eb43cf50afd7..377b3af10ac55 100644 --- a/src/query/sql/src/planner/metadata/metadata.rs +++ b/src/query/sql/src/planner/metadata/metadata.rs @@ -25,12 +25,16 @@ use databend_common_ast::ast::Literal; use databend_common_catalog::plan::DataSourcePlan; use databend_common_catalog::plan::InternalColumn; use databend_common_catalog::table::Table; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; use databend_common_expression::ColumnId; use databend_common_expression::ComputedExpr; +use databend_common_expression::Scalar; pub use databend_common_expression::Symbol; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::display::display_tuple_field_name; +use databend_common_expression::infer_schema_type; use databend_common_expression::is_stream_column_id; use databend_common_expression::types::DataType; use jsonb::keypath::OwnedKeyPaths; @@ -49,6 +53,25 @@ pub const DUMMY_TABLE_INDEX: IndexType = IndexType::MAX; /// ColumnSet represents a set of columns identified by `Symbol`. pub type ColumnSet = BTreeSet; +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct MaterializedViewAggregateDesc { + pub name: String, + pub params: Vec, + pub argument_types: Vec, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct MaterializedViewScanInfo { + pub aggregate_functions: Vec, + /// Physical types stored in the Fuse-backed MV, in canonical + /// `[aggregate states..., group keys...]` order. + pub physical_data_types: Vec, + /// Finalized storage-query aggregate results followed by group keys. This is still the + /// storage-query schema; conversion to the logical MV schema is a separate projection phase. + pub final_data_types: Vec, + pub group_data_types: Vec, +} + /// A Send & Send version of [`Metadata`]. /// /// Callers can clone this ref safely and cheaply. @@ -76,6 +99,7 @@ pub struct Metadata { /// Mappings from table index to _row_id column index. table_row_id_index: HashMap, agg_indices: HashMap>, + materialized_view_scans: HashMap, max_column_position: usize, // for CSV has_column_name_ref: bool, // for schema inference from stage files @@ -101,6 +125,21 @@ impl Metadata { self.tables.get(index).expect("metadata must contain table") } + pub fn set_materialized_view_scan( + &mut self, + table_index: IndexType, + info: MaterializedViewScanInfo, + ) { + self.materialized_view_scans.insert(table_index, info); + } + + pub fn materialized_view_scan( + &self, + table_index: IndexType, + ) -> Option<&MaterializedViewScanInfo> { + self.materialized_view_scans.get(&table_index) + } + pub fn tables(&self) -> &[TableEntry] { self.tables.as_slice() } @@ -492,6 +531,50 @@ impl Metadata { table_index } + pub fn set_materialized_view_column_types( + &mut self, + table_index: IndexType, + final_data_types: &[DataType], + ) -> Result<()> { + let final_table_types = final_data_types + .iter() + .map(infer_schema_type) + .collect::>>()?; + let column_indexes = self + .columns + .iter() + .enumerate() + .filter_map(|(index, column)| match column { + ColumnEntry::BaseTableColumn(BaseTableColumn { + table_index: column_table_index, + path_indices: None, + column_id, + virtual_expr: None, + .. + }) if *column_table_index == table_index && !is_stream_column_id(*column_id) => { + Some(index) + } + _ => None, + }) + .collect::>(); + + if column_indexes.len() != final_table_types.len() { + return Err(ErrorCode::Internal(format!( + "materialized view table {table_index} has {} top-level columns, but its finalized schema has {} columns", + column_indexes.len(), + final_table_types.len() + ))); + } + + for (column_index, data_type) in column_indexes.into_iter().zip(final_table_types) { + let ColumnEntry::BaseTableColumn(column) = &mut self.columns[column_index] else { + unreachable!("materialized view column indexes were validated") + }; + column.data_type = data_type; + } + Ok(()) + } + pub fn change_derived_column_alias(&mut self, index: Symbol, alias: String) { let derived_column = self .columns diff --git a/src/query/sql/src/planner/planner.rs b/src/query/sql/src/planner/planner.rs index 96e36d958444b..260339a74aa83 100644 --- a/src/query/sql/src/planner/planner.rs +++ b/src/query/sql/src/planner/planner.rs @@ -95,6 +95,29 @@ impl Planner { Ok((plan, extras)) } + #[async_backtrace::framed] + #[fastrace::trace] + pub async fn bind_sql(&mut self, sql: &str) -> Result<(Plan, PlanExtras)> { + let extras = self.parse_sql(sql)?; + let settings = self.ctx.get_settings(); + let name_resolution_ctx = NameResolutionContext::try_from(settings.as_ref())?; + let metadata = Arc::new(RwLock::new(Metadata::default())); + let binder = Binder::new( + self.ctx.clone(), + CatalogManager::instance(), + name_resolution_ctx, + metadata, + ) + .with_subquery_executor(self.query_executor.clone()); + let query_kind = get_query_kind(&extras.statement); + self.ctx + .attach_query_str(query_kind, extras.statement.to_mask_sql()); + let plan = binder.bind(&extras.statement).await?; + self.ctx + .attach_query_str(query_kind, extras.statement.to_mask_sql()); + Ok((plan, extras)) + } + #[fastrace::trace] pub fn parse_sql_with_params(&self, sql: &str) -> Result { self.parse_sql_inner(sql, true) diff --git a/src/query/sql/src/planner/plans/ddl/materialized_view.rs b/src/query/sql/src/planner/plans/ddl/materialized_view.rs new file mode 100644 index 0000000000000..35f2261d5ed4d --- /dev/null +++ b/src/query/sql/src/planner/plans/ddl/materialized_view.rs @@ -0,0 +1,64 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_expression::DataSchemaRef; +use databend_common_expression::TableSchemaRef; +use databend_common_meta_app::schema::CreateOption; +use databend_common_meta_app::schema::MVDefinition; +use databend_common_meta_app::tenant::Tenant; + +use crate::plans::TableOptions; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CreateMaterializedViewPlan { + pub create_option: CreateOption, + pub tenant: Tenant, + pub catalog: String, + pub database: String, + pub view_name: String, + pub schema: TableSchemaRef, + pub options: TableOptions, + pub mv_definition: MVDefinition, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DropMaterializedViewPlan { + pub if_exists: bool, + pub tenant: Tenant, + pub catalog: String, + pub database: String, + pub view_name: String, +} + +#[derive(Clone, Debug)] +pub struct DescribeMaterializedViewPlan { + pub catalog: String, + pub database: String, + pub view_name: String, + pub schema: DataSchemaRef, +} + +impl DescribeMaterializedViewPlan { + pub fn schema(&self) -> DataSchemaRef { + self.schema.clone() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RefreshMaterializedViewPlan { + pub tenant: Tenant, + pub catalog: String, + pub database: String, + pub view_name: String, +} diff --git a/src/query/sql/src/planner/plans/ddl/mod.rs b/src/query/sql/src/planner/plans/ddl/mod.rs index 8eed077e40094..864a2f600267b 100644 --- a/src/query/sql/src/planner/plans/ddl/mod.rs +++ b/src/query/sql/src/planner/plans/ddl/mod.rs @@ -21,6 +21,7 @@ mod dictionary; mod dynamic_table; mod file_format; mod index; +mod materialized_view; mod notification; mod procedure; mod sequence; @@ -44,6 +45,7 @@ pub use dictionary::*; pub use dynamic_table::*; pub use file_format::*; pub use index::*; +pub use materialized_view::*; pub use notification::*; pub use procedure::*; pub use sequence::*; diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index 0dabd1d8b5a3b..63e2af9ad31b3 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -62,6 +62,7 @@ use crate::plans::CreateDatamaskPolicyPlan; use crate::plans::CreateDynamicTablePlan; use crate::plans::CreateFileFormatPlan; use crate::plans::CreateIndexPlan; +use crate::plans::CreateMaterializedViewPlan; use crate::plans::CreateNetworkPolicyPlan; use crate::plans::CreateNotificationPlan; use crate::plans::CreatePasswordPolicyPlan; @@ -91,6 +92,7 @@ use crate::plans::DescProcedurePlan; use crate::plans::DescRowAccessPolicyPlan; use crate::plans::DescSequencePlan; use crate::plans::DescUserPlan; +use crate::plans::DescribeMaterializedViewPlan; use crate::plans::DescribeTablePlan; use crate::plans::DescribeTaskPlan; use crate::plans::DescribeViewPlan; @@ -101,6 +103,7 @@ use crate::plans::DropDatabasePlan; use crate::plans::DropDatamaskPolicyPlan; use crate::plans::DropFileFormatPlan; use crate::plans::DropIndexPlan; +use crate::plans::DropMaterializedViewPlan; use crate::plans::DropNetworkPolicyPlan; use crate::plans::DropNotificationPlan; use crate::plans::DropPasswordPolicyPlan; @@ -145,6 +148,7 @@ use crate::plans::PresignPlan; use crate::plans::ReclusterPlan; use crate::plans::RefreshDatabaseCachePlan; use crate::plans::RefreshIndexPlan; +use crate::plans::RefreshMaterializedViewPlan; use crate::plans::RefreshTableCachePlan; use crate::plans::RefreshTableIndexPlan; use crate::plans::RefreshVirtualColumnPlan; @@ -350,6 +354,12 @@ pub enum Plan { DropView(Box), DescribeView(Box), + // Materialized Views + CreateMaterializedView(Box), + DropMaterializedView(Box), + DescribeMaterializedView(Box), + RefreshMaterializedView(Box), + // Streams CreateStream(Box), DropStream(Box), @@ -584,6 +594,7 @@ impl Plan { Plan::VacuumTemporaryFiles(plan) => plan.schema(), Plan::ExistsTable(plan) => plan.schema(), Plan::DescribeView(plan) => plan.schema(), + Plan::DescribeMaterializedView(plan) => plan.schema(), Plan::ShowFileFormats(plan) => plan.schema(), Plan::Replace(plan) => plan.schema(), Plan::Presign(plan) => plan.schema(), diff --git a/src/query/sql/src/planner/semantic/materialized_view_rewriter.rs b/src/query/sql/src/planner/semantic/materialized_view_rewriter.rs new file mode 100644 index 0000000000000..5fea9d2acba6a --- /dev/null +++ b/src/query/sql/src/planner/semantic/materialized_view_rewriter.rs @@ -0,0 +1,676 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Materialized-view definition rewriter. +//! +//! The rewriter describes both schemas while converting an aggregate query into +//! the query whose outputs are persisted. Types are filled from the bound plans +//! by the materialized-view binder. + +use databend_common_ast::ast::BinaryOperator; +use databend_common_ast::ast::ColumnID; +use databend_common_ast::ast::ColumnRef; +use databend_common_ast::ast::Expr; +use databend_common_ast::ast::FunctionCall; +use databend_common_ast::ast::GroupBy; +use databend_common_ast::ast::Identifier; +use databend_common_ast::ast::Literal; +use databend_common_ast::ast::Query; +use databend_common_ast::ast::SelectStmt; +use databend_common_ast::ast::SelectTarget; +use databend_common_ast::ast::SetExpr; +use databend_common_ast::ast::TableReference; +use databend_common_ast::visit::VisitControl; +use databend_common_ast::visit::VisitResult; +use databend_common_ast::visit::Visitor; +use databend_common_ast::visit::VisitorMut; +use databend_common_ast::visit::Walk; +use databend_common_ast::visit::WalkMut; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::FunctionKind; +use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionFactory; + +use crate::planner::SUPPORTED_AGGREGATING_INDEX_FUNCTIONS; + +/// Rewrites a user MV definition into its physical storage query and records +/// physical names and logical definition expressions in their schema order. +#[derive(Debug, Clone, Default)] +pub struct MaterializedViewRewriter { + is_aggregating: bool, + source_database: String, + physical_names: Vec, + logical_names: Vec, + logical_define_exprs: Vec, + specified_columns: Vec, +} + +impl MaterializedViewRewriter { + pub fn new( + is_aggregating: bool, + source_database: impl Into, + specified_columns: Vec, + ) -> Self { + Self { + is_aggregating, + source_database: source_database.into(), + specified_columns, + ..Default::default() + } + } + + pub fn physical_names(&self) -> &[String] { + &self.physical_names + } + + pub fn logical_names(&self) -> &[String] { + &self.logical_names + } + + pub fn logical_define_exprs(&self) -> &[String] { + &self.logical_define_exprs + } + + pub fn rewrite_query(&mut self, query: &mut Query) -> Result<()> { + self.physical_names.clear(); + self.logical_names.clear(); + self.logical_define_exprs.clear(); + query.walk_mut(self)?; + if self.logical_define_exprs.is_empty() { + return Err(ErrorCode::Internal( + "materialized view rewriter produced no logical outputs".to_string(), + )); + } + Ok(()) + } + + fn output_names(&self, stmt: &SelectStmt) -> Result> { + if !self.specified_columns.is_empty() { + if self.specified_columns.len() != stmt.select_list.len() { + return Err(ErrorCode::SemanticError(format!( + "materialized view column list has {} columns, but SELECT has {} columns", + self.specified_columns.len(), + stmt.select_list.len() + ))); + } + return Ok(self.specified_columns.clone()); + } + + stmt.select_list + .iter() + .map(|target| match target { + SelectTarget::AliasedExpr { + alias: Some(alias), .. + } => Ok(alias.name.clone()), + SelectTarget::AliasedExpr { expr, alias: None } => match expr.as_ref() { + Expr::ColumnRef { column, .. } => Ok(column.column.name().to_string()), + _ => Err(ErrorCode::SemanticError( + "materialized view SELECT expressions must have aliases when no column list is specified" + .to_string(), + )), + }, + SelectTarget::StarColumns { .. } => Err(ErrorCode::SemanticError( + "materialized view does not support SELECT *".to_string(), + )), + }) + .collect() + } + + fn column_ref_expr(name: &str) -> Expr { + Expr::ColumnRef { + span: None, + column: ColumnRef { + database: None, + table: None, + column: ColumnID::Name(Identifier::from_name(None, name)), + }, + } + } + + fn facade_name(name: &str, index: usize, field_count: usize) -> String { + if field_count == 1 { + name.to_string() + } else { + format!("{name}$sys_facade${index}") + } + } + + fn aggregate_target(expr: Expr, name: &str) -> SelectTarget { + SelectTarget::AliasedExpr { + expr: Box::new(expr), + alias: Some(Identifier::from_name(None, name)), + } + } + + fn function_expr(original: &Expr, func: &FunctionCall, name: &str) -> Expr { + Expr::FunctionCall { + span: original.span(), + func: FunctionCall { + name: Identifier::from_name(original.span(), name), + ..func.clone() + }, + } + } + + fn rewrite_aggregate_expr( + &mut self, + expr: &Expr, + output_name: &str, + physical_field_count: usize, + aggregate_targets: &mut Vec, + ) -> Result { + let mut expr = expr.clone(); + let mut rewriter = AggregateExprRewriter { + aggregate_targets, + output_name, + physical_field_count, + next_field_index: 0, + }; + expr.walk_mut(&mut rewriter)?; + Ok(expr) + } + + fn rewrite_non_aggregate(&mut self, stmt: &SelectStmt) -> Result<()> { + let output_names = self.output_names(stmt)?; + for (target, name) in stmt.select_list.iter().zip(output_names) { + let SelectTarget::AliasedExpr { .. } = target else { + return Err(ErrorCode::SemanticError( + "materialized view does not support SELECT *".to_string(), + )); + }; + self.physical_names.push(name.clone()); + self.logical_names.push(name.clone()); + self.logical_define_exprs.push(name); + } + Ok(()) + } + + fn rewrite_aggregate(&mut self, stmt: &mut SelectStmt) -> Result<()> { + let original_targets = stmt.select_list.clone(); + let output_names = self.output_names(stmt)?; + self.logical_names = output_names.clone(); + let mut aggregate_targets = Vec::new(); + + for (target, output_name) in original_targets.iter().zip(&output_names) { + let SelectTarget::AliasedExpr { expr, .. } = target else { + return Err(ErrorCode::SemanticError( + "materialized view does not support SELECT *".to_string(), + )); + }; + let physical_field_count = AggregateFieldCounter::count(expr); + let default_expr = self + .rewrite_aggregate_expr( + expr, + output_name, + physical_field_count, + &mut aggregate_targets, + )? + .to_string(); + self.logical_define_exprs.push(default_expr); + } + + if let Some(GroupBy::Normal(groups)) = &stmt.group_by { + for group in groups { + let selected_group = original_targets.iter().enumerate().find(|(_, target)| { + matches!(target, SelectTarget::AliasedExpr { expr, alias } + if !MaterializedViewChecker::check(expr) + && (expr.as_ref().to_string() == group.to_string() + || alias.as_ref().is_some_and(|alias| { + matches!(group, Expr::ColumnRef { column, .. } + if column.column.name() == alias.name) + }))) + }); + let Some((output_index, _)) = selected_group else { + return Err(ErrorCode::SemanticError(format!( + "materialized view GROUP BY expression '{}' must appear in the SELECT list", + group + ))); + }; + let name = output_names[output_index].clone(); + aggregate_targets.push(Self::aggregate_target(group.clone(), &name)); + self.logical_define_exprs[output_index] = name; + } + } + + self.physical_names = aggregate_targets + .iter() + .map(|target| match target { + SelectTarget::AliasedExpr { + alias: Some(alias), .. + } => Ok(alias.name.clone()), + _ => Err(ErrorCode::Internal( + "materialized view physical target has no alias".to_string(), + )), + }) + .collect::>>()?; + stmt.select_list = aggregate_targets; + Ok(()) + } + + fn rewrite_select_stmt(&mut self, stmt: &mut SelectStmt) -> Result<()> { + if self.is_aggregating { + self.rewrite_aggregate(stmt) + } else { + self.rewrite_non_aggregate(stmt) + } + } +} + +struct AggregateFieldCounter { + count: usize, +} + +impl AggregateFieldCounter { + fn count(expr: &Expr) -> usize { + let mut counter = Self { count: 0 }; + let _ = expr.walk(&mut counter); + counter.count + } +} + +impl Visitor for AggregateFieldCounter { + fn visit_expr(&mut self, expr: &Expr) -> VisitResult { + if matches!(expr, Expr::CountAll { .. }) { + self.count += 1; + return Ok(VisitControl::SkipChildren); + } + Ok(VisitControl::Continue) + } + + fn visit_function_call(&mut self, call: &FunctionCall) -> VisitResult { + let name = call.name.name.to_ascii_lowercase(); + if SUPPORTED_AGGREGATING_INDEX_FUNCTIONS.contains(&name.as_str()) { + self.count += if name == "avg" { 2 } else { 1 }; + return Ok(VisitControl::SkipChildren); + } + Ok(VisitControl::Continue) + } +} + +struct AggregateExprRewriter<'a> { + aggregate_targets: &'a mut Vec, + output_name: &'a str, + physical_field_count: usize, + next_field_index: usize, +} + +impl AggregateExprRewriter<'_> { + fn add_target(&mut self, expr: Expr, name: &str) -> String { + if let Some(existing_name) = self + .aggregate_targets + .iter() + .find_map(|target| match target { + SelectTarget::AliasedExpr { + expr: existing_expr, + alias: Some(alias), + } if existing_expr.as_ref() == &expr => Some(alias.name.clone()), + _ => None, + }) + { + return existing_name; + } + self.aggregate_targets + .push(MaterializedViewRewriter::aggregate_target(expr, name)); + name.to_string() + } + + fn next_physical_name(&mut self) -> String { + let name = MaterializedViewRewriter::facade_name( + self.output_name, + self.next_field_index, + self.physical_field_count, + ); + self.next_field_index += 1; + name + } + + fn rewrite_avg(&mut self, expr: &Expr, func: &FunctionCall) -> Result { + if func.args.len() != 1 { + return Err(ErrorCode::SemanticError( + "materialized view avg() requires exactly one argument".to_string(), + )); + } + + let sum_expr = MaterializedViewRewriter::function_expr(expr, func, "sum"); + let sum_name = self.next_physical_name(); + let sum_name = self.add_target(sum_expr, &sum_name); + + let count_expr = MaterializedViewRewriter::function_expr(expr, func, "count"); + let count_name = self.next_physical_name(); + let count_name = self.add_target(count_expr, &count_name); + + let count_ref = MaterializedViewRewriter::column_ref_expr(&count_name); + let denominator = Expr::FunctionCall { + span: expr.span(), + func: FunctionCall { + distinct: false, + name: Identifier::from_name(expr.span(), "if"), + args: vec![ + Expr::BinaryOp { + span: expr.span(), + op: BinaryOperator::Eq, + left: Box::new(count_ref.clone()), + right: Box::new(Expr::Literal { + span: expr.span(), + value: Literal::UInt64(0), + }), + }, + Expr::Literal { + span: expr.span(), + value: Literal::UInt64(1), + }, + count_ref, + ], + params: vec![], + order_by: vec![], + filter: None, + window: None, + lambda: None, + }, + }; + Ok(Expr::BinaryOp { + span: expr.span(), + op: BinaryOperator::Divide, + left: Box::new(MaterializedViewRewriter::column_ref_expr(&sum_name)), + right: Box::new(denominator), + }) + } +} + +impl VisitorMut for AggregateExprRewriter<'_> { + type Error = ErrorCode; + + fn visit_expr(&mut self, expr: &mut Expr) -> std::result::Result { + match expr { + Expr::FunctionCall { func, .. } + if !func.distinct + && func.filter.is_none() + && func.window.is_none() + && func.lambda.is_none() + && func.order_by.is_empty() + && func.params.is_empty() + && SUPPORTED_AGGREGATING_INDEX_FUNCTIONS + .contains(&func.name.name.to_ascii_lowercase().as_str()) => + { + let original = expr.clone(); + let Expr::FunctionCall { func, .. } = &original else { + unreachable!() + }; + let func_name = func.name.name.to_ascii_lowercase(); + if func_name == "avg" { + *expr = self.rewrite_avg(&original, func)?; + } else { + let name = self.next_physical_name(); + let name = self.add_target(original, &name); + *expr = MaterializedViewRewriter::column_ref_expr(&name); + } + Ok(VisitControl::SkipChildren) + } + Expr::CountAll { + filter: None, + window: None, + .. + } => { + let original = expr.clone(); + let name = self.next_physical_name(); + let name = self.add_target(original, &name); + *expr = MaterializedViewRewriter::column_ref_expr(&name); + Ok(VisitControl::SkipChildren) + } + _ => Ok(VisitControl::Continue), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct MaterializedViewChecker { + has_aggregate: bool, + has_group_by: bool, + has_selection: bool, + not_supported: bool, +} + +impl MaterializedViewChecker { + pub fn check(expr: &Expr) -> bool { + let mut checker = Self::default(); + let _ = expr.walk(&mut checker); + checker.has_aggregate + } + + pub fn check_query(query: &Query) -> Self { + let mut checker = Self::default(); + let _ = query.walk(&mut checker); + checker + } + + pub fn is_aggregating(&self) -> bool { + self.has_aggregate || self.has_group_by + } + + pub fn is_supported(&self) -> bool { + !self.not_supported && (self.has_aggregate || self.has_group_by || self.has_selection) + } +} + +impl Visitor for MaterializedViewChecker { + fn visit_query(&mut self, query: &Query) -> VisitResult { + if query.with.is_some() + || !query.order_by.is_empty() + || !query.limit.is_empty() + || !matches!(&query.body, SetExpr::Select(_)) + { + self.not_supported = true; + } + Ok(VisitControl::Continue) + } + + fn visit_select_stmt(&mut self, stmt: &SelectStmt) -> VisitResult { + if stmt.having.is_some() + || stmt.window_list.is_some() + || stmt.qualify.is_some() + || stmt.top_n.is_some() + || !matches!(stmt.group_by, None | Some(GroupBy::Normal(_))) + || stmt.from.len() != 1 + || !matches!(stmt.from[0], TableReference::Table { .. }) + || stmt.select_list.iter().any(SelectTarget::is_star) + { + self.not_supported = true; + } + self.has_selection |= stmt.selection.is_some(); + self.has_group_by |= stmt.group_by.is_some(); + Ok(VisitControl::Continue) + } + + fn visit_expr(&mut self, expr: &Expr) -> VisitResult { + if let Expr::CountAll { filter, window, .. } = expr { + self.has_aggregate = true; + if filter.is_some() || window.is_some() { + self.not_supported = true; + } + } + Ok(VisitControl::Continue) + } + + fn visit_function_call(&mut self, call: &FunctionCall) -> VisitResult { + let name = call.name.name.to_ascii_lowercase(); + if AggregateFunctionFactory::instance().contains(&name) { + self.has_aggregate = true; + if !SUPPORTED_AGGREGATING_INDEX_FUNCTIONS.contains(&name.as_str()) + || call.distinct + || call.filter.is_some() + || call.window.is_some() + || !call.order_by.is_empty() + { + self.not_supported = true; + } + } else if let Some(property) = BUILTIN_FUNCTIONS.get_property(&name) { + if property.kind == FunctionKind::SRF || property.non_deterministic { + self.not_supported = true; + } + } else { + self.not_supported = true; + } + Ok(VisitControl::Continue) + } +} + +impl VisitorMut for MaterializedViewRewriter { + type Error = ErrorCode; + + fn visit_query(&mut self, query: &mut Query) -> std::result::Result { + if let SetExpr::Select(stmt) = &mut query.body + && let Some(TableReference::Table { table, .. }) = stmt.from.first_mut() + && table.database.is_none() + { + table.database = Some(Identifier::from_name(query.span, &self.source_database)); + } + Ok(VisitControl::Continue) + } + + fn visit_select_stmt( + &mut self, + select: &mut SelectStmt, + ) -> std::result::Result { + self.rewrite_select_stmt(select)?; + Ok(VisitControl::SkipChildren) + } +} + +#[cfg(test)] +mod tests { + use databend_common_ast::ast::Statement; + use databend_common_ast::parser::Dialect; + use databend_common_ast::parser::parse_sql; + use databend_common_ast::parser::tokenize_sql; + + use super::*; + + fn rewrite_with_columns( + sql: &str, + columns: Vec, + ) -> Result<(Query, MaterializedViewRewriter)> { + let tokens = tokenize_sql(sql)?; + let (statement, _) = parse_sql(&tokens, Dialect::PostgreSQL)?; + let Statement::Query(mut query) = statement else { + unreachable!() + }; + let checker = MaterializedViewChecker::check_query(&query); + let mut rewriter = + MaterializedViewRewriter::new(checker.is_aggregating(), "default", columns); + rewriter.rewrite_query(&mut query)?; + Ok((*query, rewriter)) + } + + fn rewrite(sql: &str) -> Result<(Query, MaterializedViewRewriter)> { + rewrite_with_columns(sql, Vec::new()) + } + + #[test] + fn test_rewrite_non_aggregate_query() -> Result<()> { + let (query, rewriter) = + rewrite("SELECT amount AS value, category FROM t WHERE amount > 0")?; + + assert_eq!(rewriter.physical_names(), ["value", "category"]); + assert_eq!(rewriter.logical_names(), ["value", "category"]); + assert_eq!(rewriter.logical_define_exprs(), ["value", "category"]); + assert_eq!( + query.to_string(), + "SELECT amount AS value, category FROM default.t WHERE amount > 0" + ); + Ok(()) + } + + #[test] + fn test_rewrite_with_specified_columns() -> Result<()> { + let (query, rewriter) = rewrite_with_columns( + "SELECT category, avg(amount) FROM t GROUP BY category", + vec!["kind".to_string(), "mean".to_string()], + )?; + + assert_eq!(rewriter.logical_names(), ["kind", "mean"]); + assert_eq!(rewriter.physical_names(), [ + "mean$sys_facade$0", + "mean$sys_facade$1", + "kind" + ]); + assert!( + query + .to_string() + .contains("sum(amount) AS mean$sys_facade$0") + ); + assert!(query.to_string().contains("category AS kind")); + Ok(()) + } + + #[test] + fn test_rewrite_rejects_invalid_output_names() { + let error = rewrite("SELECT amount + 1 FROM t").unwrap_err(); + assert!(error.message().contains("must have aliases")); + + let error = + rewrite_with_columns("SELECT amount, category FROM t", vec!["value".to_string()]) + .unwrap_err(); + assert!(error.message().contains("column list has 1 columns")); + } + + #[test] + fn test_rewrite_multiple_aggregates_with_specified_columns() -> Result<()> { + let (query, rewriter) = rewrite_with_columns( + "SELECT count(id) AS row_count_state, avg(amount) AS amount_avg_state, category FROM mv_mock_base GROUP BY category", + vec![ + "row_count_state".to_string(), + "amount_avg_state".to_string(), + "category".to_string(), + ], + )?; + + assert_eq!(rewriter.physical_names(), [ + "row_count_state", + "amount_avg_state$sys_facade$0", + "amount_avg_state$sys_facade$1", + "category", + ]); + assert_eq!( + query.to_string(), + "SELECT count(id) AS row_count_state, sum(amount) AS amount_avg_state$sys_facade$0, count(amount) AS amount_avg_state$sys_facade$1, category AS category FROM default.mv_mock_base GROUP BY category" + ); + assert_eq!(query.to_string().matches("count(amount)").count(), 1); + Ok(()) + } + + #[test] + fn test_rewrite_aggregate_query() -> Result<()> { + let (query, rewriter) = + rewrite("SELECT category, avg(amount) + 1 AS avg FROM t GROUP BY category")?; + + assert_eq!(rewriter.physical_names(), [ + "avg$sys_facade$0", + "avg$sys_facade$1", + "category" + ]); + assert_eq!(rewriter.logical_names(), ["category", "avg"]); + assert_eq!(rewriter.logical_define_exprs()[0], "category"); + let avg_expr = &rewriter.logical_define_exprs()[1]; + assert!(avg_expr.contains("avg$sys_facade$0")); + assert!(avg_expr.contains("if(avg$sys_facade$1 = 0, 1, avg$sys_facade$1)")); + assert!(avg_expr.ends_with("+ 1")); + + let physical_query = query.to_string(); + assert!(physical_query.contains("sum(amount) AS avg$sys_facade$0")); + assert!(physical_query.contains("count(amount) AS avg$sys_facade$1")); + assert!(physical_query.contains("category AS category")); + Ok(()) + } +} diff --git a/src/query/sql/src/planner/semantic/mod.rs b/src/query/sql/src/planner/semantic/mod.rs index d34381f0f1de4..d963dd2a11455 100644 --- a/src/query/sql/src/planner/semantic/mod.rs +++ b/src/query/sql/src/planner/semantic/mod.rs @@ -19,6 +19,7 @@ mod count_set_ops; mod distinct_to_groupby; mod grouping_check; mod lowering; +mod materialized_view_rewriter; mod name_resolution; mod type_check; mod types; @@ -35,6 +36,8 @@ pub use count_set_ops::CountSetOps; pub use distinct_to_groupby::DistinctToGroupBy; pub use grouping_check::GroupingChecker; pub use lowering::*; +pub use materialized_view_rewriter::MaterializedViewChecker; +pub use materialized_view_rewriter::MaterializedViewRewriter; pub use name_resolution::ClusterKeyNormalizer; pub use name_resolution::IdentifierNormalizer; pub use name_resolution::NameResolutionContext; diff --git a/src/query/storages/common/table_meta/src/table/table_keys.rs b/src/query/storages/common/table_meta/src/table/table_keys.rs index 284f1f33eafd3..4e333fe3e2080 100644 --- a/src/query/storages/common/table_meta/src/table/table_keys.rs +++ b/src/query/storages/common/table_meta/src/table/table_keys.rs @@ -20,14 +20,17 @@ use std::sync::LazyLock; use databend_common_exception::ErrorCode; use databend_common_frozen_api::FrozenAPI; -use databend_common_meta_app::schema::OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID; +// use databend_common_meta_app::schema::OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID; use crate::meta::ColumnCountMinSketch; pub const OPT_KEY_DATABASE_ID: &str = "database_id"; pub const OPT_KEY_STORAGE_PREFIX: &str = "storage_prefix"; pub const OPT_KEY_TEMP_PREFIX: &str = "temp_prefix"; pub const OPT_KEY_RECURSIVE_CTE: &str = "recursive_cte"; pub const OPT_KEY_SNAPSHOT_LOCATION: &str = "snapshot_location"; +pub const OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION: &str = + "materialized_view_source_snapshot_location"; +pub const OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID: &str = "materialized_view_source_table_id"; pub const OPT_KEY_SNAPSHOT_LOCATION_FIXED_FLAG: &str = "snapshot_location_fixed"; pub const OPT_KEY_STORAGE_FORMAT: &str = "storage_format"; pub const OPT_KEY_SEGMENT_FORMAT: &str = "segment_format"; @@ -88,10 +91,11 @@ pub static RESERVED_TABLE_OPTION_KEYS: LazyLock> = LazyLoc let mut r = HashSet::new(); r.insert(OPT_KEY_DATABASE_ID); r.insert(OPT_KEY_LEGACY_SNAPSHOT_LOC); - r.insert(OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID); r.insert(OPT_KEY_RECURSIVE_CTE); r.insert(OPT_KEY_PARTITION_BY); r.insert(OPT_KEY_CLUSTER_TYPE); + r.insert(OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION); + r.insert(OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID); r }); @@ -106,6 +110,8 @@ pub static INTERNAL_TABLE_OPTION_KEYS: LazyLock> = LazyLoc r.insert(OPT_KEY_RECURSIVE_CTE); r.insert(OPT_KEY_PARTITION_BY); r.insert(OPT_KEY_CLUSTER_TYPE); + r.insert(OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION); + r.insert(OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID); r }); diff --git a/src/query/storages/factory/src/storage_factory.rs b/src/query/storages/factory/src/storage_factory.rs index 57274a544824f..87573fe7bee22 100644 --- a/src/query/storages/factory/src/storage_factory.rs +++ b/src/query/storages/factory/src/storage_factory.rs @@ -145,6 +145,12 @@ impl StorageFactory { descriptor: Arc::new(FuseTable::description), }); + creators.insert("MATERIALIZED_VIEW".to_string(), Storage { + // TODO should get default storage specs + creator: Arc::new(FuseTableCreator::default()), + descriptor: Arc::new(FuseTable::description), + }); + // Register VIEW table engine creators.insert("VIEW".to_string(), Storage { creator: Arc::new(ViewTable::try_create), diff --git a/src/query/storages/fuse/Cargo.toml b/src/query/storages/fuse/Cargo.toml index 9484f2d628d50..9adba56fa6ed0 100644 --- a/src/query/storages/fuse/Cargo.toml +++ b/src/query/storages/fuse/Cargo.toml @@ -48,6 +48,7 @@ async-backtrace = { workspace = true } async-channel = { workspace = true } async-trait = { workspace = true } backoff = { workspace = true } +bumpalo = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } enum-as-inner = { workspace = true } diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index 3141f8c31af6b..f0b23ca53beca 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -71,6 +71,7 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; use databend_common_meta_app::schema::UpdateStreamMetaReq; use databend_common_meta_app::schema::UpsertTableCopiedFileReq; +use databend_common_meta_app::schema::is_materialized_view_engine; use databend_common_meta_app::storage::S3StorageClass; use databend_common_meta_app::storage::StorageParams; use databend_common_meta_app::storage::set_s3_storage_class; @@ -1004,7 +1005,13 @@ impl FuseTable { #[async_trait::async_trait] impl Table for FuseTable { fn distribution_level(&self) -> DistributionLevel { - DistributionLevel::Cluster + if is_materialized_view_engine(self.engine()) { + // The first MV read implementation merges persisted aggregate states in one pipeline. + // Restore cluster distribution after a hash-shuffle final merge is added. + DistributionLevel::Local + } else { + DistributionLevel::Cluster + } } fn as_any(&self) -> &dyn Any { diff --git a/src/query/storages/fuse/src/operations/common/processors/mod.rs b/src/query/storages/fuse/src/operations/common/processors/mod.rs index e9f41f911236e..d50d8cd20b4d3 100644 --- a/src/query/storages/fuse/src/operations/common/processors/mod.rs +++ b/src/query/storages/fuse/src/operations/common/processors/mod.rs @@ -16,6 +16,8 @@ mod multi_table_insert_commit; mod sink_commit; mod transform_block_writer; mod transform_constraint_verify; +mod transform_materialized_view; +mod transform_materialized_view_state_merge; mod transform_merge_commit_meta; mod transform_mutation_aggregator; mod transform_partition_by; @@ -28,6 +30,10 @@ pub use sink_commit::CommitSink; pub use transform_block_writer::TransformBlockBuilder; pub use transform_block_writer::TransformBlockWriter; pub use transform_constraint_verify::TransformConstraintVerify; +pub use transform_materialized_view::MaterializedViewTransformPlan; +pub use transform_materialized_view::TransformMaterializedView; +pub use transform_materialized_view_state_merge::MaterializedViewStateMergePlan; +pub use transform_materialized_view_state_merge::TransformMaterializedViewStateMerge; pub use transform_merge_commit_meta::TransformMergeCommitMeta; pub use transform_mutation_aggregator::TableMutationAggregator; pub use transform_partition_by::TransformPartitionBy; diff --git a/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs b/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs index a97aa6e9fe394..56eafaaee6019 100644 --- a/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs +++ b/src/query/storages/fuse/src/operations/common/processors/multi_table_insert_commit.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; @@ -40,6 +41,9 @@ use databend_storages_common_table_meta::meta::BlockTopN; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; +use databend_storages_common_table_meta::table::OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION; +use databend_storages_common_table_meta::table::OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID; +use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use log::debug; use log::error; use log::info; @@ -64,6 +68,8 @@ pub struct CommitMultiTableInsert { deduplicated_label: Option, catalog: Arc, table_meta_timestampss: HashMap, + /// Maps each materialized-view table id to its source table id. + materialized_view_sources: HashMap, } impl CommitMultiTableInsert { @@ -75,6 +81,7 @@ impl CommitMultiTableInsert { deduplicated_label: Option, catalog: Arc, table_meta_timestampss: HashMap, + materialized_view_sources: HashMap, ) -> Self { Self { commit_metas: Default::default(), @@ -86,6 +93,7 @@ impl CommitMultiTableInsert { deduplicated_label, catalog, table_meta_timestampss, + materialized_view_sources, } } } @@ -101,6 +109,7 @@ impl AsyncSink for CommitMultiTableInsert { let mut update_table_metas = Vec::with_capacity(self.commit_metas.len()); let mut update_temp_tables = Vec::with_capacity(self.commit_metas.len()); let mut snapshot_generators = HashMap::with_capacity(self.commit_metas.len()); + let mut metadata_only_mv_updates = HashSet::new(); let mut hlls = HashMap::with_capacity(self.commit_metas.len()); let mut top_ns = HashMap::with_capacity(self.commit_metas.len()); let mut imperfect_counts = HashMap::with_capacity(self.commit_metas.len()); @@ -142,10 +151,37 @@ impl AsyncSink for CommitMultiTableInsert { hlls.insert(table_id, commit_meta.hll); } + // An MV may produce no rows for an INSERT (for example, all rows are filtered out). It + // still covered the source's new snapshot, so publish a metadata-only MV update. + for mv_table_id in self.materialized_view_sources.keys() { + if update_table_metas + .iter() + .any(|(req, _)| req.table_id == *mv_table_id) + { + continue; + } + let table = self.tables.get(mv_table_id).ok_or_else(|| { + ErrorCode::Internal(format!( + "materialized view {} is absent from the multi-table commit", + mv_table_id + )) + })?; + let table_info = table.get_table_info(); + update_table_metas.push(( + build_metadata_only_update_table_meta_req(table_info), + table_info.clone(), + )); + metadata_only_mv_updates.insert(*mv_table_id); + } + let mut backoff = set_backoff(None, None, None); let mut retries = 0; loop { + attach_materialized_view_source_snapshots( + &mut update_table_metas, + &self.materialized_view_sources, + )?; let update_multi_table_meta_req = build_non_temp_update_multi_table_meta_req( update_table_metas.clone(), self.update_stream_meta.clone(), @@ -244,8 +280,15 @@ impl AsyncSink for CommitMultiTableInsert { *table = table .refresh_with_seq_meta(self.ctx.as_ref(), seq, meta) .await?; - for (req, _) in update_table_metas.iter_mut() { + for (req, table_info) in update_table_metas.iter_mut() { if req.table_id == tid { + if metadata_only_mv_updates.contains(&tid) { + *req = build_metadata_only_update_table_meta_req( + table.get_table_info(), + ); + *table_info = table.get_table_info().clone(); + break; + } let (new_req, imperfect_count) = build_update_table_meta_req( table.as_ref(), snapshot_generators.get(&tid).unwrap(), @@ -317,6 +360,58 @@ impl AsyncSink for CommitMultiTableInsert { } } +fn build_metadata_only_update_table_meta_req(table_info: &TableInfo) -> UpdateTableMetaReq { + UpdateTableMetaReq { + table_id: table_info.ident.table_id, + seq: MatchSeq::Exact(table_info.ident.seq), + new_table_meta: table_info.meta.clone(), + base_snapshot_location: table_info + .meta + .options + .get(OPT_KEY_SNAPSHOT_LOCATION) + .cloned(), + lvt_check: None, + } +} + +fn attach_materialized_view_source_snapshots( + update_table_metas: &mut [(UpdateTableMetaReq, TableInfo)], + materialized_view_sources: &HashMap, +) -> Result<()> { + let new_snapshot_locations = update_table_metas + .iter() + .filter_map(|(req, _)| { + req.new_table_meta + .options + .get(OPT_KEY_SNAPSHOT_LOCATION) + .cloned() + .map(|location| (req.table_id, location)) + }) + .collect::>(); + + for (req, _) in update_table_metas.iter_mut() { + let Some(source_table_id) = materialized_view_sources.get(&req.table_id) else { + continue; + }; + let source_snapshot_location = + new_snapshot_locations.get(source_table_id).ok_or_else(|| { + ErrorCode::Internal(format!( + "materialized view {} source table {} has no new snapshot in the atomic commit", + req.table_id, source_table_id + )) + })?; + req.new_table_meta.options.insert( + OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID.to_string(), + source_table_id.to_string(), + ); + req.new_table_meta.options.insert( + OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION.to_string(), + source_snapshot_location.clone(), + ); + } + Ok(()) +} + fn build_non_temp_update_multi_table_meta_req( update_table_metas: Vec<(UpdateTableMetaReq, TableInfo)>, update_stream_metas: Vec, @@ -465,6 +560,73 @@ async fn write_new_snapshot_and_build_table_meta( mod tests { use super::*; + fn update_req(table_id: u64, snapshot_location: Option<&str>) -> UpdateTableMetaReq { + let mut new_table_meta = TableMeta::default(); + if let Some(snapshot_location) = snapshot_location { + new_table_meta.options.insert( + OPT_KEY_SNAPSHOT_LOCATION.to_string(), + snapshot_location.to_string(), + ); + } + UpdateTableMetaReq { + table_id, + seq: MatchSeq::Exact(1), + new_table_meta, + base_snapshot_location: None, + lvt_check: None, + } + } + + #[test] + fn attach_mv_source_snapshot_from_same_atomic_commit() { + let mut updates = vec![ + ( + update_req(1, Some("base/new.snapshot")), + TableInfo::default(), + ), + (update_req(2, Some("mv/new.snapshot")), TableInfo::default()), + ]; + attach_materialized_view_source_snapshots(&mut updates, &HashMap::from([(2, 1)])).unwrap(); + + let mv_options = &updates[1].0.new_table_meta.options; + assert_eq!( + mv_options.get(OPT_KEY_MATERIALIZED_VIEW_SOURCE_TABLE_ID), + Some(&"1".to_string()) + ); + assert_eq!( + mv_options.get(OPT_KEY_MATERIALIZED_VIEW_SOURCE_SNAPSHOT_LOCATION), + Some(&"base/new.snapshot".to_string()) + ); + } + + #[test] + fn attach_mv_source_snapshot_requires_source_update() { + let mut updates = vec![(update_req(2, Some("mv/new.snapshot")), TableInfo::default())]; + let err = attach_materialized_view_source_snapshots(&mut updates, &HashMap::from([(2, 1)])) + .unwrap_err(); + assert!(err.message().contains("has no new snapshot")); + } + + #[test] + fn metadata_only_mv_update_preserves_own_snapshot() { + let mut table_info = TableInfo::default(); + table_info.ident.table_id = 2; + table_info.ident.seq = 3; + table_info.meta.options.insert( + OPT_KEY_SNAPSHOT_LOCATION.to_string(), + "mv/current.snapshot".to_string(), + ); + + let req = build_metadata_only_update_table_meta_req(&table_info); + assert_eq!(req.table_id, 2); + assert_eq!(req.seq, MatchSeq::Exact(3)); + assert_eq!( + req.base_snapshot_location.as_deref(), + Some("mv/current.snapshot") + ); + assert_eq!(req.new_table_meta, table_info.meta); + } + #[test] fn non_temp_update_req_does_not_carry_temp_table_updates() { let req = diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_materialized_view.rs b/src/query/storages/fuse/src/operations/common/processors/transform_materialized_view.rs new file mode 100644 index 0000000000000..ed32be28d6419 --- /dev/null +++ b/src/query/storages/fuse/src/operations/common/processors/transform_materialized_view.rs @@ -0,0 +1,309 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use bumpalo::Bump; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::AggregateHashTable; +use databend_common_expression::BlockEntry; +use databend_common_expression::BlockThresholds; +use databend_common_expression::DataBlock; +use databend_common_expression::Evaluator; +use databend_common_expression::Expr; +use databend_common_expression::FunctionContext; +use databend_common_expression::HashTableConfig; +use databend_common_expression::ProbeState; +use databend_common_expression::ProjectedBlock; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; +use databend_common_functions::BUILTIN_FUNCTIONS; +use databend_common_functions::aggregates::AggregateFunctionRef; +use databend_common_pipeline_transforms::AccumulatingTransform; + +/// Runtime description of the rows generated for one synchronous MV target. +/// +/// It is derived from the bound `SExpr` by `builder_materialized_view`. When MV writes +/// are moved into distributed physical plans, introduce a serializable descriptor and resolve its +/// expressions and aggregate functions into this runtime form while building the pipeline. +/// Group and aggregate argument offsets address `projection`, or the original input block when +/// `projection` is empty. +#[derive(Clone)] +pub struct MaterializedViewTransformPlan { + pub table_id: u64, + pub filter: Option, + pub projection: Vec, + pub group_columns: Vec, + pub group_data_types: Vec, + pub aggregate_functions: Vec, + pub aggregate_arguments: Vec>, + pub output_columns: usize, + pub output_data_types: Vec, +} + +impl MaterializedViewTransformPlan { + fn has_aggregation(&self) -> bool { + !self.group_columns.is_empty() || !self.aggregate_functions.is_empty() + } + + pub fn validate(&self) -> Result<()> { + if self.group_columns.len() != self.group_data_types.len() { + return Err(ErrorCode::Internal(format!( + "materialized view {} group columns and types do not match", + self.table_id + ))); + } + if self.aggregate_functions.len() != self.aggregate_arguments.len() { + return Err(ErrorCode::Internal(format!( + "materialized view {} aggregate functions and arguments do not match", + self.table_id + ))); + } + let expected_columns = if self.has_aggregation() { + self.aggregate_functions.len() + self.group_columns.len() + } else if self.projection.is_empty() { + // The input width is only available when a block arrives. + self.output_columns + } else { + self.projection.len() + }; + if self.output_columns != expected_columns { + return Err(ErrorCode::Internal(format!( + "materialized view {} state schema mismatch: expected {} columns, got {}", + self.table_id, expected_columns, self.output_columns + ))); + } + if !self.output_data_types.is_empty() && self.output_data_types.len() != self.output_columns + { + return Err(ErrorCode::Internal(format!( + "materialized view {} output types mismatch: expected {} columns, got {} types", + self.table_id, + self.output_columns, + self.output_data_types.len() + ))); + } + Ok(()) + } +} + +/// Converts all base-table blocks from one INSERT branch into MV aggregate-state blocks. +/// +/// The output is an ordinary `DataBlock`; block/index/segment writing is deliberately delegated to +/// the shared multi-table insert pipeline. +pub struct TransformMaterializedView { + plan: MaterializedViewTransformPlan, + function_context: FunctionContext, + block_thresholds: BlockThresholds, + hash_table: AggregateHashTable, + probe_state: ProbeState, +} + +impl TransformMaterializedView { + pub fn try_create( + plan: MaterializedViewTransformPlan, + function_context: FunctionContext, + block_thresholds: BlockThresholds, + ) -> Result { + plan.validate()?; + let hash_table = Self::create_hash_table(&plan); + Ok(Self { + plan, + function_context, + block_thresholds, + hash_table, + probe_state: ProbeState::default(), + }) + } + + #[allow(clippy::arc_with_non_send_sync)] + fn create_hash_table(plan: &MaterializedViewTransformPlan) -> AggregateHashTable { + AggregateHashTable::new( + plan.group_data_types.clone(), + plan.aggregate_functions.clone(), + HashTableConfig::default(), + Arc::new(Bump::new()), + ) + } + + fn evaluate_input(&self, input: &DataBlock) -> Result { + let mut block = input.clone(); + if let Some(filter) = &self.plan.filter { + let evaluator = Evaluator::new(&block, &self.function_context, &BUILTIN_FUNCTIONS); + let predicate = evaluator + .run(filter)? + .try_downcast::() + .map_err(|_| { + ErrorCode::Internal(format!( + "materialized view {} filter must return Boolean", + self.plan.table_id + )) + })?; + block = block.filter_boolean_value(&predicate)?; + } + + if self.plan.projection.is_empty() { + return Ok(block); + } + + let evaluator = Evaluator::new(&block, &self.function_context, &BUILTIN_FUNCTIONS); + let entries = self + .plan + .projection + .iter() + .map(|expr| { + let value = evaluator.run(expr)?; + Ok(BlockEntry::new(value, || { + (expr.data_type().clone(), block.num_rows()) + })) + }) + .collect::>>()?; + Ok(DataBlock::new(entries, block.num_rows())) + } + + fn validate_offsets(&self, num_columns: usize) -> Result<()> { + let invalid_group = self + .plan + .group_columns + .iter() + .find(|offset| **offset >= num_columns); + let invalid_argument = self + .plan + .aggregate_arguments + .iter() + .flatten() + .find(|offset| **offset >= num_columns); + if invalid_group.is_some() || invalid_argument.is_some() { + return Err(ErrorCode::Internal(format!( + "materialized view {} expression offset is outside its {}-column input", + self.plan.table_id, num_columns + ))); + } + Ok(()) + } + + fn should_flush(&self) -> bool { + self.hash_table.len() >= self.block_thresholds.max_rows_per_block + || self.hash_table.allocated_bytes() >= self.block_thresholds.max_bytes_per_block + } + + fn align_output_types(&self, block: DataBlock) -> Result { + if self.plan.output_data_types.is_empty() { + return Ok(block); + } + + let num_rows = block.num_rows(); + let mut entries = Vec::with_capacity(block.num_columns()); + for (offset, (entry, target_type)) in block + .take_columns() + .into_iter() + .zip(self.plan.output_data_types.iter()) + .enumerate() + { + let actual_type = entry.data_type(); + if &actual_type == target_type { + entries.push(entry); + } else if matches!(target_type, DataType::Nullable(_)) + && actual_type == target_type.remove_nullable() + { + entries.push(entry.into_nullable()); + } else { + return Err(ErrorCode::Internal(format!( + "materialized view {} column {} expects {}, got {}", + self.plan.table_id, offset, target_type, actual_type + ))); + } + } + Ok(DataBlock::new(entries, num_rows)) + } + + fn flush(&mut self) -> Result> { + if self.hash_table.len() == 0 { + return Ok(vec![]); + } + + let hash_table = + std::mem::replace(&mut self.hash_table, Self::create_hash_table(&self.plan)); + self.probe_state = ProbeState::default(); + let block = hash_table + .payload + .aggregate_flush_all()? + .consume_convert_to_full(); + block.check_valid()?; + if block.num_columns() != self.plan.output_columns { + return Err(ErrorCode::Internal(format!( + "materialized view {} generated {} columns for a {}-column state table", + self.plan.table_id, + block.num_columns(), + self.plan.output_columns + ))); + } + Ok(vec![self.align_output_types(block)?]) + } +} + +impl AccumulatingTransform for TransformMaterializedView { + const NAME: &'static str = "TransformMaterializedView"; + + fn transform(&mut self, input: DataBlock) -> Result> { + let projected = self.evaluate_input(&input)?; + if projected.num_rows() == 0 { + return Ok(vec![]); + } + + // A filter/projection-only MV is a streaming transform. It must preserve all selected + // rows instead of passing them through an empty aggregate hash table. + if !self.plan.has_aggregation() { + if projected.num_columns() != self.plan.output_columns { + return Err(ErrorCode::Internal(format!( + "materialized view {} generated {} columns for a {}-column table", + self.plan.table_id, + projected.num_columns(), + self.plan.output_columns + ))); + } + return Ok(vec![self.align_output_types(projected)?]); + } + + self.validate_offsets(projected.num_columns())?; + let group_columns = ProjectedBlock::project(&self.plan.group_columns, &projected); + let aggregate_arguments = self + .plan + .aggregate_arguments + .iter() + .map(|offsets| ProjectedBlock::project(offsets, &projected)) + .collect::>(); + self.hash_table.add_groups( + &mut self.probe_state, + group_columns, + &aggregate_arguments, + (&[]).into(), + projected.num_rows(), + )?; + + if self.should_flush() { + self.flush() + } else { + Ok(vec![]) + } + } + + fn on_finish(&mut self, output: bool) -> Result> { + if output && self.plan.has_aggregation() { + self.flush() + } else { + Ok(vec![]) + } + } +} diff --git a/src/query/storages/fuse/src/operations/common/processors/transform_materialized_view_state_merge.rs b/src/query/storages/fuse/src/operations/common/processors/transform_materialized_view_state_merge.rs new file mode 100644 index 0000000000000..6a9c36c665421 --- /dev/null +++ b/src/query/storages/fuse/src/operations/common/processors/transform_materialized_view_state_merge.rs @@ -0,0 +1,194 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use bumpalo::Bump; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::AggregateHashTable; +use databend_common_expression::BlockEntry; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::HashTableConfig; +use databend_common_expression::ProbeState; +use databend_common_expression::ProjectedBlock; +use databend_common_expression::Scalar; +use databend_common_expression::aggregate::PayloadFlushState; +use databend_common_expression::types::DataType; +use databend_common_functions::aggregates::AggregateFunctionRef; +use databend_common_pipeline_transforms::AccumulatingTransform; + +#[derive(Clone)] +pub struct MaterializedViewStateMergePlan { + pub aggregate_functions: Vec, + pub group_data_types: Vec, + /// Types of the persisted `[aggregate states..., group keys...]` block. + pub physical_data_types: Vec, + /// Types of the finalized `[aggregate results..., group keys...]` storage block. + pub final_data_types: Vec, +} + +pub struct TransformMaterializedViewStateMerge { + plan: MaterializedViewStateMergePlan, + hash_table: AggregateHashTable, + probe_state: ProbeState, +} + +impl TransformMaterializedViewStateMerge { + pub fn create(plan: MaterializedViewStateMergePlan) -> Self { + let hash_table = Self::create_hash_table(&plan); + Self { + plan, + hash_table, + probe_state: ProbeState::default(), + } + } + + #[allow(clippy::arc_with_non_send_sync)] + fn create_hash_table(plan: &MaterializedViewStateMergePlan) -> AggregateHashTable { + AggregateHashTable::new( + plan.group_data_types.clone(), + plan.aggregate_functions.clone(), + HashTableConfig::default(), + Arc::new(Bump::new()), + ) + } + + fn normalize_state_columns(&self, block: DataBlock) -> Result { + let num_states = self.plan.aggregate_functions.len(); + if block.num_columns() != self.plan.physical_data_types.len() { + return Err(ErrorCode::Internal(format!( + "materialized view state block has {} columns, expected {}", + block.num_columns(), + self.plan.physical_data_types.len() + ))); + } + + let rows = block.num_rows(); + let entries = block + .take_columns() + .into_iter() + .enumerate() + .map(|(offset, entry)| { + if offset >= num_states { + return Ok(entry); + } + match &entry { + BlockEntry::Const(Scalar::Null, _, _) => Err(ErrorCode::Internal(format!( + "materialized view aggregate state column {} contains NULL", + offset + ))), + BlockEntry::Column(Column::Nullable(column)) + if column.validity.null_count() > 0 => + { + Err(ErrorCode::Internal(format!( + "materialized view aggregate state column {} contains NULL", + offset + ))) + } + _ => Ok(entry.remove_nullable()), + } + }) + .collect::>>()?; + Ok(DataBlock::new(entries, rows)) + } + + fn align_final_output_types(&self, block: DataBlock) -> Result { + if block.num_columns() != self.plan.final_data_types.len() { + return Err(ErrorCode::Internal(format!( + "materialized view final block has {} columns, expected {}", + block.num_columns(), + self.plan.final_data_types.len() + ))); + } + + let rows = block.num_rows(); + let entries = block + .take_columns() + .into_iter() + .zip(self.plan.final_data_types.iter()) + .enumerate() + .map(|(offset, (entry, target_type))| { + let actual_type = entry.data_type(); + if &actual_type == target_type { + Ok(entry) + } else if matches!(target_type, DataType::Nullable(_)) + && actual_type == target_type.remove_nullable() + { + Ok(entry.into_nullable()) + } else { + Err(ErrorCode::Internal(format!( + "materialized view final column {} expects {}, got {}", + offset, target_type, actual_type + ))) + } + }) + .collect::>>()?; + Ok(DataBlock::new(entries, rows)) + } + + fn flush(&mut self) -> Result> { + if self.hash_table.len() == 0 { + return Ok(vec![]); + } + let mut hash_table = + std::mem::replace(&mut self.hash_table, Self::create_hash_table(&self.plan)); + self.probe_state = ProbeState::default(); + + let mut flush_state = PayloadFlushState::default(); + let mut blocks = Vec::new(); + while hash_table.merge_result(&mut flush_state)? { + let mut entries = flush_state.take_aggregate_results(); + entries.extend(flush_state.take_group_columns()); + let num_rows = entries.first().map(BlockEntry::len).unwrap_or_default(); + blocks.push(DataBlock::new(entries, num_rows)); + } + + if blocks.is_empty() { + return Ok(vec![]); + } + let block = DataBlock::concat(&blocks)?.consume_convert_to_full(); + self.align_final_output_types(block) + .map(|block| vec![block]) + } +} + +impl AccumulatingTransform for TransformMaterializedViewStateMerge { + const NAME: &'static str = "TransformMaterializedViewStateMerge"; + + fn transform(&mut self, block: DataBlock) -> Result> { + if block.num_rows() == 0 { + return Ok(vec![]); + } + let block = self.normalize_state_columns(block)?; + let num_states = self.plan.aggregate_functions.len(); + let group_offsets = (num_states..block.num_columns()).collect::>(); + let state_offsets = (0..num_states).collect::>(); + let group_columns = ProjectedBlock::project(&group_offsets, &block); + let aggregate_states = ProjectedBlock::project(&state_offsets, &block); + self.hash_table.add_groups( + &mut self.probe_state, + group_columns, + &[], + aggregate_states, + block.num_rows(), + )?; + Ok(vec![]) + } + + fn on_finish(&mut self, output: bool) -> Result> { + if output { self.flush() } else { Ok(vec![]) } + } +} diff --git a/src/query/storages/system/src/tables_table.rs b/src/query/storages/system/src/tables_table.rs index 721304a3ddf08..7c465beb8bc7d 100644 --- a/src/query/storages/system/src/tables_table.rs +++ b/src/query/storages/system/src/tables_table.rs @@ -1074,6 +1074,8 @@ where TablesTable: HistoryAware .map(|v| { if v.engine().to_uppercase() == "VIEW" { "VIEW".to_string() + } else if v.engine().to_uppercase() == "MATERIALIZED_VIEW" { + "MATERIALIZED VIEW".to_string() } else { "BASE TABLE".to_string() } diff --git a/tests/sqllogictests/suites/query/materialized_view.test b/tests/sqllogictests/suites/query/materialized_view.test new file mode 100644 index 0000000000000..9f459671ab996 --- /dev/null +++ b/tests/sqllogictests/suites/query/materialized_view.test @@ -0,0 +1,119 @@ +# Materialized view DDL and synchronous maintenance for ordinary INSERT. + +statement ok +drop database if exists test_materialized_view + +statement ok +create database test_materialized_view + +statement ok +use test_materialized_view + +statement ok +create table source ( + id int, + amount int, + category string, + active boolean +) + +# Explicit MV column names are exposed to users. +statement ok +create materialized view mv_detail (item_id, doubled_amount) as +select id, amount * 2 from source where active + +# Aggregating MVs persist states internally and expose finalized values. +statement ok +create materialized view mv_aggregate (category_name, total_amount, row_count) as +select category, sum(amount), count(*) +from source +where active +group by category + +# IF NOT EXISTS must leave the existing MV intact. +statement ok +create materialized view if not exists mv_detail (item_id, doubled_amount) as +select id, amount * 2 from source where active + +query I +select count(*) +from system.tables +where database = 'test_materialized_view' + and table_type = 'MATERIALIZED VIEW' + and name in ('mv_detail', 'mv_aggregate') +---- +2 + +# A VALUES INSERT is synchronously fanned out to the source table and both MVs. +statement ok +insert into source values + (1, 10, 'a', true), + (2, 20, 'a', false), + (3, 30, 'b', true) + +query II +select item_id, doubled_amount from mv_detail order by item_id +---- +1 20 +3 60 + +query TII +select category_name, total_amount, row_count +from mv_aggregate +order by category_name +---- +a 10 1 +b 30 1 + +# A second INSERT verifies that new MV states are merged with existing states. +statement ok +insert into source values + (4, 40, 'a', true), + (5, 50, 'b', true), + (6, 60, 'c', false) + +query II +select item_id, doubled_amount from mv_detail order by item_id +---- +1 20 +3 60 +4 80 +5 100 + +query TII +select category_name, total_amount, row_count +from mv_aggregate +order by category_name +---- +a 50 2 +b 80 2 + +# The source table and MVs are committed by the same ordinary INSERT path. +query III +select count(*), sum(amount), count_if(active) from source +---- +6 210 4 + +statement ok +drop materialized view mv_detail + +statement ok +drop materialized view if exists mv_detail + +statement ok +drop materialized view mv_aggregate + +query I +select count(*) +from system.tables +where database = 'test_materialized_view' + and table_type = 'MATERIALIZED VIEW' + and name in ('mv_detail', 'mv_aggregate') +---- +0 + +statement ok +use default + +statement ok +drop database test_materialized_view