add settings

Author: TCeason

src/query/service/tests/it/storages/system.rs

@@ -663,6 +663,10 @@ async fn test_system_columns_tolerates_broken_attached_table() -> anyhow::Result
     let catalog = ctx.get_catalog("default").await?;
     let database = catalog.get_database(&tenant, "default").await?;
 
+    // ATTACH schema refresh is opt-in; enable it so the resilience path under test is exercised.
+    ctx.get_settings()
+        .set_setting("enable_table_schema_refresh".to_string(), "1".to_string())?;
+
     execute_command(ctx.clone(), "create table default.healthy(a int)").await?;
 
     // Simulate a broken attached-table storage: any refresh from this S3 endpoint fails with 403.
@@ -739,4 +743,3 @@ async fn test_system_columns_tolerates_broken_attached_table() -> anyhow::Result
 
     Ok(())
 }
-

src/query/settings/src/settings_default.rs

@@ -1121,6 +1121,13 @@ impl DefaultSettings {
                     scope: SettingScope::Both,
                     range: Some(SettingRange::Numeric(0..=1)),
                 }),
+                ("enable_table_schema_refresh", DefaultSettingValue {
+                    value: UserSettingValue::UInt64(0),
+                    desc: "Refresh table schema from storage when listing system.columns/statistics, so schema changes invisible to the meta server (currently only read-only ATTACH tables) are reflected. Disabled by default; each refreshed table costs one storage round-trip.",
+                    mode: SettingMode::Both,
+                    scope: SettingScope::Both,
+                    range: Some(SettingRange::Numeric(0..=1)),
+                }),
                 ("enable_experimental_row_access_policy", DefaultSettingValue {
                     value: UserSettingValue::UInt64(0),
                     desc: "experiment setting enable row access policy(disable by default).",

src/query/settings/src/settings_getter_setter.rs

@@ -197,6 +197,10 @@ impl Settings {
         Ok(self.try_get_u64("enable_auto_fix_missing_bloom_index")? != 0)
     }
 
+    pub fn get_enable_table_schema_refresh(&self) -> Result<bool> {
+        Ok(self.try_get_u64("enable_table_schema_refresh")? != 0)
+    }
+
     // Get max_block_size.
     pub fn get_max_block_size(&self) -> Result<u64> {
         self.try_get_u64("max_block_size")

src/query/storages/system/src/columns_table.rs

@@ -279,22 +279,31 @@ pub(crate) async fn dump_tables(
     let (filtered_db_names, filtered_table_names) = extract_filters(&push_downs, &func_ctx)?;
 
     // Use unified visibility collection from util.rs
-    let db_with_tables =
-        collect_visible_tables(ctx, &disabled_catalog, &filtered_db_names, &filtered_table_names)
-            .await?;
+    let db_with_tables = collect_visible_tables(
+        ctx,
+        &disabled_catalog,
+        &filtered_db_names,
+        &filtered_table_names,
+    )
+    .await?;
 
     // A read-only ATTACH table keeps its current schema in the source snapshot, not on the meta
-    // server, so the disabled catalog above hands back the schema frozen at ATTACH time. Refresh
-    // each one through the original catalog to pick up source schema changes; a failure here must
-    // not drop sibling tables, so fall back to the cached handle.
+    // server, so the disabled catalog above hands back the schema frozen at ATTACH time. Refreshing
+    // each one through the original catalog picks up source schema changes, but costs one S3
+    // round-trip per ATTACH table, so it is opt-in via enable_table_schema_refresh. A refresh
+    // failure must not drop sibling tables, so fall back to the cached handle.
+    let refresh = ctx.get_settings().get_enable_table_schema_refresh()?;
     Ok(db_with_tables
         .into_iter()
         .map(|db| {
-            let tables = db
-                .tables
-                .into_iter()
-                .map(|table| refresh_attach_table(catalog, table))
-                .collect();
+            let tables = if refresh {
+                db.tables
+                    .into_iter()
+                    .map(|table| refresh_attach_table(catalog, table))
+                    .collect()
+            } else {
+                db.tables
+            };
             (db.name, tables)
         })
         .collect())

tests/suites/5_ee/04_attach_read_only/02_0005_attach_table_read_only.result

@@ -23,6 +23,8 @@ c1	VARCHAR	NO	'c1'
 c2	VARCHAR	NO	'c2'	
 expects one row, 3 columns
 0	c1	c2
+system.columns hides source-side added columns when refresh is disabled (default)
+number
 system.columns should reflect the added columns
 c1
 c2

tests/suites/5_ee/04_attach_read_only/02_0005_attach_table_read_only.sh

@@ -49,10 +49,12 @@ echo "desc attach_read_only;" | $BENDSQL_CLIENT_CONNECT
 echo "expects one row, 3 columns"
 echo "select * from attach_read_only order by number;" | $BENDSQL_CLIENT_CONNECT
 
-echo "system.columns should reflect the added columns"
+echo "system.columns hides source-side added columns when refresh is disabled (default)"
 echo "select name from system.columns where database='default' and table='attach_read_only' order by name;" | $BENDSQL_CLIENT_CONNECT
+echo "system.columns should reflect the added columns"
+echo "set enable_table_schema_refresh=1; select name from system.columns where database='default' and table='attach_read_only' order by name;" | $BENDSQL_CLIENT_CONNECT
 echo "information_schema.columns should reflect the added columns"
-echo "select column_name from information_schema.columns where table_schema='default' and table_name='attach_read_only' order by column_name;" | $BENDSQL_CLIENT_CONNECT
+echo "set enable_table_schema_refresh=1; select column_name from information_schema.columns where table_schema='default' and table_name='attach_read_only' order by column_name;" | $BENDSQL_CLIENT_CONNECT
 
 
 echo "alter table drop column"