From 58dda4cff0adccb586f8fa72ed64af5953e477a3 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 00:16:28 +0800 Subject: [PATCH 01/20] =?UTF-8?q?feat:=20add=20ZK=20connector=20interface?= =?UTF-8?q?=20draft=20=E2=80=94=20type=20definitions=20and=20empty=20imple?= =?UTF-8?q?mentation=20skeletons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the foundational type system for ZooKeeper dual-active registry sync: - Position::Zk with path_versions checkpoint, high_water_zxid, total_paths - DtData::Zk variant with ZkEntry (path, data, ZkStat, ephemeral, event_type, source_id) - ExtractorConfig::Zk / SinkerConfig::Zk with config loading from ini - ZkFilter (path prefix whitelist/blacklist) and ZkRouter (path prefix remapping) - ZkExtractor / ZkSinker skeleton structs with todo!() implementations - DataMarker ZK support (marker znode path matching and source_id refresh) - ConflictPolicyEnum::LastWriteWins for ZK conflict resolution - DbType::Zk and factory wiring in ExtractorUtil / SinkerUtil Co-Authored-By: Claude Opus 4.6 --- dt-common/src/config/config_enums.rs | 4 ++ dt-common/src/config/extractor_config.rs | 8 ++++ dt-common/src/config/mod.rs | 1 + dt-common/src/config/router_config.rs | 3 ++ dt-common/src/config/sinker_config.rs | 8 ++++ dt-common/src/config/task_config.rs | 27 +++++++++++ dt-common/src/config/zk_filter_config.rs | 6 +++ dt-common/src/lib.rs | 1 + dt-common/src/meta/dt_data.rs | 6 ++- dt-common/src/meta/mod.rs | 1 + dt-common/src/meta/position.rs | 11 ++++- dt-common/src/meta/zk/mod.rs | 3 ++ dt-common/src/meta/zk/zk_entry.rs | 19 ++++++++ dt-common/src/meta/zk/zk_event_type.rs | 9 ++++ dt-common/src/meta/zk/zk_stat.rs | 13 +++++ dt-common/src/zk_filter.rs | 43 +++++++++++++++++ dt-connector/src/data_marker.rs | 20 +++++++- dt-connector/src/extractor/mod.rs | 1 + dt-connector/src/extractor/zk/mod.rs | 1 + dt-connector/src/extractor/zk/zk_extractor.rs | 29 ++++++++++++ dt-connector/src/lib.rs | 1 + dt-connector/src/rdb_router.rs | 1 + dt-connector/src/sinker/mod.rs | 1 + dt-connector/src/sinker/zk/mod.rs | 1 + dt-connector/src/sinker/zk/zk_sinker.rs | 27 +++++++++++ dt-connector/src/zk_router.rs | 47 +++++++++++++++++++ dt-task/src/extractor_util.rs | 27 +++++++++++ dt-task/src/sinker_util.rs | 21 +++++++++ 28 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 dt-common/src/config/zk_filter_config.rs create mode 100644 dt-common/src/meta/zk/mod.rs create mode 100644 dt-common/src/meta/zk/zk_entry.rs create mode 100644 dt-common/src/meta/zk/zk_event_type.rs create mode 100644 dt-common/src/meta/zk/zk_stat.rs create mode 100644 dt-common/src/zk_filter.rs create mode 100644 dt-connector/src/extractor/zk/mod.rs create mode 100644 dt-connector/src/extractor/zk/zk_extractor.rs create mode 100644 dt-connector/src/sinker/zk/mod.rs create mode 100644 dt-connector/src/sinker/zk/zk_sinker.rs create mode 100644 dt-connector/src/zk_router.rs diff --git a/dt-common/src/config/config_enums.rs b/dt-common/src/config/config_enums.rs index 8a23ee08d..1637e12e6 100644 --- a/dt-common/src/config/config_enums.rs +++ b/dt-common/src/config/config_enums.rs @@ -36,6 +36,8 @@ pub enum DbType { Foxlake, #[strum(serialize = "tidb")] Tidb, + #[strum(serialize = "zk")] + Zk, } #[derive(Display, EnumString, IntoStaticStr, Debug, Clone, Hash, PartialEq, Eq)] @@ -114,6 +116,8 @@ pub enum ConflictPolicyEnum { #[default] #[strum(serialize = "interrupt")] Interrupt, + #[strum(serialize = "last_write_wins")] + LastWriteWins, } #[derive(Display, EnumString, IntoStaticStr, PartialEq)] diff --git a/dt-common/src/config/extractor_config.rs b/dt-common/src/config/extractor_config.rs index dce0d03f9..2c86ce39b 100644 --- a/dt-common/src/config/extractor_config.rs +++ b/dt-common/src/config/extractor_config.rs @@ -213,6 +213,14 @@ pub enum ExtractorConfig { s3_config: S3Config, batch_size: usize, }, + + Zk { + url: String, + watch_paths: Vec, + scan_interval_secs: u64, + include_ephemeral: bool, + heartbeat_interval_secs: u64, + }, } #[derive(Clone, Debug, Hash)] diff --git a/dt-common/src/config/mod.rs b/dt-common/src/config/mod.rs index 899992f3f..bc4ad3988 100644 --- a/dt-common/src/config/mod.rs +++ b/dt-common/src/config/mod.rs @@ -20,6 +20,7 @@ pub mod s3_config; pub mod sinker_config; pub mod ssl_config; pub mod task_config; +pub mod zk_filter_config; #[cfg(feature = "metrics")] pub mod metrics_config; diff --git a/dt-common/src/config/router_config.rs b/dt-common/src/config/router_config.rs index 021f5222c..6533e34d2 100644 --- a/dt-common/src/config/router_config.rs +++ b/dt-common/src/config/router_config.rs @@ -6,4 +6,7 @@ pub enum RouterConfig { col_map: String, topic_map: String, }, + Zk { + path_prefix_map: String, + }, } diff --git a/dt-common/src/config/sinker_config.rs b/dt-common/src/config/sinker_config.rs index c1ec6ad86..9344b5b1e 100644 --- a/dt-common/src/config/sinker_config.rs +++ b/dt-common/src/config/sinker_config.rs @@ -151,6 +151,14 @@ pub enum SinkerConfig { Sql { reverse: bool, }, + + Zk { + url: String, + batch_size: usize, + create_if_not_exists: bool, + sync_ephemeral_as_persistent: bool, + conflict_policy: ConflictPolicyEnum, + }, } #[derive(Clone, Debug, Hash)] diff --git a/dt-common/src/config/task_config.rs b/dt-common/src/config/task_config.rs index 3bea6c23f..2e769b6c9 100644 --- a/dt-common/src/config/task_config.rs +++ b/dt-common/src/config/task_config.rs @@ -772,6 +772,22 @@ impl TaskConfig { ack_interval_secs: loader.get_optional(EXTRACTOR, "ack_interval_secs"), }, + DbType::Zk => { + let watch_paths_str: String = loader.get_optional(EXTRACTOR, "watch_paths"); + let watch_paths = if watch_paths_str.is_empty() { + vec!["/".to_string()] + } else { + watch_paths_str.split(',').map(|s| s.trim().to_string()).collect() + }; + ExtractorConfig::Zk { + url, + watch_paths, + scan_interval_secs: loader.get_with_default(EXTRACTOR, "scan_interval_secs", 60), + include_ephemeral: loader.get_with_default(EXTRACTOR, "include_ephemeral", false), + heartbeat_interval_secs, + } + } + db_type => { bail! {Error::ConfigError(format!( "extractor db type: {} not supported", @@ -1017,6 +1033,17 @@ impl TaskConfig { _ => bail! { not_supported_err }, }, + DbType::Zk => match sink_type { + SinkType::Write => SinkerConfig::Zk { + url, + batch_size, + create_if_not_exists: loader.get_with_default(SINKER, "create_if_not_exists", true), + sync_ephemeral_as_persistent: loader.get_with_default(SINKER, "sync_ephemeral_as_persistent", false), + conflict_policy, + }, + _ => bail! { not_supported_err }, + }, + DbType::Foxlake => { let s3_config = S3Config { bucket: loader.get_optional(SINKER, "s3_bucket"), diff --git a/dt-common/src/config/zk_filter_config.rs b/dt-common/src/config/zk_filter_config.rs new file mode 100644 index 000000000..382ad991d --- /dev/null +++ b/dt-common/src/config/zk_filter_config.rs @@ -0,0 +1,6 @@ +#[derive(Clone, Default, Hash)] +pub struct ZkFilterConfig { + pub do_paths: String, + pub ignore_paths: String, + pub include_ephemeral: bool, +} diff --git a/dt-common/src/lib.rs b/dt-common/src/lib.rs index 208dd78dd..e5a3ef899 100644 --- a/dt-common/src/lib.rs +++ b/dt-common/src/lib.rs @@ -7,5 +7,6 @@ pub mod meta; pub mod monitor; pub mod rdb_filter; pub mod system_dbs; +pub mod zk_filter; pub mod time_filter; pub mod utils; diff --git a/dt-common/src/meta/dt_data.rs b/dt-common/src/meta/dt_data.rs index 10540d58b..6eb9e5d47 100644 --- a/dt-common/src/meta/dt_data.rs +++ b/dt-common/src/meta/dt_data.rs @@ -3,7 +3,7 @@ use serde_json::json; use super::{ ddl_meta::ddl_data::DdlData, foxlake::s3_file_meta::S3FileMeta, row_data::RowData, - struct_meta::struct_data::StructData, + struct_meta::struct_data::StructData, zk::zk_entry::ZkEntry, }; use crate::meta::dcl_meta::dcl_data::DclData; use crate::meta::row_type::RowSqlType; @@ -64,6 +64,9 @@ pub enum DtData { Foxlake { file_meta: S3FileMeta, }, + Zk { + entry: ZkEntry, + }, } impl DtData { @@ -90,6 +93,7 @@ impl DtData { DtData::Ddl { ddl_data } => ddl_data.get_malloc_size(), DtData::Redis { entry } => entry.get_data_malloc_size() as u64, DtData::Foxlake { file_meta } => file_meta.data_size as u64, + DtData::Zk { entry } => entry.get_data_size(), // ignore other item types _ => 0, } diff --git a/dt-common/src/meta/mod.rs b/dt-common/src/meta/mod.rs index 29525a72d..74835ef69 100644 --- a/dt-common/src/meta/mod.rs +++ b/dt-common/src/meta/mod.rs @@ -24,3 +24,4 @@ pub mod struct_meta; pub mod syncer; pub mod tagged_col_value_map; pub mod time; +pub mod zk; diff --git a/dt-common/src/meta/position.rs b/dt-common/src/meta/position.rs index cf1fda43b..52bfdab82 100644 --- a/dt-common/src/meta/position.rs +++ b/dt-common/src/meta/position.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{collections::HashMap, str::FromStr}; use anyhow::Context; use chrono::{DateTime, NaiveDateTime}; @@ -61,6 +61,12 @@ pub enum Position { tb: String, s3_meta_file: String, }, + Zk { + path_versions: HashMap, + last_scan_timestamp: i64, + high_water_zxid: i64, + total_paths: usize, + }, } impl Position { @@ -128,6 +134,9 @@ impl Position { } 0 } + Position::Zk { + last_scan_timestamp, .. + } => *last_scan_timestamp as u64, _ => 0, } } diff --git a/dt-common/src/meta/zk/mod.rs b/dt-common/src/meta/zk/mod.rs new file mode 100644 index 000000000..73d0a90be --- /dev/null +++ b/dt-common/src/meta/zk/mod.rs @@ -0,0 +1,3 @@ +pub mod zk_entry; +pub mod zk_event_type; +pub mod zk_stat; diff --git a/dt-common/src/meta/zk/zk_entry.rs b/dt-common/src/meta/zk/zk_entry.rs new file mode 100644 index 000000000..9a741fa7a --- /dev/null +++ b/dt-common/src/meta/zk/zk_entry.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +use super::{zk_event_type::ZkEventType, zk_stat::ZkStat}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ZkEntry { + pub path: String, + pub data: Option>, + pub stat: ZkStat, + pub ephemeral: bool, + pub event_type: ZkEventType, + pub source_id: String, +} + +impl ZkEntry { + pub fn get_data_size(&self) -> u64 { + self.data.as_ref().map_or(0, |d| d.len() as u64) + } +} diff --git a/dt-common/src/meta/zk/zk_event_type.rs b/dt-common/src/meta/zk/zk_event_type.rs new file mode 100644 index 000000000..a7563dd9c --- /dev/null +++ b/dt-common/src/meta/zk/zk_event_type.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum ZkEventType { + Created, + Updated, + Deleted, + ChildrenChanged, +} diff --git a/dt-common/src/meta/zk/zk_stat.rs b/dt-common/src/meta/zk/zk_stat.rs new file mode 100644 index 000000000..b46c826d2 --- /dev/null +++ b/dt-common/src/meta/zk/zk_stat.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ZkStat { + pub version: i32, + pub cversion: i32, + pub aversion: i32, + pub mtime: i64, + pub ctime: i64, + pub czxid: i64, + pub mzxid: i64, + pub ephemeral_owner: i64, +} diff --git a/dt-common/src/zk_filter.rs b/dt-common/src/zk_filter.rs new file mode 100644 index 000000000..5873c94e6 --- /dev/null +++ b/dt-common/src/zk_filter.rs @@ -0,0 +1,43 @@ +use std::collections::HashSet; + +use crate::config::zk_filter_config::ZkFilterConfig; + +#[derive(Debug, Clone)] +pub struct ZkFilter { + pub do_paths: HashSet, + pub ignore_paths: HashSet, + pub include_ephemeral: bool, +} + +impl ZkFilter { + pub fn from_config(config: &ZkFilterConfig) -> anyhow::Result { + let do_paths = Self::parse_paths(&config.do_paths); + let ignore_paths = Self::parse_paths(&config.ignore_paths); + Ok(Self { + do_paths, + ignore_paths, + include_ephemeral: config.include_ephemeral, + }) + } + + pub fn filter_path(&self, path: &str) -> bool { + if self.ignore_paths.iter().any(|p| path.starts_with(p)) { + return true; + } + if self.do_paths.is_empty() { + return false; + } + !self.do_paths.iter().any(|p| path.starts_with(p)) + } + + pub fn filter_ephemeral(&self, ephemeral: bool) -> bool { + ephemeral && !self.include_ephemeral + } + + fn parse_paths(config_str: &str) -> HashSet { + if config_str.trim().is_empty() { + return HashSet::new(); + } + config_str.split(',').map(|s| s.trim().to_string()).collect() + } +} diff --git a/dt-connector/src/data_marker.rs b/dt-connector/src/data_marker.rs index 2cd78b182..c7df65748 100644 --- a/dt-connector/src/data_marker.rs +++ b/dt-connector/src/data_marker.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use dt_common::{ config::{config_enums::DbType, data_marker_config::DataMarkerConfig}, - meta::{dt_data::DtData, redis::redis_entry::RedisEntry}, + meta::{dt_data::DtData, redis::redis_entry::RedisEntry, zk::zk_entry::ZkEntry}, }; #[derive(Debug, Clone, Default)] @@ -58,6 +58,9 @@ impl DataMarker { me.marker_schema = marker_info[0].to_string(); me.marker_tb = marker_info[1].to_string(); } + DbType::Zk => { + me.marker = config.marker.clone(); + } _ => me.marker = config.marker.clone(), } @@ -76,6 +79,7 @@ impl DataMarker { match dt_data { DtData::Dml { row_data } => self.is_rdb_marker_info(&row_data.schema, &row_data.tb), DtData::Redis { entry } => self.is_redis_marker_info(entry), + DtData::Zk { entry } => self.is_zk_marker_info(entry), _ => false, } } @@ -95,6 +99,10 @@ impl DataMarker { self.marker_schema == schema && self.marker_tb == tb } + pub fn is_zk_marker_info(&self, entry: &ZkEntry) -> bool { + entry.path == self.marker || entry.path.starts_with(&format!("{}/", self.marker)) + } + pub fn refresh(&mut self, dt_data: &DtData) { match dt_data { DtData::Dml { row_data } => { @@ -113,6 +121,16 @@ impl DataMarker { self.data_origin_node = entry.cmd.get_str_arg(2); } + DtData::Zk { entry } => { + if let Some(data) = &entry.data { + if let Ok(marker_value) = serde_json::from_slice::(data) { + if let Some(source_id) = marker_value.get("source_id").and_then(|v| v.as_str()) { + self.data_origin_node = source_id.to_string(); + } + } + } + } + _ => {} } diff --git a/dt-connector/src/extractor/mod.rs b/dt-connector/src/extractor/mod.rs index 4d71de928..696ab68b6 100644 --- a/dt-connector/src/extractor/mod.rs +++ b/dt-connector/src/extractor/mod.rs @@ -10,6 +10,7 @@ pub mod pg; pub mod rdb_snapshot_extract_statement; pub mod redis; pub mod resumer; +pub mod zk; pub mod snapshot_chunk_id_generator; pub mod snapshot_dispatcher; pub mod snapshot_types; diff --git a/dt-connector/src/extractor/zk/mod.rs b/dt-connector/src/extractor/zk/mod.rs new file mode 100644 index 000000000..88ed885de --- /dev/null +++ b/dt-connector/src/extractor/zk/mod.rs @@ -0,0 +1 @@ +pub mod zk_extractor; diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs new file mode 100644 index 000000000..6acda024c --- /dev/null +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -0,0 +1,29 @@ +use async_trait::async_trait; + +use dt_common::zk_filter::ZkFilter; + +use crate::extractor::base_extractor::BaseExtractor; +use crate::extractor::extractor_monitor::ExtractorMonitor; +use crate::Extractor; + +pub struct ZkExtractor { + pub url: String, + pub watch_paths: Vec, + pub scan_interval_secs: u64, + pub include_ephemeral: bool, + pub heartbeat_interval_secs: u64, + pub filter: ZkFilter, + pub base_extractor: BaseExtractor, + pub monitor: ExtractorMonitor, +} + +#[async_trait] +impl Extractor for ZkExtractor { + async fn extract(&mut self) -> anyhow::Result<()> { + todo!("ZkExtractor::extract not yet implemented") + } + + async fn close(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/dt-connector/src/lib.rs b/dt-connector/src/lib.rs index 3e959aaaf..7f5d6dfd6 100644 --- a/dt-connector/src/lib.rs +++ b/dt-connector/src/lib.rs @@ -10,6 +10,7 @@ pub mod meta_fetcher; pub mod rdb_query_builder; pub mod rdb_router; pub mod sinker; +pub mod zk_router; use async_trait::async_trait; use checker::check_log::CheckLog; diff --git a/dt-connector/src/rdb_router.rs b/dt-connector/src/rdb_router.rs index 96738a357..ed4056b5c 100644 --- a/dt-connector/src/rdb_router.rs +++ b/dt-connector/src/rdb_router.rs @@ -187,6 +187,7 @@ impl RdbRouterInner { col_map, }) } + _ => Ok(Self::default()), } } diff --git a/dt-connector/src/sinker/mod.rs b/dt-connector/src/sinker/mod.rs index 224c234f2..98e19f4a9 100644 --- a/dt-connector/src/sinker/mod.rs +++ b/dt-connector/src/sinker/mod.rs @@ -11,3 +11,4 @@ pub mod pg; pub mod redis; pub mod sql_sinker; pub mod starrocks; +pub mod zk; diff --git a/dt-connector/src/sinker/zk/mod.rs b/dt-connector/src/sinker/zk/mod.rs new file mode 100644 index 000000000..b46861cc3 --- /dev/null +++ b/dt-connector/src/sinker/zk/mod.rs @@ -0,0 +1 @@ +pub mod zk_sinker; diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs new file mode 100644 index 000000000..43abd9923 --- /dev/null +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -0,0 +1,27 @@ +use async_trait::async_trait; + +use dt_common::config::config_enums::ConflictPolicyEnum; +use dt_common::meta::dt_data::DtItem; + +use crate::sinker::base_sinker::BaseSinker; +use crate::Sinker; + +pub struct ZkSinker { + pub url: String, + pub batch_size: usize, + pub create_if_not_exists: bool, + pub sync_ephemeral_as_persistent: bool, + pub conflict_policy: ConflictPolicyEnum, + pub base_sinker: BaseSinker, +} + +#[async_trait] +impl Sinker for ZkSinker { + async fn sink_raw(&mut self, _data: Vec, _batch: bool) -> anyhow::Result<()> { + todo!("ZkSinker::sink_raw not yet implemented") + } + + async fn close(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/dt-connector/src/zk_router.rs b/dt-connector/src/zk_router.rs new file mode 100644 index 000000000..fa1e4522a --- /dev/null +++ b/dt-connector/src/zk_router.rs @@ -0,0 +1,47 @@ +use std::collections::HashMap; + +use dt_common::config::router_config::RouterConfig; + +type PathPrefixMap = HashMap; + +#[derive(Debug, Clone, Default)] +pub struct ZkRouter { + pub path_prefix_map: PathPrefixMap, +} + +impl ZkRouter { + pub fn from_config(config: &RouterConfig) -> anyhow::Result { + match config { + RouterConfig::Zk { path_prefix_map } => { + let map = Self::parse_path_prefix_map(path_prefix_map)?; + Ok(Self { + path_prefix_map: map, + }) + } + _ => Ok(Self::default()), + } + } + + pub fn route_path(&self, path: &str) -> String { + for (src_prefix, dst_prefix) in &self.path_prefix_map { + if path.starts_with(src_prefix.as_str()) { + return format!("{}{}", dst_prefix, &path[src_prefix.len()..]); + } + } + path.to_string() + } + + fn parse_path_prefix_map(config_str: &str) -> anyhow::Result { + let mut map = PathPrefixMap::new(); + if config_str.trim().is_empty() { + return Ok(map); + } + for pair in config_str.split(',') { + let parts: Vec<&str> = pair.split(':').collect(); + if parts.len() == 2 { + map.insert(parts[0].trim().to_string(), parts[1].trim().to_string()); + } + } + Ok(map) + } +} diff --git a/dt-task/src/extractor_util.rs b/dt-task/src/extractor_util.rs index 93edb21f5..b5f37e54b 100644 --- a/dt-task/src/extractor_util.rs +++ b/dt-task/src/extractor_util.rs @@ -57,6 +57,7 @@ use dt_connector::{ redis_snapshot_file_extractor::RedisSnapshotFileExtractor, }, resumer::recovery::Recovery, + zk::zk_extractor::ZkExtractor, }, rdb_router::RdbRouter, Extractor, @@ -744,6 +745,32 @@ impl ExtractorUtil { }; Box::new(extractor) } + + ExtractorConfig::Zk { + url, + watch_paths, + scan_interval_secs, + include_ephemeral, + heartbeat_interval_secs, + } => { + let zk_filter_config = dt_common::config::zk_filter_config::ZkFilterConfig { + do_paths: watch_paths.join(","), + ignore_paths: "/zookeeper".to_string(), + include_ephemeral, + }; + let zk_filter = dt_common::zk_filter::ZkFilter::from_config(&zk_filter_config)?; + let extractor = ZkExtractor { + url, + watch_paths, + scan_interval_secs, + include_ephemeral, + heartbeat_interval_secs, + filter: zk_filter, + base_extractor, + monitor: extract_state.monitor, + }; + Box::new(extractor) + } }; Ok(extractor) } diff --git a/dt-task/src/sinker_util.rs b/dt-task/src/sinker_util.rs index 4e26ec60e..6b84bd2ac 100644 --- a/dt-task/src/sinker_util.rs +++ b/dt-task/src/sinker_util.rs @@ -53,6 +53,7 @@ use dt_connector::{ starrocks::{ starrocks_sinker::StarRocksSinker, starrocks_struct_sinker::StarrocksStructSinker, }, + zk::zk_sinker::ZkSinker, }, Sinker, }; @@ -735,6 +736,26 @@ impl SinkerUtil { }; Self::push_sinker(&mut sub_sinkers, sinker); } + + SinkerConfig::Zk { + url, + batch_size, + create_if_not_exists, + sync_ephemeral_as_persistent, + conflict_policy, + } => { + for _ in 0..parallel_size { + let sinker = ZkSinker { + url: url.clone(), + batch_size, + create_if_not_exists, + sync_ephemeral_as_persistent, + conflict_policy: conflict_policy.clone(), + base_sinker: BaseSinker::new(monitor.clone(), monitor_interval), + }; + Self::push_sinker(&mut sub_sinkers, sinker); + } + } }; Ok(sub_sinkers) } From 49fea0312b4e53b9149d3617c49b38658b552eaa Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 00:26:06 +0800 Subject: [PATCH 02/20] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20add=20ExtractState/Syncer/Recovery=20to=20ZkExtract?= =?UTF-8?q?or,=20ZkRouter=20to=20ZkSinker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blocker fixes: - ZkExtractor: add extract_state, syncer, recovery fields (matching MongoCdcExtractor pattern); remove standalone monitor field - ZkSinker: add ZkRouter field for path prefix remapping Should-fix: - ZkFilter config loaded from [zk_filter] ini section instead of hardcoded; added zk_filter field to TaskConfig with load_zk_filter_config() - ZkRouter: changed from HashMap to Vec<(String, String)> sorted by prefix length descending for deterministic longest-prefix-match - ZkEntry.get_data_size(): include path length in size calculation Nit: - Removed redundant DbType::Zk arm in DataMarker::from_config Co-Authored-By: Claude Opus 4.6 --- dt-common/src/config/task_config.rs | 16 ++++++++++++++++ dt-common/src/meta/zk/zk_entry.rs | 2 +- dt-connector/src/data_marker.rs | 3 --- dt-connector/src/extractor/zk/zk_extractor.rs | 12 +++++++++--- dt-connector/src/sinker/zk/zk_sinker.rs | 2 ++ dt-connector/src/zk_router.rs | 14 ++++++-------- dt-task/src/extractor_util.rs | 13 ++++++------- dt-task/src/sinker_util.rs | 3 +++ 8 files changed, 43 insertions(+), 22 deletions(-) diff --git a/dt-common/src/config/task_config.rs b/dt-common/src/config/task_config.rs index 2e769b6c9..a4dc75bad 100644 --- a/dt-common/src/config/task_config.rs +++ b/dt-common/src/config/task_config.rs @@ -42,6 +42,7 @@ use super::{ runtime_config::RuntimeConfig, s3_config::S3Config, sinker_config::{BasicSinkerConfig, SinkerConfig}, + zk_filter_config::ZkFilterConfig, }; #[derive(Clone)] @@ -61,6 +62,7 @@ pub struct TaskConfig { pub meta_center: Option, pub data_marker: Option, pub processor: Option, + pub zk_filter: ZkFilterConfig, #[cfg(feature = "metrics")] pub metrics: MetricsConfig, } @@ -80,6 +82,7 @@ const FILTER: &str = "filter"; const ROUTER: &str = "router"; const RESUMER: &str = "resumer"; const DATA_MARKER: &str = "data_marker"; +const ZK_FILTER: &str = "zk_filter"; const PROCESSOR: &str = "processor"; const CHECKER: &str = "checker"; const META_CENTER: &str = "metacenter"; @@ -256,6 +259,7 @@ impl TaskConfig { checker, data_marker: Self::load_data_marker_config(&loader)?, processor: Self::load_processor_config(&loader)?, + zk_filter: Self::load_zk_filter_config(&loader), meta_center: Self::load_meta_center_config(&loader)?, #[cfg(feature = "metrics")] metrics: Self::load_metrics_config(&loader)?, @@ -1424,6 +1428,18 @@ impl TaskConfig { } } + fn load_zk_filter_config(loader: &IniLoader) -> ZkFilterConfig { + ZkFilterConfig { + do_paths: loader.get_optional(ZK_FILTER, "do_paths"), + ignore_paths: loader.get_with_default( + ZK_FILTER, + "ignore_paths", + "/zookeeper".to_string(), + ), + include_ephemeral: loader.get_with_default(ZK_FILTER, "include_ephemeral", false), + } + } + fn load_data_marker_config(loader: &IniLoader) -> anyhow::Result> { if !loader.ini.sections().contains(&DATA_MARKER.to_string()) { return Ok(None); diff --git a/dt-common/src/meta/zk/zk_entry.rs b/dt-common/src/meta/zk/zk_entry.rs index 9a741fa7a..37b227222 100644 --- a/dt-common/src/meta/zk/zk_entry.rs +++ b/dt-common/src/meta/zk/zk_entry.rs @@ -14,6 +14,6 @@ pub struct ZkEntry { impl ZkEntry { pub fn get_data_size(&self) -> u64 { - self.data.as_ref().map_or(0, |d| d.len() as u64) + self.path.len() as u64 + self.data.as_ref().map_or(0, |d| d.len() as u64) } } diff --git a/dt-connector/src/data_marker.rs b/dt-connector/src/data_marker.rs index c7df65748..a1ac32b3c 100644 --- a/dt-connector/src/data_marker.rs +++ b/dt-connector/src/data_marker.rs @@ -58,9 +58,6 @@ impl DataMarker { me.marker_schema = marker_info[0].to_string(); me.marker_tb = marker_info[1].to_string(); } - DbType::Zk => { - me.marker = config.marker.clone(); - } _ => me.marker = config.marker.clone(), } diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index 6acda024c..3d5785462 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -1,9 +1,13 @@ +use std::sync::Arc; + use async_trait::async_trait; +use tokio::sync::Mutex; +use dt_common::meta::syncer::Syncer; use dt_common::zk_filter::ZkFilter; -use crate::extractor::base_extractor::BaseExtractor; -use crate::extractor::extractor_monitor::ExtractorMonitor; +use crate::extractor::base_extractor::{BaseExtractor, ExtractState}; +use crate::extractor::resumer::recovery::Recovery; use crate::Extractor; pub struct ZkExtractor { @@ -14,7 +18,9 @@ pub struct ZkExtractor { pub heartbeat_interval_secs: u64, pub filter: ZkFilter, pub base_extractor: BaseExtractor, - pub monitor: ExtractorMonitor, + pub extract_state: ExtractState, + pub syncer: Arc>, + pub recovery: Option>, } #[async_trait] diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index 43abd9923..fc21803ba 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -4,6 +4,7 @@ use dt_common::config::config_enums::ConflictPolicyEnum; use dt_common::meta::dt_data::DtItem; use crate::sinker::base_sinker::BaseSinker; +use crate::zk_router::ZkRouter; use crate::Sinker; pub struct ZkSinker { @@ -12,6 +13,7 @@ pub struct ZkSinker { pub create_if_not_exists: bool, pub sync_ephemeral_as_persistent: bool, pub conflict_policy: ConflictPolicyEnum, + pub router: ZkRouter, pub base_sinker: BaseSinker, } diff --git a/dt-connector/src/zk_router.rs b/dt-connector/src/zk_router.rs index fa1e4522a..244f961ef 100644 --- a/dt-connector/src/zk_router.rs +++ b/dt-connector/src/zk_router.rs @@ -1,12 +1,9 @@ -use std::collections::HashMap; - use dt_common::config::router_config::RouterConfig; -type PathPrefixMap = HashMap; - #[derive(Debug, Clone, Default)] pub struct ZkRouter { - pub path_prefix_map: PathPrefixMap, + // sorted by src prefix length descending for longest-prefix-match + pub path_prefix_map: Vec<(String, String)>, } impl ZkRouter { @@ -31,17 +28,18 @@ impl ZkRouter { path.to_string() } - fn parse_path_prefix_map(config_str: &str) -> anyhow::Result { - let mut map = PathPrefixMap::new(); + fn parse_path_prefix_map(config_str: &str) -> anyhow::Result> { + let mut map = Vec::new(); if config_str.trim().is_empty() { return Ok(map); } for pair in config_str.split(',') { let parts: Vec<&str> = pair.split(':').collect(); if parts.len() == 2 { - map.insert(parts[0].trim().to_string(), parts[1].trim().to_string()); + map.push((parts[0].trim().to_string(), parts[1].trim().to_string())); } } + map.sort_by(|a, b| b.0.len().cmp(&a.0.len())); Ok(map) } } diff --git a/dt-task/src/extractor_util.rs b/dt-task/src/extractor_util.rs index b5f37e54b..ccb761429 100644 --- a/dt-task/src/extractor_util.rs +++ b/dt-task/src/extractor_util.rs @@ -753,12 +753,9 @@ impl ExtractorUtil { include_ephemeral, heartbeat_interval_secs, } => { - let zk_filter_config = dt_common::config::zk_filter_config::ZkFilterConfig { - do_paths: watch_paths.join(","), - ignore_paths: "/zookeeper".to_string(), - include_ephemeral, - }; - let zk_filter = dt_common::zk_filter::ZkFilter::from_config(&zk_filter_config)?; + let zk_filter = dt_common::zk_filter::ZkFilter::from_config( + &config.zk_filter, + )?; let extractor = ZkExtractor { url, watch_paths, @@ -767,7 +764,9 @@ impl ExtractorUtil { heartbeat_interval_secs, filter: zk_filter, base_extractor, - monitor: extract_state.monitor, + extract_state, + syncer, + recovery, }; Box::new(extractor) } diff --git a/dt-task/src/sinker_util.rs b/dt-task/src/sinker_util.rs index 6b84bd2ac..1406823fa 100644 --- a/dt-task/src/sinker_util.rs +++ b/dt-task/src/sinker_util.rs @@ -55,6 +55,7 @@ use dt_connector::{ }, zk::zk_sinker::ZkSinker, }, + zk_router::ZkRouter, Sinker, }; @@ -744,6 +745,7 @@ impl SinkerUtil { sync_ephemeral_as_persistent, conflict_policy, } => { + let zk_router = ZkRouter::from_config(&config.router)?; for _ in 0..parallel_size { let sinker = ZkSinker { url: url.clone(), @@ -751,6 +753,7 @@ impl SinkerUtil { create_if_not_exists, sync_ephemeral_as_persistent, conflict_policy: conflict_policy.clone(), + router: zk_router.clone(), base_sinker: BaseSinker::new(monitor.clone(), monitor_interval), }; Self::push_sinker(&mut sub_sinkers, sinker); From 99804e1a2cc5f3fd230884ba3c740039c093ce53 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 00:44:11 +0800 Subject: [PATCH 03/20] feat(zk): implement ZkSinker sink_raw with full ZK write logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the todo!() stub with complete sink implementation: - Lazy ZK client connection via zookeeper-client crate - Serial processing: route path → LWW conflict check → apply event - Event dispatch: Created (upsert), Updated (auto-create), Deleted (idempotent) - LWW conflict policy: compare target stat.mtime vs source mtime - ensure_parent_paths for recursive znode creation - DataMarker writing after each batch (JSON source_id in marker znode) - Ephemeral-to-persistent conversion via sync_ephemeral_as_persistent flag - Metrics tracking via base_sinker.update_serial_monitor Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 1 + dt-connector/Cargo.toml | 1 + dt-connector/src/sinker/zk/zk_sinker.rs | 208 +++++++++++++++++++++++- dt-task/src/sinker_util.rs | 2 + 4 files changed, 209 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82acdac8d..b47a8113a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ opendal = { version = "0.54.1", features = ["services-s3"] } governor = "0.10.4" indexmap = "2" libc = "0.2" +zookeeper-client = "0.8" [profile.release] panic = 'unwind' diff --git a/dt-connector/Cargo.toml b/dt-connector/Cargo.toml index 1ce6ed133..b12ead36d 100644 --- a/dt-connector/Cargo.toml +++ b/dt-connector/Cargo.toml @@ -50,3 +50,4 @@ opendal = { workspace = true } dashmap = {workspace = true} indexmap = {workspace = true} openssl = { workspace = true } +zookeeper-client = { workspace = true } diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index fc21803ba..00979e53b 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -1,8 +1,17 @@ +use std::sync::Arc; + +use anyhow::bail; use async_trait::async_trait; +use tokio::sync::RwLock; +use zookeeper_client as zk; use dt_common::config::config_enums::ConflictPolicyEnum; -use dt_common::meta::dt_data::DtItem; +use dt_common::log_info; +use dt_common::meta::dt_data::{DtData, DtItem}; +use dt_common::meta::zk::zk_entry::ZkEntry; +use dt_common::meta::zk::zk_event_type::ZkEventType; +use crate::data_marker::DataMarker; use crate::sinker::base_sinker::BaseSinker; use crate::zk_router::ZkRouter; use crate::Sinker; @@ -15,15 +24,208 @@ pub struct ZkSinker { pub conflict_policy: ConflictPolicyEnum, pub router: ZkRouter, pub base_sinker: BaseSinker, + pub data_marker: Option>>, + pub client: Option, } #[async_trait] impl Sinker for ZkSinker { - async fn sink_raw(&mut self, _data: Vec, _batch: bool) -> anyhow::Result<()> { - todo!("ZkSinker::sink_raw not yet implemented") + async fn sink_raw(&mut self, data: Vec, _batch: bool) -> anyhow::Result<()> { + if data.is_empty() { + return Ok(()); + } + + let client = self.get_or_connect().await?; + self.serial_sink_raw(&client, data).await } async fn close(&mut self) -> anyhow::Result<()> { + self.client = None; Ok(()) } } + +impl ZkSinker { + async fn get_or_connect(&mut self) -> anyhow::Result { + if let Some(client) = &self.client { + return Ok(client.clone()); + } + let client = zk::Client::connect(&self.url) + .await + .map_err(|e| anyhow::anyhow!("ZK connect to {} failed: {}", self.url, e))?; + log_info!("ZkSinker connected to {}", self.url); + self.client = Some(client.clone()); + Ok(client) + } + + async fn serial_sink_raw( + &mut self, + client: &zk::Client, + data: Vec, + ) -> anyhow::Result<()> { + let mut data_size = 0u64; + let mut record_count = 0u64; + + for dt_item in data.iter() { + if let DtData::Zk { entry } = &dt_item.dt_data { + data_size += entry.get_data_size(); + let routed_path = self.router.route_path(&entry.path); + + if self.conflict_policy == ConflictPolicyEnum::LastWriteWins { + if self.should_skip_by_lww(client, &routed_path, entry).await? { + continue; + } + } + + self.apply_zk_event(client, &routed_path, entry).await?; + record_count += 1; + } + } + + self.write_data_marker(client).await?; + + self.base_sinker + .update_serial_monitor(record_count, data_size) + .await + } + + async fn should_skip_by_lww( + &self, + client: &zk::Client, + path: &str, + entry: &ZkEntry, + ) -> anyhow::Result { + match client.check_stat(path).await { + Ok(Some(stat)) => Ok(stat.mtime > entry.stat.mtime), + Ok(None) => Ok(false), + Err(_) => Ok(false), + } + } + + async fn apply_zk_event( + &self, + client: &zk::Client, + path: &str, + entry: &ZkEntry, + ) -> anyhow::Result<()> { + let data = entry.data.as_deref().unwrap_or(&[]); + + match entry.event_type { + ZkEventType::Created => { + let options = self.create_options(entry); + + if self.create_if_not_exists { + self.ensure_parent_paths(client, path).await?; + } + + match client.create(path, data, &options).await { + Ok(_) => Ok(()), + Err(ref e) if is_node_exists(e) => { + client + .set_data(path, data, None) + .await + .map_err(|e| { + anyhow::anyhow!("ZK set_data (upsert) {} failed: {}", path, e) + })?; + Ok(()) + } + Err(e) => bail!("ZK create {} failed: {}", path, e), + } + } + + ZkEventType::Updated => match client.set_data(path, data, None).await { + Ok(_) => Ok(()), + Err(ref e) if is_no_node(e) && self.create_if_not_exists => { + self.ensure_parent_paths(client, path).await?; + let options = self.create_options(entry); + client.create(path, data, &options).await.map_err(|e| { + anyhow::anyhow!("ZK create (auto) {} failed: {}", path, e) + })?; + Ok(()) + } + Err(e) => bail!("ZK set_data {} failed: {}", path, e), + }, + + ZkEventType::Deleted => match client.delete(path, None).await { + Ok(()) => Ok(()), + Err(ref e) if is_no_node(e) => Ok(()), + Err(e) => bail!("ZK delete {} failed: {}", path, e), + }, + + ZkEventType::ChildrenChanged => Ok(()), + } + } + + fn create_options(&self, entry: &ZkEntry) -> zk::CreateOptions<'static> { + let mode = if entry.ephemeral && !self.sync_ephemeral_as_persistent { + zk::CreateMode::Ephemeral + } else { + zk::CreateMode::Persistent + }; + mode.with_acls(zk::Acls::anyone_all()) + } + + fn persistent_options(&self) -> zk::CreateOptions<'static> { + zk::CreateMode::Persistent.with_acls(zk::Acls::anyone_all()) + } + + async fn ensure_parent_paths(&self, client: &zk::Client, path: &str) -> anyhow::Result<()> { + let parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + if parts.len() <= 1 { + return Ok(()); + } + let options = self.persistent_options(); + let mut current = String::new(); + for part in &parts[..parts.len() - 1] { + current.push('/'); + current.push_str(part); + match client.create(¤t, &[], &options).await { + Ok(_) => {} + Err(ref e) if is_node_exists(e) => {} + Err(e) => bail!("ZK create parent {} failed: {}", current, e), + } + } + Ok(()) + } + + async fn write_data_marker(&self, client: &zk::Client) -> anyhow::Result<()> { + if let Some(data_marker) = &self.data_marker { + let data_marker = data_marker.read().await; + let marker_data = serde_json::to_vec(&serde_json::json!({ + "source_id": data_marker.data_origin_node + }))?; + + match client + .set_data(&data_marker.marker, &marker_data, None) + .await + { + Ok(_) => {} + Err(ref e) if is_no_node(e) => { + self.ensure_parent_paths(client, &data_marker.marker) + .await?; + let options = self.persistent_options(); + client + .create(&data_marker.marker, &marker_data, &options) + .await + .map_err(|e| { + anyhow::anyhow!( + "ZK create marker {} failed: {}", + data_marker.marker, + e + ) + })?; + } + Err(e) => bail!("ZK write marker {} failed: {}", data_marker.marker, e), + } + } + Ok(()) + } +} + +fn is_node_exists(e: &zk::Error) -> bool { + matches!(e, zk::Error::NodeExists) +} + +fn is_no_node(e: &zk::Error) -> bool { + matches!(e, zk::Error::NoNode) +} diff --git a/dt-task/src/sinker_util.rs b/dt-task/src/sinker_util.rs index 1406823fa..4a3f855c6 100644 --- a/dt-task/src/sinker_util.rs +++ b/dt-task/src/sinker_util.rs @@ -755,6 +755,8 @@ impl SinkerUtil { conflict_policy: conflict_policy.clone(), router: zk_router.clone(), base_sinker: BaseSinker::new(monitor.clone(), monitor_interval), + data_marker: data_marker.clone(), + client: None, }; Self::push_sinker(&mut sub_sinkers, sinker); } From 1b3faf4950ea813cdfa5e19bf994c95e779f9ea5 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 00:48:48 +0800 Subject: [PATCH 04/20] fix(zk-sinker): address review - LWW tie-break, NotEmpty, marker src_node 1. LWW deterministic tie-break: read target marker znode once per batch to get existing source_id; when mtime equal, larger source_id wins; if target has no source_id (first sync), default apply 2. Delete NotEmpty handling: log_warn and skip instead of bail, avoids force-deleting parent nodes that have new children from peer 3. write_data_marker uses src_node (fixed identity) instead of data_origin_node (dynamic, refreshed per-event) Co-Authored-By: Claude Opus 4.6 --- dt-connector/src/sinker/zk/zk_sinker.rs | 55 +++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index 00979e53b..71705838a 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -7,6 +7,7 @@ use zookeeper_client as zk; use dt_common::config::config_enums::ConflictPolicyEnum; use dt_common::log_info; +use dt_common::log_warn; use dt_common::meta::dt_data::{DtData, DtItem}; use dt_common::meta::zk::zk_entry::ZkEntry; use dt_common::meta::zk::zk_event_type::ZkEventType; @@ -66,13 +67,22 @@ impl ZkSinker { let mut data_size = 0u64; let mut record_count = 0u64; + let target_source_id = if self.conflict_policy == ConflictPolicyEnum::LastWriteWins { + self.read_target_source_id(client).await + } else { + None + }; + for dt_item in data.iter() { if let DtData::Zk { entry } = &dt_item.dt_data { data_size += entry.get_data_size(); let routed_path = self.router.route_path(&entry.path); if self.conflict_policy == ConflictPolicyEnum::LastWriteWins { - if self.should_skip_by_lww(client, &routed_path, entry).await? { + if self + .should_skip_by_lww(client, &routed_path, entry, &target_source_id) + .await? + { continue; } } @@ -89,14 +99,42 @@ impl ZkSinker { .await } + async fn read_target_source_id(&self, client: &zk::Client) -> Option { + if let Some(data_marker) = &self.data_marker { + let data_marker = data_marker.read().await; + if let Ok((data, _)) = client.get_data(&data_marker.marker).await { + if let Ok(val) = serde_json::from_slice::(&data) { + return val + .get("source_id") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + } + } + } + None + } + async fn should_skip_by_lww( &self, client: &zk::Client, path: &str, entry: &ZkEntry, + target_source_id: &Option, ) -> anyhow::Result { match client.check_stat(path).await { - Ok(Some(stat)) => Ok(stat.mtime > entry.stat.mtime), + Ok(Some(stat)) => { + if stat.mtime > entry.stat.mtime { + return Ok(true); + } + if stat.mtime == entry.stat.mtime { + if let Some(existing_id) = target_source_id { + if *existing_id > entry.source_id { + return Ok(true); + } + } + } + Ok(false) + } Ok(None) => Ok(false), Err(_) => Ok(false), } @@ -149,6 +187,13 @@ impl ZkSinker { ZkEventType::Deleted => match client.delete(path, None).await { Ok(()) => Ok(()), Err(ref e) if is_no_node(e) => Ok(()), + Err(ref e) if is_not_empty(e) => { + log_warn!( + "ZK delete {} skipped: node has children (may have new data from peer)", + path + ); + Ok(()) + } Err(e) => bail!("ZK delete {} failed: {}", path, e), }, @@ -192,7 +237,7 @@ impl ZkSinker { if let Some(data_marker) = &self.data_marker { let data_marker = data_marker.read().await; let marker_data = serde_json::to_vec(&serde_json::json!({ - "source_id": data_marker.data_origin_node + "source_id": data_marker.src_node }))?; match client @@ -229,3 +274,7 @@ fn is_node_exists(e: &zk::Error) -> bool { fn is_no_node(e: &zk::Error) -> bool { matches!(e, zk::Error::NoNode) } + +fn is_not_empty(e: &zk::Error) -> bool { + matches!(e, zk::Error::NotEmpty) +} From 7eb102793b399683a8293c54e6cc7c1e2761338a Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 01:03:01 +0800 Subject: [PATCH 05/20] feat(zk): add ZkExtractor implementation (cherry-pick from Elizabeth) Full ZkExtractor implementation (~340 lines): - full_scan: recursive znode traversal with path/ephemeral filtering - incremental_rescan: fresh map scan + diff for delete detection, old_versions for version comparison to skip unchanged znodes - Event type correctly set to Created vs Updated based on old_versions - source_id from DataMarker.src_node (not random UUID) for anti-loop - Checkpoint recovery via Position::Zk (path_versions + high_water_zxid) - Heartbeat task writing source_id + position to /__ape_dts_heartbeat Cherry-picked from Elizabeth feat/zk-extractor-impl branch (d33512de). Co-Authored-By: Claude Opus 4.6 --- dt-connector/src/extractor/zk/zk_extractor.rs | 325 +++++++++++++++++- 1 file changed, 321 insertions(+), 4 deletions(-) diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index 3d5785462..4fa5bc7a8 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -1,10 +1,25 @@ -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; use async_trait::async_trait; use tokio::sync::Mutex; +use zookeeper_client::{Client as ZkClient, CreateMode, Acls, Stat}; -use dt_common::meta::syncer::Syncer; -use dt_common::zk_filter::ZkFilter; +use dt_common::{ + log_error, log_info, log_warn, + meta::{ + dt_data::DtData, + position::Position, + syncer::Syncer, + zk::{zk_entry::ZkEntry, zk_event_type::ZkEventType, zk_stat::ZkStat}, + }, + zk_filter::ZkFilter, +}; use crate::extractor::base_extractor::{BaseExtractor, ExtractState}; use crate::extractor::resumer::recovery::Recovery; @@ -23,10 +38,312 @@ pub struct ZkExtractor { pub recovery: Option>, } +impl ZkExtractor { + fn stat_to_zk_stat(stat: &Stat) -> ZkStat { + ZkStat { + version: stat.version, + cversion: stat.cversion, + aversion: stat.aversion, + mtime: stat.mtime, + ctime: stat.ctime, + czxid: stat.czxid, + mzxid: stat.mzxid, + ephemeral_owner: stat.ephemeral_owner, + } + } + + fn is_ephemeral(stat: &Stat) -> bool { + stat.ephemeral_owner != 0 + } + + async fn recover_position(&mut self) -> anyhow::Result<(HashMap, i64)> { + if let Some(recovery) = &self.recovery { + if let Some(position) = recovery.get_cdc_resume_position().await { + if let Position::Zk { + path_versions, + high_water_zxid, + last_scan_timestamp, + total_paths, + } = &position + { + log_info!( + "ZkExtractor recovery: {} paths, high_water_zxid={}, last_scan={}", + total_paths, high_water_zxid, last_scan_timestamp + ); + self.base_extractor + .push_dt_data(&mut self.extract_state, DtData::Heartbeat {}, position) + .await?; + return Ok((path_versions.clone(), *high_water_zxid)); + } + } + } + Ok((HashMap::new(), 0)) + } + + async fn full_scan( + &mut self, + client: &ZkClient, + path_versions: &mut HashMap, + high_water_zxid: &mut i64, + source_id: &str, + ) -> anyhow::Result<()> { + log_info!("ZkExtractor starting full scan"); + let old_versions = path_versions.clone(); + for watch_path in &self.watch_paths.clone() { + if self.base_extractor.shut_down.load(Ordering::Relaxed) { + break; + } + self.scan_node_recursive( + client, watch_path, &old_versions, path_versions, high_water_zxid, source_id, + ) + .await?; + } + log_info!( + "ZkExtractor full scan complete: {} paths, high_water_zxid={}", + path_versions.len(), high_water_zxid + ); + Ok(()) + } + + async fn scan_node_recursive( + &mut self, + client: &ZkClient, + path: &str, + old_versions: &HashMap, + current_versions: &mut HashMap, + high_water_zxid: &mut i64, + source_id: &str, + ) -> anyhow::Result<()> { + if self.base_extractor.shut_down.load(Ordering::Relaxed) { + return Ok(()); + } + + if self.filter.filter_path(path) { + return Ok(()); + } + + let (data, stat) = match client.get_data(path).await { + Ok(r) => r, + Err(e) => { + log_warn!("ZkExtractor: get_data failed for {}: {}", path, e); + return Ok(()); + } + }; + + let ephemeral = Self::is_ephemeral(&stat); + if self.filter.filter_ephemeral(ephemeral) { + return Ok(()); + } + + let (already_synced, is_update) = match old_versions.get(path) { + Some((v, _)) => (*v >= stat.version, true), + None => (false, false), + }; + + if !already_synced { + let event_type = if is_update { + ZkEventType::Updated + } else { + ZkEventType::Created + }; + let entry = ZkEntry { + path: path.to_string(), + data: Some(data), + stat: Self::stat_to_zk_stat(&stat), + ephemeral, + event_type, + source_id: source_id.to_string(), + }; + let position = self.build_position(current_versions, *high_water_zxid); + self.extract_state + .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position) + .await?; + } + + current_versions.insert(path.to_string(), (stat.version, stat.mzxid)); + if stat.mzxid > *high_water_zxid { + *high_water_zxid = stat.mzxid; + } + + let children = match client.list_children(path).await { + Ok(c) => c, + Err(e) => { + log_warn!("ZkExtractor: list_children failed for {}: {}", path, e); + return Ok(()); + } + }; + + for child in children { + let child_path = if path == "/" { + format!("/{}", child) + } else { + format!("{}/{}", path, child) + }; + Box::pin(self.scan_node_recursive( + client, &child_path, old_versions, current_versions, high_water_zxid, source_id, + )) + .await?; + } + + Ok(()) + } + + async fn incremental_rescan( + &mut self, + client: &ZkClient, + path_versions: &mut HashMap, + high_water_zxid: &mut i64, + source_id: &str, + ) -> anyhow::Result<()> { + log_info!("ZkExtractor starting incremental rescan"); + let old_versions = path_versions.clone(); + + let mut current_versions: HashMap = HashMap::new(); + let mut current_zxid = *high_water_zxid; + + for watch_path in &self.watch_paths.clone() { + if self.base_extractor.shut_down.load(Ordering::Relaxed) { + break; + } + self.scan_node_recursive( + client, watch_path, &old_versions, &mut current_versions, &mut current_zxid, + source_id, + ) + .await?; + } + + // Detect deletes: paths in old_versions but missing from current scan + for (path, _) in &old_versions { + if current_versions.contains_key(path) { + continue; + } + if self.filter.filter_path(path) { + continue; + } + let entry = ZkEntry { + path: path.clone(), + data: None, + stat: ZkStat::default(), + ephemeral: false, + event_type: ZkEventType::Deleted, + source_id: source_id.to_string(), + }; + let position = self.build_position(¤t_versions, current_zxid); + self.extract_state + .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position) + .await?; + } + + *path_versions = current_versions; + *high_water_zxid = current_zxid; + + log_info!( + "ZkExtractor incremental rescan complete: {} paths", + path_versions.len() + ); + Ok(()) + } + + fn build_position( + &self, + path_versions: &HashMap, + high_water_zxid: i64, + ) -> Position { + Position::Zk { + path_versions: path_versions.clone(), + last_scan_timestamp: chrono::Utc::now().timestamp_millis(), + high_water_zxid, + total_paths: path_versions.len(), + } + } + + fn start_heartbeat( + &self, + shut_down: Arc, + syncer: Arc>, + url: String, + heartbeat_interval_secs: u64, + source_id: String, + ) { + if heartbeat_interval_secs == 0 { + return; + } + tokio::spawn(async move { + let hb_client = match ZkClient::connect(&url).await { + Ok(c) => c, + Err(e) => { + log_error!("ZkExtractor heartbeat: connect failed: {}", e); + return; + } + }; + + let marker_path = "/__ape_dts_heartbeat"; + let _ = hb_client + .create(marker_path, b"", &CreateMode::Persistent.with_acls(Acls::anyone_all())) + .await; + + loop { + if shut_down.load(Ordering::Relaxed) { + break; + } + tokio::time::sleep(tokio::time::Duration::from_secs(heartbeat_interval_secs)).await; + if shut_down.load(Ordering::Relaxed) { + break; + } + + let syncer_guard = syncer.lock().await; + let hb_data = serde_json::json!({ + "source_id": source_id, + "received_position": syncer_guard.received_position.to_string(), + "committed_position": syncer_guard.committed_position.to_string(), + "timestamp": chrono::Utc::now().to_rfc3339(), + }); + drop(syncer_guard); + + if let Err(e) = hb_client.set_data(marker_path, hb_data.to_string().as_bytes(), None).await { + log_warn!("ZkExtractor heartbeat write failed: {}", e); + } + } + }); + } +} + #[async_trait] impl Extractor for ZkExtractor { async fn extract(&mut self) -> anyhow::Result<()> { - todo!("ZkExtractor::extract not yet implemented") + let source_id = self + .extract_state + .data_marker + .as_ref() + .map(|dm| dm.src_node.clone()) + .unwrap_or_else(|| format!("ape_dts_{}", self.url)); + + let (mut path_versions, mut high_water_zxid) = self.recover_position().await?; + + let client = ZkClient::connect(&self.url).await?; + log_info!("ZkExtractor connected to {}", self.url); + + self.start_heartbeat( + self.base_extractor.shut_down.clone(), + self.syncer.clone(), + self.url.clone(), + self.heartbeat_interval_secs, + source_id.clone(), + ); + + self.full_scan(&client, &mut path_versions, &mut high_water_zxid, &source_id) + .await?; + + while !self.base_extractor.shut_down.load(Ordering::Relaxed) { + tokio::time::sleep(tokio::time::Duration::from_secs(self.scan_interval_secs)).await; + if self.base_extractor.shut_down.load(Ordering::Relaxed) { + break; + } + self.incremental_rescan(&client, &mut path_versions, &mut high_water_zxid, &source_id) + .await?; + } + + self.base_extractor.wait_task_finish(&mut self.extract_state).await } async fn close(&mut self) -> anyhow::Result<()> { From 9fefa89c35f9f0f5f58c11a6d139533df99b5490 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 15:46:59 +0800 Subject: [PATCH 06/20] fix: resolve compilation errors from ZK connector additions - ZkExtractor: clone position before move to fix borrow checker error - base_struct_sinker, clickhouse_struct_sinker, starrocks_struct_sinker: add wildcard arm for ConflictPolicyEnum::LastWriteWins - base_pipeline: add DtData::Zk to SinkMethod::Raw dispatch All changes pass cargo check --workspace. Co-Authored-By: Claude Opus 4.6 --- dt-connector/src/extractor/zk/zk_extractor.rs | 2 +- dt-connector/src/sinker/base_struct_sinker.rs | 1 + dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs | 1 + dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs | 1 + dt-pipeline/src/base_pipeline.rs | 2 +- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index 4fa5bc7a8..e32ba7f62 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -71,7 +71,7 @@ impl ZkExtractor { total_paths, high_water_zxid, last_scan_timestamp ); self.base_extractor - .push_dt_data(&mut self.extract_state, DtData::Heartbeat {}, position) + .push_dt_data(&mut self.extract_state, DtData::Heartbeat {}, position.clone()) .await?; return Ok((path_versions.clone(), *high_water_zxid)); } diff --git a/dt-connector/src/sinker/base_struct_sinker.rs b/dt-connector/src/sinker/base_struct_sinker.rs index ac1ae0c7c..13ae21403 100644 --- a/dt-connector/src/sinker/base_struct_sinker.rs +++ b/dt-connector/src/sinker/base_struct_sinker.rs @@ -46,6 +46,7 @@ impl BaseStructSinker { match conflict_policy { ConflictPolicyEnum::Interrupt => bail! {error}, ConflictPolicyEnum::Ignore => {} + _ => {} } } } diff --git a/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs b/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs index 194734678..23cd5e485 100644 --- a/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs +++ b/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs @@ -289,6 +289,7 @@ impl ClickhouseStructSinker { match self.conflict_policy { ConflictPolicyEnum::Interrupt => bail! {error}, ConflictPolicyEnum::Ignore => {} + _ => {} } } } diff --git a/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs b/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs index 3347a245c..7e54eadc5 100644 --- a/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs +++ b/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs @@ -388,6 +388,7 @@ impl StarrocksStructSinker { match self.conflict_policy { ConflictPolicyEnum::Interrupt => bail! {error}, ConflictPolicyEnum::Ignore => {} + _ => {} } } } diff --git a/dt-pipeline/src/base_pipeline.rs b/dt-pipeline/src/base_pipeline.rs index c055bb09f..546f6645a 100644 --- a/dt-pipeline/src/base_pipeline.rs +++ b/dt-pipeline/src/base_pipeline.rs @@ -503,7 +503,7 @@ impl BasePipeline { | SinkerConfig::Foxlake { .. } => return SinkMethod::Raw, _ => return SinkMethod::Dml, }, - DtData::Redis { .. } | DtData::Foxlake { .. } => return SinkMethod::Raw, + DtData::Redis { .. } | DtData::Foxlake { .. } | DtData::Zk { .. } => return SinkMethod::Raw, DtData::Begin {} | DtData::Commit { .. } | DtData::Heartbeat {} => continue, } } From 769981dfe095c9243e88f3dd091cfd85882898e5 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 20:43:18 +0800 Subject: [PATCH 07/20] test(zk): add dual ZK docker-compose and basic test config - docker-compose.zk.yml: two standalone ZK 3.9 instances (ports 2181, 2182) - zk_to_zk/cdc/basic_test/task_config.ini: ZK-to-ZK sync task config Co-Authored-By: Claude Opus 4.6 --- dt-tests/docker-compose.zk.yml | 30 +++++++++++++++ .../zk_to_zk/cdc/basic_test/task_config.ini | 38 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 dt-tests/docker-compose.zk.yml create mode 100644 dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini diff --git a/dt-tests/docker-compose.zk.yml b/dt-tests/docker-compose.zk.yml new file mode 100644 index 000000000..15dea2c17 --- /dev/null +++ b/dt-tests/docker-compose.zk.yml @@ -0,0 +1,30 @@ +services: + zk-src: + image: zookeeper:3.9 + container_name: zk-src-it + ports: + - "2181:2181" + environment: + ZOO_MY_ID: 1 + ZOO_STANDALONE_ENABLED: "true" + healthcheck: + test: ["CMD-SHELL", "echo srvr | nc localhost 2181 | grep standalone"] + timeout: 5s + retries: 5 + start_period: 10s + interval: 5s + + zk-dst: + image: zookeeper:3.9 + container_name: zk-dst-it + ports: + - "2182:2181" + environment: + ZOO_MY_ID: 2 + ZOO_STANDALONE_ENABLED: "true" + healthcheck: + test: ["CMD-SHELL", "echo srvr | nc localhost 2181 | grep standalone"] + timeout: 5s + retries: 5 + start_period: 10s + interval: 5s diff --git a/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini b/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini new file mode 100644 index 000000000..cec9c1d4b --- /dev/null +++ b/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini @@ -0,0 +1,38 @@ +[extractor] +db_type=zk +extract_type=cdc +url=localhost:2181 +watch_paths=/app,/config +scan_interval_secs=5 +include_ephemeral=false +heartbeat_interval_secs=10 + +[zk_filter] +do_paths=* +ignore_paths=/zookeeper +include_ephemeral=false + +[sinker] +db_type=zk +sink_type=write +url=localhost:2182 +batch_size=1 +create_if_not_exists=true +sync_ephemeral_as_persistent=true +conflict_policy=last_write_wins + +[router] +path_prefix_map= + +[pipeline] +buffer_size=4 +checkpoint_interval_secs=1 + +[parallelizer] +parallel_type=serial +parallel_size=1 + +[runtime] +log_level=info +log4rs_file=./log4rs.yaml +log_dir=./logs From 819e349071ed7d2c823e90429861ce61fa209448 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 21:07:26 +0800 Subject: [PATCH 08/20] fix(zk-sinker): skip write when target data is unchanged Prevents infinite sync loop in dual-active mode by comparing data bytes before writing. If the target znode already has identical data, the write is skipped, avoiding version bumps that would trigger the peer extractor. Co-Authored-By: Claude Opus 4.6 --- dt-connector/src/sinker/zk/zk_sinker.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index 71705838a..e7bea7a35 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -148,6 +148,14 @@ impl ZkSinker { ) -> anyhow::Result<()> { let data = entry.data.as_deref().unwrap_or(&[]); + if matches!(entry.event_type, ZkEventType::Created | ZkEventType::Updated) { + if let Ok((existing_data, _)) = client.get_data(path).await { + if existing_data == data { + return Ok(()); + } + } + } + match entry.event_type { ZkEventType::Created => { let options = self.create_options(entry); From 9fd589e1692bc286b07b24927dcd3808f6aa46e4 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 21:19:28 +0800 Subject: [PATCH 09/20] fix(test): fix zk-dst ZOO_MY_ID and add dual-direction test configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZOO_MY_ID=2 caused config mismatch in standalone ZK 3.9 (only server.1 defined). Changed to ZOO_MY_ID=1 for both independent instances. Added topo1 node1↔node2 test configs with data_marker anti-loop. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 886 +++++++++++------- dt-tests/docker-compose.zk.yml | 2 +- .../topo1_node1_to_node2/task_config.ini | 49 + .../topo1_node2_to_node1/task_config.ini | 49 + 4 files changed, 652 insertions(+), 334 deletions(-) create mode 100644 dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini create mode 100644 dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini diff --git a/Cargo.lock b/Cargo.lock index 6e221471e..4d62e7dca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ dependencies = [ "brotli", "bytes", "bytestring", - "derive_more", + "derive_more 2.0.1", "encoding_rs", "flate2", "foldhash 0.1.5", @@ -65,7 +65,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -149,7 +149,7 @@ dependencies = [ "bytestring", "cfg-if", "cookie", - "derive_more", + "derive_more 2.0.1", "encoding_rs", "foldhash 0.1.5", "futures-core", @@ -182,7 +182,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -351,8 +351,8 @@ dependencies = [ "regex-lite", "serde", "serde_json", - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.3", "thiserror 1.0.69", "typed-builder 0.16.2", "uuid", @@ -368,7 +368,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -530,7 +530,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -586,7 +586,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -619,6 +619,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backon" version = "1.6.0" @@ -697,7 +719,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -781,10 +803,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" dependencies = [ "once_cell", - "proc-macro-crate", + "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -905,6 +927,15 @@ dependencies = [ "bytes", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.27" @@ -987,7 +1018,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -1008,7 +1039,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -1052,7 +1083,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -1084,6 +1115,17 @@ dependencies = [ "tokio-util 0.7.15", ] +[[package]] +name = "compact_str" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b5c3ee2b4ffa00ac2b00d1645cd9229ade668139bccf95f15fadcf374127b" +dependencies = [ + "castaway", + "itoa", + "ryu", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1125,6 +1167,33 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const_format" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +dependencies = [ + "const_format_proc_macros", + "konst", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "cookie" version = "0.16.2" @@ -1191,7 +1260,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ - "rustc_version", + "rustc_version 0.4.1", ] [[package]] @@ -1203,30 +1272,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "critical-section" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1270,72 +1315,72 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.11" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", + "darling_core 0.13.4", + "darling_macro 0.13.4", ] [[package]] name = "darling" -version = "0.21.3" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] name = "darling_core" -version = "0.20.11" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim", - "syn 2.0.117", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] name = "darling_core" -version = "0.21.3" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim", - "syn 2.0.117", + "strsim 0.11.1", + "syn 2.0.103", ] [[package]] name = "darling_macro" -version = "0.20.11" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core 0.20.11", + "darling_core 0.13.4", "quote", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -1396,25 +1441,27 @@ dependencies = [ ] [[package]] -name = "derive-syn-parse" -version = "0.2.0" +name = "derive-where" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] -name = "derive-where" -version = "1.6.1" +name = "derive_more" +version = "0.99.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" dependencies = [ + "convert_case", "proc-macro2", "quote", - "syn 2.0.117", + "rustc_version 0.4.1", + "syn 2.0.103", ] [[package]] @@ -1434,7 +1481,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", "unicode-xid", ] @@ -1479,7 +1526,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -1538,7 +1585,7 @@ dependencies = [ "serde_json", "serde_yaml", "sqlx", - "strum", + "strum 0.25.0", "thiserror 1.0.69", "tokio", "url", @@ -1585,12 +1632,13 @@ dependencies = [ "serde", "serde_json", "sqlx", - "strum", + "strum 0.25.0", "thiserror 1.0.69", "tokio", "tokio-postgres", "url", "uuid", + "zookeeper-client", ] [[package]] @@ -1658,7 +1706,7 @@ dependencies = [ "redis", "regex", "sqlx", - "strum", + "strum 0.25.0", "tokio", ] @@ -1695,7 +1743,7 @@ dependencies = [ "serde_yaml", "serial_test", "sqlx", - "strum", + "strum 0.25.0", "tokio", ] @@ -1761,9 +1809,15 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.15.0" @@ -1784,14 +1838,14 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.6.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -1988,7 +2042,7 @@ checksum = "a0b4095fc99e1d858e5b8c7125d2638372ec85aa0fe6c807105cf10b0265ca6c" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -2000,7 +2054,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -2012,9 +2066,15 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.117", + "syn 2.0.103", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -2116,7 +2176,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -2310,6 +2370,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "hashlink" version = "0.10.0" @@ -2319,6 +2388,15 @@ dependencies = [ "hashbrown 0.15.4", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.1" @@ -2349,52 +2427,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hickory-proto" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna", - "ipnet", - "once_cell", - "rand 0.9.2", - "ring", - "thiserror 2.0.12", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "hickory-resolver" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" -dependencies = [ - "cfg-if", - "futures-util", - "hickory-proto", - "ipconfig", - "moka", - "once_cell", - "parking_lot 0.12.4", - "rand 0.9.2", - "resolv-conf", - "smallvec", - "thiserror 2.0.12", - "tokio", - "tracing", -] - [[package]] name = "hkdf" version = "0.12.4" @@ -2558,10 +2590,10 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls", + "rustls 0.23.34", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", "webpki-roots 1.0.3", ] @@ -2719,6 +2751,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "1.0.3" @@ -2740,6 +2783,12 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "665ff4dce8edd10d490641ccb78949832f1ddbff02c584fb1f85ab888fe0e50c" + [[package]] name = "impl-more" version = "0.1.9" @@ -2873,6 +2922,21 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kv-log-macro" version = "1.0.7" @@ -2965,6 +3029,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -3060,6 +3130,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -3102,52 +3181,10 @@ dependencies = [ ] [[package]] -name = "macro_magic" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" -dependencies = [ - "macro_magic_core", - "macro_magic_macros", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "macro_magic_core" -version = "0.5.1" +name = "matches" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" -dependencies = [ - "const-random", - "derive-syn-parse", - "macro_magic_core_macros", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "macro_magic_core_macros" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "macro_magic_macros" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" -dependencies = [ - "macro_magic_core", - "quote", - "syn 2.0.117", -] +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "md-5" @@ -3244,97 +3281,51 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "moka" -version = "0.12.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" -dependencies = [ - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "parking_lot 0.12.4", - "portable-atomic", - "smallvec", - "tagptr", - "uuid", -] - -[[package]] -name = "mongocrypt" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" -dependencies = [ - "bson", - "mongocrypt-sys", - "once_cell", - "serde", -] - -[[package]] -name = "mongocrypt-sys" -version = "0.1.5+1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" - [[package]] name = "mongodb" -version = "3.6.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ef2c933617431ad0246fb5b43c425ebdae18c7f7259c87de0726d93b0e7e91b" +checksum = "ef206acb1b72389b49bc9985efe7eb1f8a9bb18e5680d262fac26c07f44025f1" dependencies = [ - "base64 0.22.1", - "bitflags 2.9.1", + "async-trait", + "base64 0.13.1", + "bitflags 1.3.2", "bson", - "derive-where", - "derive_more", + "chrono", + "derivative", + "derive_more 0.99.20", "futures-core", + "futures-executor", "futures-io", "futures-util", "hex", - "hickory-proto", - "hickory-resolver", "hmac 0.12.1", - "macro_magic", + "lazy_static", "md-5 0.10.6", - "mongocrypt", - "mongodb-internal-macros", "pbkdf2", "percent-encoding", - "rand 0.9.2", + "rand 0.8.5", "rustc_version_runtime", - "rustls", - "rustversion", + "rustls 0.21.12", + "rustls-pemfile", "serde", "serde_bytes", "serde_with", - "sha1", + "sha-1", "sha2 0.10.9", - "socket2 0.6.1", + "socket2 0.4.10", "stringprep", - "strsim", + "strsim 0.10.0", "take_mut", - "thiserror 2.0.12", + "thiserror 1.0.69", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tokio-util 0.7.15", - "typed-builder 0.22.0", + "trust-dns-proto", + "trust-dns-resolver", + "typed-builder 0.10.0", "uuid", - "webpki-roots 1.0.3", -] - -[[package]] -name = "mongodb-internal-macros" -version = "3.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5758dc828eb2d02ec30563cba365609d56ddd833190b192beaee2b475a7bb3" -dependencies = [ - "macro_magic", - "proc-macro2", - "quote", - "syn 2.0.117", + "webpki-roots 0.25.4", ] [[package]] @@ -3351,7 +3342,7 @@ dependencies = [ "lazy_static", "log", "mysql_common", - "num_enum", + "num_enum 0.7.3", "openssl", "percent-encoding", "serde", @@ -3374,11 +3365,11 @@ dependencies = [ "darling 0.20.11", "heck 0.5.0", "num-bigint", - "proc-macro-crate", + "proc-macro-crate 3.3.0", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", "termcolor", "thiserror 1.0.69", ] @@ -3530,13 +3521,34 @@ dependencies = [ "libm", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive 0.5.11", +] + [[package]] name = "num_enum" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" dependencies = [ - "num_enum_derive", + "num_enum_derive 0.7.3", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -3545,10 +3557,10 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -3565,10 +3577,6 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -dependencies = [ - "critical-section", - "portable-atomic", -] [[package]] name = "once_cell_polyfill" @@ -3633,7 +3641,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -3752,9 +3760,9 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.7", ] @@ -3942,13 +3950,23 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit 0.19.15", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ - "toml_edit", + "toml_edit 0.22.27", ] [[package]] @@ -3970,7 +3988,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -4133,7 +4151,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.34", "socket2 0.6.1", "thiserror 2.0.12", "tokio", @@ -4153,7 +4171,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls", + "rustls 0.23.34", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -4292,7 +4310,7 @@ dependencies = [ "cmake", "libc", "libz-sys", - "num_enum", + "num_enum 0.7.3", "pkg-config", ] @@ -4481,14 +4499,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.34", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util 0.7.15", "tower", "tower-http", @@ -4614,23 +4632,32 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver", + "semver 1.0.26", ] [[package]] name = "rustc_version_runtime" -version = "0.3.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +checksum = "d31b7153270ebf48bf91c65ae5b0c00e749c4cfad505f66530ac74950249582f" dependencies = [ - "rustc_version", - "semver", + "rustc_version 0.2.3", + "semver 0.9.0", ] [[package]] @@ -4673,21 +4700,43 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pki-types" version = "1.13.0" @@ -4698,12 +4747,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4751,6 +4811,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sdd" version = "3.0.8" @@ -4771,7 +4841,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -4797,12 +4867,27 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.219" @@ -4839,7 +4924,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -4850,7 +4935,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -4880,25 +4965,24 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" dependencies = [ "serde", - "serde_derive", "serde_with_macros", ] [[package]] name = "serde_with_macros" -version = "3.14.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling 0.21.3", + "darling 0.13.4", "proc-macro2", "quote", - "syn 2.0.117", + "syn 1.0.109", ] [[package]] @@ -4936,7 +5020,18 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", +] + +[[package]] +name = "sha-1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", ] [[package]] @@ -5141,7 +5236,7 @@ dependencies = [ "futures-io", "futures-util", "hashbrown 0.15.4", - "hashlink", + "hashlink 0.10.0", "indexmap", "ipnetwork", "log", @@ -5149,7 +5244,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls", + "rustls 0.23.34", "serde", "serde_json", "sha2 0.10.9", @@ -5172,7 +5267,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5196,7 +5291,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.117", + "syn 2.0.103", "url", ] @@ -5341,19 +5436,47 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" +dependencies = [ + "strum_macros 0.23.1", +] + [[package]] name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", +] + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", ] [[package]] @@ -5366,7 +5489,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5398,9 +5521,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" dependencies = [ "proc-macro2", "quote", @@ -5424,7 +5547,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5448,12 +5571,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - [[package]] name = "take_mut" version = "0.2.2" @@ -5514,7 +5631,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5525,7 +5642,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5628,7 +5745,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5674,13 +5791,23 @@ dependencies = [ "tokio-util 0.6.10", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.34", "tokio", ] @@ -5718,6 +5845,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow 0.5.40", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -5726,7 +5864,7 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "toml_datetime", - "winnow", + "winnow 0.7.11", ] [[package]] @@ -5794,7 +5932,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5806,6 +5944,51 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trust-dns-proto" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "log", + "rand 0.8.5", + "smallvec", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "parking_lot 0.12.4", + "resolv-conf", + "smallvec", + "thiserror 1.0.69", + "tokio", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -5825,20 +6008,22 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" dependencies = [ - "typed-builder-macro 0.16.2", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] name = "typed-builder" -version = "0.22.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" dependencies = [ - "typed-builder-macro 0.22.0", + "typed-builder-macro", ] [[package]] @@ -5849,18 +6034,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "typed-builder-macro" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -5905,6 +6079,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -5939,7 +6119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna", + "idna 1.0.3", "percent-encoding", ] @@ -6052,7 +6232,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", "wasm-bindgen-shared", ] @@ -6087,7 +6267,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6134,6 +6314,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "webpki-roots" version = "0.26.11" @@ -6244,7 +6430,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -6255,7 +6441,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -6518,6 +6704,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.11" @@ -6587,7 +6782,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", "synstructure", ] @@ -6608,7 +6803,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", ] [[package]] @@ -6628,7 +6823,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", "synstructure", ] @@ -6668,7 +6863,32 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.103", +] + +[[package]] +name = "zookeeper-client" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dfa35623e95f8c27cc64ee268d338cb1a8ae8fedfdc7b2b7aa8deb83d2cde14" +dependencies = [ + "bytes", + "compact_str", + "const_format", + "derive-where", + "either", + "fastrand 2.3.0", + "hashbrown 0.12.3", + "hashlink 0.8.4", + "ignore-result", + "num_enum 0.5.11", + "static_assertions", + "strum 0.23.0", + "thiserror 1.0.69", + "tokio", + "tokio-rustls 0.26.4", + "tracing", + "uuid", ] [[package]] diff --git a/dt-tests/docker-compose.zk.yml b/dt-tests/docker-compose.zk.yml index 15dea2c17..20d81726f 100644 --- a/dt-tests/docker-compose.zk.yml +++ b/dt-tests/docker-compose.zk.yml @@ -20,7 +20,7 @@ services: ports: - "2182:2181" environment: - ZOO_MY_ID: 2 + ZOO_MY_ID: 1 ZOO_STANDALONE_ENABLED: "true" healthcheck: test: ["CMD-SHELL", "echo srvr | nc localhost 2181 | grep standalone"] diff --git a/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini new file mode 100644 index 000000000..5cddcf250 --- /dev/null +++ b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini @@ -0,0 +1,49 @@ +[extractor] +db_type=zk +extract_type=cdc +url=localhost:2181 +watch_paths=/app +scan_interval_secs=5 +include_ephemeral=false +heartbeat_interval_secs=10 + +[zk_filter] +do_paths=/app +ignore_paths=/zookeeper,/__ape_dts_heartbeat,/__ape_dts_marker +include_ephemeral=false + +[data_marker] +topo_name=topo1 +topo_nodes=node1,node2 +src_node=node1 +dst_node=node2 +do_nodes=node1 +ignore_nodes=node2 +marker=/__ape_dts_marker + +[sinker] +db_type=zk +sink_type=write +url=localhost:2182 +batch_size=200 +create_if_not_exists=true +sync_ephemeral_as_persistent=false +conflict_policy=last_write_wins + +[router] +db_map= +col_map= +tb_map= + +[parallelizer] +parallel_type=serial +parallel_size=1 + +[pipeline] +buffer_size=100 +checkpoint_interval_secs=1 + +[runtime] +log_level=info +log4rs_file=./log4rs.yaml +log_dir=./logs diff --git a/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini new file mode 100644 index 000000000..43b9b07c1 --- /dev/null +++ b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini @@ -0,0 +1,49 @@ +[extractor] +db_type=zk +extract_type=cdc +url=localhost:2182 +watch_paths=/app +scan_interval_secs=5 +include_ephemeral=false +heartbeat_interval_secs=10 + +[zk_filter] +do_paths=/app +ignore_paths=/zookeeper,/__ape_dts_heartbeat,/__ape_dts_marker +include_ephemeral=false + +[data_marker] +topo_name=topo1 +topo_nodes=node1,node2 +src_node=node2 +dst_node=node1 +do_nodes=node2 +ignore_nodes=node1 +marker=/__ape_dts_marker + +[sinker] +db_type=zk +sink_type=write +url=localhost:2181 +batch_size=200 +create_if_not_exists=true +sync_ephemeral_as_persistent=false +conflict_policy=last_write_wins + +[router] +db_map= +col_map= +tb_map= + +[parallelizer] +parallel_type=serial +parallel_size=1 + +[pipeline] +buffer_size=100 +checkpoint_interval_secs=1 + +[runtime] +log_level=info +log4rs_file=./log4rs.yaml +log_dir=./logs From 943269760c94871b5c3c088d6054ab3b2019c104 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 21:41:59 +0800 Subject: [PATCH 10/20] feat(zk-sinker): add per-znode shadow metadata for proper LWW Replace target-stat-based LWW with shadow metadata at /__ape_dts_shadow/ storing (source_id, source_mtime, version). This compares original source mtimes instead of sinker write times, enabling correct conflict resolution in dual-active mode. Also write/delete shadow alongside data operations and add /__ape_dts_shadow to ignore_paths in all test configs. Co-Authored-By: Claude Opus 4.6 --- dt-connector/src/sinker/zk/zk_sinker.rs | 173 +++++++++++------- .../zk_to_zk/cdc/basic_test/task_config.ini | 2 +- .../topo1_node1_to_node2/task_config.ini | 2 +- .../topo1_node2_to_node1/task_config.ini | 2 +- 4 files changed, 105 insertions(+), 74 deletions(-) diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index e7bea7a35..b9bde26c4 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -67,24 +67,17 @@ impl ZkSinker { let mut data_size = 0u64; let mut record_count = 0u64; - let target_source_id = if self.conflict_policy == ConflictPolicyEnum::LastWriteWins { - self.read_target_source_id(client).await - } else { - None - }; - for dt_item in data.iter() { if let DtData::Zk { entry } = &dt_item.dt_data { data_size += entry.get_data_size(); let routed_path = self.router.route_path(&entry.path); - if self.conflict_policy == ConflictPolicyEnum::LastWriteWins { - if self - .should_skip_by_lww(client, &routed_path, entry, &target_source_id) + if self.conflict_policy == ConflictPolicyEnum::LastWriteWins + && self + .should_skip_by_lww(client, &routed_path, entry) .await? - { - continue; - } + { + continue; } self.apply_zk_event(client, &routed_path, entry).await?; @@ -99,45 +92,32 @@ impl ZkSinker { .await } - async fn read_target_source_id(&self, client: &zk::Client) -> Option { - if let Some(data_marker) = &self.data_marker { - let data_marker = data_marker.read().await; - if let Ok((data, _)) = client.get_data(&data_marker.marker).await { - if let Ok(val) = serde_json::from_slice::(&data) { - return val - .get("source_id") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - } - } - } - None - } - async fn should_skip_by_lww( &self, client: &zk::Client, path: &str, entry: &ZkEntry, - target_source_id: &Option, ) -> anyhow::Result { - match client.check_stat(path).await { - Ok(Some(stat)) => { - if stat.mtime > entry.stat.mtime { + let shadow = shadow_path(path); + if let Ok((data, _)) = client.get_data(&shadow).await { + if let Ok(val) = serde_json::from_slice::(&data) { + let source_mtime = val + .get("source_mtime") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let source_id = val + .get("source_id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + if source_mtime > entry.stat.mtime { return Ok(true); } - if stat.mtime == entry.stat.mtime { - if let Some(existing_id) = target_source_id { - if *existing_id > entry.source_id { - return Ok(true); - } - } + if source_mtime == entry.stat.mtime && *source_id > *entry.source_id { + return Ok(true); } - Ok(false) } - Ok(None) => Ok(false), - Err(_) => Ok(false), } + Ok(false) } async fn apply_zk_event( @@ -165,45 +145,50 @@ impl ZkSinker { } match client.create(path, data, &options).await { - Ok(_) => Ok(()), + Ok(_) => {} Err(ref e) if is_node_exists(e) => { - client - .set_data(path, data, None) - .await - .map_err(|e| { - anyhow::anyhow!("ZK set_data (upsert) {} failed: {}", path, e) - })?; - Ok(()) + client.set_data(path, data, None).await.map_err(|e| { + anyhow::anyhow!("ZK set_data (upsert) {} failed: {}", path, e) + })?; } Err(e) => bail!("ZK create {} failed: {}", path, e), } + self.write_shadow(client, path, entry).await?; + Ok(()) } - ZkEventType::Updated => match client.set_data(path, data, None).await { - Ok(_) => Ok(()), - Err(ref e) if is_no_node(e) && self.create_if_not_exists => { - self.ensure_parent_paths(client, path).await?; - let options = self.create_options(entry); - client.create(path, data, &options).await.map_err(|e| { - anyhow::anyhow!("ZK create (auto) {} failed: {}", path, e) - })?; - Ok(()) + ZkEventType::Updated => { + match client.set_data(path, data, None).await { + Ok(_) => {} + Err(ref e) if is_no_node(e) && self.create_if_not_exists => { + self.ensure_parent_paths(client, path).await?; + let options = self.create_options(entry); + client.create(path, data, &options).await.map_err(|e| { + anyhow::anyhow!("ZK create (auto) {} failed: {}", path, e) + })?; + } + Err(e) => bail!("ZK set_data {} failed: {}", path, e), } - Err(e) => bail!("ZK set_data {} failed: {}", path, e), - }, - - ZkEventType::Deleted => match client.delete(path, None).await { - Ok(()) => Ok(()), - Err(ref e) if is_no_node(e) => Ok(()), - Err(ref e) if is_not_empty(e) => { - log_warn!( - "ZK delete {} skipped: node has children (may have new data from peer)", - path - ); - Ok(()) + self.write_shadow(client, path, entry).await?; + Ok(()) + } + + ZkEventType::Deleted => { + match client.delete(path, None).await { + Ok(()) => {} + Err(ref e) if is_no_node(e) => {} + Err(ref e) if is_not_empty(e) => { + log_warn!( + "ZK delete {} skipped: node has children (may have new data from peer)", + path + ); + return Ok(()); + } + Err(e) => bail!("ZK delete {} failed: {}", path, e), } - Err(e) => bail!("ZK delete {} failed: {}", path, e), - }, + self.delete_shadow(client, path).await?; + Ok(()) + } ZkEventType::ChildrenChanged => Ok(()), } @@ -241,6 +226,46 @@ impl ZkSinker { Ok(()) } + async fn write_shadow( + &self, + client: &zk::Client, + path: &str, + entry: &ZkEntry, + ) -> anyhow::Result<()> { + let shadow = shadow_path(path); + let shadow_data = serde_json::to_vec(&serde_json::json!({ + "source_id": entry.source_id, + "source_mtime": entry.stat.mtime, + "version": entry.stat.version, + }))?; + match client.set_data(&shadow, &shadow_data, None).await { + Ok(_) => {} + Err(ref e) if is_no_node(e) => { + self.ensure_parent_paths(client, &shadow).await?; + let options = self.persistent_options(); + match client.create(&shadow, &shadow_data, &options).await { + Ok(_) => {} + Err(ref e) if is_node_exists(e) => { + client.set_data(&shadow, &shadow_data, None).await.ok(); + } + Err(e) => bail!("ZK create shadow {} failed: {}", shadow, e), + } + } + Err(e) => bail!("ZK write shadow {} failed: {}", shadow, e), + } + Ok(()) + } + + async fn delete_shadow(&self, client: &zk::Client, path: &str) -> anyhow::Result<()> { + let shadow = shadow_path(path); + match client.delete(&shadow, None).await { + Ok(()) => {} + Err(ref e) if is_no_node(e) => {} + Err(e) => bail!("ZK delete shadow {} failed: {}", shadow, e), + } + Ok(()) + } + async fn write_data_marker(&self, client: &zk::Client) -> anyhow::Result<()> { if let Some(data_marker) = &self.data_marker { let data_marker = data_marker.read().await; @@ -275,6 +300,12 @@ impl ZkSinker { } } +const SHADOW_PREFIX: &str = "/__ape_dts_shadow"; + +fn shadow_path(path: &str) -> String { + format!("{}{}", SHADOW_PREFIX, path) +} + fn is_node_exists(e: &zk::Error) -> bool { matches!(e, zk::Error::NodeExists) } diff --git a/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini b/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini index cec9c1d4b..dac900d59 100644 --- a/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini +++ b/dt-tests/tests/zk_to_zk/cdc/basic_test/task_config.ini @@ -9,7 +9,7 @@ heartbeat_interval_secs=10 [zk_filter] do_paths=* -ignore_paths=/zookeeper +ignore_paths=/zookeeper,/__ape_dts_shadow include_ephemeral=false [sinker] diff --git a/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini index 5cddcf250..30457c00c 100644 --- a/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini +++ b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node1_to_node2/task_config.ini @@ -9,7 +9,7 @@ heartbeat_interval_secs=10 [zk_filter] do_paths=/app -ignore_paths=/zookeeper,/__ape_dts_heartbeat,/__ape_dts_marker +ignore_paths=/zookeeper,/__ape_dts_heartbeat,/__ape_dts_marker,/__ape_dts_shadow include_ephemeral=false [data_marker] diff --git a/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini index 43b9b07c1..82011b423 100644 --- a/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini +++ b/dt-tests/tests/zk_to_zk/cdc/basic_test/topo1_node2_to_node1/task_config.ini @@ -9,7 +9,7 @@ heartbeat_interval_secs=10 [zk_filter] do_paths=/app -ignore_paths=/zookeeper,/__ape_dts_heartbeat,/__ape_dts_marker +ignore_paths=/zookeeper,/__ape_dts_heartbeat,/__ape_dts_marker,/__ape_dts_shadow include_ephemeral=false [data_marker] From b8381fad1b76b24c88a08bbe5a7e8b071e7181cd Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 6 Jun 2026 21:38:28 +0800 Subject: [PATCH 11/20] fix(zk): version+mzxid sync check for delete+recreate, cargo fmt - Extractor now checks both version AND mzxid when determining if a path is already synced. Fixes missed sync after delete+recreate where version resets to 0 but mzxid advances. - Run cargo fmt to fix CI lint gate. Co-Authored-By: Claude Opus 4.6 --- dt-common/src/config/task_config.rs | 29 ++++- dt-common/src/lib.rs | 2 +- dt-common/src/meta/position.rs | 3 +- dt-common/src/zk_filter.rs | 5 +- dt-connector/src/data_marker.rs | 4 +- dt-connector/src/extractor/mod.rs | 2 +- dt-connector/src/extractor/zk/zk_extractor.rs | 70 +++++++--- dt-connector/src/sinker/zk/zk_sinker.rs | 20 +-- dt-pipeline/src/base_pipeline.rs | 4 +- dt-task/src/extractor_util.rs | 4 +- .../zk_to_zk/node1_to_node2_verify.log | 113 ++++++++++++++++ dt-tests/scripts/zk_integration_test.sh | 121 ++++++++++++++++++ 12 files changed, 335 insertions(+), 42 deletions(-) create mode 100644 dt-tests/evidence/zk_to_zk/node1_to_node2_verify.log create mode 100755 dt-tests/scripts/zk_integration_test.sh diff --git a/dt-common/src/config/task_config.rs b/dt-common/src/config/task_config.rs index a4dc75bad..7ce863020 100644 --- a/dt-common/src/config/task_config.rs +++ b/dt-common/src/config/task_config.rs @@ -781,13 +781,24 @@ impl TaskConfig { let watch_paths = if watch_paths_str.is_empty() { vec!["/".to_string()] } else { - watch_paths_str.split(',').map(|s| s.trim().to_string()).collect() + watch_paths_str + .split(',') + .map(|s| s.trim().to_string()) + .collect() }; ExtractorConfig::Zk { url, watch_paths, - scan_interval_secs: loader.get_with_default(EXTRACTOR, "scan_interval_secs", 60), - include_ephemeral: loader.get_with_default(EXTRACTOR, "include_ephemeral", false), + scan_interval_secs: loader.get_with_default( + EXTRACTOR, + "scan_interval_secs", + 60, + ), + include_ephemeral: loader.get_with_default( + EXTRACTOR, + "include_ephemeral", + false, + ), heartbeat_interval_secs, } } @@ -1041,8 +1052,16 @@ impl TaskConfig { SinkType::Write => SinkerConfig::Zk { url, batch_size, - create_if_not_exists: loader.get_with_default(SINKER, "create_if_not_exists", true), - sync_ephemeral_as_persistent: loader.get_with_default(SINKER, "sync_ephemeral_as_persistent", false), + create_if_not_exists: loader.get_with_default( + SINKER, + "create_if_not_exists", + true, + ), + sync_ephemeral_as_persistent: loader.get_with_default( + SINKER, + "sync_ephemeral_as_persistent", + false, + ), conflict_policy, }, _ => bail! { not_supported_err }, diff --git a/dt-common/src/lib.rs b/dt-common/src/lib.rs index e5a3ef899..151624bd4 100644 --- a/dt-common/src/lib.rs +++ b/dt-common/src/lib.rs @@ -7,6 +7,6 @@ pub mod meta; pub mod monitor; pub mod rdb_filter; pub mod system_dbs; -pub mod zk_filter; pub mod time_filter; pub mod utils; +pub mod zk_filter; diff --git a/dt-common/src/meta/position.rs b/dt-common/src/meta/position.rs index 52bfdab82..877636400 100644 --- a/dt-common/src/meta/position.rs +++ b/dt-common/src/meta/position.rs @@ -135,7 +135,8 @@ impl Position { 0 } Position::Zk { - last_scan_timestamp, .. + last_scan_timestamp, + .. } => *last_scan_timestamp as u64, _ => 0, } diff --git a/dt-common/src/zk_filter.rs b/dt-common/src/zk_filter.rs index 5873c94e6..4c5cbd908 100644 --- a/dt-common/src/zk_filter.rs +++ b/dt-common/src/zk_filter.rs @@ -38,6 +38,9 @@ impl ZkFilter { if config_str.trim().is_empty() { return HashSet::new(); } - config_str.split(',').map(|s| s.trim().to_string()).collect() + config_str + .split(',') + .map(|s| s.trim().to_string()) + .collect() } } diff --git a/dt-connector/src/data_marker.rs b/dt-connector/src/data_marker.rs index a1ac32b3c..2443e0806 100644 --- a/dt-connector/src/data_marker.rs +++ b/dt-connector/src/data_marker.rs @@ -121,7 +121,9 @@ impl DataMarker { DtData::Zk { entry } => { if let Some(data) = &entry.data { if let Ok(marker_value) = serde_json::from_slice::(data) { - if let Some(source_id) = marker_value.get("source_id").and_then(|v| v.as_str()) { + if let Some(source_id) = + marker_value.get("source_id").and_then(|v| v.as_str()) + { self.data_origin_node = source_id.to_string(); } } diff --git a/dt-connector/src/extractor/mod.rs b/dt-connector/src/extractor/mod.rs index 696ab68b6..707192e51 100644 --- a/dt-connector/src/extractor/mod.rs +++ b/dt-connector/src/extractor/mod.rs @@ -10,10 +10,10 @@ pub mod pg; pub mod rdb_snapshot_extract_statement; pub mod redis; pub mod resumer; -pub mod zk; pub mod snapshot_chunk_id_generator; pub mod snapshot_dispatcher; pub mod snapshot_types; +pub mod zk; fn estimated_sample_limit(sample_rate: Option, estimated_count: u64) -> Option { let sample_rate = sample_rate.filter(|rate| (1..100).contains(rate))?; diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index e32ba7f62..f810a9841 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -8,7 +8,7 @@ use std::{ use async_trait::async_trait; use tokio::sync::Mutex; -use zookeeper_client::{Client as ZkClient, CreateMode, Acls, Stat}; +use zookeeper_client::{Acls, Client as ZkClient, CreateMode, Stat}; use dt_common::{ log_error, log_info, log_warn, @@ -68,10 +68,16 @@ impl ZkExtractor { { log_info!( "ZkExtractor recovery: {} paths, high_water_zxid={}, last_scan={}", - total_paths, high_water_zxid, last_scan_timestamp + total_paths, + high_water_zxid, + last_scan_timestamp ); self.base_extractor - .push_dt_data(&mut self.extract_state, DtData::Heartbeat {}, position.clone()) + .push_dt_data( + &mut self.extract_state, + DtData::Heartbeat {}, + position.clone(), + ) .await?; return Ok((path_versions.clone(), *high_water_zxid)); } @@ -94,13 +100,19 @@ impl ZkExtractor { break; } self.scan_node_recursive( - client, watch_path, &old_versions, path_versions, high_water_zxid, source_id, + client, + watch_path, + &old_versions, + path_versions, + high_water_zxid, + source_id, ) .await?; } log_info!( "ZkExtractor full scan complete: {} paths, high_water_zxid={}", - path_versions.len(), high_water_zxid + path_versions.len(), + high_water_zxid ); Ok(()) } @@ -136,7 +148,7 @@ impl ZkExtractor { } let (already_synced, is_update) = match old_versions.get(path) { - Some((v, _)) => (*v >= stat.version, true), + Some((v, old_mzxid)) => (*v >= stat.version && *old_mzxid >= stat.mzxid, true), None => (false, false), }; @@ -180,7 +192,12 @@ impl ZkExtractor { format!("{}/{}", path, child) }; Box::pin(self.scan_node_recursive( - client, &child_path, old_versions, current_versions, high_water_zxid, source_id, + client, + &child_path, + old_versions, + current_versions, + high_water_zxid, + source_id, )) .await?; } @@ -206,7 +223,11 @@ impl ZkExtractor { break; } self.scan_node_recursive( - client, watch_path, &old_versions, &mut current_versions, &mut current_zxid, + client, + watch_path, + &old_versions, + &mut current_versions, + &mut current_zxid, source_id, ) .await?; @@ -279,7 +300,11 @@ impl ZkExtractor { let marker_path = "/__ape_dts_heartbeat"; let _ = hb_client - .create(marker_path, b"", &CreateMode::Persistent.with_acls(Acls::anyone_all())) + .create( + marker_path, + b"", + &CreateMode::Persistent.with_acls(Acls::anyone_all()), + ) .await; loop { @@ -300,7 +325,10 @@ impl ZkExtractor { }); drop(syncer_guard); - if let Err(e) = hb_client.set_data(marker_path, hb_data.to_string().as_bytes(), None).await { + if let Err(e) = hb_client + .set_data(marker_path, hb_data.to_string().as_bytes(), None) + .await + { log_warn!("ZkExtractor heartbeat write failed: {}", e); } } @@ -331,19 +359,31 @@ impl Extractor for ZkExtractor { source_id.clone(), ); - self.full_scan(&client, &mut path_versions, &mut high_water_zxid, &source_id) - .await?; + self.full_scan( + &client, + &mut path_versions, + &mut high_water_zxid, + &source_id, + ) + .await?; while !self.base_extractor.shut_down.load(Ordering::Relaxed) { tokio::time::sleep(tokio::time::Duration::from_secs(self.scan_interval_secs)).await; if self.base_extractor.shut_down.load(Ordering::Relaxed) { break; } - self.incremental_rescan(&client, &mut path_versions, &mut high_water_zxid, &source_id) - .await?; + self.incremental_rescan( + &client, + &mut path_versions, + &mut high_water_zxid, + &source_id, + ) + .await?; } - self.base_extractor.wait_task_finish(&mut self.extract_state).await + self.base_extractor + .wait_task_finish(&mut self.extract_state) + .await } async fn close(&mut self) -> anyhow::Result<()> { diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index b9bde26c4..1a444150f 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -73,9 +73,7 @@ impl ZkSinker { let routed_path = self.router.route_path(&entry.path); if self.conflict_policy == ConflictPolicyEnum::LastWriteWins - && self - .should_skip_by_lww(client, &routed_path, entry) - .await? + && self.should_skip_by_lww(client, &routed_path, entry).await? { continue; } @@ -105,10 +103,7 @@ impl ZkSinker { .get("source_mtime") .and_then(|v| v.as_i64()) .unwrap_or(0); - let source_id = val - .get("source_id") - .and_then(|v| v.as_str()) - .unwrap_or(""); + let source_id = val.get("source_id").and_then(|v| v.as_str()).unwrap_or(""); if source_mtime > entry.stat.mtime { return Ok(true); } @@ -128,7 +123,10 @@ impl ZkSinker { ) -> anyhow::Result<()> { let data = entry.data.as_deref().unwrap_or(&[]); - if matches!(entry.event_type, ZkEventType::Created | ZkEventType::Updated) { + if matches!( + entry.event_type, + ZkEventType::Created | ZkEventType::Updated + ) { if let Ok((existing_data, _)) = client.get_data(path).await { if existing_data == data { return Ok(()); @@ -286,11 +284,7 @@ impl ZkSinker { .create(&data_marker.marker, &marker_data, &options) .await .map_err(|e| { - anyhow::anyhow!( - "ZK create marker {} failed: {}", - data_marker.marker, - e - ) + anyhow::anyhow!("ZK create marker {} failed: {}", data_marker.marker, e) })?; } Err(e) => bail!("ZK write marker {} failed: {}", data_marker.marker, e), diff --git a/dt-pipeline/src/base_pipeline.rs b/dt-pipeline/src/base_pipeline.rs index 546f6645a..211c23f4f 100644 --- a/dt-pipeline/src/base_pipeline.rs +++ b/dt-pipeline/src/base_pipeline.rs @@ -503,7 +503,9 @@ impl BasePipeline { | SinkerConfig::Foxlake { .. } => return SinkMethod::Raw, _ => return SinkMethod::Dml, }, - DtData::Redis { .. } | DtData::Foxlake { .. } | DtData::Zk { .. } => return SinkMethod::Raw, + DtData::Redis { .. } | DtData::Foxlake { .. } | DtData::Zk { .. } => { + return SinkMethod::Raw + } DtData::Begin {} | DtData::Commit { .. } | DtData::Heartbeat {} => continue, } } diff --git a/dt-task/src/extractor_util.rs b/dt-task/src/extractor_util.rs index ccb761429..1d43f40b8 100644 --- a/dt-task/src/extractor_util.rs +++ b/dt-task/src/extractor_util.rs @@ -753,9 +753,7 @@ impl ExtractorUtil { include_ephemeral, heartbeat_interval_secs, } => { - let zk_filter = dt_common::zk_filter::ZkFilter::from_config( - &config.zk_filter, - )?; + let zk_filter = dt_common::zk_filter::ZkFilter::from_config(&config.zk_filter)?; let extractor = ZkExtractor { url, watch_paths, diff --git a/dt-tests/evidence/zk_to_zk/node1_to_node2_verify.log b/dt-tests/evidence/zk_to_zk/node1_to_node2_verify.log new file mode 100644 index 000000000..1b6857463 --- /dev/null +++ b/dt-tests/evidence/zk_to_zk/node1_to_node2_verify.log @@ -0,0 +1,113 @@ +Connecting to localhost:2181 +2026-06-06 12:59:07,389 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:zookeeper.version=3.9.5-293c895a8d966a3ecb92872be4a1daf87d725da2, built on 2026-02-11 20:18 UTC +2026-06-06 12:59:07,390 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:host.name=56cf11fd6120 +2026-06-06 12:59:07,390 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.version=17.0.19 +2026-06-06 12:59:07,390 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.vendor=Eclipse Adoptium +2026-06-06 12:59:07,390 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.home=/opt/java/openjdk +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.class.path=/apache-zookeeper-3.9.5-bin/bin/../zookeeper-metrics-providers/zookeeper-prometheus-metrics/target/classes:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/target/classes:/apache-zookeeper-3.9.5-bin/bin/../build/classes:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-metrics-providers/zookeeper-prometheus-metrics/target/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/target/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../build/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-prometheus-metrics-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-jute-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/snappy-java-1.1.10.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/slf4j-api-2.0.13.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_servlet-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_hotspot-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_common-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-native-unix-common-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-classes-epoll-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-classes-2.0.74.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-resolver-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-handler-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-common-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-codec-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-buffer-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/metrics-core-4.1.12.1.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/logback-core-1.3.15.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/logback-classic-1.3.15.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jline-3.25.1.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-util-ajax-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-util-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-servlet-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-server-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-security-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-io-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-http-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/javax.servlet-api-3.1.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-databind-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-core-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-annotations-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/commons-io-2.17.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/commons-cli-1.5.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/audience-annotations-0.12.0.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-*.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/src/main/resources/lib/*.jar:/conf: +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.io.tmpdir=/tmp +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.compiler= +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.name=Linux +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.arch=aarch64 +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.version=6.12.76-linuxkit +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.name=root +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.home=/root +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.dir=/apache-zookeeper-3.9.5-bin +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.free=114MB +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.max=256MB +2026-06-06 12:59:07,391 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.total=128MB +2026-06-06 12:59:07,392 [myid:] - INFO [main:o.a.z.ZooKeeper@635] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@25b485ba +2026-06-06 12:59:07,395 [myid:] - INFO [main:o.a.z.c.X509Util@93] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation +2026-06-06 12:59:07,460 [myid:] - INFO [main:o.a.z.c.X509Util@115] - Default TLS protocol is TLSv1.3, supported TLS protocols are [TLSv1.3, TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2Hello] +2026-06-06 12:59:07,463 [myid:] - INFO [main:o.a.z.ClientCnxnSocket@235] - jute.maxbuffer value is 1048575 Bytes +2026-06-06 12:59:07,466 [myid:] - INFO [main:o.a.z.ClientCnxn@1734] - zookeeper.request.timeout value is 0. feature enabled=false +2026-06-06 12:59:07,467 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1170] - Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:2181. +2026-06-06 12:59:07,467 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1172] - SASL config status: Will not attempt to authenticate using SASL (unknown error) +2026-06-06 12:59:07,470 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1006] - Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:44318, server: localhost/[0:0:0:0:0:0:0:1]:2181 +2026-06-06 12:59:07,472 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1438] - Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:2181, session id = 0x1002ba28b500009, negotiated timeout = 30000 + +WATCHER:: + +WatchedEvent state:SyncConnected type:None path:null zxid: -1 +/ +/__ape_dts_marker +/app +/zookeeper +/app/service-a +/app/service-b +/app/service1 +/app/service-a/config +/zookeeper/config +/zookeeper/quota +2026-06-06 12:59:07,478 [myid:] - INFO [main:o.a.z.u.ServiceUtils@45] - Exiting JVM with code 0 +---service-a--- +Connecting to localhost:2181 +2026-06-06 12:59:08,100 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:zookeeper.version=3.9.5-293c895a8d966a3ecb92872be4a1daf87d725da2, built on 2026-02-11 20:18 UTC +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:host.name=56cf11fd6120 +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.version=17.0.19 +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.vendor=Eclipse Adoptium +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.home=/opt/java/openjdk +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.class.path=/apache-zookeeper-3.9.5-bin/bin/../zookeeper-metrics-providers/zookeeper-prometheus-metrics/target/classes:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/target/classes:/apache-zookeeper-3.9.5-bin/bin/../build/classes:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-metrics-providers/zookeeper-prometheus-metrics/target/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/target/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../build/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-prometheus-metrics-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-jute-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/snappy-java-1.1.10.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/slf4j-api-2.0.13.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_servlet-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_hotspot-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_common-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-native-unix-common-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-classes-epoll-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-classes-2.0.74.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-resolver-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-handler-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-common-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-codec-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-buffer-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/metrics-core-4.1.12.1.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/logback-core-1.3.15.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/logback-classic-1.3.15.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jline-3.25.1.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-util-ajax-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-util-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-servlet-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-server-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-security-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-io-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-http-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/javax.servlet-api-3.1.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-databind-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-core-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-annotations-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/commons-io-2.17.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/commons-cli-1.5.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/audience-annotations-0.12.0.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-*.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/src/main/resources/lib/*.jar:/conf: +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.io.tmpdir=/tmp +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.compiler= +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.name=Linux +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.arch=aarch64 +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.version=6.12.76-linuxkit +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.name=root +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.home=/root +2026-06-06 12:59:08,101 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.dir=/apache-zookeeper-3.9.5-bin +2026-06-06 12:59:08,102 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.free=114MB +2026-06-06 12:59:08,102 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.max=256MB +2026-06-06 12:59:08,102 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.total=128MB +2026-06-06 12:59:08,102 [myid:] - INFO [main:o.a.z.ZooKeeper@635] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@25b485ba +2026-06-06 12:59:08,105 [myid:] - INFO [main:o.a.z.c.X509Util@93] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation +2026-06-06 12:59:08,170 [myid:] - INFO [main:o.a.z.c.X509Util@115] - Default TLS protocol is TLSv1.3, supported TLS protocols are [TLSv1.3, TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2Hello] +2026-06-06 12:59:08,172 [myid:] - INFO [main:o.a.z.ClientCnxnSocket@235] - jute.maxbuffer value is 1048575 Bytes +2026-06-06 12:59:08,175 [myid:] - INFO [main:o.a.z.ClientCnxn@1734] - zookeeper.request.timeout value is 0. feature enabled=false +2026-06-06 12:59:08,177 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1170] - Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:2181. +2026-06-06 12:59:08,177 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1172] - SASL config status: Will not attempt to authenticate using SASL (unknown error) +2026-06-06 12:59:08,179 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1006] - Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:44328, server: localhost/[0:0:0:0:0:0:0:1]:2181 +2026-06-06 12:59:08,181 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1438] - Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:2181, session id = 0x1002ba28b50000a, negotiated timeout = 30000 + +WATCHER:: + +WatchedEvent state:SyncConnected type:None path:null zxid: -1 +host1:8080 +2026-06-06 12:59:08,183 [myid:] - INFO [main:o.a.z.u.ServiceUtils@45] - Exiting JVM with code 0 +---service-b--- +Connecting to localhost:2181 +2026-06-06 12:59:08,819 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:zookeeper.version=3.9.5-293c895a8d966a3ecb92872be4a1daf87d725da2, built on 2026-02-11 20:18 UTC +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:host.name=56cf11fd6120 +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.version=17.0.19 +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.vendor=Eclipse Adoptium +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.home=/opt/java/openjdk +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.class.path=/apache-zookeeper-3.9.5-bin/bin/../zookeeper-metrics-providers/zookeeper-prometheus-metrics/target/classes:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/target/classes:/apache-zookeeper-3.9.5-bin/bin/../build/classes:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-metrics-providers/zookeeper-prometheus-metrics/target/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/target/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../build/lib/*.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-prometheus-metrics-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-jute-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/zookeeper-3.9.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/snappy-java-1.1.10.5.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/slf4j-api-2.0.13.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_servlet-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_hotspot-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient_common-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/simpleclient-0.9.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-native-unix-common-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-classes-epoll-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-transport-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-classes-2.0.74.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-tcnative-boringssl-static-2.0.74.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-resolver-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-handler-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-common-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-codec-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/netty-buffer-4.1.130.Final.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/metrics-core-4.1.12.1.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/logback-core-1.3.15.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/logback-classic-1.3.15.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jline-3.25.1.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-util-ajax-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-util-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-servlet-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-server-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-security-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-io-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jetty-http-9.4.58.v20250814.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/javax.servlet-api-3.1.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-databind-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-core-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/jackson-annotations-2.15.2.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/commons-io-2.17.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/commons-cli-1.5.0.jar:/apache-zookeeper-3.9.5-bin/bin/../lib/audience-annotations-0.12.0.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-*.jar:/apache-zookeeper-3.9.5-bin/bin/../zookeeper-server/src/main/resources/lib/*.jar:/conf: +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.io.tmpdir=/tmp +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:java.compiler= +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.name=Linux +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.arch=aarch64 +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.version=6.12.76-linuxkit +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.name=root +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.home=/root +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:user.dir=/apache-zookeeper-3.9.5-bin +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.free=114MB +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.max=256MB +2026-06-06 12:59:08,820 [myid:] - INFO [main:o.a.z.Environment@98] - Client environment:os.memory.total=128MB +2026-06-06 12:59:08,821 [myid:] - INFO [main:o.a.z.ZooKeeper@635] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@25b485ba +2026-06-06 12:59:08,824 [myid:] - INFO [main:o.a.z.c.X509Util@93] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation +2026-06-06 12:59:08,888 [myid:] - INFO [main:o.a.z.c.X509Util@115] - Default TLS protocol is TLSv1.3, supported TLS protocols are [TLSv1.3, TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2Hello] +2026-06-06 12:59:08,891 [myid:] - INFO [main:o.a.z.ClientCnxnSocket@235] - jute.maxbuffer value is 1048575 Bytes +2026-06-06 12:59:08,893 [myid:] - INFO [main:o.a.z.ClientCnxn@1734] - zookeeper.request.timeout value is 0. feature enabled=false +2026-06-06 12:59:08,895 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1170] - Opening socket connection to server localhost/127.0.0.1:2181. +2026-06-06 12:59:08,895 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1172] - SASL config status: Will not attempt to authenticate using SASL (unknown error) +2026-06-06 12:59:08,897 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1006] - Socket connection established, initiating session, client: /127.0.0.1:60954, server: localhost/127.0.0.1:2181 +2026-06-06 12:59:08,900 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):o.a.z.ClientCnxn$SendThread@1438] - Session establishment complete on server localhost/127.0.0.1:2181, session id = 0x1002ba28b50000b, negotiated timeout = 30000 + +WATCHER:: + +WatchedEvent state:SyncConnected type:None path:null zxid: -1 +host2:9090 +2026-06-06 12:59:08,902 [myid:] - INFO [main:o.a.z.u.ServiceUtils@45] - Exiting JVM with code 0 diff --git a/dt-tests/scripts/zk_integration_test.sh b/dt-tests/scripts/zk_integration_test.sh new file mode 100755 index 000000000..ddf5a9d8a --- /dev/null +++ b/dt-tests/scripts/zk_integration_test.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DT_TESTS_DIR="$(dirname "$SCRIPT_DIR")" +PROJECT_ROOT="$(dirname "$DT_TESTS_DIR")" + +COMPOSE_FILE="$DT_TESTS_DIR/docker-compose.zk.yml" +EVIDENCE_DIR="$DT_TESTS_DIR/evidence/zk_to_zk" +TASK_DIR="$DT_TESTS_DIR/tests/zk_to_zk/cdc/basic_test" + +ZK1_HOST="localhost:2181" +ZK2_HOST="localhost:2182" + +mkdir -p "$EVIDENCE_DIR" + +log() { echo "[$(date '+%H:%M:%S')] $*"; } + +cleanup() { + log "Stopping containers..." + docker-compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true +} + +wait_for_zk() { + local host=$1 name=$2 + log "Waiting for $name ($host)..." + for i in $(seq 1 30); do + if echo ruok | nc -w 2 "${host%%:*}" "${host##*:}" 2>/dev/null | grep -q imok; then + log "$name ready" + return 0 + fi + sleep 1 + done + log "ERROR: $name not ready after 30s" + return 1 +} + +phase_1_start_zk() { + log "=== Phase 1: Start dual ZK ensembles ===" + docker-compose -f "$COMPOSE_FILE" up -d + wait_for_zk "$ZK1_HOST" "zk-node1" + wait_for_zk "$ZK2_HOST" "zk-node2" + log "Phase 1 PASS" +} + +phase_2_seed_data() { + log "=== Phase 2: Seed test data on zk-node1 ===" + docker exec zk-node1 /apache-zookeeper-*/bin/zkCli.sh -server localhost:2181 <<'ZK_CMDS' +create /app "" +create /app/service-a "host1:8080" +create /app/service-b "host2:9090" +create /app/service-a/config "timeout=30" +quit +ZK_CMDS + log "Seeded: /app/service-a, /app/service-b, /app/service-a/config" + log "Phase 2 PASS" +} + +phase_3_run_sync() { + log "=== Phase 3: Run ape-dts sync (node1 → node2) ===" + log "NOTE: Build and run manually:" + log " cargo build --release -p dt-main" + log " ./target/release/dt-main --task_conf $TASK_DIR/topo1_node1_to_node2/task_config.ini" + log "" + log "After sync runs for ~10 seconds, Ctrl+C and proceed to phase 4." +} + +phase_4_verify() { + log "=== Phase 4: Verify sync results on zk-node2 ===" + local result_file="$EVIDENCE_DIR/verify_$(date +%Y%m%d_%H%M%S).log" + + { + echo "=== zk-node2 data after sync ===" + docker exec zk-node2 /apache-zookeeper-*/bin/zkCli.sh -server localhost:2181 <<'ZK_CMDS' +ls /app +get /app/service-a +get /app/service-b +get /app/service-a/config +quit +ZK_CMDS + } 2>&1 | tee "$result_file" + + log "Evidence saved to: $result_file" + + if grep -q "host1:8080" "$result_file" && grep -q "host2:9090" "$result_file"; then + log "Phase 4 PASS — data synced correctly" + else + log "Phase 4 FAIL — data not found on zk-node2" + fi +} + +phase_5_verify_bidirectional() { + log "=== Phase 5: Verify bidirectional sync ===" + log "1. Write new data on zk-node2:" + log " docker exec zk-node2 zkCli.sh create /app/service-c 'host3:7070'" + log "2. Run reverse sync task:" + log " ./target/release/dt-main --task_conf $TASK_DIR/topo1_node2_to_node1/task_config.ini" + log "3. Verify on zk-node1:" + log " docker exec zk-node1 zkCli.sh get /app/service-c" +} + +case "${1:-all}" in + start) phase_1_start_zk ;; + seed) phase_2_seed_data ;; + sync) phase_3_run_sync ;; + verify) phase_4_verify ;; + bidir) phase_5_verify_bidirectional ;; + cleanup) cleanup ;; + all) + trap cleanup EXIT + phase_1_start_zk + phase_2_seed_data + phase_3_run_sync + phase_4_verify + phase_5_verify_bidirectional + ;; + *) + echo "Usage: $0 {start|seed|sync|verify|bidir|cleanup|all}" + exit 1 + ;; +esac From 767d4bc8ae86326b472b00667dced2f033f2e153 Mon Sep 17 00:00:00 2001 From: wei Date: Sun, 7 Jun 2026 20:12:28 +0800 Subject: [PATCH 12/20] fix(zk-sinker): update shadow metadata on skip-if-unchanged path The same-data early return skipped write_shadow, leaving shadow source_mtime stale. LWW decisions then used an outdated watermark. Co-Authored-By: Claude Opus 4.6 --- dt-connector/src/sinker/zk/zk_sinker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index 1a444150f..dd487c2de 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -129,6 +129,7 @@ impl ZkSinker { ) { if let Ok((existing_data, _)) = client.get_data(path).await { if existing_data == data { + self.write_shadow(client, path, entry).await?; return Ok(()); } } From 0488c398591808e24579d6a0e415a7f9ba21721c Mon Sep 17 00:00:00 2001 From: wei Date: Thu, 11 Jun 2026 21:42:29 +0800 Subject: [PATCH 13/20] fix: handle zk config in rdb topic router --- dt-connector/src/rdb_router.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dt-connector/src/rdb_router.rs b/dt-connector/src/rdb_router.rs index ed4056b5c..c15e3a273 100644 --- a/dt-connector/src/rdb_router.rs +++ b/dt-connector/src/rdb_router.rs @@ -470,6 +470,7 @@ impl RdbTopicRouterInner { RouterConfig::Rdb { topic_map, .. } => Ok(Self { topic_map: Self::parse_topic_map(topic_map, db_type)?, }), + _ => Ok(Self::default()), } } From 1834569d833efb431dedb8cceb7898ccfad3b99f Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 20 Jun 2026 05:11:58 +0800 Subject: [PATCH 14/20] fix(zk): add ordered tombstone shadow handling --- dt-common/src/meta/zk/zk_entry.rs | 22 ++ dt-connector/src/extractor/zk/zk_extractor.rs | 22 +- dt-connector/src/sinker/zk/zk_sinker.rs | 214 ++++++++++++++++-- 3 files changed, 231 insertions(+), 27 deletions(-) diff --git a/dt-common/src/meta/zk/zk_entry.rs b/dt-common/src/meta/zk/zk_entry.rs index 37b227222..41228c9b9 100644 --- a/dt-common/src/meta/zk/zk_entry.rs +++ b/dt-common/src/meta/zk/zk_entry.rs @@ -2,6 +2,14 @@ use serde::{Deserialize, Serialize}; use super::{zk_event_type::ZkEventType, zk_stat::ZkStat}; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub enum ZkOrderOrigin { + #[default] + SourceStatMtime, + WatcherObserved, + ReconcileObserved, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ZkEntry { pub path: String, @@ -10,10 +18,24 @@ pub struct ZkEntry { pub ephemeral: bool, pub event_type: ZkEventType, pub source_id: String, + #[serde(default)] + pub source_order_millis: i64, + #[serde(default)] + pub source_zxid: i64, + #[serde(default)] + pub order_origin: ZkOrderOrigin, } impl ZkEntry { pub fn get_data_size(&self) -> u64 { self.path.len() as u64 + self.data.as_ref().map_or(0, |d| d.len() as u64) } + + pub fn source_order_millis(&self) -> i64 { + if self.source_order_millis != 0 { + self.source_order_millis + } else { + self.stat.mtime + } + } } diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index f810a9841..6859b6b49 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -16,7 +16,11 @@ use dt_common::{ dt_data::DtData, position::Position, syncer::Syncer, - zk::{zk_entry::ZkEntry, zk_event_type::ZkEventType, zk_stat::ZkStat}, + zk::{ + zk_entry::{ZkEntry, ZkOrderOrigin}, + zk_event_type::ZkEventType, + zk_stat::ZkStat, + }, }, zk_filter::ZkFilter, }; @@ -165,6 +169,9 @@ impl ZkExtractor { ephemeral, event_type, source_id: source_id.to_string(), + source_order_millis: stat.mtime, + source_zxid: stat.mzxid, + order_origin: ZkOrderOrigin::SourceStatMtime, }; let position = self.build_position(current_versions, *high_water_zxid); self.extract_state @@ -234,20 +241,29 @@ impl ZkExtractor { } // Detect deletes: paths in old_versions but missing from current scan - for (path, _) in &old_versions { + for (path, (old_version, old_mzxid)) in &old_versions { if current_versions.contains_key(path) { continue; } if self.filter.filter_path(path) { continue; } + let delete_order_millis = chrono::Utc::now().timestamp_millis(); let entry = ZkEntry { path: path.clone(), data: None, - stat: ZkStat::default(), + stat: ZkStat { + version: *old_version, + mzxid: *old_mzxid, + mtime: delete_order_millis, + ..Default::default() + }, ephemeral: false, event_type: ZkEventType::Deleted, source_id: source_id.to_string(), + source_order_millis: delete_order_millis, + source_zxid: *old_mzxid, + order_origin: ZkOrderOrigin::ReconcileObserved, }; let position = self.build_position(¤t_versions, current_zxid); self.extract_state diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index dd487c2de..d2eadfb4e 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -9,7 +9,7 @@ use dt_common::config::config_enums::ConflictPolicyEnum; use dt_common::log_info; use dt_common::log_warn; use dt_common::meta::dt_data::{DtData, DtItem}; -use dt_common::meta::zk::zk_entry::ZkEntry; +use dt_common::meta::zk::zk_entry::{ZkEntry, ZkOrderOrigin}; use dt_common::meta::zk::zk_event_type::ZkEventType; use crate::data_marker::DataMarker; @@ -98,23 +98,21 @@ impl ZkSinker { ) -> anyhow::Result { let shadow = shadow_path(path); if let Ok((data, _)) = client.get_data(&shadow).await { - if let Ok(val) = serde_json::from_slice::(&data) { - let source_mtime = val - .get("source_mtime") - .and_then(|v| v.as_i64()) - .unwrap_or(0); - let source_id = val.get("source_id").and_then(|v| v.as_str()).unwrap_or(""); - if source_mtime > entry.stat.mtime { - return Ok(true); - } - if source_mtime == entry.stat.mtime && *source_id > *entry.source_id { - return Ok(true); - } + if let Ok(shadow_meta) = serde_json::from_slice::(&data) { + return Ok(Self::should_skip_by_shadow_meta(&shadow_meta, entry)); } } Ok(false) } + fn should_skip_by_shadow_meta(shadow: &ZkShadowMeta, entry: &ZkEntry) -> bool { + let shadow_order = shadow.source_order_millis(); + let entry_order = entry.source_order_millis(); + + shadow_order > entry_order + || (shadow_order == entry_order && shadow.source_id.as_str() > entry.source_id.as_str()) + } + async fn apply_zk_event( &self, client: &zk::Client, @@ -185,7 +183,7 @@ impl ZkSinker { } Err(e) => bail!("ZK delete {} failed: {}", path, e), } - self.delete_shadow(client, path).await?; + self.write_tombstone_shadow(client, path, entry).await?; Ok(()) } @@ -232,11 +230,7 @@ impl ZkSinker { entry: &ZkEntry, ) -> anyhow::Result<()> { let shadow = shadow_path(path); - let shadow_data = serde_json::to_vec(&serde_json::json!({ - "source_id": entry.source_id, - "source_mtime": entry.stat.mtime, - "version": entry.stat.version, - }))?; + let shadow_data = Self::shadow_data(entry, false)?; match client.set_data(&shadow, &shadow_data, None).await { Ok(_) => {} Err(ref e) if is_no_node(e) => { @@ -255,16 +249,38 @@ impl ZkSinker { Ok(()) } - async fn delete_shadow(&self, client: &zk::Client, path: &str) -> anyhow::Result<()> { + async fn write_tombstone_shadow( + &self, + client: &zk::Client, + path: &str, + entry: &ZkEntry, + ) -> anyhow::Result<()> { let shadow = shadow_path(path); - match client.delete(&shadow, None).await { - Ok(()) => {} - Err(ref e) if is_no_node(e) => {} - Err(e) => bail!("ZK delete shadow {} failed: {}", shadow, e), + let shadow_data = Self::shadow_data(entry, true)?; + match client.set_data(&shadow, &shadow_data, None).await { + Ok(_) => {} + Err(ref e) if is_no_node(e) => { + self.ensure_parent_paths(client, &shadow).await?; + let options = self.persistent_options(); + match client.create(&shadow, &shadow_data, &options).await { + Ok(_) => {} + Err(ref e) if is_node_exists(e) => { + client.set_data(&shadow, &shadow_data, None).await.ok(); + } + Err(e) => bail!("ZK create tombstone shadow {} failed: {}", shadow, e), + } + } + Err(e) => bail!("ZK write tombstone shadow {} failed: {}", shadow, e), } Ok(()) } + fn shadow_data(entry: &ZkEntry, deleted: bool) -> anyhow::Result> { + Ok(serde_json::to_vec(&ZkShadowMeta::from_entry( + entry, deleted, + ))?) + } + async fn write_data_marker(&self, client: &zk::Client) -> anyhow::Result<()> { if let Some(data_marker) = &self.data_marker { let data_marker = data_marker.read().await; @@ -297,6 +313,45 @@ impl ZkSinker { const SHADOW_PREFIX: &str = "/__ape_dts_shadow"; +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +struct ZkShadowMeta { + pub source_id: String, + #[serde(default)] + pub source_order_millis: i64, + #[serde(default)] + pub source_mtime: i64, + #[serde(default)] + pub source_zxid: Option, + #[serde(default)] + pub version: i32, + #[serde(default)] + pub deleted: bool, + #[serde(default)] + pub order_origin: ZkOrderOrigin, +} + +impl ZkShadowMeta { + fn from_entry(entry: &ZkEntry, deleted: bool) -> Self { + Self { + source_id: entry.source_id.clone(), + source_order_millis: entry.source_order_millis(), + source_mtime: entry.stat.mtime, + source_zxid: (entry.source_zxid != 0).then_some(entry.source_zxid), + version: entry.stat.version, + deleted, + order_origin: entry.order_origin.clone(), + } + } + + fn source_order_millis(&self) -> i64 { + if self.source_order_millis != 0 { + self.source_order_millis + } else { + self.source_mtime + } + } +} + fn shadow_path(path: &str) -> String { format!("{}{}", SHADOW_PREFIX, path) } @@ -312,3 +367,114 @@ fn is_no_node(e: &zk::Error) -> bool { fn is_not_empty(e: &zk::Error) -> bool { matches!(e, zk::Error::NotEmpty) } + +#[cfg(test)] +mod tests { + use super::*; + use dt_common::meta::zk::zk_stat::ZkStat; + + fn entry(order: i64, source_id: &str, event_type: ZkEventType) -> ZkEntry { + ZkEntry { + path: "/app/service".to_string(), + data: Some(b"value".to_vec()), + stat: ZkStat { + version: 1, + mtime: order, + mzxid: order, + ..Default::default() + }, + ephemeral: false, + event_type, + source_id: source_id.to_string(), + source_order_millis: order, + source_zxid: order, + order_origin: ZkOrderOrigin::SourceStatMtime, + } + } + + #[test] + fn newer_tombstone_skips_older_upsert() { + let old_upsert = entry(100, "node-a", ZkEventType::Updated); + let tombstone = ZkShadowMeta { + source_id: "node-b".to_string(), + source_order_millis: 200, + source_mtime: 200, + source_zxid: Some(200), + version: 1, + deleted: true, + order_origin: ZkOrderOrigin::ReconcileObserved, + }; + + assert!(ZkSinker::should_skip_by_shadow_meta( + &tombstone, + &old_upsert + )); + } + + #[test] + fn newer_upsert_can_pass_older_tombstone() { + let new_upsert = entry(300, "node-a", ZkEventType::Updated); + let tombstone = ZkShadowMeta { + source_id: "node-b".to_string(), + source_order_millis: 200, + source_mtime: 200, + source_zxid: Some(200), + version: 1, + deleted: true, + order_origin: ZkOrderOrigin::ReconcileObserved, + }; + + assert!(!ZkSinker::should_skip_by_shadow_meta( + &tombstone, + &new_upsert + )); + } + + #[test] + fn equal_order_uses_source_id_tie_break() { + let entry_from_a = entry(100, "node-a", ZkEventType::Updated); + let shadow_from_b = ZkShadowMeta { + source_id: "node-b".to_string(), + source_order_millis: 100, + source_mtime: 100, + source_zxid: Some(100), + version: 1, + deleted: false, + order_origin: ZkOrderOrigin::SourceStatMtime, + }; + + assert!(ZkSinker::should_skip_by_shadow_meta( + &shadow_from_b, + &entry_from_a + )); + } + + #[test] + fn delete_shadow_data_is_tombstone() { + let delete = ZkEntry { + data: None, + event_type: ZkEventType::Deleted, + order_origin: ZkOrderOrigin::ReconcileObserved, + ..entry(400, "node-a", ZkEventType::Deleted) + }; + + let bytes = ZkSinker::shadow_data(&delete, true).unwrap(); + let shadow: ZkShadowMeta = serde_json::from_slice(&bytes).unwrap(); + + assert!(shadow.deleted); + assert_eq!(shadow.source_order_millis, 400); + assert_eq!(shadow.source_mtime, 400); + assert_eq!(shadow.source_zxid, Some(400)); + assert_eq!(shadow.order_origin, ZkOrderOrigin::ReconcileObserved); + } + + #[test] + fn old_shadow_json_falls_back_to_source_mtime() { + let old_json = br#"{"source_id":"node-b","source_mtime":200,"version":1}"#; + let shadow: ZkShadowMeta = serde_json::from_slice(old_json).unwrap(); + let old_upsert = entry(100, "node-a", ZkEventType::Updated); + + assert_eq!(shadow.source_order_millis(), 200); + assert!(ZkSinker::should_skip_by_shadow_meta(&shadow, &old_upsert)); + } +} From 7411bfb5e23f4f17bb37ac106f441fb800c36aff Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 20 Jun 2026 05:16:23 +0800 Subject: [PATCH 15/20] fix(zk): tighten tombstone lww ordering --- dt-connector/src/sinker/zk/zk_sinker.rs | 127 ++++++++++++++++++++++-- 1 file changed, 121 insertions(+), 6 deletions(-) diff --git a/dt-connector/src/sinker/zk/zk_sinker.rs b/dt-connector/src/sinker/zk/zk_sinker.rs index d2eadfb4e..213c10daa 100644 --- a/dt-connector/src/sinker/zk/zk_sinker.rs +++ b/dt-connector/src/sinker/zk/zk_sinker.rs @@ -98,19 +98,32 @@ impl ZkSinker { ) -> anyhow::Result { let shadow = shadow_path(path); if let Ok((data, _)) = client.get_data(&shadow).await { - if let Ok(shadow_meta) = serde_json::from_slice::(&data) { - return Ok(Self::should_skip_by_shadow_meta(&shadow_meta, entry)); + match serde_json::from_slice::(&data) { + Ok(shadow_meta) => { + return Ok(Self::should_skip_by_shadow_meta(&shadow_meta, entry)); + } + Err(e) => { + log_warn!( + "ZK shadow {} has invalid metadata, ignoring for LWW: {}", + shadow, + e + ); + } } } Ok(false) } fn should_skip_by_shadow_meta(shadow: &ZkShadowMeta, entry: &ZkEntry) -> bool { - let shadow_order = shadow.source_order_millis(); - let entry_order = entry.source_order_millis(); + shadow.order_key() > Self::entry_order_key(entry) + } - shadow_order > entry_order - || (shadow_order == entry_order && shadow.source_id.as_str() > entry.source_id.as_str()) + fn entry_order_key(entry: &ZkEntry) -> (i64, i64, &str) { + ( + entry.source_order_millis(), + entry.source_zxid, + entry.source_id.as_str(), + ) } async fn apply_zk_event( @@ -276,11 +289,30 @@ impl ZkSinker { } fn shadow_data(entry: &ZkEntry, deleted: bool) -> anyhow::Result> { + if deleted { + Self::validate_tombstone_entry(entry)?; + } Ok(serde_json::to_vec(&ZkShadowMeta::from_entry( entry, deleted, ))?) } + fn validate_tombstone_entry(entry: &ZkEntry) -> anyhow::Result<()> { + if entry.source_order_millis() <= 0 { + bail!( + "ZK tombstone for {} must have positive source_order_millis", + entry.path + ); + } + if entry.source_zxid <= 0 { + bail!( + "ZK tombstone for {} must have positive source_zxid", + entry.path + ); + } + Ok(()) + } + async fn write_data_marker(&self, client: &zk::Client) -> anyhow::Result<()> { if let Some(data_marker) = &self.data_marker { let data_marker = data_marker.read().await; @@ -350,6 +382,18 @@ impl ZkShadowMeta { self.source_mtime } } + + fn source_zxid(&self) -> i64 { + self.source_zxid.unwrap_or_default() + } + + fn order_key(&self) -> (i64, i64, &str) { + ( + self.source_order_millis(), + self.source_zxid(), + self.source_id.as_str(), + ) + } } fn shadow_path(path: &str) -> String { @@ -449,6 +493,46 @@ mod tests { )); } + #[test] + fn same_source_same_millis_shadow_zxid_newer_skips_old_upsert() { + let mut old_upsert = entry(100, "node-a", ZkEventType::Updated); + old_upsert.source_zxid = 10; + let newer_tombstone = ZkShadowMeta { + source_id: "node-a".to_string(), + source_order_millis: 100, + source_mtime: 100, + source_zxid: Some(11), + version: 1, + deleted: true, + order_origin: ZkOrderOrigin::ReconcileObserved, + }; + + assert!(ZkSinker::should_skip_by_shadow_meta( + &newer_tombstone, + &old_upsert + )); + } + + #[test] + fn same_source_same_millis_entry_zxid_newer_passes_tombstone() { + let mut new_upsert = entry(100, "node-a", ZkEventType::Updated); + new_upsert.source_zxid = 12; + let older_tombstone = ZkShadowMeta { + source_id: "node-a".to_string(), + source_order_millis: 100, + source_mtime: 100, + source_zxid: Some(11), + version: 1, + deleted: true, + order_origin: ZkOrderOrigin::ReconcileObserved, + }; + + assert!(!ZkSinker::should_skip_by_shadow_meta( + &older_tombstone, + &new_upsert + )); + } + #[test] fn delete_shadow_data_is_tombstone() { let delete = ZkEntry { @@ -468,6 +552,37 @@ mod tests { assert_eq!(shadow.order_origin, ZkOrderOrigin::ReconcileObserved); } + #[test] + fn delete_shadow_rejects_zero_order() { + let delete = ZkEntry { + data: None, + event_type: ZkEventType::Deleted, + order_origin: ZkOrderOrigin::ReconcileObserved, + source_order_millis: 0, + stat: ZkStat { + mtime: 0, + ..Default::default() + }, + source_zxid: 1, + ..entry(400, "node-a", ZkEventType::Deleted) + }; + + assert!(ZkSinker::shadow_data(&delete, true).is_err()); + } + + #[test] + fn delete_shadow_rejects_zero_zxid() { + let delete = ZkEntry { + data: None, + event_type: ZkEventType::Deleted, + order_origin: ZkOrderOrigin::ReconcileObserved, + source_zxid: 0, + ..entry(400, "node-a", ZkEventType::Deleted) + }; + + assert!(ZkSinker::shadow_data(&delete, true).is_err()); + } + #[test] fn old_shadow_json_falls_back_to_source_mtime() { let old_json = br#"{"source_id":"node-b","source_mtime":200,"version":1}"#; From 298e0807a2c57b6df2e9d2ffc2274460ba719749 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 20 Jun 2026 05:26:33 +0800 Subject: [PATCH 16/20] feat(zk): add watch-triggered reconciliation loop --- dt-common/src/zk_filter.rs | 34 ++ dt-connector/src/extractor/zk/zk_extractor.rs | 406 ++++++++++++++---- 2 files changed, 361 insertions(+), 79 deletions(-) diff --git a/dt-common/src/zk_filter.rs b/dt-common/src/zk_filter.rs index 4c5cbd908..4424912b1 100644 --- a/dt-common/src/zk_filter.rs +++ b/dt-common/src/zk_filter.rs @@ -10,6 +10,12 @@ pub struct ZkFilter { } impl ZkFilter { + pub const INTERNAL_PATHS: [&'static str; 3] = [ + "/__ape_dts_marker", + "/__ape_dts_shadow", + "/__ape_dts_heartbeat", + ]; + pub fn from_config(config: &ZkFilterConfig) -> anyhow::Result { let do_paths = Self::parse_paths(&config.do_paths); let ignore_paths = Self::parse_paths(&config.ignore_paths); @@ -21,6 +27,9 @@ impl ZkFilter { } pub fn filter_path(&self, path: &str) -> bool { + if Self::INTERNAL_PATHS.iter().any(|p| path.starts_with(p)) { + return true; + } if self.ignore_paths.iter().any(|p| path.starts_with(p)) { return true; } @@ -44,3 +53,28 @@ impl ZkFilter { .collect() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::zk_filter_config::ZkFilterConfig; + + fn filter(do_paths: &str, ignore_paths: &str) -> ZkFilter { + ZkFilter::from_config(&ZkFilterConfig { + do_paths: do_paths.to_string(), + ignore_paths: ignore_paths.to_string(), + include_ephemeral: false, + }) + .unwrap() + } + + #[test] + fn internal_paths_are_always_filtered() { + let filter = filter("/__ape_dts_shadow,/app", ""); + + assert!(filter.filter_path("/__ape_dts_marker")); + assert!(filter.filter_path("/__ape_dts_shadow/app")); + assert!(filter.filter_path("/__ape_dts_heartbeat")); + assert!(!filter.filter_path("/app/service")); + } +} diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index 6859b6b49..2933f4e23 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -4,11 +4,15 @@ use std::{ atomic::{AtomicBool, Ordering}, Arc, }, + time::Duration, }; +use anyhow::bail; use async_trait::async_trait; -use tokio::sync::Mutex; -use zookeeper_client::{Acls, Client as ZkClient, CreateMode, Stat}; +use tokio::sync::{mpsc, Mutex}; +use zookeeper_client::{ + Acls, AddWatchMode, Client as ZkClient, CreateMode, EventType, SessionState, Stat, WatchedEvent, +}; use dt_common::{ log_error, log_info, log_warn, @@ -42,6 +46,18 @@ pub struct ZkExtractor { pub recovery: Option>, } +#[derive(Debug, Clone)] +struct ZkWatchEvent { + watch_path: String, + event: WatchedEvent, +} + +#[derive(Debug, Clone)] +enum ReconcileScope { + All, + Path(String), +} + impl ZkExtractor { fn stat_to_zk_stat(stat: &Stat) -> ZkStat { ZkStat { @@ -90,37 +106,143 @@ impl ZkExtractor { Ok((HashMap::new(), 0)) } - async fn full_scan( + async fn reconcile( &mut self, client: &ZkClient, path_versions: &mut HashMap, high_water_zxid: &mut i64, source_id: &str, + scope: ReconcileScope, + reason: &str, ) -> anyhow::Result<()> { - log_info!("ZkExtractor starting full scan"); let old_versions = path_versions.clone(); - for watch_path in &self.watch_paths.clone() { + let mut current_versions = match scope { + ReconcileScope::All => HashMap::new(), + ReconcileScope::Path(_) => old_versions.clone(), + }; + let mut current_zxid = *high_water_zxid; + let roots = self.reconcile_roots(&scope); + + if matches!(scope, ReconcileScope::Path(_)) { + current_versions.retain(|path, _| !Self::scope_contains_path(&scope, path)); + } + + log_info!( + "ZkExtractor reconciliation start: reason={}, scope={:?}, roots={:?}", + reason, + scope, + roots + ); + + for root in roots { if self.base_extractor.shut_down.load(Ordering::Relaxed) { break; } self.scan_node_recursive( client, - watch_path, + &root, &old_versions, - path_versions, - high_water_zxid, + &mut current_versions, + &mut current_zxid, source_id, ) .await?; } + + for (path, (old_version, old_mzxid)) in &old_versions { + if !Self::scope_contains_path(&scope, path) { + continue; + } + if current_versions.contains_key(path) { + continue; + } + if self.filter.filter_path(path) { + continue; + } + let delete_order_millis = chrono::Utc::now().timestamp_millis(); + let delete_zxid = if *old_mzxid > 0 { + *old_mzxid + } else { + delete_order_millis + }; + let entry = ZkEntry { + path: path.clone(), + data: None, + stat: ZkStat { + version: *old_version, + mzxid: delete_zxid, + mtime: delete_order_millis, + ..Default::default() + }, + ephemeral: false, + event_type: ZkEventType::Deleted, + source_id: source_id.to_string(), + source_order_millis: delete_order_millis, + source_zxid: delete_zxid, + order_origin: ZkOrderOrigin::ReconcileObserved, + }; + let position = self.build_position(¤t_versions, current_zxid); + self.extract_state + .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position) + .await?; + } + + *path_versions = current_versions; + *high_water_zxid = current_zxid; + log_info!( - "ZkExtractor full scan complete: {} paths, high_water_zxid={}", + "ZkExtractor reconciliation complete: reason={}, {} paths, high_water_zxid={}", + reason, path_versions.len(), high_water_zxid ); Ok(()) } + fn reconcile_roots(&self, scope: &ReconcileScope) -> Vec { + match scope { + ReconcileScope::All => self.watch_paths.clone(), + ReconcileScope::Path(path) => { + if self.watch_paths.iter().any(|watch_path| { + Self::path_contains(watch_path, path) || Self::path_contains(path, watch_path) + }) { + vec![path.clone()] + } else { + Vec::new() + } + } + } + } + + fn scope_contains_path(scope: &ReconcileScope, path: &str) -> bool { + match scope { + ReconcileScope::All => true, + ReconcileScope::Path(scope_path) => Self::path_contains(scope_path, path), + } + } + + fn path_contains(root: &str, path: &str) -> bool { + root == "/" || path == root || path.starts_with(&format!("{}/", root.trim_end_matches('/'))) + } + + async fn full_scan( + &mut self, + client: &ZkClient, + path_versions: &mut HashMap, + high_water_zxid: &mut i64, + source_id: &str, + ) -> anyhow::Result<()> { + self.reconcile( + client, + path_versions, + high_water_zxid, + source_id, + ReconcileScope::All, + "initial", + ) + .await + } + async fn scan_node_recursive( &mut self, client: &ZkClient, @@ -219,66 +341,15 @@ impl ZkExtractor { high_water_zxid: &mut i64, source_id: &str, ) -> anyhow::Result<()> { - log_info!("ZkExtractor starting incremental rescan"); - let old_versions = path_versions.clone(); - - let mut current_versions: HashMap = HashMap::new(); - let mut current_zxid = *high_water_zxid; - - for watch_path in &self.watch_paths.clone() { - if self.base_extractor.shut_down.load(Ordering::Relaxed) { - break; - } - self.scan_node_recursive( - client, - watch_path, - &old_versions, - &mut current_versions, - &mut current_zxid, - source_id, - ) - .await?; - } - - // Detect deletes: paths in old_versions but missing from current scan - for (path, (old_version, old_mzxid)) in &old_versions { - if current_versions.contains_key(path) { - continue; - } - if self.filter.filter_path(path) { - continue; - } - let delete_order_millis = chrono::Utc::now().timestamp_millis(); - let entry = ZkEntry { - path: path.clone(), - data: None, - stat: ZkStat { - version: *old_version, - mzxid: *old_mzxid, - mtime: delete_order_millis, - ..Default::default() - }, - ephemeral: false, - event_type: ZkEventType::Deleted, - source_id: source_id.to_string(), - source_order_millis: delete_order_millis, - source_zxid: *old_mzxid, - order_origin: ZkOrderOrigin::ReconcileObserved, - }; - let position = self.build_position(¤t_versions, current_zxid); - self.extract_state - .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position) - .await?; - } - - *path_versions = current_versions; - *high_water_zxid = current_zxid; - - log_info!( - "ZkExtractor incremental rescan complete: {} paths", - path_versions.len() - ); - Ok(()) + self.reconcile( + client, + path_versions, + high_water_zxid, + source_id, + ReconcileScope::All, + "periodic", + ) + .await } fn build_position( @@ -350,6 +421,135 @@ impl ZkExtractor { } }); } + + async fn start_watchers( + &self, + client: &ZkClient, + ) -> anyhow::Result> { + if self.watch_paths.is_empty() { + bail!("ZkExtractor requires at least one watch_path"); + } + + let (tx, rx) = mpsc::unbounded_channel(); + for watch_path in self.watch_paths.clone() { + let mut watcher = client + .watch(&watch_path, AddWatchMode::PersistentRecursive) + .await + .map_err(|e| anyhow::anyhow!("ZK register watcher {} failed: {}", watch_path, e))?; + let tx = tx.clone(); + let shut_down = self.base_extractor.shut_down.clone(); + + log_info!( + "ZkExtractor registered persistent recursive watcher on {}", + watch_path + ); + tokio::spawn(async move { + loop { + if shut_down.load(Ordering::Relaxed) { + break; + } + + let event = watcher.changed().await; + let terminal = event.event_type == EventType::Session + && event.session_state.is_terminated(); + if tx + .send(ZkWatchEvent { + watch_path: watch_path.clone(), + event, + }) + .is_err() + { + break; + } + if terminal { + break; + } + } + }); + } + drop(tx); + Ok(rx) + } + + async fn handle_watch_event( + &mut self, + client: &ZkClient, + path_versions: &mut HashMap, + high_water_zxid: &mut i64, + source_id: &str, + watch_event: ZkWatchEvent, + session_uncertain: &mut bool, + ) -> anyhow::Result<()> { + let event = watch_event.event; + log_info!( + "ZkExtractor watcher event: watch_path={}, event_type={}, state={}, path={}, zxid={}", + watch_event.watch_path, + event.event_type, + event.session_state, + event.path, + event.zxid + ); + + match event.event_type { + EventType::Session => match event.session_state { + SessionState::Disconnected => { + *session_uncertain = true; + log_warn!( + "ZkExtractor watcher disconnected; next connected event will force reconciliation" + ); + Ok(()) + } + SessionState::SyncConnected | SessionState::ConnectedReadOnly => { + if *session_uncertain { + *session_uncertain = false; + self.reconcile( + client, + path_versions, + high_water_zxid, + source_id, + ReconcileScope::All, + "watch-reconnect", + ) + .await?; + } + Ok(()) + } + SessionState::AuthFailed | SessionState::Expired | SessionState::Closed => { + bail!( + "ZkExtractor watcher terminal session state: {}", + event.session_state + ) + } + }, + EventType::NodeCreated + | EventType::NodeDeleted + | EventType::NodeDataChanged + | EventType::NodeChildrenChanged => { + if event.path.is_empty() { + return Ok(()); + } + if self.filter.filter_path(&event.path) { + return Ok(()); + } + if *session_uncertain { + log_warn!( + "ZkExtractor treats watcher event {} as hint while session is uncertain", + event.path + ); + return Ok(()); + } + self.reconcile( + client, + path_versions, + high_water_zxid, + source_id, + ReconcileScope::Path(event.path.clone()), + "watch-event", + ) + .await + } + } + } } #[async_trait] @@ -375,6 +575,8 @@ impl Extractor for ZkExtractor { source_id.clone(), ); + let mut watch_rx = self.start_watchers(&client).await?; + self.full_scan( &client, &mut path_versions, @@ -383,18 +585,42 @@ impl Extractor for ZkExtractor { ) .await?; + let mut reconciliation_timer = + tokio::time::interval(Duration::from_secs(self.scan_interval_secs.max(1))); + reconciliation_timer.tick().await; + let mut session_uncertain = false; + let mut watch_rx_closed = false; + while !self.base_extractor.shut_down.load(Ordering::Relaxed) { - tokio::time::sleep(tokio::time::Duration::from_secs(self.scan_interval_secs)).await; - if self.base_extractor.shut_down.load(Ordering::Relaxed) { - break; + tokio::select! { + maybe_event = watch_rx.recv(), if !watch_rx_closed => { + if let Some(watch_event) = maybe_event { + self.handle_watch_event( + &client, + &mut path_versions, + &mut high_water_zxid, + &source_id, + watch_event, + &mut session_uncertain, + ).await?; + } else { + log_warn!("ZkExtractor watcher channel closed; forcing periodic reconciliation only"); + watch_rx_closed = true; + } + } + _ = reconciliation_timer.tick() => { + if self.base_extractor.shut_down.load(Ordering::Relaxed) { + break; + } + self.incremental_rescan( + &client, + &mut path_versions, + &mut high_water_zxid, + &source_id, + ) + .await?; + } } - self.incremental_rescan( - &client, - &mut path_versions, - &mut high_water_zxid, - &source_id, - ) - .await?; } self.base_extractor @@ -406,3 +632,25 @@ impl Extractor for ZkExtractor { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn path_contains_matches_exact_and_descendants() { + assert!(ZkExtractor::path_contains("/", "/app")); + assert!(ZkExtractor::path_contains("/app", "/app")); + assert!(ZkExtractor::path_contains("/app", "/app/service")); + assert!(!ZkExtractor::path_contains("/app", "/application")); + } + + #[test] + fn scoped_reconcile_only_matches_path_subtree() { + let scope = ReconcileScope::Path("/app".to_string()); + + assert!(ZkExtractor::scope_contains_path(&scope, "/app")); + assert!(ZkExtractor::scope_contains_path(&scope, "/app/service")); + assert!(!ZkExtractor::scope_contains_path(&scope, "/config")); + } +} From a60f822c5acdd1d108e4607f9718b665f4120b4d Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 20 Jun 2026 05:35:10 +0800 Subject: [PATCH 17/20] fix(zk): abort reconciliation on read uncertainty --- dt-common/src/zk_filter.rs | 25 +++- dt-connector/src/extractor/zk/zk_extractor.rs | 117 ++++++++++++------ 2 files changed, 97 insertions(+), 45 deletions(-) diff --git a/dt-common/src/zk_filter.rs b/dt-common/src/zk_filter.rs index 4424912b1..a3072c6e7 100644 --- a/dt-common/src/zk_filter.rs +++ b/dt-common/src/zk_filter.rs @@ -27,16 +27,26 @@ impl ZkFilter { } pub fn filter_path(&self, path: &str) -> bool { - if Self::INTERNAL_PATHS.iter().any(|p| path.starts_with(p)) { + if Self::INTERNAL_PATHS + .iter() + .any(|p| Self::path_matches_prefix(p, path)) + { return true; } - if self.ignore_paths.iter().any(|p| path.starts_with(p)) { + if self + .ignore_paths + .iter() + .any(|p| Self::path_matches_prefix(p, path)) + { return true; } if self.do_paths.is_empty() { return false; } - !self.do_paths.iter().any(|p| path.starts_with(p)) + !self + .do_paths + .iter() + .any(|p| Self::path_matches_prefix(p, path)) } pub fn filter_ephemeral(&self, ephemeral: bool) -> bool { @@ -52,6 +62,12 @@ impl ZkFilter { .map(|s| s.trim().to_string()) .collect() } + + fn path_matches_prefix(prefix: &str, path: &str) -> bool { + prefix == "/" + || path == prefix + || path.starts_with(&format!("{}/", prefix.trim_end_matches('/'))) + } } #[cfg(test)] @@ -70,11 +86,12 @@ mod tests { #[test] fn internal_paths_are_always_filtered() { - let filter = filter("/__ape_dts_shadow,/app", ""); + let filter = filter("/", ""); assert!(filter.filter_path("/__ape_dts_marker")); assert!(filter.filter_path("/__ape_dts_shadow/app")); assert!(filter.filter_path("/__ape_dts_heartbeat")); + assert!(!filter.filter_path("/__ape_dts_shadow2")); assert!(!filter.filter_path("/app/service")); } } diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index 2933f4e23..cac2f2f9b 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -11,7 +11,8 @@ use anyhow::bail; use async_trait::async_trait; use tokio::sync::{mpsc, Mutex}; use zookeeper_client::{ - Acls, AddWatchMode, Client as ZkClient, CreateMode, EventType, SessionState, Stat, WatchedEvent, + Acls, AddWatchMode, Client as ZkClient, CreateMode, Error as ZkError, EventType, SessionState, + Stat, WatchedEvent, }; use dt_common::{ @@ -58,6 +59,13 @@ enum ReconcileScope { Path(String), } +#[derive(Debug, Clone)] +struct ZkScanResult { + versions: HashMap, + high_water_zxid: i64, + entries: Vec, +} + impl ZkExtractor { fn stat_to_zk_stat(stat: &Stat) -> ZkStat { ZkStat { @@ -116,15 +124,20 @@ impl ZkExtractor { reason: &str, ) -> anyhow::Result<()> { let old_versions = path_versions.clone(); - let mut current_versions = match scope { - ReconcileScope::All => HashMap::new(), - ReconcileScope::Path(_) => old_versions.clone(), + let mut scan_result = ZkScanResult { + versions: match scope { + ReconcileScope::All => HashMap::new(), + ReconcileScope::Path(_) => old_versions.clone(), + }, + high_water_zxid: *high_water_zxid, + entries: Vec::new(), }; - let mut current_zxid = *high_water_zxid; let roots = self.reconcile_roots(&scope); if matches!(scope, ReconcileScope::Path(_)) { - current_versions.retain(|path, _| !Self::scope_contains_path(&scope, path)); + scan_result + .versions + .retain(|path, _| !Self::scope_contains_path(&scope, path)); } log_info!( @@ -138,22 +151,15 @@ impl ZkExtractor { if self.base_extractor.shut_down.load(Ordering::Relaxed) { break; } - self.scan_node_recursive( - client, - &root, - &old_versions, - &mut current_versions, - &mut current_zxid, - source_id, - ) - .await?; + self.scan_node_recursive(client, &root, &old_versions, &mut scan_result, source_id) + .await?; } for (path, (old_version, old_mzxid)) in &old_versions { if !Self::scope_contains_path(&scope, path) { continue; } - if current_versions.contains_key(path) { + if scan_result.versions.contains_key(path) { continue; } if self.filter.filter_path(path) { @@ -181,14 +187,18 @@ impl ZkExtractor { source_zxid: delete_zxid, order_origin: ZkOrderOrigin::ReconcileObserved, }; - let position = self.build_position(¤t_versions, current_zxid); + scan_result.entries.push(entry); + } + + let position = self.build_position(&scan_result.versions, scan_result.high_water_zxid); + for entry in scan_result.entries { self.extract_state - .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position) + .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position.clone()) .await?; } - *path_versions = current_versions; - *high_water_zxid = current_zxid; + *path_versions = scan_result.versions; + *high_water_zxid = scan_result.high_water_zxid; log_info!( "ZkExtractor reconciliation complete: reason={}, {} paths, high_water_zxid={}", @@ -248,8 +258,7 @@ impl ZkExtractor { client: &ZkClient, path: &str, old_versions: &HashMap, - current_versions: &mut HashMap, - high_water_zxid: &mut i64, + scan_result: &mut ZkScanResult, source_id: &str, ) -> anyhow::Result<()> { if self.base_extractor.shut_down.load(Ordering::Relaxed) { @@ -262,10 +271,12 @@ impl ZkExtractor { let (data, stat) = match client.get_data(path).await { Ok(r) => r, - Err(e) => { - log_warn!("ZkExtractor: get_data failed for {}: {}", path, e); + Err(ZkError::NoNode) => { return Ok(()); } + Err(e) => { + bail!("ZkExtractor: get_data failed for {}: {}", path, e); + } }; let ephemeral = Self::is_ephemeral(&stat); @@ -273,6 +284,16 @@ impl ZkExtractor { return Ok(()); } + let children = match client.list_children(path).await { + Ok(c) => c, + Err(ZkError::NoNode) => { + return Ok(()); + } + Err(e) => { + bail!("ZkExtractor: list_children failed for {}: {}", path, e); + } + }; + let (already_synced, is_update) = match old_versions.get(path) { Some((v, old_mzxid)) => (*v >= stat.version && *old_mzxid >= stat.mzxid, true), None => (false, false), @@ -295,25 +316,16 @@ impl ZkExtractor { source_zxid: stat.mzxid, order_origin: ZkOrderOrigin::SourceStatMtime, }; - let position = self.build_position(current_versions, *high_water_zxid); - self.extract_state - .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position) - .await?; + scan_result.entries.push(entry); } - current_versions.insert(path.to_string(), (stat.version, stat.mzxid)); - if stat.mzxid > *high_water_zxid { - *high_water_zxid = stat.mzxid; + scan_result + .versions + .insert(path.to_string(), (stat.version, stat.mzxid)); + if stat.mzxid > scan_result.high_water_zxid { + scan_result.high_water_zxid = stat.mzxid; } - let children = match client.list_children(path).await { - Ok(c) => c, - Err(e) => { - log_warn!("ZkExtractor: list_children failed for {}: {}", path, e); - return Ok(()); - } - }; - for child in children { let child_path = if path == "/" { format!("/{}", child) @@ -324,8 +336,7 @@ impl ZkExtractor { client, &child_path, old_versions, - current_versions, - high_water_zxid, + scan_result, source_id, )) .await?; @@ -612,6 +623,12 @@ impl Extractor for ZkExtractor { if self.base_extractor.shut_down.load(Ordering::Relaxed) { break; } + if session_uncertain { + log_warn!( + "ZkExtractor skips periodic reconciliation while session is uncertain" + ); + continue; + } self.incremental_rescan( &client, &mut path_versions, @@ -653,4 +670,22 @@ mod tests { assert!(ZkExtractor::scope_contains_path(&scope, "/app/service")); assert!(!ZkExtractor::scope_contains_path(&scope, "/config")); } + + #[test] + fn aborted_scan_result_does_not_replace_old_versions_or_emit_deletes() { + let old_versions = HashMap::from([("/app/service".to_string(), (1, 10))]); + let scan_result = ZkScanResult { + versions: HashMap::new(), + high_water_zxid: 0, + entries: Vec::new(), + }; + let scan_error: anyhow::Result<()> = Err(anyhow::anyhow!( + "ZkExtractor: list_children failed for /app" + )); + + assert!(scan_error.is_err()); + assert_eq!(old_versions.get("/app/service"), Some(&(1, 10))); + assert!(scan_result.versions.is_empty()); + assert!(scan_result.entries.is_empty()); + } } From 6b41e7c4d21232f1bf228ff72edfba2a6b1d5c7a Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 20 Jun 2026 05:43:18 +0800 Subject: [PATCH 18/20] fix(zk): checkpoint reconciliation after staged entries --- dt-connector/src/extractor/zk/zk_extractor.rs | 27 +++++++-- dt-pipeline/src/base_pipeline.rs | 56 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/dt-connector/src/extractor/zk/zk_extractor.rs b/dt-connector/src/extractor/zk/zk_extractor.rs index cac2f2f9b..dd9789e5a 100644 --- a/dt-connector/src/extractor/zk/zk_extractor.rs +++ b/dt-connector/src/extractor/zk/zk_extractor.rs @@ -190,21 +190,31 @@ impl ZkExtractor { scan_result.entries.push(entry); } - let position = self.build_position(&scan_result.versions, scan_result.high_water_zxid); + let final_position = + self.build_position(&scan_result.versions, scan_result.high_water_zxid); + let emitted_entries = scan_result.entries.len(); for entry in scan_result.entries { self.extract_state - .push_dt_data(&self.base_extractor, DtData::Zk { entry }, position.clone()) + .push_dt_data( + &self.base_extractor, + DtData::Zk { entry }, + Self::staged_entry_position(), + ) .await?; } + self.extract_state + .push_dt_data(&self.base_extractor, DtData::Heartbeat {}, final_position) + .await?; *path_versions = scan_result.versions; *high_water_zxid = scan_result.high_water_zxid; log_info!( - "ZkExtractor reconciliation complete: reason={}, {} paths, high_water_zxid={}", + "ZkExtractor reconciliation complete: reason={}, {} paths, high_water_zxid={}, emitted_entries={}", reason, path_versions.len(), - high_water_zxid + high_water_zxid, + emitted_entries ); Ok(()) } @@ -235,6 +245,10 @@ impl ZkExtractor { root == "/" || path == root || path.starts_with(&format!("{}/", root.trim_end_matches('/'))) } + fn staged_entry_position() -> Position { + Position::None + } + async fn full_scan( &mut self, client: &ZkClient, @@ -688,4 +702,9 @@ mod tests { assert!(scan_result.versions.is_empty()); assert!(scan_result.entries.is_empty()); } + + #[test] + fn staged_reconciliation_entries_do_not_carry_final_position() { + assert_eq!(ZkExtractor::staged_entry_position(), Position::None); + } } diff --git a/dt-pipeline/src/base_pipeline.rs b/dt-pipeline/src/base_pipeline.rs index 211c23f4f..6664b48f3 100644 --- a/dt-pipeline/src/base_pipeline.rs +++ b/dt-pipeline/src/base_pipeline.rs @@ -663,6 +663,7 @@ mod tests { dt_data::{DtData, DtItem}, position::Position, redis::redis_entry::RedisEntry, + zk::{zk_entry::ZkEntry, zk_event_type::ZkEventType, zk_stat::ZkStat}, }; use dt_connector::extractor::resumer::utils::ResumerUtil; @@ -690,6 +691,34 @@ mod tests { } } + fn zk_item(position: Position) -> DtItem { + DtItem { + dt_data: DtData::Zk { + entry: ZkEntry { + path: "/app/service".to_string(), + data: Some(b"value".to_vec()), + stat: ZkStat::default(), + ephemeral: false, + event_type: ZkEventType::Updated, + source_id: "node-a".to_string(), + source_order_millis: 1, + source_zxid: 1, + order_origin: Default::default(), + }, + }, + position, + data_origin_node: String::new(), + } + } + + fn heartbeat_item(position: Position) -> DtItem { + DtItem { + dt_data: DtData::Heartbeat {}, + position, + data_origin_node: String::new(), + } + } + #[test] fn fetch_raw_collects_latest_position_per_redis_node() { let mut pending_snapshot_finished = HashMap::new(); @@ -713,4 +742,31 @@ mod tests { assert_eq!(by_key.get("redis-node-node-1"), Some(&node_1_new)); assert_eq!(by_key.get("redis-node-node-2"), Some(&node_2)); } + + #[test] + fn fetch_raw_advances_zk_position_only_on_trailing_heartbeat() { + let mut pending_snapshot_finished = HashMap::new(); + let final_position = Position::Zk { + path_versions: HashMap::from([("/app/service".to_string(), (1, 10))]), + last_scan_timestamp: 100, + high_water_zxid: 10, + total_paths: 1, + }; + let partial_data = vec![zk_item(Position::None)]; + + let (_, partial_position, partial_commits) = + BasePipeline::fetch_raw(&partial_data, &mut pending_snapshot_finished); + assert_eq!(partial_position, Some(Position::None)); + assert!(partial_commits.is_empty()); + + let complete_data = vec![ + zk_item(Position::None), + heartbeat_item(final_position.clone()), + ]; + let (_, complete_position, complete_commits) = + BasePipeline::fetch_raw(&complete_data, &mut pending_snapshot_finished); + + assert_eq!(complete_position, Some(final_position.clone())); + assert_eq!(complete_commits, vec![final_position]); + } } From c1b4ea3a7555d1007ae9a32510507fb6f3274169 Mon Sep 17 00:00:00 2001 From: wei Date: Wed, 24 Jun 2026 01:43:21 +0800 Subject: [PATCH 19/20] fix(ci): restore rust 1.85 compatibility --- Cargo.lock | 541 +++++++++--------- dt-connector/src/sinker/base_struct_sinker.rs | 5 +- .../clickhouse/clickhouse_struct_sinker.rs | 5 +- .../src/sinker/mongo/mongo_struct_sinker.rs | 4 +- .../starrocks/starrocks_struct_sinker.rs | 5 +- 5 files changed, 283 insertions(+), 277 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d62e7dca..7b06ca913 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ dependencies = [ "brotli", "bytes", "bytestring", - "derive_more 2.0.1", + "derive_more", "encoding_rs", "flate2", "foldhash 0.1.5", @@ -149,7 +149,7 @@ dependencies = [ "bytestring", "cfg-if", "cookie", - "derive_more 2.0.1", + "derive_more", "encoding_rs", "foldhash 0.1.5", "futures-core", @@ -1018,7 +1018,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] @@ -1188,12 +1188,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "cookie" version = "0.16.2" @@ -1260,7 +1254,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ - "rustc_version 0.4.1", + "rustc_version", ] [[package]] @@ -1272,6 +1266,30 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1315,70 +1333,70 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] name = "darling" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] name = "darling_core" -version = "0.13.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 1.0.109", + "strsim", + "syn 2.0.103", ] [[package]] name = "darling_core" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.11.1", + "strsim", "syn 2.0.103", ] [[package]] name = "darling_macro" -version = "0.13.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.13.4", + "darling_core 0.20.11", "quote", - "syn 1.0.109", + "syn 2.0.103", ] [[package]] name = "darling_macro" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core 0.20.11", + "darling_core 0.21.3", "quote", "syn 2.0.103", ] @@ -1441,10 +1459,10 @@ dependencies = [ ] [[package]] -name = "derive-where" -version = "1.6.1" +name = "derive-syn-parse" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", @@ -1452,15 +1470,13 @@ dependencies = [ ] [[package]] -name = "derive_more" -version = "0.99.20" +name = "derive-where" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ - "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.1", "syn 2.0.103", ] @@ -1838,14 +1854,14 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.4.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.103", ] [[package]] @@ -2427,6 +2443,52 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.2", + "ring", + "thiserror 2.0.12", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "moka", + "once_cell", + "parking_lot 0.12.4", + "rand 0.9.2", + "resolv-conf", + "smallvec", + "thiserror 2.0.12", + "tokio", + "tracing", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -2590,10 +2652,10 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.34", + "rustls", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower-service", "webpki-roots 1.0.3", ] @@ -2751,17 +2813,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.0.3" @@ -3029,12 +3080,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -3130,15 +3175,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -3181,10 +3217,52 @@ dependencies = [ ] [[package]] -name = "matches" -version = "0.1.10" +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.103", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.103", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.103", +] [[package]] name = "md-5" @@ -3281,51 +3359,97 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot 0.12.4", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "mongocrypt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.5+1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" + [[package]] name = "mongodb" -version = "2.8.2" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef206acb1b72389b49bc9985efe7eb1f8a9bb18e5680d262fac26c07f44025f1" +checksum = "1ef2c933617431ad0246fb5b43c425ebdae18c7f7259c87de0726d93b0e7e91b" dependencies = [ - "async-trait", - "base64 0.13.1", - "bitflags 1.3.2", + "base64 0.22.1", + "bitflags 2.9.1", "bson", - "chrono", - "derivative", - "derive_more 0.99.20", + "derive-where", + "derive_more", "futures-core", - "futures-executor", "futures-io", "futures-util", "hex", + "hickory-proto", + "hickory-resolver", "hmac 0.12.1", - "lazy_static", + "macro_magic", "md-5 0.10.6", + "mongocrypt", + "mongodb-internal-macros", "pbkdf2", "percent-encoding", - "rand 0.8.5", + "rand 0.9.2", "rustc_version_runtime", - "rustls 0.21.12", - "rustls-pemfile", + "rustls", + "rustversion", "serde", "serde_bytes", "serde_with", - "sha-1", + "sha1", "sha2 0.10.9", - "socket2 0.4.10", + "socket2 0.6.1", "stringprep", - "strsim 0.10.0", + "strsim", "take_mut", - "thiserror 1.0.69", + "thiserror 2.0.12", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util 0.7.15", - "trust-dns-proto", - "trust-dns-resolver", - "typed-builder 0.10.0", + "typed-builder 0.22.0", "uuid", - "webpki-roots 0.25.4", + "webpki-roots 1.0.3", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5758dc828eb2d02ec30563cba365609d56ddd833190b192beaee2b475a7bb3" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.103", ] [[package]] @@ -3577,6 +3701,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -3760,9 +3888,9 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.11.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", ] @@ -4151,7 +4279,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.34", + "rustls", "socket2 0.6.1", "thiserror 2.0.12", "tokio", @@ -4171,7 +4299,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls 0.23.34", + "rustls", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -4499,14 +4627,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.34", + "rustls", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tokio-util 0.7.15", "tower", "tower-http", @@ -4632,32 +4760,23 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.26", + "semver", ] [[package]] name = "rustc_version_runtime" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d31b7153270ebf48bf91c65ae5b0c00e749c4cfad505f66530ac74950249582f" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" dependencies = [ - "rustc_version 0.2.3", - "semver 0.9.0", + "rustc_version", + "semver", ] [[package]] @@ -4700,18 +4819,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.34" @@ -4723,20 +4830,11 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pki-types" version = "1.13.0" @@ -4747,16 +4845,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.103.8" @@ -4811,16 +4899,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sdd" version = "3.0.8" @@ -4867,27 +4945,12 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" version = "1.0.219" @@ -4965,24 +5028,25 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ "serde", + "serde_derive", "serde_with_macros", ] [[package]] name = "serde_with_macros" -version = "1.5.2" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ - "darling 0.13.4", + "darling 0.21.3", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.103", ] [[package]] @@ -5023,17 +5087,6 @@ dependencies = [ "syn 2.0.103", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", -] - [[package]] name = "sha1" version = "0.10.6" @@ -5244,7 +5297,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.34", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -5436,12 +5489,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -5571,6 +5618,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "take_mut" version = "0.2.2" @@ -5791,23 +5844,13 @@ dependencies = [ "tokio-util 0.6.10", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.34", + "rustls", "tokio", ] @@ -5944,51 +5987,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "trust-dns-proto" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.2.3", - "ipnet", - "lazy_static", - "log", - "rand 0.8.5", - "smallvec", - "thiserror 1.0.69", - "tinyvec", - "tokio", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" -dependencies = [ - "cfg-if", - "futures-util", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "parking_lot 0.12.4", - "resolv-conf", - "smallvec", - "thiserror 1.0.69", - "tokio", - "trust-dns-proto", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -6008,22 +6006,20 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.10.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "typed-builder-macro 0.16.2", ] [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.22.0", ] [[package]] @@ -6037,6 +6033,17 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "typemap-ors" version = "1.0.0" @@ -6119,7 +6126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 1.0.3", + "idna", "percent-encoding", ] @@ -6314,12 +6321,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "webpki-roots" version = "0.26.11" @@ -6886,7 +6887,7 @@ dependencies = [ "strum 0.23.0", "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tracing", "uuid", ] diff --git a/dt-connector/src/sinker/base_struct_sinker.rs b/dt-connector/src/sinker/base_struct_sinker.rs index 13ae21403..8c60f0c21 100644 --- a/dt-connector/src/sinker/base_struct_sinker.rs +++ b/dt-connector/src/sinker/base_struct_sinker.rs @@ -44,9 +44,10 @@ impl BaseStructSinker { Err(error) => { log_error!("ddl failed, error: {}", error); match conflict_policy { - ConflictPolicyEnum::Interrupt => bail! {error}, + ConflictPolicyEnum::Interrupt | ConflictPolicyEnum::LastWriteWins => { + bail! {error} + } ConflictPolicyEnum::Ignore => {} - _ => {} } } } diff --git a/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs b/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs index 23cd5e485..12227d03f 100644 --- a/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs +++ b/dt-connector/src/sinker/clickhouse/clickhouse_struct_sinker.rs @@ -287,9 +287,10 @@ impl ClickhouseStructSinker { Err(error) => { log_error!("ddl failed, error: {}", error); match self.conflict_policy { - ConflictPolicyEnum::Interrupt => bail! {error}, + ConflictPolicyEnum::Interrupt | ConflictPolicyEnum::LastWriteWins => { + bail! {error} + } ConflictPolicyEnum::Ignore => {} - _ => {} } } } diff --git a/dt-connector/src/sinker/mongo/mongo_struct_sinker.rs b/dt-connector/src/sinker/mongo/mongo_struct_sinker.rs index 2d80d863b..4caccc823 100644 --- a/dt-connector/src/sinker/mongo/mongo_struct_sinker.rs +++ b/dt-connector/src/sinker/mongo/mongo_struct_sinker.rs @@ -53,7 +53,9 @@ impl Sinker for MongoStructSinker { Err(error) => { log_error!("mongo struct failed, error: {}", error); match self.conflict_policy { - ConflictPolicyEnum::Interrupt => return Err(error), + ConflictPolicyEnum::Interrupt | ConflictPolicyEnum::LastWriteWins => { + return Err(error) + } ConflictPolicyEnum::Ignore => {} } } diff --git a/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs b/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs index 7e54eadc5..777cfc3ed 100644 --- a/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs +++ b/dt-connector/src/sinker/starrocks/starrocks_struct_sinker.rs @@ -386,9 +386,10 @@ impl StarrocksStructSinker { Err(error) => { log_error!("ddl failed, error: {}", error); match self.conflict_policy { - ConflictPolicyEnum::Interrupt => bail! {error}, + ConflictPolicyEnum::Interrupt | ConflictPolicyEnum::LastWriteWins => { + bail! {error} + } ConflictPolicyEnum::Ignore => {} - _ => {} } } } From de88a916afd6c4e6b84cfc3fb359133069bc9cfc Mon Sep 17 00:00:00 2001 From: wei Date: Thu, 9 Jul 2026 18:24:37 +0800 Subject: [PATCH 20/20] fix: format pr504 conflict resolution --- dt-common/src/config/task_config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/dt-common/src/config/task_config.rs b/dt-common/src/config/task_config.rs index aaddf2743..ad923528e 100644 --- a/dt-common/src/config/task_config.rs +++ b/dt-common/src/config/task_config.rs @@ -1044,7 +1044,6 @@ impl TaskConfig { }, _ => bail! { not_supported_err }, }, - }; Ok((basic, sinker)) }