Skip to content

Commit 89071f5

Browse files
farhan-syahfarhan-syah
committed
test(nodedb): add DDL replication correctness tests
Cover the full create/drop lifecycle for collections, sequences, triggers, and schedules across a 3-node test cluster. Each test verifies that DDL executed on the leader becomes visible on all followers within a bounded window, and that IF [NOT] EXISTS branches complete without error.
1 parent a451bc5 commit 89071f5

1 file changed

Lines changed: 265 additions & 0 deletions

File tree

‎nodedb/tests/sql_ddl_cluster.rs‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
//! DDL replication correctness matrix.
2+
//!
3+
//! For every DDL variant that flows through the replicated metadata
4+
//! path, this file tests:
5+
//!
6+
//! 1. Execute DDL on the leader → visible on every follower.
7+
//! 2. Execute the inverse DDL → removal visible on every node.
8+
//! 3. `IF NOT EXISTS` / `IF EXISTS` branches handled without error.
9+
//!
10+
//! Uses the 3-node `TestCluster` harness from `common/cluster_harness`.
11+
12+
mod common;
13+
14+
use std::time::Duration;
15+
16+
use common::cluster_harness::{TestCluster, wait_for};
17+
18+
// ── Collection ───────────────────────────────────────────────────
19+
20+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
21+
async fn ddl_create_drop_collection_replicates() {
22+
let cluster = TestCluster::spawn_three().await.expect("cluster");
23+
cluster
24+
.exec_ddl_on_any_leader("CREATE COLLECTION ddl_test_coll")
25+
.await
26+
.expect("create");
27+
wait_for(
28+
"collection visible on all nodes",
29+
Duration::from_secs(10),
30+
Duration::from_millis(50),
31+
|| {
32+
cluster
33+
.nodes
34+
.iter()
35+
.all(|n| n.cached_collection_count() >= 1)
36+
},
37+
)
38+
.await;
39+
40+
cluster
41+
.exec_ddl_on_any_leader("DROP COLLECTION ddl_test_coll")
42+
.await
43+
.expect("drop");
44+
wait_for(
45+
"collection removed on all nodes",
46+
Duration::from_secs(10),
47+
Duration::from_millis(50),
48+
|| {
49+
cluster
50+
.nodes
51+
.iter()
52+
.all(|n| n.cached_collection_count() == 0)
53+
},
54+
)
55+
.await;
56+
cluster.shutdown().await;
57+
}
58+
59+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
60+
async fn ddl_create_collection_if_not_exists() {
61+
let cluster = TestCluster::spawn_three().await.expect("cluster");
62+
cluster
63+
.exec_ddl_on_any_leader("CREATE COLLECTION ine_coll")
64+
.await
65+
.expect("first create");
66+
wait_for(
67+
"collection visible",
68+
Duration::from_secs(10),
69+
Duration::from_millis(50),
70+
|| {
71+
cluster
72+
.nodes
73+
.iter()
74+
.all(|n| n.cached_collection_count() >= 1)
75+
},
76+
)
77+
.await;
78+
// Second CREATE IF NOT EXISTS must succeed without error.
79+
cluster
80+
.exec_ddl_on_any_leader("CREATE COLLECTION IF NOT EXISTS ine_coll")
81+
.await
82+
.expect("if not exists must not error");
83+
cluster.shutdown().await;
84+
}
85+
86+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
87+
async fn ddl_drop_collection_if_exists_missing() {
88+
let cluster = TestCluster::spawn_three().await.expect("cluster");
89+
// DROP IF EXISTS on a nonexistent collection must succeed.
90+
cluster
91+
.exec_ddl_on_any_leader("DROP COLLECTION IF EXISTS no_such_coll")
92+
.await
93+
.expect("if exists on missing must not error");
94+
cluster.shutdown().await;
95+
}
96+
97+
// ── Sequence ─────────────────────────────────────────────────────
98+
99+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
100+
async fn ddl_create_drop_sequence_replicates() {
101+
let cluster = TestCluster::spawn_three().await.expect("cluster");
102+
cluster
103+
.exec_ddl_on_any_leader("CREATE SEQUENCE ddl_test_seq START 1")
104+
.await
105+
.expect("create seq");
106+
wait_for(
107+
"sequence visible on all nodes",
108+
Duration::from_secs(10),
109+
Duration::from_millis(50),
110+
|| {
111+
cluster
112+
.nodes
113+
.iter()
114+
.all(|n| n.has_sequence(1, "ddl_test_seq"))
115+
},
116+
)
117+
.await;
118+
119+
cluster
120+
.exec_ddl_on_any_leader("DROP SEQUENCE ddl_test_seq")
121+
.await
122+
.expect("drop seq");
123+
wait_for(
124+
"sequence removed on all nodes",
125+
Duration::from_secs(10),
126+
Duration::from_millis(50),
127+
|| cluster.nodes.iter().all(|n| n.sequence_count(1) == 0),
128+
)
129+
.await;
130+
cluster.shutdown().await;
131+
}
132+
133+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
134+
async fn ddl_create_sequence_if_not_exists() {
135+
let cluster = TestCluster::spawn_three().await.expect("cluster");
136+
cluster
137+
.exec_ddl_on_any_leader("CREATE SEQUENCE ine_seq START 1")
138+
.await
139+
.expect("first create");
140+
wait_for(
141+
"seq visible",
142+
Duration::from_secs(10),
143+
Duration::from_millis(50),
144+
|| cluster.nodes.iter().all(|n| n.has_sequence(1, "ine_seq")),
145+
)
146+
.await;
147+
cluster
148+
.exec_ddl_on_any_leader("CREATE SEQUENCE IF NOT EXISTS ine_seq START 1")
149+
.await
150+
.expect("if not exists must not error");
151+
cluster.shutdown().await;
152+
}
153+
154+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
155+
async fn ddl_drop_sequence_if_exists_missing() {
156+
let cluster = TestCluster::spawn_three().await.expect("cluster");
157+
cluster
158+
.exec_ddl_on_any_leader("DROP SEQUENCE IF EXISTS no_such_seq")
159+
.await
160+
.expect("if exists on missing must not error");
161+
cluster.shutdown().await;
162+
}
163+
164+
// ── Trigger ──────────────────────────────────────────────────────
165+
166+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
167+
async fn ddl_create_drop_trigger_replicates() {
168+
let cluster = TestCluster::spawn_three().await.expect("cluster");
169+
cluster
170+
.exec_ddl_on_any_leader("CREATE COLLECTION trig_coll")
171+
.await
172+
.expect("create coll for trigger");
173+
wait_for(
174+
"coll visible",
175+
Duration::from_secs(10),
176+
Duration::from_millis(50),
177+
|| {
178+
cluster
179+
.nodes
180+
.iter()
181+
.all(|n| n.cached_collection_count() >= 1)
182+
},
183+
)
184+
.await;
185+
186+
cluster
187+
.exec_ddl_on_any_leader(
188+
"CREATE TRIGGER ddl_test_trig AFTER INSERT ON trig_coll FOR EACH ROW BEGIN RETURN 1; END",
189+
)
190+
.await
191+
.expect("create trigger");
192+
wait_for(
193+
"trigger visible on all nodes",
194+
Duration::from_secs(10),
195+
Duration::from_millis(50),
196+
|| {
197+
cluster
198+
.nodes
199+
.iter()
200+
.all(|n| n.has_trigger(1, "ddl_test_trig"))
201+
},
202+
)
203+
.await;
204+
205+
cluster
206+
.exec_ddl_on_any_leader("DROP TRIGGER ddl_test_trig ON trig_coll")
207+
.await
208+
.expect("drop trigger");
209+
wait_for(
210+
"trigger removed on all nodes",
211+
Duration::from_secs(10),
212+
Duration::from_millis(50),
213+
|| {
214+
cluster
215+
.nodes
216+
.iter()
217+
.all(|n| !n.has_trigger(1, "ddl_test_trig"))
218+
},
219+
)
220+
.await;
221+
cluster.shutdown().await;
222+
}
223+
224+
// ── Schedule ─────────────────────────────────────────────────────
225+
226+
#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
227+
async fn ddl_create_drop_schedule_replicates() {
228+
let cluster = TestCluster::spawn_three().await.expect("cluster");
229+
cluster
230+
.exec_ddl_on_any_leader(
231+
"CREATE SCHEDULE ddl_test_sched CRON '0 0 * * *' AS BEGIN RETURN 1; END",
232+
)
233+
.await
234+
.expect("create schedule");
235+
wait_for(
236+
"schedule visible on all nodes",
237+
Duration::from_secs(10),
238+
Duration::from_millis(50),
239+
|| {
240+
cluster
241+
.nodes
242+
.iter()
243+
.all(|n| n.has_schedule(1, "ddl_test_sched"))
244+
},
245+
)
246+
.await;
247+
248+
cluster
249+
.exec_ddl_on_any_leader("DROP SCHEDULE ddl_test_sched")
250+
.await
251+
.expect("drop schedule");
252+
wait_for(
253+
"schedule removed on all nodes",
254+
Duration::from_secs(10),
255+
Duration::from_millis(50),
256+
|| {
257+
cluster
258+
.nodes
259+
.iter()
260+
.all(|n| !n.has_schedule(1, "ddl_test_sched"))
261+
},
262+
)
263+
.await;
264+
cluster.shutdown().await;
265+
}

0 commit comments

Comments
 (0)