Skip to content

Commit b41a473

Browse files
bobhan1Copilot
bobhan1
and
Copilot
committed
Improve FE unit tests for table-level event-driven warmup
- Rewrite CloudWarmUpJobTableFilterTest (13 tests): add SHOW WARM UP JOB column verification (all 15 columns), matched tables string output, dynamic table ID tracking (create/drop/rename scenarios) - Create CacheHotspotManagerTableFilterTest (13 tests): test resolveTableIds with mocked Env/InternalCatalog, dynamic table changes (new table, drop, rename), refreshAllTableFilters with running jobs, cluster-level job skip - Fix unused import in OnTablesFilterTest Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 2ca6f50 commit b41a473

3 files changed

Lines changed: 470 additions & 27 deletions

File tree

‎fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java‎

Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.cloud;
19+
20+
import org.apache.doris.catalog.DatabaseIf;
21+
import org.apache.doris.catalog.Env;
22+
import org.apache.doris.catalog.TableIf;
23+
import org.apache.doris.cloud.OnTablesFilter.TableFilterRule;
24+
import org.apache.doris.cloud.OnTablesFilter.TableFilterRule.RuleType;
25+
import org.apache.doris.cloud.system.CloudSystemInfoService;
26+
import org.apache.doris.datasource.InternalCatalog;
27+
28+
import org.junit.jupiter.api.AfterEach;
29+
import org.junit.jupiter.api.Assertions;
30+
import org.junit.jupiter.api.BeforeEach;
31+
import org.junit.jupiter.api.Test;
32+
import org.mockito.MockedStatic;
33+
import org.mockito.Mockito;
34+
35+
import java.util.ArrayList;
36+
import java.util.Arrays;
37+
import java.util.HashSet;
38+
import java.util.List;
39+
import java.util.Set;
40+
41+
/**
42+
* Tests for CacheHotspotManager's table filter methods:
43+
* resolveTableIds() and refreshAllTableFilters().
44+
* Uses Mockito to mock Env.getCurrentInternalCatalog() with fake databases/tables.
45+
*/
46+
public class CacheHotspotManagerTableFilterTest {
47+
48+
private MockedStatic<Env> envMock;
49+
private InternalCatalog mockCatalog;
50+
private CacheHotspotManager manager;
51+
private List<DatabaseIf<? extends TableIf>> databases;
52+
53+
@BeforeEach
54+
public void setUp() {
55+
mockCatalog = Mockito.mock(InternalCatalog.class);
56+
envMock = Mockito.mockStatic(Env.class);
57+
envMock.when(Env::getCurrentInternalCatalog).thenReturn(mockCatalog);
58+
59+
databases = new ArrayList<>();
60+
Mockito.when(mockCatalog.getAllDbs()).thenAnswer(inv -> databases);
61+
62+
manager = new CacheHotspotManager(Mockito.mock(CloudSystemInfoService.class));
63+
}
64+
65+
@AfterEach
66+
public void tearDown() {
67+
envMock.close();
68+
}
69+
70+
@SuppressWarnings("unchecked")
71+
private DatabaseIf<TableIf> mockDb(String name, TableIf... tables) {
72+
DatabaseIf<TableIf> db = Mockito.mock(DatabaseIf.class);
73+
Mockito.when(db.getFullName()).thenReturn(name);
74+
Mockito.when(db.getTables()).thenReturn(Arrays.asList(tables));
75+
return db;
76+
}
77+
78+
private TableIf mockTable(long id, String name) {
79+
TableIf table = Mockito.mock(TableIf.class);
80+
Mockito.when(table.getId()).thenReturn(id);
81+
Mockito.when(table.getName()).thenReturn(name);
82+
return table;
83+
}
84+
85+
private OnTablesFilter buildFilter(TableFilterRule... rules) {
86+
return new OnTablesFilter(Arrays.asList(rules));
87+
}
88+
89+
// ===== resolveTableIds() =====
90+
91+
@Test
92+
public void testResolveTableIdsBasicMatching() {
93+
// Scenario: INCLUDE 'ods.*' matches all tables in ods database
94+
databases.add(mockDb("ods",
95+
mockTable(1001, "orders"),
96+
mockTable(1002, "users"),
97+
mockTable(1003, "tmp_staging")));
98+
databases.add(mockDb("dw",
99+
mockTable(2001, "fact_sales")));
100+
101+
OnTablesFilter filter = buildFilter(
102+
new TableFilterRule(RuleType.INCLUDE, "ods.*"));
103+
Set<Long> ids = manager.resolveTableIds(filter);
104+
105+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1001L, 1002L, 1003L)), ids);
106+
}
107+
108+
@Test
109+
public void testResolveTableIdsWithExclude() {
110+
// Scenario: INCLUDE 'ods.*' EXCLUDE 'ods.tmp_*' — exclude tmp tables
111+
databases.add(mockDb("ods",
112+
mockTable(1001, "orders"),
113+
mockTable(1002, "tmp_staging"),
114+
mockTable(1003, "tmp_data")));
115+
116+
OnTablesFilter filter = buildFilter(
117+
new TableFilterRule(RuleType.INCLUDE, "ods.*"),
118+
new TableFilterRule(RuleType.EXCLUDE, "ods.tmp_*"));
119+
Set<Long> ids = manager.resolveTableIds(filter);
120+
121+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1001L)), ids);
122+
}
123+
124+
@Test
125+
public void testResolveTableIdsMultipleDatabases() {
126+
// Scenario: INCLUDE 'ods.*', INCLUDE 'dw.fact_*' — match across two databases
127+
databases.add(mockDb("ods",
128+
mockTable(1001, "orders"),
129+
mockTable(1002, "users")));
130+
databases.add(mockDb("dw",
131+
mockTable(2001, "fact_sales"),
132+
mockTable(2002, "dim_product"),
133+
mockTable(2003, "fact_orders")));
134+
135+
OnTablesFilter filter = buildFilter(
136+
new TableFilterRule(RuleType.INCLUDE, "ods.*"),
137+
new TableFilterRule(RuleType.INCLUDE, "dw.fact_*"));
138+
Set<Long> ids = manager.resolveTableIds(filter);
139+
140+
Assertions.assertEquals(
141+
new HashSet<>(Arrays.asList(1001L, 1002L, 2001L, 2003L)), ids);
142+
}
143+
144+
@Test
145+
public void testResolveTableIdsNoMatch() {
146+
// Scenario: pattern matches nothing → empty set
147+
databases.add(mockDb("ods", mockTable(1001, "orders")));
148+
149+
OnTablesFilter filter = buildFilter(
150+
new TableFilterRule(RuleType.INCLUDE, "nonexistent.*"));
151+
Set<Long> ids = manager.resolveTableIds(filter);
152+
153+
Assertions.assertTrue(ids.isEmpty());
154+
}
155+
156+
@Test
157+
public void testResolveTableIdsNullFilter() {
158+
Set<Long> ids = manager.resolveTableIds(null);
159+
Assertions.assertTrue(ids.isEmpty());
160+
}
161+
162+
@Test
163+
public void testResolveTableIdsDbNameWithPrefix() {
164+
// CacheHotspotManager strips "default_cluster:" prefix from db name
165+
databases.add(mockDb("default_cluster:ods",
166+
mockTable(1001, "orders")));
167+
168+
OnTablesFilter filter = buildFilter(
169+
new TableFilterRule(RuleType.INCLUDE, "ods.*"));
170+
Set<Long> ids = manager.resolveTableIds(filter);
171+
172+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1001L)), ids);
173+
}
174+
175+
// ===== resolveTableIds() with dynamic table changes =====
176+
177+
@Test
178+
public void testResolveTableIdsAfterNewTableCreated() {
179+
// Initial: ods has orders. After new table created, re-resolve picks it up.
180+
DatabaseIf<TableIf> odsDb = mockDb("ods", mockTable(1001, "orders"));
181+
databases.add(odsDb);
182+
183+
OnTablesFilter filter = buildFilter(
184+
new TableFilterRule(RuleType.INCLUDE, "ods.*"));
185+
186+
Set<Long> ids1 = manager.resolveTableIds(filter);
187+
Assertions.assertEquals(1, ids1.size());
188+
189+
// Simulate new table created: replace the db mock to include new table
190+
databases.clear();
191+
databases.add(mockDb("ods",
192+
mockTable(1001, "orders"),
193+
mockTable(1004, "payments")));
194+
195+
Set<Long> ids2 = manager.resolveTableIds(filter);
196+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1001L, 1004L)), ids2);
197+
}
198+
199+
@Test
200+
public void testResolveTableIdsAfterTableDropped() {
201+
// Initial: ods has orders and users. After orders dropped, re-resolve removes it.
202+
databases.add(mockDb("ods",
203+
mockTable(1001, "orders"),
204+
mockTable(1002, "users")));
205+
206+
OnTablesFilter filter = buildFilter(
207+
new TableFilterRule(RuleType.INCLUDE, "ods.*"));
208+
209+
Set<Long> ids1 = manager.resolveTableIds(filter);
210+
Assertions.assertEquals(2, ids1.size());
211+
212+
databases.clear();
213+
databases.add(mockDb("ods", mockTable(1002, "users")));
214+
215+
Set<Long> ids2 = manager.resolveTableIds(filter);
216+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1002L)), ids2);
217+
}
218+
219+
@Test
220+
public void testResolveTableIdsAfterTableRenamed() {
221+
// Scenario from user guide: INCLUDE 'db.order_*', rename order_2024→archive_2024 → stops matching
222+
databases.add(mockDb("db",
223+
mockTable(1001, "order_2024"),
224+
mockTable(1002, "order_2025")));
225+
226+
OnTablesFilter filter = buildFilter(
227+
new TableFilterRule(RuleType.INCLUDE, "db.order_*"));
228+
229+
Set<Long> ids1 = manager.resolveTableIds(filter);
230+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1001L, 1002L)), ids1);
231+
232+
// Rename order_2024 → archive_2024 (no longer matches order_*)
233+
databases.clear();
234+
databases.add(mockDb("db",
235+
mockTable(1001, "archive_2024"),
236+
mockTable(1002, "order_2025")));
237+
238+
Set<Long> ids2 = manager.resolveTableIds(filter);
239+
Assertions.assertEquals(new HashSet<>(Arrays.asList(1002L)), ids2);
240+
}
241+
242+
@Test
243+
public void testResolveTableIdsAfterAllTablesDropped() {
244+
// User guide: all matched tables dropped → empty set, Job stays RUNNING
245+
databases.add(mockDb("ods", mockTable(1001, "orders")));
246+
247+
OnTablesFilter filter = buildFilter(
248+
new TableFilterRule(RuleType.INCLUDE, "ods.*"));
249+
250+
Set<Long> ids1 = manager.resolveTableIds(filter);
251+
Assertions.assertEquals(1, ids1.size());
252+
253+
databases.clear();
254+
databases.add(mockDb("ods")); // empty database
255+
256+
Set<Long> ids2 = manager.resolveTableIds(filter);
257+
Assertions.assertTrue(ids2.isEmpty());
258+
}
259+
260+
// ===== refreshAllTableFilters() =====
261+
262+
@Test
263+
public void testRefreshAllTableFiltersUpdatesJobTableIds() throws Exception {
264+
// Setup: create a job with table filter, add it via replayCloudWarmUpJob
265+
databases.add(mockDb("ods",
266+
mockTable(1001, "orders"),
267+
mockTable(1002, "users")));
268+
269+
CloudWarmUpJob.PersistedTableFilterRule rule = new CloudWarmUpJob.PersistedTableFilterRule();
270+
rule.ruleType = "INCLUDE";
271+
rule.pattern = "ods.*";
272+
273+
CloudWarmUpJob job = new CloudWarmUpJob.Builder()
274+
.setJobId(100L)
275+
.setSrcClusterName("write_cg")
276+
.setDstClusterName("read_cg")
277+
.setJobType(CloudWarmUpJob.JobType.CLUSTER)
278+
.setSyncMode(CloudWarmUpJob.SyncMode.EVENT_DRIVEN)
279+
.setTableFilterRules(Arrays.asList(rule))
280+
.setTableFilterExpr("{\"include\":[\"ods.*\"]}")
281+
.build();
282+
283+
manager.replayCloudWarmUpJob(job);
284+
285+
// Verify initial resolution picked up 2 tables
286+
Assertions.assertEquals(
287+
new HashSet<>(Arrays.asList(1001L, 1002L)),
288+
job.getCurrentTableIds());
289+
290+
// Simulate new table created
291+
databases.clear();
292+
databases.add(mockDb("ods",
293+
mockTable(1001, "orders"),
294+
mockTable(1002, "users"),
295+
mockTable(1003, "payments")));
296+
297+
manager.refreshAllTableFilters();
298+
299+
// Verify job now has 3 table IDs
300+
Assertions.assertEquals(
301+
new HashSet<>(Arrays.asList(1001L, 1002L, 1003L)),
302+
job.getCurrentTableIds());
303+
}
304+
305+
@Test
306+
public void testRefreshAllTableFiltersSkipsClusterLevelJob() throws Exception {
307+
// Cluster-level job (no table filter) should not be affected by refresh
308+
databases.add(mockDb("ods", mockTable(1001, "orders")));
309+
310+
CloudWarmUpJob clusterJob = new CloudWarmUpJob.Builder()
311+
.setJobId(200L)
312+
.setSrcClusterName("write_cg")
313+
.setDstClusterName("read_cg")
314+
.setJobType(CloudWarmUpJob.JobType.CLUSTER)
315+
.setSyncMode(CloudWarmUpJob.SyncMode.EVENT_DRIVEN)
316+
.build();
317+
318+
manager.replayCloudWarmUpJob(clusterJob);
319+
320+
// currentTableIds should be empty (no table filter)
321+
Assertions.assertTrue(clusterJob.getCurrentTableIds().isEmpty());
322+
323+
manager.refreshAllTableFilters();
324+
325+
// Still empty after refresh — cluster-level jobs are skipped
326+
Assertions.assertTrue(clusterJob.getCurrentTableIds().isEmpty());
327+
}
328+
329+
@Test
330+
public void testRefreshAllTableFiltersHandlesTableDrop() throws Exception {
331+
// Setup: job matching ods.*, initially 2 tables
332+
databases.add(mockDb("ods",
333+
mockTable(1001, "orders"),
334+
mockTable(1002, "users")));
335+
336+
CloudWarmUpJob.PersistedTableFilterRule rule = new CloudWarmUpJob.PersistedTableFilterRule();
337+
rule.ruleType = "INCLUDE";
338+
rule.pattern = "ods.*";
339+
340+
CloudWarmUpJob job = new CloudWarmUpJob.Builder()
341+
.setJobId(300L)
342+
.setSrcClusterName("write_cg")
343+
.setDstClusterName("read_cg")
344+
.setJobType(CloudWarmUpJob.JobType.CLUSTER)
345+
.setSyncMode(CloudWarmUpJob.SyncMode.EVENT_DRIVEN)
346+
.setTableFilterRules(Arrays.asList(rule))
347+
.setTableFilterExpr("{\"include\":[\"ods.*\"]}")
348+
.build();
349+
350+
manager.replayCloudWarmUpJob(job);
351+
Assertions.assertEquals(2, job.getCurrentTableIds().size());
352+
353+
// Drop one table
354+
databases.clear();
355+
databases.add(mockDb("ods", mockTable(1002, "users")));
356+
357+
manager.refreshAllTableFilters();
358+
359+
Assertions.assertEquals(
360+
new HashSet<>(Arrays.asList(1002L)),
361+
job.getCurrentTableIds());
362+
}
363+
}

0 commit comments

Comments
 (0)