Skip to content

Commit 94c9bd6

Browse files
youngsofunyoungsofun
committed
test(query): cover lineage history traversal
1 parent 1583555 commit 94c9bd6

17 files changed

Lines changed: 743 additions & 0 deletions

‎tests/logging/history_table/check_logs_table.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
set -e
44

5+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6+
source "$SCRIPT_DIR/lineage_sqllogic.sh"
7+
58
# Source query IDs from setup script
69
if [ -z "$QUERY_ID" ]; then
710
echo "Error: Query IDs not set. Run setup_test_data.sh first."
@@ -71,6 +74,9 @@ check_query_log "basic-10" "$CREATE_VIEW_QUERY_ID" "select query_text from syste
7174

7275
check_query_log "basic-11" null "SELECT count(*) FROM system_history.login_history WHERE session_id = '$SELECT_SESSION_ID' " "1"
7376

77+
# Lineage assertions mirror the grouped setup suite and clean up its dedicated database.
78+
run_lineage_suite check
79+
7480
check_query_log "resource-1" "$SELECT_QUERY_ID" "SELECT count(*) FROM system_history.query_history WHERE resource_usage IS NOT NULL AND resource_usage['read_count']::UInt64 > 0 AND resource_usage['read_bytes']::UInt64 > 0 AND resource_usage['read_duration_ms'] IS NOT NULL AND resource_usage['cpu_time_ms'] IS NOT NULL AND resource_usage['wait_time_ms'] IS NOT NULL AND" "1"
7581

7682
check_query_log "resource-2" "$INSERT_QUERY_ID" "SELECT count(*) FROM system_history.query_history WHERE resource_usage IS NOT NULL AND resource_usage['write_count']::UInt64 > 0 AND resource_usage['write_bytes']::UInt64 > 0 AND resource_usage['write_duration_ms'] IS NOT NULL AND resource_usage['cpu_time_ms'] IS NOT NULL AND resource_usage['wait_time_ms'] IS NOT NULL AND" "1"

‎tests/logging/history_table/history_table.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@ table_name = "login_history"
1313

1414
[[log.history.tables]]
1515
table_name = "access_history"
16+
17+
[[log.history.tables]]
18+
table_name = "lineage_unresolved"

‎tests/logging/history_table/lineage_sqllogic.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
# Shared sqllogictest runner for lineage history setup and assertions.
4+
LINEAGE_HISTORY_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
LINEAGE_HISTORY_REPO_ROOT="$(cd "$LINEAGE_HISTORY_SCRIPT_DIR/../../.." && pwd)"
6+
BUILD_PROFILE="${BUILD_PROFILE:-debug}"
7+
8+
run_lineage_suite() {
9+
local suite_dir="$1"
10+
(
11+
cd "$LINEAGE_HISTORY_REPO_ROOT"
12+
"target/${BUILD_PROFILE}/databend-sqllogictests" \
13+
--suites tests/logging/history_table/sqllogic \
14+
--run_dir "$suite_dir" \
15+
--handlers http \
16+
--parallel 1
17+
)
18+
}

‎tests/logging/history_table/setup_test_data.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
set -e
44

5+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6+
source "$SCRIPT_DIR/lineage_sqllogic.sh"
7+
58
execute_query() {
69
local sql="$1"
710
local extra_headers="$2"
@@ -45,6 +48,10 @@ select_session_id=$(echo $response | jq -r '.session_id')
4548
echo "Select Query ID: $select_query_id"
4649
echo "Select Session ID: $select_session_id"
4750

51+
# Lineage setup is grouped in a dedicated sqllogictest suite so scenarios remain reviewable and
52+
# can grow without adding SQL assertions to this shell script.
53+
run_lineage_suite setup
54+
4855
execute_query_silent "drop user if exists wrong_pass_user"
4956

5057
execute_query_silent "create user wrong_pass_user identified by 'secure_password'"
@@ -58,6 +65,33 @@ for _ in {1..3}; do
5865
sleep 3
5966
done
6067

68+
# History ingestion is asynchronous. Poll the transformed edge table instead of assuming the fixed
69+
# delay above is sufficient on every CI runner.
70+
lineage_ready=false
71+
for _ in {1..30}; do
72+
lineage_response=$(execute_query "SELECT count_if(source_database IN ('lineage_history_objects', 'lineage_history_columns', 'lineage_history_views', 'lineage_history_statements') OR target_database IN ('lineage_history_objects', 'lineage_history_columns', 'lineage_history_views', 'lineage_history_statements')) AS captured_edges, count_if(target_database = 'lineage_history_lifecycle') AS lifecycle_edges, count_if(source_catalog = 'lineage_history_iceberg_catalog' AND source_database = 'lineage_db' AND target_database = 'lineage_history_iceberg') AS iceberg_edges, count_if(target_database = 'lineage_history_views' AND target_name IN ('src_view', 'view_dst')) AS view_edges FROM system_history.lineage_unresolved")
73+
if [ "$(echo "$lineage_response" | jq -r '.state')" = "Failed" ]; then
74+
echo "Lineage readiness query failed"
75+
echo "$lineage_response" | jq '.error'
76+
exit 1
77+
fi
78+
lineage_count=$(echo "$lineage_response" | jq -r '.data[0][0] // 0')
79+
lifecycle_count=$(echo "$lineage_response" | jq -r '.data[0][1] // 0')
80+
iceberg_count=$(echo "$lineage_response" | jq -r '.data[0][2] // 0')
81+
view_count=$(echo "$lineage_response" | jq -r '.data[0][3] // 0')
82+
if [ "$lineage_count" -ge 16 ] 2>/dev/null && [ "$lifecycle_count" -eq 3 ] 2>/dev/null && [ "$iceberg_count" -ge 1 ] 2>/dev/null && [ "$view_count" -ge 2 ] 2>/dev/null; then
83+
lineage_ready=true
84+
break
85+
fi
86+
sleep 1
87+
done
88+
89+
if [ "$lineage_ready" = false ]; then
90+
echo "Lineage history was not transformed within 30 seconds"
91+
echo "$lineage_response"
92+
exit 1
93+
fi
94+
6195
# Export query IDs for use in other scripts
6296
export QUERY_ID="$query_id"
6397
export CREATE_QUERY_ID="$create_query_id"

‎tests/logging/history_table/sqllogic/check/01_object_patterns.test‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Object patterns correspond to setup/01_object_patterns.test.
2+
3+
query I
4+
SELECT count(*)
5+
FROM system_history.lineage_unresolved
6+
WHERE source_database = 'lineage_history_objects'
7+
AND target_database = 'lineage_history_objects'
8+
AND source_name = 'src'
9+
AND target_name = 'mid'
10+
AND lineage_kind IN ('CTAS', 'DML')
11+
----
12+
2
13+
14+
query ITT
15+
SELECT distance, source_object_name, target_object_name
16+
FROM get_lineage('lineage_history_objects.mid', 'TABLE', 'UPSTREAM', 1)
17+
ORDER BY distance, source_object_name, target_object_name
18+
----
19+
1 default.lineage_history_objects.src default.lineage_history_objects.mid
20+
21+
statement ok
22+
DROP DATABASE lineage_history_objects

‎tests/logging/history_table/sqllogic/check/02_column_multihop.test‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Column patterns and multi-hop traversal correspond to setup/02_column_multihop.test.
2+
3+
query ITTTT rowsort
4+
SELECT distance, source_object_name, source_column_name, target_object_name, target_column_name
5+
FROM get_lineage('lineage_history_columns.mid.x', 'COLUMN', 'UPSTREAM', 1)
6+
----
7+
1 default.lineage_history_columns.src a default.lineage_history_columns.mid x
8+
1 default.lineage_history_columns.src b default.lineage_history_columns.mid x
9+
10+
query ITTTT rowsort
11+
SELECT distance, source_object_name, source_column_name, target_object_name, target_column_name
12+
FROM get_lineage('lineage_history_columns.dst.z', 'COLUMN', 'UPSTREAM', 2)
13+
----
14+
1 default.lineage_history_columns.mid x default.lineage_history_columns.dst z
15+
2 default.lineage_history_columns.src a default.lineage_history_columns.mid x
16+
2 default.lineage_history_columns.src b default.lineage_history_columns.mid x
17+
18+
# Masking applies to a concrete target column, not to the whole object or traversal path.
19+
query ITTTTT rowsort
20+
SELECT distance, source_object_name, source_column_name, target_object_name, target_column_name, target_status
21+
FROM get_lineage('lineage_history_columns.dst.z', 'COLUMN', 'UPSTREAM', 2)
22+
----
23+
1 default.lineage_history_columns.mid x default.lineage_history_columns.dst z MASKED
24+
2 default.lineage_history_columns.src a default.lineage_history_columns.mid x ACTIVE
25+
2 default.lineage_history_columns.src b default.lineage_history_columns.mid x ACTIVE
26+
27+
query T
28+
SELECT target_status
29+
FROM get_lineage('lineage_history_columns.dst', 'TABLE', 'UPSTREAM', 1)
30+
----
31+
ACTIVE
32+
33+
query ITTTT rowsort
34+
SELECT distance, source_object_name, source_column_name, target_object_name, target_column_name
35+
FROM get_lineage('lineage_history_columns.src.a', 'COLUMN', 'DOWNSTREAM', 2)
36+
----
37+
1 default.lineage_history_columns.src a default.lineage_history_columns.mid x
38+
1 default.lineage_history_columns.src a default.lineage_history_columns.mid y
39+
2 default.lineage_history_columns.mid x default.lineage_history_columns.dst z
40+
41+
statement ok
42+
ALTER TABLE lineage_history_columns.dst MODIFY COLUMN z UNSET MASKING POLICY
43+
44+
statement ok
45+
DROP MASKING POLICY lineage_history_columns_mask
46+
47+
statement ok
48+
DROP DATABASE lineage_history_columns

‎tests/logging/history_table/sqllogic/check/03_view_boundary.test‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# View traversal corresponds to setup/03_view_boundary.test.
2+
3+
query ITTTT rowsort
4+
SELECT distance, source_object_name, source_column_name, target_object_name, target_column_name
5+
FROM get_lineage('lineage_history_views.view_dst.z', 'COLUMN', 'UPSTREAM', 2)
6+
----
7+
1 default.lineage_history_views.src_view v default.lineage_history_views.view_dst z
8+
2 default.lineage_history_views.src a default.lineage_history_views.src_view v
9+
2 default.lineage_history_views.src b default.lineage_history_views.src_view v
10+
11+
statement ok
12+
DROP DATABASE lineage_history_views

‎tests/logging/history_table/sqllogic/check/04_statement_coverage.test‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Statement coverage corresponds to setup/04_statement_coverage.test.
2+
3+
query ITT
4+
SELECT distance, source_object_name, target_object_name
5+
FROM get_lineage('lineage_history_statements.self_insert_dst', 'TABLE', 'UPSTREAM', 1)
6+
----
7+
1 default.lineage_history_statements.src default.lineage_history_statements.self_insert_dst
8+
9+
query ISS
10+
SELECT distance, source_column_name, target_column_name
11+
FROM get_lineage('lineage_history_statements.self_insert_dst.a', 'COLUMN', 'UPSTREAM', 1)
12+
----
13+
1 a a
14+
15+
query I
16+
SELECT count(*)
17+
FROM system_history.lineage_unresolved
18+
WHERE target_database = 'lineage_history_statements'
19+
AND source_lineage_key = target_lineage_key
20+
----
21+
0
22+
23+
query ITT
24+
SELECT distance, source_object_name, target_object_name
25+
FROM get_lineage('lineage_history_statements.insert_dst', 'TABLE', 'UPSTREAM', 1)
26+
----
27+
1 default.lineage_history_statements.src default.lineage_history_statements.insert_dst
28+
29+
query ITT
30+
SELECT distance, source_object_name, target_object_name
31+
FROM get_lineage('lineage_history_statements.multi_a', 'TABLE', 'UPSTREAM', 1)
32+
----
33+
1 default.lineage_history_statements.src default.lineage_history_statements.multi_a
34+
35+
query ITT
36+
SELECT distance, source_object_name, target_object_name
37+
FROM get_lineage('lineage_history_statements.multi_b', 'TABLE', 'UPSTREAM', 1)
38+
----
39+
1 default.lineage_history_statements.src default.lineage_history_statements.multi_b
40+
41+
query ITT
42+
SELECT distance, source_object_name, target_object_name
43+
FROM get_lineage('lineage_history_statements.replace_dst', 'TABLE', 'UPSTREAM', 1)
44+
----
45+
1 default.lineage_history_statements.src default.lineage_history_statements.replace_dst
46+
47+
# Stream data lineage passes through to the backing table.
48+
query ITT
49+
SELECT distance, source_object_name, target_object_name
50+
FROM get_lineage('lineage_history_statements.stream_dst', 'TABLE', 'UPSTREAM', 1)
51+
----
52+
1 default.lineage_history_statements.stream_src default.lineage_history_statements.stream_dst
53+
54+
query ISS
55+
SELECT distance, source_column_name, target_column_name
56+
FROM get_lineage('lineage_history_statements.stream_dst.x', 'COLUMN', 'UPSTREAM', 1)
57+
----
58+
1 a x
59+
60+
# The Stream wrapper itself must not be persisted as a lineage endpoint.
61+
query I
62+
SELECT count(*)
63+
FROM system_history.lineage_unresolved
64+
WHERE source_database = 'lineage_history_statements'
65+
AND source_name = 'src_stream'
66+
AND target_database = 'lineage_history_statements'
67+
AND target_name = 'stream_dst'
68+
----
69+
0
70+
71+
query ITT
72+
SELECT distance, source_object_name, target_object_name
73+
FROM get_lineage('lineage_history_statements.copy_dst', 'TABLE', 'UPSTREAM', 2)
74+
ORDER BY distance, source_object_name, target_object_name
75+
----
76+
1 lineage_history_statements_stage default.lineage_history_statements.copy_dst
77+
2 default.lineage_history_statements.src lineage_history_statements_stage
78+
79+
query ITT
80+
SELECT distance, source_object_name, target_object_name
81+
FROM get_lineage('lineage_history_statements_stage', 'STAGE', 'DOWNSTREAM', 1)
82+
ORDER BY target_object_name
83+
----
84+
1 lineage_history_statements_stage default.lineage_history_statements.copy_dst
85+
1 lineage_history_statements_stage default.lineage_history_statements.insert_select_stage_dst
86+
1 lineage_history_statements_stage default.lineage_history_statements.insert_stage_dst
87+
88+
query ITT
89+
SELECT distance, source_object_name, target_object_name
90+
FROM get_lineage('lineage_history_statements.insert_select_stage_dst', 'TABLE', 'UPSTREAM', 1)
91+
----
92+
1 lineage_history_statements_stage default.lineage_history_statements.insert_select_stage_dst
93+
94+
query ITT
95+
SELECT distance, source_object_name, target_object_name
96+
FROM get_lineage('lineage_history_statements.insert_stage_dst', 'TABLE', 'UPSTREAM', 1)
97+
----
98+
1 lineage_history_statements_stage default.lineage_history_statements.insert_stage_dst
99+
100+
# Stage file fields are not stable columns, so Stage-to-table edges stay object-only.
101+
query I
102+
SELECT count(*)
103+
FROM get_lineage('lineage_history_statements.copy_dst.a', 'COLUMN', 'UPSTREAM', 1)
104+
----
105+
0
106+
107+
query I
108+
SELECT count(*)
109+
FROM get_lineage('lineage_history_statements.insert_select_stage_dst.a', 'COLUMN', 'UPSTREAM', 1)
110+
----
111+
0
112+
113+
query I
114+
SELECT count(*)
115+
FROM get_lineage('lineage_history_statements.insert_stage_dst.a', 'COLUMN', 'UPSTREAM', 1)
116+
----
117+
0
118+
119+
statement ok
120+
DROP DATABASE lineage_history_statements
121+
122+
statement ok
123+
DROP STAGE lineage_history_statements_stage

0 commit comments

Comments
 (0)