Skip to content

Commit ab1823a

Browse files
RazvanLiviuVarzaruRazvanLiviuVarzaru
committed
MDBF-1185: Modernize MSAN/UBSAN CI with Clang 22
Phase in Clang 22 MSAN/UBSAN builders and update their worker assignments. Remove the obsolete Clang 20 builders and the MSAN debug configuration. Add an experimental mode to evaluate new suites and options with the Clang 22 builders. Extend Spider coverage to the non-debug MSAN build after it proved quick and stable locally. Evaluate cursor and view protocols across MSAN/UBSAN debug and non-debug builds. UBSAN mainly reproduced known minor failures. MSAN view exposed a failure that was also reproducible without MSAN, so run cursor and view tests on the faster non-debug MSAN builder. Use -Og for debug sanitizer builds to reduce instrumentation and improve CI runtime while retaining useful debugging information. Recognize DuckDB as a plugin, but disable it under MSAN/UBSAN until its linking issue is fixed.
1 parent 663de4c commit ab1823a

6 files changed

Lines changed: 224 additions & 33 deletions

File tree

‎configuration/builders/sequences/helpers.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,102 @@ def get_mtr_normal_steps(
6565
return steps
6666

6767

68+
def get_mtr_cursor_steps(
69+
jobs,
70+
path_to_test_runner: PurePath,
71+
halt_on_failure: bool = True,
72+
step_wrapping_fn=lambda step: step,
73+
additional_mtr_options: list[MTROption] = [],
74+
env_vars: list[tuple] = [],
75+
):
76+
steps = []
77+
steps.append(
78+
step_wrapping_fn(
79+
ShellStep(
80+
MTRTest(
81+
name="cursor",
82+
save_logs_path=MTR_PATH_TO_SAVE_LOGS / "cursor",
83+
workdir=path_to_test_runner,
84+
testcase=MTRGenerator(
85+
flags=[
86+
MTROption(MTR.VERBOSE_RESTART, True),
87+
MTROption(MTR.FORCE, True),
88+
MTROption(MTR.CURSOR_PROTOCOL, True),
89+
MTROption(MTR.MAX_SAVE_CORE, 2),
90+
MTROption(MTR.MAX_SAVE_DATADIR, 1),
91+
MTROption(MTR.MAX_TEST_FAIL, 20),
92+
MTROption(MTR.PARALLEL, jobs * 2),
93+
MTROption(MTR.VARDIR, "/dev/shm/cursor"),
94+
MTROption(
95+
MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "cursor.xml"
96+
),
97+
]
98+
+ additional_mtr_options,
99+
suite_collection=TestSuiteCollection(
100+
[
101+
SUITE.MAIN,
102+
]
103+
),
104+
),
105+
),
106+
options=StepOptions(
107+
haltOnFailure=halt_on_failure, descriptionDone="MTR cursor"
108+
),
109+
env_vars=env_vars,
110+
)
111+
)
112+
)
113+
return steps
114+
115+
116+
def get_mtr_view_steps(
117+
jobs,
118+
path_to_test_runner: PurePath,
119+
halt_on_failure: bool = True,
120+
step_wrapping_fn=lambda step: step,
121+
additional_mtr_options: list[MTROption] = [],
122+
env_vars: list[tuple] = [],
123+
):
124+
steps = []
125+
steps.append(
126+
step_wrapping_fn(
127+
ShellStep(
128+
MTRTest(
129+
name="view",
130+
save_logs_path=MTR_PATH_TO_SAVE_LOGS / "view",
131+
workdir=path_to_test_runner,
132+
testcase=MTRGenerator(
133+
flags=[
134+
MTROption(MTR.VERBOSE_RESTART, True),
135+
MTROption(MTR.FORCE, True),
136+
MTROption(MTR.VIEW_PROTOCOL, True),
137+
MTROption(MTR.MAX_SAVE_CORE, 2),
138+
MTROption(MTR.MAX_SAVE_DATADIR, 1),
139+
MTROption(MTR.MAX_TEST_FAIL, 20),
140+
MTROption(MTR.PARALLEL, jobs * 2),
141+
MTROption(MTR.VARDIR, "/dev/shm/view"),
142+
MTROption(
143+
MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "view.xml"
144+
),
145+
]
146+
+ additional_mtr_options,
147+
suite_collection=TestSuiteCollection(
148+
[
149+
SUITE.MAIN,
150+
]
151+
),
152+
),
153+
),
154+
options=StepOptions(
155+
haltOnFailure=halt_on_failure, descriptionDone="MTR view"
156+
),
157+
env_vars=env_vars,
158+
)
159+
)
160+
)
161+
return steps
162+
163+
68164
def get_mtr_rocksdb_steps(
69165
jobs,
70166
path_to_test_runner: PurePath,

‎configuration/builders/sequences/sanitizers.py‎

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
InContainer,
77
)
88
from configuration.builders.sequences.helpers import (
9+
get_mtr_cursor_steps,
910
get_mtr_normal_steps,
1011
get_mtr_s3_steps,
1112
get_mtr_spider_steps,
13+
get_mtr_view_steps,
1214
mtr_reporter,
1315
save_mtr_logs,
1416
)
@@ -34,6 +36,7 @@ def asan_ubsan(
3436
config: DockerConfig,
3537
jobs: int,
3638
isDebugBuildType: bool,
39+
isExperimental: bool,
3740
):
3841
sequence = BuildSequence()
3942

@@ -73,10 +76,17 @@ def asan_ubsan(
7376
CMakeOption(WITH.UBSAN, True),
7477
CMakeOption(WITH.UNIT_TESTS, False),
7578
CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False),
79+
CMakeOption(PLUGIN.DUCKDB_STORAGE_ENGINE, False),
7680
]
7781
if isDebugBuildType:
78-
flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG))
79-
flags.append(CMakeOption(WITH.DBUG_TRACE, False))
82+
flags.extend(
83+
[
84+
CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG),
85+
CMakeOption(WITH.DBUG_TRACE, False),
86+
CMakeOption(CMAKE.C_FLAGS, "-Og"),
87+
CMakeOption(CMAKE.CXX_FLAGS, "-Og"),
88+
]
89+
)
8090

8191
sequence.add_step(
8292
InContainer(
@@ -164,28 +174,33 @@ def asan_ubsan(
164174
docker_environment=config, step=step
165175
),
166176
)
167-
+ [
168-
save_mtr_logs(
169-
step_wrapping_fn=lambda step: InContainer(
170-
docker_environment=config, step=step
171-
),
172-
),
173-
mtr_reporter(
174-
step_wrapping_fn=lambda step: InContainer(
175-
docker_environment=config, step=step
176-
),
177-
),
178-
]
179177
):
180178
sequence.add_step(step)
181179

180+
sequence.add_step(
181+
save_mtr_logs(
182+
step_wrapping_fn=lambda step: InContainer(
183+
docker_environment=config, step=step
184+
),
185+
)
186+
)
187+
188+
sequence.add_step(
189+
mtr_reporter(
190+
step_wrapping_fn=lambda step: InContainer(
191+
docker_environment=config, step=step
192+
),
193+
),
194+
)
195+
182196
return sequence
183197

184198

185199
def msan(
186200
config: DockerConfig,
187201
jobs: int,
188202
isDebugBuildType: bool,
203+
isExperimental: bool,
189204
):
190205
sequence = BuildSequence()
191206

@@ -214,13 +229,20 @@ def msan(
214229
CMakeOption(WITH.ZLIB, "bundled"),
215230
CMakeOption(WITH.SYSTEMD, "no"),
216231
CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False),
217-
CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType),
232+
CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType or isExperimental),
218233
CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False),
219234
CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False),
235+
CMakeOption(PLUGIN.DUCKDB_STORAGE_ENGINE, False),
220236
]
221237
if isDebugBuildType:
222-
flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG))
223-
flags.append(CMakeOption(WITH.DBUG_TRACE, False))
238+
flags.extend(
239+
[
240+
CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG),
241+
CMakeOption(WITH.DBUG_TRACE, False),
242+
CMakeOption(CMAKE.C_FLAGS, "-Og"),
243+
CMakeOption(CMAKE.CXX_FLAGS, "-Og"),
244+
]
245+
)
224246

225247
sequence.add_step(
226248
InContainer(
@@ -279,7 +301,7 @@ def msan(
279301
path_to_test_runner=PurePath("bld", "mysql-test"),
280302
step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step),
281303
)
282-
if isDebugBuildType:
304+
if isDebugBuildType or isExperimental:
283305
steps += get_mtr_spider_steps(
284306
jobs=jobs,
285307
env_vars=env_vars,
@@ -290,6 +312,26 @@ def msan(
290312
docker_environment=config, step=step
291313
),
292314
)
315+
if isExperimental and not isDebugBuildType:
316+
steps += get_mtr_cursor_steps(
317+
jobs=jobs,
318+
env_vars=env_vars,
319+
halt_on_failure=False,
320+
path_to_test_runner=PurePath("bld", "mysql-test"),
321+
step_wrapping_fn=lambda step: InContainer(
322+
docker_environment=config, step=step
323+
),
324+
)
325+
steps += get_mtr_view_steps(
326+
jobs=jobs,
327+
env_vars=env_vars,
328+
halt_on_failure=False,
329+
path_to_test_runner=PurePath("bld", "mysql-test"),
330+
step_wrapping_fn=lambda step: InContainer(
331+
docker_environment=config, step=step
332+
),
333+
)
334+
# Save results
293335
steps += [
294336
save_mtr_logs(
295337
step_wrapping_fn=lambda step: InContainer(

‎configuration/steps/generators/cmake/options.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ class PLUGIN(StrEnum):
7373
ARCHIVE_STORAGE_ENGINE = "ARCHIVE"
7474
COLUMNSTORE_STORAGE_ENGINE = "COLUMNSTORE"
7575
CONNECT_STORAGE_ENGINE = "CONNECT"
76+
DUCKDB_STORAGE_ENGINE = "DUCKDB"
7677
FEDERATED_STORAGE_ENGINE = "FEDERATED"
7778
FEDERATEDX_STORAGE_ENGINE = "FEDERATEDX"
7879
FEEDBACK = "FEEDBACK"

‎constants.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@
136136
"amd64-fedora-44-valgrind",
137137
"amd64-freebsd-14",
138138
"amd64-msan-clang-20",
139+
"amd64-msan-clang-22",
139140
"amd64-openeuler-2403",
140141
"amd64-openssl3-fips",
141142
"amd64-rhel-7",
@@ -185,10 +186,9 @@
185186
"amd64-debian-12",
186187
"amd64-debian-12-deb-autobake-migration",
187188
"amd64-debian-12-debug-embedded",
188-
"amd64-msan-clang-20-debug",
189189
"amd64-rhel-10",
190-
"amd64-ubasan-clang-20",
191-
"amd64-ubasan-clang-20-debug",
190+
"amd64-ubasan-clang-22",
191+
"amd64-ubasan-clang-22-debug",
192192
"amd64-ubuntu-2404",
193193
"ppc64le-centos-stream10",
194194
"ppc64le-rhel-10",

0 commit comments

Comments
 (0)