Skip to content

Commit 5d07cfb

Browse files
grooverdangrooverdan
committed
MDBF-1185 msan/ubsan/asan with clang 22 (experiments)
With the new clang 22 msan, ubasan builders there is the opportunity to test new suites/options. As new versions generally allow a little time to experiment with different options, added an experimental option to helpers. The spider suite is enabled in Debug but not RelWithDeb info Debug is proved stable, so check if the non-debug is also. This proved quick and stable locally. On testing cursor and view protocols on msan and ubasan in both debug and non-debug, the ubasan non-debug was showing the same few trivial tests to correct as elsewhere and ran significantly faster. msan view showed a test failure that occurred without MSAN. This was fast enough in a non-debug mode.
1 parent ff553bf commit 5d07cfb

3 files changed

Lines changed: 164 additions & 14 deletions

File tree

‎configuration/builders/sequences/helpers.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,102 @@ def get_mtr_normal_steps(
6565
return steps
6666

6767

68+
def get_mtr_cursor_steps(
69+
jobs,
70+
path_to_test_runner: PurePath,
71+
halt_on_failure: bool = True,
72+
step_wrapping_fn=lambda step: step,
73+
additional_mtr_options: list[MTROption] = [],
74+
env_vars: list[tuple] = [],
75+
):
76+
steps = []
77+
steps.append(
78+
step_wrapping_fn(
79+
ShellStep(
80+
MTRTest(
81+
name="cursor",
82+
save_logs_path=MTR_PATH_TO_SAVE_LOGS / "cursor",
83+
workdir=path_to_test_runner,
84+
testcase=MTRGenerator(
85+
flags=[
86+
MTROption(MTR.VERBOSE_RESTART, True),
87+
MTROption(MTR.FORCE, True),
88+
MTROption(MTR.CURSOR_PROTOCOL, True),
89+
MTROption(MTR.MAX_SAVE_CORE, 2),
90+
MTROption(MTR.MAX_SAVE_DATADIR, 1),
91+
MTROption(MTR.MAX_TEST_FAIL, 20),
92+
MTROption(MTR.PARALLEL, jobs * 2),
93+
MTROption(MTR.VARDIR, "/dev/shm/cursor"),
94+
MTROption(
95+
MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "cursor.xml"
96+
),
97+
]
98+
+ additional_mtr_options,
99+
suite_collection=TestSuiteCollection(
100+
[
101+
SUITE.MAIN,
102+
]
103+
),
104+
),
105+
),
106+
options=StepOptions(
107+
haltOnFailure=halt_on_failure, descriptionDone="MTR cursor"
108+
),
109+
env_vars=env_vars,
110+
)
111+
)
112+
)
113+
return steps
114+
115+
116+
def get_mtr_view_steps(
117+
jobs,
118+
path_to_test_runner: PurePath,
119+
halt_on_failure: bool = True,
120+
step_wrapping_fn=lambda step: step,
121+
additional_mtr_options: list[MTROption] = [],
122+
env_vars: list[tuple] = [],
123+
):
124+
steps = []
125+
steps.append(
126+
step_wrapping_fn(
127+
ShellStep(
128+
MTRTest(
129+
name="view",
130+
save_logs_path=MTR_PATH_TO_SAVE_LOGS / "view",
131+
workdir=path_to_test_runner,
132+
testcase=MTRGenerator(
133+
flags=[
134+
MTROption(MTR.VERBOSE_RESTART, True),
135+
MTROption(MTR.FORCE, True),
136+
MTROption(MTR.VIEW_PROTOCOL, True),
137+
MTROption(MTR.MAX_SAVE_CORE, 2),
138+
MTROption(MTR.MAX_SAVE_DATADIR, 1),
139+
MTROption(MTR.MAX_TEST_FAIL, 20),
140+
MTROption(MTR.PARALLEL, jobs * 2),
141+
MTROption(MTR.VARDIR, "/dev/shm/view"),
142+
MTROption(
143+
MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "view.xml"
144+
),
145+
]
146+
+ additional_mtr_options,
147+
suite_collection=TestSuiteCollection(
148+
[
149+
SUITE.MAIN,
150+
]
151+
),
152+
),
153+
),
154+
options=StepOptions(
155+
haltOnFailure=halt_on_failure, descriptionDone="MTR view"
156+
),
157+
env_vars=env_vars,
158+
)
159+
)
160+
)
161+
return steps
162+
163+
68164
def get_mtr_rocksdb_steps(
69165
jobs,
70166
path_to_test_runner: PurePath,

‎configuration/builders/sequences/sanitizers.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
InContainer,
77
)
88
from configuration.builders.sequences.helpers import (
9+
get_mtr_cursor_steps,
910
get_mtr_normal_steps,
1011
get_mtr_s3_steps,
1112
get_mtr_spider_steps,
13+
get_mtr_view_steps,
1214
mtr_reporter,
1315
save_mtr_logs,
1416
)
@@ -34,6 +36,7 @@ def asan_ubsan(
3436
config: DockerConfig,
3537
jobs: int,
3638
isDebugBuildType: bool,
39+
isExperimental: bool,
3740
):
3841
sequence = BuildSequence()
3942

@@ -179,13 +182,34 @@ def asan_ubsan(
179182
):
180183
sequence.add_step(step)
181184

185+
if isExperimental and not isDebugBuildType:
186+
for step in get_mtr_cursor_steps(
187+
jobs=jobs,
188+
env_vars=env_vars,
189+
halt_on_failure=False,
190+
path_to_test_runner=PurePath("bld", "mysql-test"),
191+
step_wrapping_fn=lambda step: InContainer(
192+
docker_environment=config, step=step
193+
),
194+
) + get_mtr_view_steps(
195+
jobs=jobs,
196+
env_vars=env_vars,
197+
halt_on_failure=False,
198+
path_to_test_runner=PurePath("bld", "mysql-test"),
199+
step_wrapping_fn=lambda step: InContainer(
200+
docker_environment=config, step=step
201+
),
202+
):
203+
sequence.add_step(step)
204+
182205
return sequence
183206

184207

185208
def msan(
186209
config: DockerConfig,
187210
jobs: int,
188211
isDebugBuildType: bool,
212+
isExperimental: bool,
189213
):
190214
sequence = BuildSequence()
191215

@@ -214,7 +238,7 @@ def msan(
214238
CMakeOption(WITH.ZLIB, "bundled"),
215239
CMakeOption(WITH.SYSTEMD, "no"),
216240
CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False),
217-
CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType),
241+
CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType or isExperimental),
218242
CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False),
219243
CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False),
220244
]
@@ -279,7 +303,7 @@ def msan(
279303
path_to_test_runner=PurePath("bld", "mysql-test"),
280304
step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step),
281305
)
282-
if isDebugBuildType:
306+
if isDebugBuildType or isExperimental:
283307
steps += get_mtr_spider_steps(
284308
jobs=jobs,
285309
env_vars=env_vars,
@@ -290,6 +314,7 @@ def msan(
290314
docker_environment=config, step=step
291315
),
292316
)
317+
# Save results
293318
steps += [
294319
save_mtr_logs(
295320
step_wrapping_fn=lambda step: InContainer(

‎master-migration/master.cfg‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,11 @@ c["builders"].append(
160160

161161

162162
def ubasan_builder(
163-
name: str, image: str, debug: bool, workers: Iterable[WorkerBase]
163+
name: str,
164+
image: str,
165+
debug: bool,
166+
experimental: bool,
167+
workers: Iterable[WorkerBase],
164168
) -> GenericBuilder:
165169
tags_ubasan = ("Debian", "clang", "asan", "ubsan", "big")
166170
jobs = 12
@@ -174,6 +178,7 @@ def ubasan_builder(
174178
jobs=jobs,
175179
config=docker_config(image=image, shm_size="24g"),
176180
isDebugBuildType=debug,
181+
isExperimental=experimental,
177182
)
178183
],
179184
).get_config(
@@ -185,25 +190,39 @@ def ubasan_builder(
185190

186191
sg_bbw1 = WORKER_POOL.get_workers_for_arch(arch="amd64", names=["sg-bbw1"])
187192

188-
for builder, image, workers in [
189-
("amd64-ubasan-clang-20", "debian12-msan-clang-20", DEFAULT_AMD64_WORKER_POOL),
193+
for builder, image, experimental, workers in [
194+
(
195+
"amd64-ubasan-clang-20",
196+
"debian12-msan-clang-20",
197+
False,
198+
DEFAULT_AMD64_WORKER_POOL,
199+
),
190200
(
191201
"amd64-ubasan-clang-20-debug",
192202
"debian12-msan-clang-20",
203+
False,
193204
DEFAULT_AMD64_WORKER_POOL,
194205
),
195-
("amd64-ubasan-clang-22", "debian13-msan-clang-22", sg_bbw1),
196-
("amd64-ubasan-clang-22-debug", "debian13-msan-clang-22", sg_bbw1),
206+
("amd64-ubasan-clang-22", "debian13-msan-clang-22", True, sg_bbw1),
207+
("amd64-ubasan-clang-22-debug", "debian13-msan-clang-22", True, sg_bbw1),
197208
]:
198209
c["builders"].append(
199210
ubasan_builder(
200-
name=builder, image=image, debug=builder.endswith("debug"), workers=workers
211+
name=builder,
212+
image=image,
213+
debug=builder.endswith("debug"),
214+
experimental=experimental,
215+
workers=workers,
201216
)
202217
)
203218

204219

205220
def msan_builder(
206-
name: str, image: str, debug: bool, workers: Iterable[WorkerBase]
221+
name: str,
222+
image: str,
223+
debug: bool,
224+
experimental: bool,
225+
workers: Iterable[WorkerBase],
207226
) -> GenericBuilder:
208227
tags_msan = ("Debian", "clang", "msan", "big")
209228
jobs = 12
@@ -218,6 +237,7 @@ def msan_builder(
218237
jobs=jobs,
219238
config=docker_config(image=image, shm_size="24g"),
220239
isDebugBuildType=debug,
240+
isExperimental=experimental,
221241
)
222242
],
223243
).get_config(
@@ -227,14 +247,23 @@ def msan_builder(
227247
)
228248

229249

230-
for builder, image, workers in [
231-
("amd64-msan-clang-20-debug", "debian12-msan-clang-20", DEFAULT_AMD64_WORKER_POOL),
232-
("amd64-msan-clang-22", "debian13-msan-clang-22", sg_bbw1),
233-
("amd64-msan-clang-22-debug", "debian13-msan-clang-22", sg_bbw1),
250+
for builder, image, experimental, workers in [
251+
(
252+
"amd64-msan-clang-20-debug",
253+
"debian12-msan-clang-20",
254+
False,
255+
DEFAULT_AMD64_WORKER_POOL,
256+
),
257+
("amd64-msan-clang-22", "debian13-msan-clang-22", True, sg_bbw1),
258+
("amd64-msan-clang-22-debug", "debian13-msan-clang-22", True, sg_bbw1),
234259
]:
235260
c["builders"].append(
236261
msan_builder(
237-
name=builder, image=image, debug=builder.endswith("debug"), workers=workers
262+
name=builder,
263+
image=image,
264+
debug=builder.endswith("debug"),
265+
experimental=experimental,
266+
workers=workers,
238267
)
239268
)
240269

0 commit comments

Comments
 (0)