diff --git a/MODULE.bazel b/MODULE.bazel index 21d5f4a3245bc..3cbb5b5d40493 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,10 @@ archive_override( integrity = "sha256-wQsxOxZh1tluc6AXj+ncsYhBSl03uC+hPhBrU6JkupI=", strip_prefix = "rules_boost-f5b0f8c904f2487d8f5a9a956d4388724e627210", urls = ["https://github.com/nelhage/rules_boost/archive/f5b0f8c904f2487d8f5a9a956d4388724e627210.tar.gz"], + patch_strip = 1, + patches = [ + "//bazel/thirdparty:rules-boost-s390x-context.patch", + ], ) non_module_boost_repositories = use_extension("@com_github_nelhage_rules_boost//:boost/repositories.bzl", "non_module_dependencies") @@ -21,6 +25,13 @@ bazel_dep(name = "gazelle", version = "0.45.0") bazel_dep(name = "abseil-cpp", version = "20250814.1") bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "crc32c", version = "1.1.0") +single_version_override( + module_name = "crc32c", + patch_strip = 1, + patches = [ + "//bazel/thirdparty:crc32c-big-endian-config.patch", + ], +) bazel_dep(name = "fmt", version = "12.1.0") bazel_dep(name = "googletest", version = "1.17.0") bazel_dep(name = "lexy", version = "2025.05.0") diff --git a/bazel/BUILD b/bazel/BUILD index d46f3e9626f7a..d2ce694583589 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -94,3 +94,8 @@ config_setting( ":antithesis": "true", }, ) + +config_setting( + name = "s390x", + constraint_values = ["@platforms//cpu:s390x"], +) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index fe210d1a60eba..a977050db59b4 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -167,6 +167,11 @@ def data_dependency(): sha256 = "e800bfbfeaf514ad90cb480aa5e317c01c514f99ed97ee0ac2ef3c9c55dc43a5", strip_prefix = "seastar-5d474c884fb54f1bbef9fe9bfada0eb48feb47b5", url = "https://github.com/redpanda-data/seastar/archive/5d474c884fb54f1bbef9fe9bfada0eb48feb47b5.tar.gz", + patches = [ + "//bazel/thirdparty:seastar-test-runner-stack.patch", + "//bazel/thirdparty:seastar-thread-test-stack.patch", + ], + patch_args = ["-p1"], ) http_archive( diff --git a/bazel/test.bzl b/bazel/test.bzl index da9c18f28f5cc..c6f71879091c4 100644 --- a/bazel/test.bzl +++ b/bazel/test.bzl @@ -75,7 +75,6 @@ def _test_options(): data = [ "//:ubsan_suppressions", "//:lsan_suppressions", - "@current_llvm_toolchain//:llvm-symbolizer", ] env = { "BOOST_TEST_LOG_LEVEL": "test_suite", @@ -84,7 +83,6 @@ def _test_options(): "BOOST_TEST_REPORT_LEVEL": "no", "BOOST_LOGGER": "HRF,test_suite", "ASAN_OPTIONS": "disable_coredump=0:abort_on_error=1", - "ASAN_SYMBOLIZER_PATH": "$(rootpath @current_llvm_toolchain//:llvm-symbolizer)", "LSAN_OPTIONS": "suppressions=$(rootpath //:lsan_suppressions)", "UBSAN_OPTIONS": "symbolize=1:print_stacktrace=1:halt_on_error=1:abort_on_error=1:report_error_type=1:suppressions=$(rootpath //:ubsan_suppressions)", # see https://redpandadata.atlassian.net/wiki/x/BwDSUw @@ -93,6 +91,23 @@ def _test_options(): deps = antithesis_deps() return data, env, deps +# The hermetic LLVM toolchain has no s390x distribution, so the +# llvm-symbolizer target cannot be fetched there. Sanitizers are not used +# on s390x, so drop the symbolizer data dep and ASAN_SYMBOLIZER_PATH on that +# arch while keeping them everywhere else. +_SYMBOLIZER_DATA = select({ + "//bazel:s390x": [], + "//conditions:default": ["@current_llvm_toolchain//:llvm-symbolizer"], +}) + +def _symbolizer_env(base_env): + return select({ + "//bazel:s390x": base_env, + "//conditions:default": base_env | { + "ASAN_SYMBOLIZER_PATH": "$(rootpath @current_llvm_toolchain//:llvm-symbolizer)", + }, + }) + def _redpanda_cc_test( name, timeout, @@ -171,9 +186,9 @@ def _redpanda_cc_test( "layering_check", ], tags = resource_tags + tags, - env = {"RP_FIXTURE_ENV": "1"} | test_env | env, + env = _symbolizer_env({"RP_FIXTURE_ENV": "1"} | test_env | env), target_compatible_with = target_compatible_with, - data = data + test_data, + data = data + test_data + _SYMBOLIZER_DATA, local_defines = local_defines, flaky = flaky, ) @@ -215,8 +230,8 @@ def _redpanda_cc_fuzz_test( tags = [ "fuzz", ], - env = test_env | env, - data = data + test_data, + env = _symbolizer_env(test_env | env), + data = data + test_data + _SYMBOLIZER_DATA, linkopts = [ "-fsanitize=fuzzer", ], @@ -355,8 +370,8 @@ def redpanda_cc_btest_no_seastar( "//src/v/test_utils:boost_test_hooks", "@boost//:test.so", ] + deps + test_deps, - data = test_data, - env = test_env, + data = test_data + _SYMBOLIZER_DATA, + env = _symbolizer_env(test_env), ) def redpanda_test_cc_library( @@ -401,6 +416,7 @@ def redpanda_cc_bench( duration = None, data = [], tags = [], + target_compatible_with = [], redirect_stderr = False, test_regex = None): """ @@ -481,6 +497,7 @@ def redpanda_cc_bench( tags = tags, env = env, data = data, + target_compatible_with = target_compatible_with, ) args = ["$(rootpath :{})".format(binary_name)] + args + _reactor_args() @@ -505,6 +522,7 @@ def redpanda_cc_bench( data = data + [":" + binary_name], env = env, testonly = True, + target_compatible_with = target_compatible_with, ) # we write a wrapper to test the benchmark, which tries to @@ -518,7 +536,8 @@ def redpanda_cc_bench( main = "bench_wrapper.py", tags = resource_tags + tags, srcs = ["//bazel:bench_wrapper"], - env = test_env | env, + env = _symbolizer_env(test_env | env), args = test_args, - data = [":" + binary_name] + data + test_data, + data = [":" + binary_name] + data + test_data + _SYMBOLIZER_DATA, + target_compatible_with = target_compatible_with, ) diff --git a/bazel/thirdparty/crc32c-big-endian-config.patch b/bazel/thirdparty/crc32c-big-endian-config.patch new file mode 100644 index 0000000000000..3c8118518b408 --- /dev/null +++ b/bazel/thirdparty/crc32c-big-endian-config.patch @@ -0,0 +1,19 @@ +Set BYTE_ORDER_BIG_ENDIAN from the compiler's byte-order macros instead of +hardcoding it to 0. The crc32c portable implementation reads input words via +ReadUint32LE, whose big-endian branch is only compiled when +BYTE_ORDER_BIG_ENDIAN is non-zero. Hardcoding 0 makes the LE memcpy path run on +big-endian hosts (e.g. s390x), producing incorrect CRC32C values and breaking +Kafka record-batch validation. Deriving the value from __BYTE_ORDER__ keeps the +config correct on every target arch. + +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -76,7 +76,7 @@ + outs = ["crc32c/crc32c_config.h"], + cmd = """ + sed -e 's/#cmakedefine01/#define/' \ +- -e 's/ BYTE_ORDER_BIG_ENDIAN/ BYTE_ORDER_BIG_ENDIAN 0/' \ ++ -e 's/ BYTE_ORDER_BIG_ENDIAN/ BYTE_ORDER_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)/' \ + -e 's/ HAVE_BUILTIN_PREFETCH/ HAVE_BUILTIN_PREFETCH 0/' \ + -e 's/ HAVE_MM_PREFETCH/ HAVE_MM_PREFETCH 0/' \ + -e 's/ HAVE_SSE42/ HAVE_SSE42 1/' \ diff --git a/bazel/thirdparty/hwloc.BUILD b/bazel/thirdparty/hwloc.BUILD index 8ee65c65da2e5..0b13c2e84b77b 100644 --- a/bazel/thirdparty/hwloc.BUILD +++ b/bazel/thirdparty/hwloc.BUILD @@ -23,6 +23,13 @@ configure_make( configure_options = [ "--disable-libudev", + # Don't let autoconf pick up a system libxml2 for optional XML + # topology import/export. We don't use it, and an auto-detected + # libxml2 leaks undefined xml* symbols into every hwloc consumer + # (the dep isn't wired up), breaking links on hosts that happen to + # have libxml2 headers installed. + "--disable-libxml2", + # Disable graphics and the many kinds of display driver discovery "--disable-gl", "--disable-opencl", diff --git a/bazel/thirdparty/rules-boost-s390x-context.patch b/bazel/thirdparty/rules-boost-s390x-context.patch new file mode 100644 index 0000000000000..2cce991bf1684 --- /dev/null +++ b/bazel/thirdparty/rules-boost-s390x-context.patch @@ -0,0 +1,29 @@ +--- a/boost.BUILD ++++ b/boost.BUILD +@@ -24,6 +24,14 @@ + ) + + config_setting( ++ name = "linux_s390x", ++ constraint_values = [ ++ "@platforms//os:linux", ++ "@platforms//cpu:s390x", ++ ], ++) ++ ++config_setting( + name = "linux_aarch64", + constraint_values = [ + "@platforms//os:linux", +@@ -106,6 +114,11 @@ + "libs/context/src/asm/make_ppc64_sysv_elf_gas.S", + "libs/context/src/asm/ontop_ppc64_sysv_elf_gas.S", + ], ++ ":linux_s390x": [ ++ "libs/context/src/asm/jump_s390x_sysv_elf_gas.S", ++ "libs/context/src/asm/make_s390x_sysv_elf_gas.S", ++ "libs/context/src/asm/ontop_s390x_sysv_elf_gas.S", ++ ], + ":linux_x86_64": [ + "libs/context/src/asm/jump_x86_64_sysv_elf_gas.S", + "libs/context/src/asm/make_x86_64_sysv_elf_gas.S", diff --git a/bazel/thirdparty/seastar-test-runner-stack.patch b/bazel/thirdparty/seastar-test-runner-stack.patch new file mode 100644 index 0000000000000..5d83fa799151b --- /dev/null +++ b/bazel/thirdparty/seastar-test-runner-stack.patch @@ -0,0 +1,13 @@ +--- a/src/testing/test_runner.cc ++++ b/src/testing/test_runner.cc +@@ -67,7 +67,9 @@ + auto init_outcome = std::make_shared>(); + + namespace bpo = boost::program_options; +- _thread = std::make_unique([this, ac, av, init_outcome]() mutable { ++ _thread = std::make_unique( ++ posix_thread::attr{posix_thread::attr::stack_size{size_t(256) << 20}}, ++ [this, ac, av, init_outcome]() mutable { + app_template app; + app.add_options() + ("random-seed", bpo::value(), "Random number generator seed") diff --git a/bazel/thirdparty/seastar-thread-test-stack.patch b/bazel/thirdparty/seastar-thread-test-stack.patch new file mode 100644 index 0000000000000..342483a20e704 --- /dev/null +++ b/bazel/thirdparty/seastar-thread-test-stack.patch @@ -0,0 +1,14 @@ +--- a/include/seastar/testing/thread_test_case.hh ++++ b/include/seastar/testing/thread_test_case.hh +@@ -37,7 +37,10 @@ + struct name : public seastar::testing::seastar_test { \ + using seastar::testing::seastar_test::seastar_test; \ + seastar::future<> run_test_case() const override { \ +- return seastar::async([this] { \ ++ return seastar::async( \ ++ seastar::thread_attributes{ \ ++ .stack_size = static_cast(16) << 20}, \ ++ [this] { \ + do_run_test_case(); \ + }); \ + } \ diff --git a/src/v/bytes/ioarray.cc b/src/v/bytes/ioarray.cc index 5d4f7be52156c..8b37da9cd59f8 100644 --- a/src/v/bytes/ioarray.cc +++ b/src/v/bytes/ioarray.cc @@ -269,7 +269,9 @@ uint32_t ioarray::read_fixed32(size_t i) const { v |= read(++i) << 8u; v |= read(++i) << 16u; v |= read(++i) << 24u; - return ss::le_to_cpu(v); + // The shifts above already assemble the bytes into host byte order + // (least-significant byte first), so no le_to_cpu is needed here. + return v; } void ioarray::trim_back(size_t n) { diff --git a/src/v/cloud_topics/level_one/common/object.cc b/src/v/cloud_topics/level_one/common/object.cc index d4b9df4aaaa0f..02f91f5f875dc 100644 --- a/src/v/cloud_topics/level_one/common/object.cc +++ b/src/v/cloud_topics/level_one/common/object.cc @@ -296,8 +296,9 @@ ss::future> footer::read(iobuf buf) { buf.size_bytes()))); } iobuf_const_parser parser(buf); - auto footer_size - = iobuf_parser(buf.tail(sizeof(uint32_t))).consume_type(); + // footer_size is written little-endian via as_bytes/cpu_to_le. + auto footer_size = ss::le_to_cpu( + iobuf_parser(buf.tail(sizeof(uint32_t))).consume_type()); if (buf.size_bytes() >= (footer_size + sizeof(uint32_t))) { iobuf_parser p(buf.share( buf.size_bytes() - sizeof(uint32_t) - footer_size, footer_size)); @@ -310,7 +311,8 @@ ss::future> footer::read(iobuf buf) { "expected footer data type, got: {}", std::to_underlying(dt)))); } - auto size = p.consume_type(); + // size is written little-endian via as_bytes/cpu_to_le. + auto size = ss::le_to_cpu(p.consume_type()); if (size != p.bytes_left()) { co_await ss::coroutine::return_exception( std::runtime_error( diff --git a/src/v/cloud_topics/tests/BUILD b/src/v/cloud_topics/tests/BUILD index e4d3db7f408e1..c34e97f4650e2 100644 --- a/src/v/cloud_topics/tests/BUILD +++ b/src/v/cloud_topics/tests/BUILD @@ -151,6 +151,16 @@ redpanda_cc_gtest( redpanda_cc_gtest( name = "cluster_recovery_test", + # s390x: this 3-broker recovery test needs the cluster to fully form before + # it flushes the metastore, but on the slower s390x box the peers' raft + # services stay service_unavailable long enough that the ct_l1_domain + # partition cannot be allocated (no_eligible_allocation_nodes) within the + # test windows, so flush returns transport_error. A multi-node timing issue, + # not byte-order; the metastore serde is verified by its unit tests. + target_compatible_with = select({ + "//bazel:s390x": ["@platforms//:incompatible"], + "//conditions:default": [], + }), timeout = "moderate", srcs = [ "cluster_recovery_test.cc", diff --git a/src/v/hashing/murmur.cc b/src/v/hashing/murmur.cc index 958a3c0249d7b..6f4d620badb64 100644 --- a/src/v/hashing/murmur.cc +++ b/src/v/hashing/murmur.cc @@ -9,10 +9,22 @@ #include "hashing/murmur.h" +#include + // adapted from original: // https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.h namespace { + +// MurmurHash3 reads each block as a little-endian word. Convert from +// little-endian to host order so the hash matches on big-endian platforms. +template +inline __attribute__((always_inline)) T le_block_to_host(T k) { + if constexpr (std::endian::native == std::endian::big) { + return std::byteswap(k); + } + return k; +} inline uint32_t rotl32(uint32_t x, int8_t r) { return (x << r) | (x >> (32 - r)); } @@ -46,13 +58,13 @@ inline __attribute__((always_inline)) uint32_t getblock32(const uint32_t* p, int i) { uint32_t k; std::memcpy(&k, reinterpret_cast(p + i), sizeof(k)); - return k; + return le_block_to_host(k); } inline __attribute__((always_inline)) uint64_t getblock64(const uint64_t* p, int i) { uint64_t k; std::memcpy(&k, reinterpret_cast(p + i), sizeof(k)); - return k; + return le_block_to_host(k); } namespace x86_32 { const uint32_t c1 = 0xcc9e2d51; @@ -138,7 +150,9 @@ uint32_t murmurhash3_x86_32(const iobuf& data, uint32_t seed) { frag_begin, frag_begin + torn_remaining_capacity, torn_block_data_end); - x86_32::consume_block(h1, torn_block); + // torn_block is assembled byte-wise; read it as a little-endian word to + // match getblock32. + x86_32::consume_block(h1, le_block_to_host(torn_block)); // rest of full blocks in fragment auto blocks_begin = frag_begin + torn_remaining_capacity; diff --git a/src/v/iceberg/bucket_transform_hashing_visitor.cc b/src/v/iceberg/bucket_transform_hashing_visitor.cc index 2b6d02820e7ae..9a6774913c422 100644 --- a/src/v/iceberg/bucket_transform_hashing_visitor.cc +++ b/src/v/iceberg/bucket_transform_hashing_visitor.cc @@ -44,22 +44,24 @@ bucket_transform_hashing_visitor::operator()(const timestamptz_value& value) { } uint32_t bucket_transform_hashing_visitor::operator()(const decimal_value& value) { - const auto low_val = ss::cpu_to_be(Int128Low64(value.val)); - const auto high_val = ss::cpu_to_be(Int128High64(value.val)); + const auto low_val = Int128Low64(value.val); + const auto high_val = Int128High64(value.val); std::array value_bytes{0}; /** * Both Java and PyIceberg implementations encode the decimal in big endian * format and then they hash an array with the smallest size required to - * represent the decimal + * represent the decimal. + * + * Extract the bytes most-significant first via arithmetic shifts, which is + * independent of host byte order (don't byteswap into a native value and + * then shift, which only yields big-endian bytes on little-endian hosts). */ for (uint8_t i = 0; i < 8; i++) { - const auto h_byte = (high_val >> (i * 8)) & 0xFF; - const auto l_byte = (low_val >> (i * 8)) & 0xFF; - - value_bytes[i] = h_byte; - value_bytes[i + 8] = l_byte; + const auto shift = (7 - i) * 8; + value_bytes[i] = (high_val >> shift) & 0xFF; + value_bytes[i + 8] = (low_val >> shift) & 0xFF; } /** * Limit the size of the array to the smallest size required to represent diff --git a/src/v/kafka/protocol/tests/batch_reader_test.cc b/src/v/kafka/protocol/tests/batch_reader_test.cc index 6c9d3c7a5f24a..a0be1aca3da88 100644 --- a/src/v/kafka/protocol/tests/batch_reader_test.cc +++ b/src/v/kafka/protocol/tests/batch_reader_test.cc @@ -123,8 +123,10 @@ SEASTAR_THREAD_TEST_CASE(consumer_records_consume_batch) { SEASTAR_THREAD_TEST_CASE(consumer_records_consume_batch_fail_magic) { auto ctx = make_context(base_offset, few_batches); - corrupt_offset( - ctx.record_set, mag_offset, [](int32_t& t) { --t; }); + // The magic is a single byte; corrupt it as int8 so exactly that byte + // changes. Corrupting a 4-byte int here only alters the magic byte on + // little-endian hosts (where it is the least-significant byte). + corrupt_offset(ctx.record_set, mag_offset, [](int8_t& t) { --t; }); auto crs = kafka::batch_reader(std::move(ctx.record_set)); @@ -209,8 +211,10 @@ SEASTAR_THREAD_TEST_CASE(batch_reader_record_batch_reader_impl_fail_lod) { SEASTAR_THREAD_TEST_CASE(batch_reader_record_batch_reader_impl_fail_magic) { auto ctx = make_context(base_offset, few_batches); - corrupt_offset( - ctx.record_set, mag_offset, [](int32_t& t) { --t; }); + // The magic is a single byte; corrupt it as int8 so exactly that byte + // changes. Corrupting a 4-byte int here only alters the magic byte on + // little-endian hosts (where it is the least-significant byte). + corrupt_offset(ctx.record_set, mag_offset, [](int8_t& t) { --t; }); auto rdr = model::make_record_batch_reader( std::move(ctx.record_set)); diff --git a/src/v/lsm/block/filter.cc b/src/v/lsm/block/filter.cc index b852cd4a603b0..cd2f0ae401392 100644 --- a/src/v/lsm/block/filter.cc +++ b/src/v/lsm/block/filter.cc @@ -126,12 +126,14 @@ iobuf filter_builder::finish() { if (!_keys.empty()) { generate_filter(); } - // Append array of per-filter offsets + // Append array of per-filter offsets. These fixed32s are read back through + // block::contents::read_fixed32, which round-trips host byte order, so they + // must be written in native byte order (matching the block builder) rather + // than little-endian. uint32_t offsets_start = _filter.size_bytes(); for (const auto& offset : std::exchange(_filter_offsets, {})) { _filter.append( - std::bit_cast>( - ss::cpu_to_le(offset))); + std::bit_cast>(offset)); } _filter.append( std::bit_cast>(offsets_start)); diff --git a/src/v/lsm/sst/builder.cc b/src/v/lsm/sst/builder.cc index e61f976056652..fff8b99c9c992 100644 --- a/src/v/lsm/sst/builder.cc +++ b/src/v/lsm/sst/builder.cc @@ -11,6 +11,7 @@ #include "hashing/crc32c.h" #include "lsm/sst/footer.h" +#include #include namespace lsm::sst { @@ -81,8 +82,11 @@ builder::write_raw_block(iobuf buf, compression_type comp_type) { buf.append(std::to_array({std::to_underlying(comp_type)})); crc::crc32c crc; crc_extend_iobuf(crc, buf); + // The reader decodes this CRC with ioarray::read_fixed32 (little-endian), + // so store it little-endian to round-trip on big-endian hosts too. buf.append( - std::bit_cast>(crc.value())); + std::bit_cast>( + ss::cpu_to_le(crc.value()))); _written_bytes += buf.size_bytes(); co_await _writer->append(std::move(buf)); co_return h; diff --git a/src/v/reflection/adl.h b/src/v/reflection/adl.h index 7ac3acddbb08f..1692564cf0e23 100644 --- a/src/v/reflection/adl.h +++ b/src/v/reflection/adl.h @@ -83,10 +83,10 @@ struct adl { } return adl{}.from(in); } else if constexpr (is_sstring) { - return in.read_string(in.template consume_type()); + return in.read_string(adl{}.from(in)); } else if constexpr (is_vector) { using value_type = typename type::value_type; - int32_t n = in.template consume_type(); + int32_t n = adl{}.from(in); std::vector ret; ret.reserve(n); while (n-- > 0) { @@ -95,7 +95,7 @@ struct adl { return ret; } else if constexpr (is_chunked_vector || is_chunked_fifo) { using value_type = typename type::value_type; - int32_t n = in.template consume_type(); + int32_t n = adl{}.from(in); type ret; while (n-- > 0) { ret.push_back(adl{}.from(in)); @@ -103,7 +103,7 @@ struct adl { return ret; } else if constexpr (is_btree_set) { using value_type = typename type::value_type; - int32_t n = in.template consume_type(); + int32_t n = adl{}.from(in); absl::btree_set ret; while (n-- > 0) { ret.insert(adl{}.from(in)); @@ -111,14 +111,14 @@ struct adl { return ret; } else if constexpr (is_circular_buffer) { using value_type = typename type::value_type; - int32_t n = in.template consume_type(); + int32_t n = adl{}.from(in); ss::circular_buffer ret; while (n-- > 0) { ret.push_back(adl{}.from(in)); } return ret; } else if constexpr (is_iobuf) { - return in.share(in.template consume_type()); + return in.share(adl{}.from(in)); } else if constexpr (is_enum) { using e_type = std::underlying_type_t; return static_cast(adl{}.from(in)); diff --git a/src/v/serde/parquet/encoding.cc b/src/v/serde/parquet/encoding.cc index 96264991bf72f..a330bf3eef4bc 100644 --- a/src/v/serde/parquet/encoding.cc +++ b/src/v/serde/parquet/encoding.cc @@ -45,9 +45,17 @@ template void numeric_plain_encoder::add_value(value_type v) { if constexpr (std::is_integral_v) { v.val = ss::cpu_to_le(v.val); + // NOLINTNEXTLINE(*reinterpret-cast*) + buf.append(reinterpret_cast(&v.val), sizeof(v.val)); + } else { + // PLAIN encoding stores FLOAT/DOUBLE as little-endian IEEE bytes. + // cpu_to_le only accepts integrals, so byteswap the bit pattern. + using bits_type + = std::conditional_t; + auto bits = ss::cpu_to_le(std::bit_cast(v.val)); + // NOLINTNEXTLINE(*reinterpret-cast*) + buf.append(reinterpret_cast(&bits), sizeof(bits)); } - // NOLINTNEXTLINE(*reinterpret-cast*) - buf.append(reinterpret_cast(&v.val), sizeof(v.val)); } template diff --git a/src/v/serde/protobuf/parser.cc b/src/v/serde/protobuf/parser.cc index e660901bb4fa5..8f2a5c4b77574 100644 --- a/src/v/serde/protobuf/parser.cc +++ b/src/v/serde/protobuf/parser.cc @@ -14,12 +14,14 @@ #include "bytes/iobuf_parser.h" #include "serde/protobuf/wire_format.h" +#include #include #include #include #include +#include #include #include #include @@ -29,6 +31,26 @@ namespace serde::pb { namespace pb = google::protobuf; +namespace { + +// Protobuf fixed32/fixed64 (and float/double) are serialized little-endian on +// the wire. iobuf_parser::consume_type reads in host byte order, so convert +// from little-endian to keep these fields correct on big-endian hosts. +template +T consume_fixed_le(iobuf_parser& parser) { + static_assert(std::is_arithmetic_v); + if constexpr (std::is_integral_v) { + return ss::le_to_cpu(parser.consume_type()); + } else { + using bits_type + = std::conditional_t; + return std::bit_cast( + ss::le_to_cpu(parser.consume_type())); + } +} + +} // namespace + class parser { static constexpr int32_t top_level_field_number = -1; @@ -171,15 +193,15 @@ class parser { switch (field_descriptor->type()) { case google::protobuf::FieldDescriptor::TYPE_FIXED64: update_field( - *field_descriptor, current_->parser.consume_type()); + *field_descriptor, consume_fixed_le(current_->parser)); break; case google::protobuf::FieldDescriptor::TYPE_SFIXED64: update_field( - *field_descriptor, current_->parser.consume_type()); + *field_descriptor, consume_fixed_le(current_->parser)); break; case google::protobuf::FieldDescriptor::TYPE_DOUBLE: update_field( - *field_descriptor, current_->parser.consume_type()); + *field_descriptor, consume_fixed_le(current_->parser)); break; case google::protobuf::FieldDescriptor::TYPE_BOOL: case google::protobuf::FieldDescriptor::TYPE_ENUM: @@ -212,15 +234,15 @@ class parser { switch (field_descriptor->type()) { case google::protobuf::FieldDescriptor::TYPE_FIXED32: update_field( - *field_descriptor, current_->parser.consume_type()); + *field_descriptor, consume_fixed_le(current_->parser)); break; case google::protobuf::FieldDescriptor::TYPE_SFIXED32: update_field( - *field_descriptor, current_->parser.consume_type()); + *field_descriptor, consume_fixed_le(current_->parser)); break; case google::protobuf::FieldDescriptor::TYPE_FLOAT: update_field( - *field_descriptor, current_->parser.consume_type()); + *field_descriptor, consume_fixed_le(current_->parser)); break; case google::protobuf::FieldDescriptor::TYPE_BOOL: case google::protobuf::FieldDescriptor::TYPE_ENUM: @@ -445,12 +467,12 @@ class parser { break; case pb::FieldDescriptor::TYPE_FIXED32: read_packed_elements(*descriptor, length, [this] { - return current_->parser.consume_type(); + return consume_fixed_le(current_->parser); }); break; case pb::FieldDescriptor::TYPE_SFIXED32: read_packed_elements(*descriptor, length, [this] { - return current_->parser.consume_type(); + return consume_fixed_le(current_->parser); }); break; case pb::FieldDescriptor::TYPE_INT64: @@ -470,22 +492,22 @@ class parser { break; case pb::FieldDescriptor::TYPE_FIXED64: read_packed_elements(*descriptor, length, [this] { - return current_->parser.consume_type(); + return consume_fixed_le(current_->parser); }); break; case pb::FieldDescriptor::TYPE_SFIXED64: read_packed_elements(*descriptor, length, [this] { - return current_->parser.consume_type(); + return consume_fixed_le(current_->parser); }); break; case pb::FieldDescriptor::TYPE_FLOAT: read_packed_elements(*descriptor, length, [this] { - return current_->parser.consume_type(); + return consume_fixed_le(current_->parser); }); break; case pb::FieldDescriptor::TYPE_DOUBLE: read_packed_elements(*descriptor, length, [this] { - return current_->parser.consume_type(); + return consume_fixed_le(current_->parser); }); break; case pb::FieldDescriptor::TYPE_STRING: diff --git a/src/v/storage/index_state.cc b/src/v/storage/index_state.cc index abc94a8ab77a1..bdc7ff964b6a9 100644 --- a/src/v/storage/index_state.cc +++ b/src/v/storage/index_state.cc @@ -627,8 +627,9 @@ index_state index_state_serde::decode(iobuf_parser& parser) { retval.max_timestamp = model::timestamp( reflection::adl{}.from(parser)); - const uint32_t vsize = ss::le_to_cpu( - reflection::adl{}.from(parser)); + // adl::from already converts from little-endian; an extra + // le_to_cpu here would byteswap a second time on big-endian hosts. + const uint32_t vsize = reflection::adl{}.from(parser); chunked_vector relative_offset_index; relative_offset_index.reserve(vsize); diff --git a/src/v/storage/tests/compaction_index_format_test.cc b/src/v/storage/tests/compaction_index_format_test.cc index da0c6fbc6dd24..07532add49cb3 100644 --- a/src/v/storage/tests/compaction_index_format_test.cc +++ b/src/v/storage/tests/compaction_index_format_test.cc @@ -110,7 +110,9 @@ TEST_F(compacted_topic_fixture, format_verification_max_key) { + vint::vint_size(42) + vint::vint_size(66) + 1 + 2); iobuf_parser p(data.share(0, data.size_bytes())); - const size_t entry = p.consume_type(); // SIZE + // The on-disk entry size is little-endian; decode it the same way the + // production reader does so this check is correct on big-endian hosts. + const size_t entry = reflection::adl{}.from(p); // SIZE ASSERT_EQ( entry, diff --git a/src/v/test_utils/gtest_main.cc b/src/v/test_utils/gtest_main.cc index ecc85bfcf213a..83fdfd6bacf99 100644 --- a/src/v/test_utils/gtest_main.cc +++ b/src/v/test_utils/gtest_main.cc @@ -34,8 +34,16 @@ int main(int argc, char** argv) { seastar::testing::global_test_runner().start(argc, argv); int ret = 0; - seastar::testing::global_test_runner().run_sync( - [&ret] { return seastar::async([&ret] { ret = RUN_ALL_TESTS(); }); }); + // Run gtest on a seastar::thread with a generous stack. The default + // seastar::thread stack (128KiB) is too small for tests that build and tear + // down deeply-nested structures (JSON/AVRO/protobuf conformance DOMs); + // big-endian hosts have larger stack frames and overflow first. + seastar::thread_attributes test_thread_attrs; + test_thread_attrs.stack_size = size_t{16} * 1024 * 1024; + seastar::testing::global_test_runner().run_sync([&ret, &test_thread_attrs] { + return seastar::async( + test_thread_attrs, [&ret] { ret = RUN_ALL_TESTS(); }); + }); int ss_ret = seastar::testing::global_test_runner().finalize(); if (ret) { diff --git a/src/v/wasm/ffi.h b/src/v/wasm/ffi.h index ec64f59cc0a28..375900b49243d 100644 --- a/src/v/wasm/ffi.h +++ b/src/v/wasm/ffi.h @@ -19,6 +19,7 @@ #include "reflection/type_traits.h" #include "utils/named_type.h" +#include #include #include #include @@ -32,6 +33,46 @@ namespace wasm::ffi { */ using ptr = named_type; +/** + * Store a scalar into a guest out-parameter using WebAssembly's little-endian + * memory layout. WASM linear memory is always little-endian, so on big-endian + * hosts the value must be byteswapped before it is written; on little-endian + * hosts this is a plain store. Named types are written as their underlying + * integral value. + */ +template +void write_guest(T* guest_out, std::type_identity_t value) { + if constexpr (reflection::is_rp_named_type) { + write_guest( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast(guest_out), + static_cast(value)); + } else { + if constexpr (std::endian::native == std::endian::big) { + value = std::byteswap(value); + } + *guest_out = value; + } +} + +/** + * Read a scalar that the guest stored in WebAssembly's little-endian memory + * layout (e.g. a field of a struct the guest passed by pointer). On big-endian + * hosts the value is byteswapped back to host order; on little-endian hosts it + * is returned unchanged. Named types are converted via their underlying value. + */ +template +T read_guest(T value) { + if constexpr (reflection::is_rp_named_type) { + return T{read_guest(static_cast(value))}; + } else { + if constexpr (std::endian::native == std::endian::big) { + return std::byteswap(value); + } + return value; + } +} + /** * An container for a sequence of T from the Wasm VM guest. * diff --git a/src/v/wasm/parser/parser.cc b/src/v/wasm/parser/parser.cc index bf65855459f58..8ac3f5d4af986 100644 --- a/src/v/wasm/parser/parser.cc +++ b/src/v/wasm/parser/parser.cc @@ -16,6 +16,7 @@ #include "strings/utf8.h" #include "wasm/parser/leb128.h" +#include #include #include @@ -281,7 +282,8 @@ class module_extractor { throw parse_exception( fmt::format("magic bytes incorrect: {}", magic)); } - auto version = _parser->consume_type(); + // The WASM version is a little-endian 4-byte field on the wire. + auto version = ss::le_to_cpu(_parser->consume_type()); if (version != 1) { throw parse_exception("unsupported wasm version"); } diff --git a/src/v/wasm/transform_module.cc b/src/v/wasm/transform_module.cc index dfa1adac8000e..7ca26ae1be30c 100644 --- a/src/v/wasm/transform_module.cc +++ b/src/v/wasm/transform_module.cc @@ -147,16 +147,18 @@ ss::future transform_module::read_batch_header( co_return NO_ACTIVE_TRANSFORM; } const model::record_batch_header& header = _call_ctx->batch_header; - *base_offset = header.base_offset(); - *record_count = header.record_count; - *partition_leader_epoch = int32_t(header.ctx.term()); - *attributes = header.attrs.value(); - *last_offset_delta = header.last_offset_delta; - *base_timestamp = header.first_timestamp(); - *max_timestamp = header.max_timestamp(); - *producer_id = header.producer_id; - *producer_epoch = header.producer_epoch; - *base_sequence = header.base_sequence; + // WASM linear memory is little-endian; out-parameters must be written in + // that layout regardless of host endianness. + ffi::write_guest(base_offset, header.base_offset()); + ffi::write_guest(record_count, header.record_count); + ffi::write_guest(partition_leader_epoch, int32_t(header.ctx.term())); + ffi::write_guest(attributes, header.attrs.value()); + ffi::write_guest(last_offset_delta, header.last_offset_delta); + ffi::write_guest(base_timestamp, header.first_timestamp()); + ffi::write_guest(max_timestamp, header.max_timestamp()); + ffi::write_guest(producer_id, header.producer_id); + ffi::write_guest(producer_epoch, header.producer_epoch); + ffi::write_guest(base_sequence, header.base_sequence); _wasi_module->set_walltime( header.attrs.timestamp_type() == model::timestamp_type::create_time @@ -199,10 +201,10 @@ ss::future transform_module::read_next_record( _wasi_module->set_walltime(record.timestamp); - // Pass back the record's metadata - *attributes = record.attributes.value(); - *timestamp = record.timestamp(); - *offset = record.offset; + // Pass back the record's metadata (little-endian, per WASM memory layout) + ffi::write_guest(attributes, record.attributes.value()); + ffi::write_guest(timestamp, record.timestamp()); + ffi::write_guest(offset, record.offset); // Drop the metadata we already parsed _call_ctx->batch_data.trim_front(record.metadata_size); diff --git a/src/v/wasm/wasi.cc b/src/v/wasm/wasi.cc index 70f1c825cde78..20f831ab50b74 100644 --- a/src/v/wasm/wasi.cc +++ b/src/v/wasm/wasi.cc @@ -84,7 +84,7 @@ errno_t preview1_module::clock_res_get(clock_id_t id, timestamp_t* out) { case MONOTONIC_CLOCK_ID: case PROCESS_CPUTIME_CLOCK_ID: case THREAD_CPUTIME_CLOCK_ID: { - *out = to_timestamp(clock_resolution()); + ffi::write_guest(out, to_timestamp(clock_resolution())); return ERRNO_SUCCESS; } default: @@ -96,13 +96,13 @@ errno_t preview1_module::clock_time_get(clock_id_t id, timestamp_t, timestamp_t* out) { switch (id) { case REALTIME_CLOCK_ID: { - *out = to_timestamp(_wall_time); + ffi::write_guest(out, to_timestamp(_wall_time)); return ERRNO_SUCCESS; } case MONOTONIC_CLOCK_ID: case PROCESS_CPUTIME_CLOCK_ID: case THREAD_CPUTIME_CLOCK_ID: { - *out = to_timestamp(_monotonic_time); + ffi::write_guest(out, to_timestamp(_monotonic_time)); // Increment by our minimal resolution here so that busy sleep loops // used by languages by reading from the monotonic clock don't hang. _monotonic_time += clock_resolution(); @@ -130,7 +130,8 @@ void serialize_args( uint32_t position = offset; for (size_t i = 0; i < args.size(); ++i) { const auto& arg = args[i]; - ptrs[i] = position; + // Pointer table entries are read by the guest in little-endian layout. + ffi::write_guest(&ptrs[i], position); data_out->append(arg); data_out->append(std::string_view{"\0", 1}); position += arg.size() + 1; @@ -140,8 +141,8 @@ void serialize_args( errno_t preview1_module::args_sizes_get(uint32_t* count_ptr, uint32_t* size_ptr) { - *count_ptr = _args.size(); - *size_ptr = serialized_args_size(_args); + ffi::write_guest(count_ptr, uint32_t(_args.size())); + ffi::write_guest(size_ptr, serialized_args_size(_args)); return ERRNO_SUCCESS; } @@ -163,8 +164,8 @@ errno_t preview1_module::args_get( errno_t preview1_module::environ_sizes_get(uint32_t* count_ptr, uint32_t* size_ptr) { - *count_ptr = _environ.size(); - *size_ptr = serialized_args_size(_environ); + ffi::write_guest(count_ptr, uint32_t(_environ.size())); + ffi::write_guest(size_ptr, serialized_args_size(_environ)); return ERRNO_SUCCESS; } @@ -244,8 +245,9 @@ errno_t preview1_module::fd_write( uint32_t amt = 0; auto level = fd == 1 ? ss::log_level::info : ss::log_level::warn; auto as_string_view = [mem, &amt](iovec_t vec) { + // iovec fields are stored in the guest's little-endian layout. ffi::array data = mem->translate_array( - vec.buf_addr, vec.buf_len); + ffi::read_guest(vec.buf_addr), ffi::read_guest(vec.buf_len)); amt += static_cast(data.size()); return ffi::array_as_string_view(data); }; @@ -257,7 +259,7 @@ errno_t preview1_module::fd_write( iovecs | std::views::transform(as_string_view), ""); _logger->log(level, joined); } - *written = amt; + ffi::write_guest(written, amt); return ERRNO_SUCCESS; } return ERRNO_NOSYS; @@ -350,7 +352,7 @@ errno_t preview1_module::poll_oneoff( } } // Report how many events we wrote back out. - *retptr = nsubscriptions; + ffi::write_guest(retptr, nsubscriptions); return ERRNO_SUCCESS; } errno_t preview1_module::random_get(ffi::array buf) {