tests: Add the cargo-fuzz suite and all of its supporting changes

Squashes every fuzzing-infrastructure change in this branch into one commit,
separate from the individual bug fixes the fuzzing surfaced (one commit each):

* The cargo-fuzz crates under `src/*/fuzz` — targets, seed corpora,
  dictionaries, and `prepare-corpus.sh` scripts — covering the SQL parser /
  pretty-printer, repr (strconv, jsonb, Row codec/proto, arithmetic oracles),
  the expr optimizer transforms, Avro/Protobuf/CSV/pgwire/pgcopy decoders,
  pgrepr/pgtz, the upsert state machine, persist durable-state decode, and the
  proto round-trips across storage-types/persist/catalog/external table descs.
* The harness and runner wiring: `--profile fruitful`, `--jobs auto`,
  per-crate sharding, artifact-based crash detection, `.repro.txt` sidecars,
  a time-capped post-fuzz corpus minimize/upload, and the auto-generated
  `buf.yaml` fuzz-crate excludes.
* CI: move cargo-fuzz from nightly to release qualification (24h, 48-core).
* The production-side enablement the targets require: the `fuzzing` Cargo
  feature and the `#[doc(hidden)]` / `cfg`-gated re-exports that expose upsert,
  persist-client, and pgwire internals to the fuzz crates.
* The macOS build fix those exports necessitated: switching the affected
  storage `Stream::inspect` calls to `InspectCore::inspect_container`, which
  avoids the objc2-driven trait-solver overflow the `Inspect` bound triggers
  on macOS.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: def-

.gitignore

@@ -68,3 +68,4 @@ uv.lock
 /plots/
 /test/testdrive/types.parquet*
 /test/mz-deploy/**/target/
+target-fuzz/

Cargo.toml

@@ -260,6 +260,11 @@ exclude = [
     "misc/wasm/*",
     # Ignore any Rust dependencies that python packages might pull in.
     "misc/python/venv/*",
+    # The `src/*/fuzz` cargo-fuzz crates need no entry here: each sets
+    # `package.workspace = "../../../test/cargo-fuzz"`, attaching it to the fuzz
+    # workspace, and a crate nested under a workspace member is never
+    # auto-included in the root workspace. They build on a nightly toolchain
+    # (libFuzzer) via `cargo +nightly fuzz run ...` or `ci/test/cargo-fuzz.sh`.
 ]
 
 # Use Cargo's new feature resolver, which can handle target-specific features,

ci/builder/Dockerfile

@@ -247,6 +247,14 @@ RUN mkdir rust \
     && cargo install --root /usr/local --version "=0.1.61" --locked --features=vendored-openssl cargo-udeps \
     && cargo install --root /usr/local --version "=0.4.0" --locked cargo-binutils \
     && cargo install --root /usr/local --version "=0.13.1" --locked wasm-pack \
+    && if [ "$RUST_VERSION" = "nightly" ]; then \
+           # NOTE: no --locked, unlike the installs above. cargo-fuzz 0.13.1's \
+           # bundled Cargo.lock pins deps that fail on the pinned nightly \
+           # (yanked futures-util/zip, plus a crate using the perma-unstable \
+           # `rustc_layout_scalar_valid_range_*` attribute). Let cargo resolve \
+           # compatible versions instead. \
+           cargo install --root /usr/local --version "=0.13.1" cargo-fuzz; \
+       fi \
     && rm -rf /cargo/registry /cargo/git
 
 # Shims for sanitizers

ci/plugins/mzcompose/hooks/command

@@ -353,6 +353,7 @@ cleanup() {
   killall -9 -q clusterd || true # There might be remaining processes from a cargo-test run
 
   if [ ! -s services.log ] \
+    && [ "$BUILDKITE_LABEL" != ":rust: cargo-fuzz" ] \
     && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of persist" ] \
     && [ "$BUILDKITE_LABEL" != "Long single-node Maelstrom coverage of persist" ] \
     && [ "$BUILDKITE_LABEL" != "Maelstrom coverage of txn-wal" ] \

ci/plugins/mzcompose/plugin.yml

@@ -21,5 +21,7 @@ configuration:
       type: string
     composition:
       type: string
+    ci_builder:
+      type: string
   required: ["composition"]
   additionalProperties: false

ci/release-qualification/pipeline.template.yml

@@ -254,6 +254,29 @@ steps:
             composition: sqlsmith
             args: [--max-joins=15, --explain-only, --runtime=6000]
 
+  - id: cargo-fuzz
+    label: ":rust: cargo-fuzz"
+    depends_on: []
+    timeout_in_minutes: 1440
+    agents:
+      queue: hetzner-x86-64-dedi-48cpu-192gb
+    sanitizer: skip
+    plugins:
+      - ./ci/plugins/mzcompose:
+          composition: cargo-fuzz
+          ci_builder: nightly
+          args:
+            - --profile=fruitful
+            - --max-seconds=86400
+            - --wall-budget=84600
+            # Step hard-times out at 1440min (86400s). --wall-budget ends fuzzing
+            # at 84600s, leaving 1800s; cap minimize at 1200s so the corpus
+            # upload has ~600s of headroom before the kill.
+            - --minimize-timeout=1200
+            - --corpus-sync
+    artifact_paths:
+      - src/*/fuzz/artifacts/**/*
+
   - id: test-preflight-check-rollback
     label: Test with preflight check and rollback
     depends_on: []

ci/test/lint-buf/generate-buf-config.py

@@ -18,6 +18,11 @@
 
 SOURCE_DIR = "src/"
 PROTO_FILE_GLOB = f"{SOURCE_DIR}**/*.proto"
+# Each fuzz crate (`src/<crate>/fuzz`) is its own cargo `[workspace]`, so
+# building it standalone creates `src/<crate>/fuzz/target/`. Build-script deps
+# (e.g. `protobuf-src`) extract `.proto` files into that tree, which buf must
+# not scan. We exclude every fuzz crate's `target/` instead of hand-listing them.
+FUZZ_CRATE_GLOB = f"{SOURCE_DIR}*/fuzz"
 
 GENERATION_COMMENT = "File generated by generate-buf-config.py - DO NOT EDIT"
 BUF_INSTRUCTION_PREFIX = "// buf breaking:"
@@ -37,6 +42,11 @@ def is_ignore(self) -> bool:
 def collect_proto_files() -> list[ProtoFile]:
     print(f"Working dir: {os.getcwd()}")
     proto_file_paths = glob.glob(PROTO_FILE_GLOB, recursive=True)
+    # Filter out build artifacts: each fuzz crate is its own `[workspace]`, so
+    # building it standalone creates `src/<crate>/fuzz/target/`. A build-script
+    # dep (`protobuf-src`) vendors protoc's bundled `.proto` files (Google's
+    # well-known types) into that tree, which is not source we want buf to scan.
+    proto_file_paths = [p for p in proto_file_paths if "/target/" not in p]
     return [ProtoFile(path) for path in proto_file_paths]
 
 
@@ -82,6 +92,20 @@ def generate_buf_ignore_section(ignored_files: list[ProtoFile]) -> str:
     return "\n".join(ignore_entry_lines).strip()
 
 
+def generate_fuzz_target_excludes() -> str:
+    fuzz_crate_dirs = sorted(d for d in glob.glob(FUZZ_CRATE_GLOB) if os.path.isdir(d))
+    exclude_lines = []
+    for fuzz_dir in fuzz_crate_dirs:
+        # e.g. "src/transform/fuzz" -> "transform/fuzz/target"
+        relative_path = fuzz_dir.removeprefix(SOURCE_DIR)
+        exclude_lines.append(f"    - {relative_path}/target")
+
+    if len(exclude_lines) == 0:
+        exclude_lines.append("    # none")
+
+    return "\n".join(exclude_lines).strip()
+
+
 def write_buf_configuration(
     template_path: str, target_path: str, ignored_files: list[ProtoFile]
 ) -> None:
@@ -92,6 +116,9 @@ def write_buf_configuration(
         content = content.replace(
             "${ignore-entries}", generate_buf_ignore_section(ignored_files)
         )
+        content = content.replace(
+            "${fuzz-target-excludes}", generate_fuzz_target_excludes()
+        )
 
     with open(target_path, "w") as output_file:
         output_file.write(content)

misc/python/materialize/cli/ci_annotate_errors.py

@@ -100,6 +100,9 @@
     # \s\S is any character including newlines, so this matches multiline strings
     # non-greedy using ? so that we don't match all the result comparison issues into one block
     | ----------\ RESULT\ COMPARISON\ ISSUE\ START\ ----------[\s\S]*?----------\ RESULT\ COMPARISON\ ISSUE\ END\ ------------
+    # cargo-fuzz crash, emitted by the cargo-fuzz mzcompose runner (one block
+    # per failing target, with the crash input and a reproduce command)
+    | ----------\ CARGO-FUZZ\ FAILURE\ START\ ----------[\s\S]*?----------\ CARGO-FUZZ\ FAILURE\ END\ ----------
     # output consistency tests
     # | possibly\ invalid\ operation\ specification # disabled
     # for miri test summary

src/avro/fuzz/.gitignore

@@ -0,0 +1,5 @@
+target/
+corpus/
+artifacts/
+coverage/
+Cargo.lock

src/avro/fuzz/Cargo.toml

@@ -0,0 +1,43 @@
+# Fuzz crate for mz-avro decoders. Avro bytes arrive from Kafka, so a
+# decoder bug here is a crash/poisoning risk for source ingestion.
+#
+# Excluded from the main workspace because libFuzzer requires nightly Rust.
+# Run via the repo-wide runner: `bin/ci-builder run nightly ci/test/cargo-fuzz.sh`,
+# or locally:
+#     cd src/avro/fuzz
+#     cargo +nightly fuzz run reader_decode -- -max_total_time=60
+
+[package]
+workspace = "../../../test/cargo-fuzz"
+name = "mz-avro-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+mz-avro = { path = ".." }
+
+[[bin]]
+name = "reader_decode"
+path = "fuzz_targets/reader_decode.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "schema_resolve"
+path = "fuzz_targets/schema_resolve.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "avro_schema_parse"
+path = "fuzz_targets/avro_schema_parse.rs"
+test = false
+doc = false
+bench = false