Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/fuzzing-docker-avm-build-private.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ jobs:
secrets: |
github_token=${{ secrets.GITHUB_TOKEN }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=private

- name: Install cosign
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/fuzzing-docker-avm-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ jobs:
build-args: |
COMMIT=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.commit || github.sha }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=public

- name: Install cosign
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/fuzzing-docker-build-private.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ jobs:
secrets: |
github_token=${{ secrets.GITHUB_TOKEN }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=private

- name: Install oras
Expand All @@ -65,7 +65,7 @@ jobs:
docker cp manifest-extract:/home/fuzzer/aztec-packages/barretenberg/cpp/fuzzer_manifest.json ./fuzzer_manifest.json
docker rm manifest-extract
oras attach ghcr.io/aztecprotocol/fuzzing-container-private@${{ steps.build.outputs.digest }} \
--artifact-type application/vnd.aztec.fuzzer-manifest+json \
--artifact-type application/vnd.contfuzzer.manifest+json \
fuzzer_manifest.json

- name: Install cosign
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/fuzzing-docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ jobs:
build-args: |
COMMIT=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.commit || github.sha }}
labels: |
com.aztec.source-repo=${{ github.repository }}
com.aztec.source-branch=${{ github.ref_name }}
com.aztec.commit=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.revision.branch=${{ github.ref_name }}
com.aztec.visibility=public

- name: Install oras
Expand All @@ -59,7 +59,7 @@ jobs:
docker cp manifest-extract:/home/fuzzer/aztec-packages/barretenberg/cpp/fuzzer_manifest.json ./fuzzer_manifest.json
docker rm manifest-extract
oras attach ghcr.io/aztecprotocol/fuzzing-container@${{ steps.build.outputs.digest }} \
--artifact-type application/vnd.aztec.fuzzer-manifest+json \
--artifact-type application/vnd.contfuzzer.manifest+json \
fuzzer_manifest.json

- name: Install cosign
Expand Down
141 changes: 141 additions & 0 deletions barretenberg/cpp/scripts/flatten_and_manifest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Flatten variant binaries into /targets/ and generate a unified v2 manifest.

Reads per-preset manifests (produced by generate_fuzzer_manifest.py via CMake)
to discover binaries, suffixes, and source paths. No variant knowledge is
hardcoded here — it all comes from the manifests.

Usage (from barretenberg/cpp/ during Docker build):
python3 scripts/flatten_and_manifest.py /targets /tmp/fuzzer_manifest.json \
bin-fuzzing/fuzzer_manifest.json \
bin-fuzzing-noasm/fuzzer_manifest.json \
bin-fuzzing-avm/fuzzer_manifest.json \
--copy-only bin-fuzzing-asan/fuzzer_manifest.json \
--copy-only build-fuzzing-cov/bin/fuzzer_manifest.json

Positional manifests: binaries are copied AND added to the unified manifest.
--copy-only manifests: binaries are copied but NOT added (companion builds
like asan/coverage that the entrypoint discovers by suffix convention).
"""

import argparse
import json
import os
import shutil
import sys


def _load_manifest(path: str) -> dict:
with open(path) as f:
return json.load(f)


def _copy_executables(bin_dir: str, dest_dir: str, suffix: str) -> list[str]:
"""Copy executable files from bin_dir to dest_dir/<name><suffix>. Returns names."""
copied = []
if not os.path.isdir(bin_dir):
print(f"Warning: {bin_dir} not found, skipping", file=sys.stderr)
return copied
for name in sorted(os.listdir(bin_dir)):
src = os.path.join(bin_dir, name)
if os.path.isfile(src) and os.access(src, os.X_OK):
shutil.copy2(src, os.path.join(dest_dir, f"{name}{suffix}"))
copied.append(name)
return copied


def main():
parser = argparse.ArgumentParser(description="Flatten binaries and generate v2 manifest")
parser.add_argument("targets_dir", help="Destination for flattened binaries")
parser.add_argument("manifest_out", help="Output path for unified v2 manifest")
parser.add_argument("manifests", nargs="*", help="Per-preset manifests (schedulable)")
parser.add_argument("--copy-only", action="append", default=[], dest="companions",
help="Per-preset manifests for companion builds (copy only, not scheduled)")
parser.add_argument("--cpus", type=int, default=4, help="Default CPUs per target")
parser.add_argument("--memory-gb", type=int, default=8, help="Default memory GB per target")
args = parser.parse_args()

os.makedirs(args.targets_dir, exist_ok=True)

# Collect coverage fuzzer names (from whichever manifest has preset fuzzing-coverage)
cov_names: set[str] = set()
for path in args.manifests + args.companions:
data = _load_manifest(path)
if data.get("preset") == "fuzzing-coverage":
cov_names = {fz["name"] for fz in data.get("fuzzers", [])}

targets = []
total_copied = 0

# The fuzzing-avm preset enables FUZZING_AVM in cmake, which adds the
# AVM-specific fuzzers (avm_fuzzer_*, harness_*). However cmake also
# rebuilds every base fuzzer target (stdlib_*, ecc_*, translator_*, etc.)
# as a side effect. Those collateral rebuilds are identical in behaviour
# to the base preset and should NOT be scheduled as separate jobs.
#
# We collect fuzzer names from the first (base) manifest, then for the
# AVM preset we only schedule names that are new (the actual AVM fuzzers).
# All binaries are still copied to /targets/ — the entrypoint may need them.
#
# TODO: if cmake learns to build only AVM-specific targets in the
# fuzzing-avm preset, this filter can be removed.
base_names: set[str] = set()

for i, path in enumerate(args.manifests):
data = _load_manifest(path)
suffix = data.get("suffix", "")
preset = data.get("preset", "")
label = preset.removeprefix("fuzzing-") or "asm"
bin_dir = os.path.dirname(path)

copied = _copy_executables(bin_dir, args.targets_dir, suffix)
total_copied += len(copied)

fuzzers_meta = {fz["name"]: fz["source_path"] for fz in data.get("fuzzers", [])}

if i == 0:
base_names = set(copied)

# See docstring above — only the AVM preset needs dedup.
dedup = preset == "fuzzing-avm" and bool(base_names)

scheduled = 0
for name in copied:
if dedup and name in base_names:
continue
scheduled += 1
source_path = fuzzers_meta.get(name, "")
modes = ["fuzz", "minimize", "reproduce", "regress"]
if name in cov_names:
modes.append("coverage")
targets.append({
"name": f"{name}{suffix}",
"display_name": f"{name} ({label})",
"language": "c++",
"fuzzer_engine": "libfuzzer",
"source_path": f"barretenberg/cpp/src/barretenberg/{source_path}" if source_path else "",
"modes": modes,
"resources": {"cpus": args.cpus, "memory_gb": args.memory_gb},
})
if scheduled < len(copied):
print(f" {preset}: {scheduled} scheduled, {len(copied) - scheduled} skipped (in base preset)")

# Process companion presets: copy binaries only
for path in args.companions:
data = _load_manifest(path)
suffix = data.get("suffix", "")
bin_dir = os.path.dirname(path)
copied = _copy_executables(bin_dir, args.targets_dir, suffix)
total_copied += len(copied)

manifest = {"schema": "2", "targets": targets}
os.makedirs(os.path.dirname(args.manifest_out) or ".", exist_ok=True)
with open(args.manifest_out, "w") as f:
json.dump(manifest, f, indent=2)

print(f"Copied {total_copied} binaries to {args.targets_dir}, "
f"wrote {len(targets)} targets to {args.manifest_out}")


if __name__ == "__main__":
main()
8 changes: 7 additions & 1 deletion barretenberg/cpp/scripts/generate_fuzzer_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,15 @@ def main():
name, source_path = line.split("|", 1)
fuzzers.append({"name": name, "source_path": source_path})

# Derive binary suffix from preset name (e.g. fuzzing-noasm -> _noasm)
suffix = ""
if preset and preset != "fuzzing":
tag = preset.removeprefix("fuzzing-").replace("coverage", "cov")
suffix = f"_{tag}"

os.makedirs(os.path.dirname(output_file), exist_ok=True)
with open(output_file, "w") as f:
json.dump({"version": 1, "preset": preset, "fuzzers": fuzzers}, f, indent=2)
json.dump({"version": 1, "preset": preset, "suffix": suffix, "fuzzers": fuzzers}, f, indent=2)

print(f"Wrote {len(fuzzers)} fuzzers to {output_file} (preset: {preset})")

Expand Down
62 changes: 0 additions & 62 deletions barretenberg/cpp/scripts/merge_fuzzer_manifests.py

This file was deleted.

Loading
Loading