|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Flatten variant binaries into /targets/ and generate a unified v2 manifest. |
| 3 | +
|
| 4 | +Reads per-preset manifests (produced by generate_fuzzer_manifest.py via CMake) |
| 5 | +to discover binaries, suffixes, and source paths. No variant knowledge is |
| 6 | +hardcoded here — it all comes from the manifests. |
| 7 | +
|
| 8 | +Usage (from barretenberg/cpp/ during Docker build): |
| 9 | + python3 scripts/flatten_and_manifest.py /targets /tmp/fuzzer_manifest.json \ |
| 10 | + bin-fuzzing/fuzzer_manifest.json \ |
| 11 | + bin-fuzzing-noasm/fuzzer_manifest.json \ |
| 12 | + bin-fuzzing-avm/fuzzer_manifest.json \ |
| 13 | + --copy-only bin-fuzzing-asan/fuzzer_manifest.json \ |
| 14 | + --copy-only build-fuzzing-cov/bin/fuzzer_manifest.json |
| 15 | +
|
| 16 | +Positional manifests: binaries are copied AND added to the unified manifest. |
| 17 | +--copy-only manifests: binaries are copied but NOT added (companion builds |
| 18 | +like asan/coverage that the entrypoint discovers by suffix convention). |
| 19 | +""" |
| 20 | + |
| 21 | +import argparse |
| 22 | +import json |
| 23 | +import os |
| 24 | +import shutil |
| 25 | +import sys |
| 26 | + |
| 27 | + |
| 28 | +def _load_manifest(path: str) -> dict: |
| 29 | + with open(path) as f: |
| 30 | + return json.load(f) |
| 31 | + |
| 32 | + |
| 33 | +def _copy_executables(bin_dir: str, dest_dir: str, suffix: str) -> list[str]: |
| 34 | + """Copy executable files from bin_dir to dest_dir/<name><suffix>. Returns names.""" |
| 35 | + copied = [] |
| 36 | + if not os.path.isdir(bin_dir): |
| 37 | + print(f"Warning: {bin_dir} not found, skipping", file=sys.stderr) |
| 38 | + return copied |
| 39 | + for name in sorted(os.listdir(bin_dir)): |
| 40 | + src = os.path.join(bin_dir, name) |
| 41 | + if os.path.isfile(src) and os.access(src, os.X_OK): |
| 42 | + shutil.copy2(src, os.path.join(dest_dir, f"{name}{suffix}")) |
| 43 | + copied.append(name) |
| 44 | + return copied |
| 45 | + |
| 46 | + |
| 47 | +def main(): |
| 48 | + parser = argparse.ArgumentParser(description="Flatten binaries and generate v2 manifest") |
| 49 | + parser.add_argument("targets_dir", help="Destination for flattened binaries") |
| 50 | + parser.add_argument("manifest_out", help="Output path for unified v2 manifest") |
| 51 | + parser.add_argument("manifests", nargs="*", help="Per-preset manifests (schedulable)") |
| 52 | + parser.add_argument("--copy-only", action="append", default=[], dest="companions", |
| 53 | + help="Per-preset manifests for companion builds (copy only, not scheduled)") |
| 54 | + parser.add_argument("--cpus", type=int, default=4, help="Default CPUs per target") |
| 55 | + parser.add_argument("--memory-gb", type=int, default=8, help="Default memory GB per target") |
| 56 | + args = parser.parse_args() |
| 57 | + |
| 58 | + os.makedirs(args.targets_dir, exist_ok=True) |
| 59 | + |
| 60 | + # Collect coverage fuzzer names (from whichever manifest has preset fuzzing-coverage) |
| 61 | + cov_names: set[str] = set() |
| 62 | + for path in args.manifests + args.companions: |
| 63 | + data = _load_manifest(path) |
| 64 | + if data.get("preset") == "fuzzing-coverage": |
| 65 | + cov_names = {fz["name"] for fz in data.get("fuzzers", [])} |
| 66 | + |
| 67 | + targets = [] |
| 68 | + total_copied = 0 |
| 69 | + |
| 70 | + # The fuzzing-avm preset enables FUZZING_AVM in cmake, which adds the |
| 71 | + # AVM-specific fuzzers (avm_fuzzer_*, harness_*). However cmake also |
| 72 | + # rebuilds every base fuzzer target (stdlib_*, ecc_*, translator_*, etc.) |
| 73 | + # as a side effect. Those collateral rebuilds are identical in behaviour |
| 74 | + # to the base preset and should NOT be scheduled as separate jobs. |
| 75 | + # |
| 76 | + # We collect fuzzer names from the first (base) manifest, then for the |
| 77 | + # AVM preset we only schedule names that are new (the actual AVM fuzzers). |
| 78 | + # All binaries are still copied to /targets/ — the entrypoint may need them. |
| 79 | + # |
| 80 | + # TODO: if cmake learns to build only AVM-specific targets in the |
| 81 | + # fuzzing-avm preset, this filter can be removed. |
| 82 | + base_names: set[str] = set() |
| 83 | + |
| 84 | + for i, path in enumerate(args.manifests): |
| 85 | + data = _load_manifest(path) |
| 86 | + suffix = data.get("suffix", "") |
| 87 | + preset = data.get("preset", "") |
| 88 | + label = preset.removeprefix("fuzzing-") or "asm" |
| 89 | + bin_dir = os.path.dirname(path) |
| 90 | + |
| 91 | + copied = _copy_executables(bin_dir, args.targets_dir, suffix) |
| 92 | + total_copied += len(copied) |
| 93 | + |
| 94 | + fuzzers_meta = {fz["name"]: fz["source_path"] for fz in data.get("fuzzers", [])} |
| 95 | + |
| 96 | + if i == 0: |
| 97 | + base_names = set(copied) |
| 98 | + |
| 99 | + # See docstring above — only the AVM preset needs dedup. |
| 100 | + dedup = preset == "fuzzing-avm" and bool(base_names) |
| 101 | + |
| 102 | + scheduled = 0 |
| 103 | + for name in copied: |
| 104 | + if dedup and name in base_names: |
| 105 | + continue |
| 106 | + scheduled += 1 |
| 107 | + source_path = fuzzers_meta.get(name, "") |
| 108 | + modes = ["fuzz", "minimize", "reproduce", "regress"] |
| 109 | + if name in cov_names: |
| 110 | + modes.append("coverage") |
| 111 | + targets.append({ |
| 112 | + "name": f"{name}{suffix}", |
| 113 | + "display_name": f"{name} ({label})", |
| 114 | + "language": "c++", |
| 115 | + "fuzzer_engine": "libfuzzer", |
| 116 | + "source_path": f"barretenberg/cpp/src/barretenberg/{source_path}" if source_path else "", |
| 117 | + "modes": modes, |
| 118 | + "resources": {"cpus": args.cpus, "memory_gb": args.memory_gb}, |
| 119 | + }) |
| 120 | + if scheduled < len(copied): |
| 121 | + print(f" {preset}: {scheduled} scheduled, {len(copied) - scheduled} skipped (in base preset)") |
| 122 | + |
| 123 | + # Process companion presets: copy binaries only |
| 124 | + for path in args.companions: |
| 125 | + data = _load_manifest(path) |
| 126 | + suffix = data.get("suffix", "") |
| 127 | + bin_dir = os.path.dirname(path) |
| 128 | + copied = _copy_executables(bin_dir, args.targets_dir, suffix) |
| 129 | + total_copied += len(copied) |
| 130 | + |
| 131 | + manifest = {"schema": "2", "targets": targets} |
| 132 | + os.makedirs(os.path.dirname(args.manifest_out) or ".", exist_ok=True) |
| 133 | + with open(args.manifest_out, "w") as f: |
| 134 | + json.dump(manifest, f, indent=2) |
| 135 | + |
| 136 | + print(f"Copied {total_copied} binaries to {args.targets_dir}, " |
| 137 | + f"wrote {len(targets)} targets to {args.manifest_out}") |
| 138 | + |
| 139 | + |
| 140 | +if __name__ == "__main__": |
| 141 | + main() |
0 commit comments