|
| 1 | +"""Rebuild and push weblog base images whose content-hash tag is missing from Docker Hub. |
| 2 | +
|
| 3 | +Run from a system-tests checkout, with the runner venv active: |
| 4 | +
|
| 5 | + python utils/scripts/build_base_images.py |
| 6 | +
|
| 7 | +For every library with a `utils/build/docker/<library>/weblog_metadata.yml` |
| 8 | +declaring a `base_image_dependencies` section, and for every docker-bake.hcl target |
| 9 | +listed there: |
| 10 | +
|
| 11 | + 1. Resolve the target's bake config (context/dockerfile/args) via |
| 12 | + `docker buildx bake --print`. |
| 13 | + 2. Compute a content hash from: the resolved bake config (tags excluded), the |
| 14 | + content of the target's Dockerfile, and the content of every git-tracked file |
| 15 | + under the paths listed in `base_image_dependencies`. |
| 16 | + 3. Take the base tag declared in the bake file (e.g. "datadog/system-tests:express4.base") |
| 17 | + and append "-<hash12>" to get the final tag. |
| 18 | + 4. Skip the build if that tag already exists on Docker Hub (`docker manifest inspect`); |
| 19 | + otherwise build and push it with that tag. |
| 20 | +
|
| 21 | +This is idempotent and safe to run on every push: it never overwrites an existing |
| 22 | +tag, it only creates new ones when the relevant files change. |
| 23 | +
|
| 24 | +Pass --dry-run to only print the computed tag and whether it already exists, |
| 25 | +without ever building or pushing (useful to find the tag to put in a weblog |
| 26 | +Dockerfile's FROM clause after dependencies change). |
| 27 | +""" |
| 28 | + |
| 29 | +import argparse |
| 30 | +import hashlib |
| 31 | +import json |
| 32 | +import subprocess |
| 33 | +import sys |
| 34 | +from pathlib import Path |
| 35 | + |
| 36 | +# Make `utils` importable and resolve paths regardless of the caller's cwd, so a |
| 37 | +# plain `python utils/scripts/build_base_images.py` works. |
| 38 | +REPO_ROOT = Path(__file__).resolve().parents[2] |
| 39 | +sys.path.insert(0, str(REPO_ROOT)) |
| 40 | + |
| 41 | +from utils._context.weblog_metadata import WeblogMetaData # noqa: E402 |
| 42 | +from utils.const import COMPONENT_GROUPS # noqa: E402 |
| 43 | + |
| 44 | + |
| 45 | +def _bake_file(library: str) -> Path: |
| 46 | + return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" |
| 47 | + |
| 48 | + |
| 49 | +def _bake_config(bake_file: Path, target: str) -> dict: |
| 50 | + """Resolved bake config (context, dockerfile, args, tags) for a single target.""" |
| 51 | + result = subprocess.run( |
| 52 | + ["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target], |
| 53 | + cwd=REPO_ROOT, |
| 54 | + check=True, |
| 55 | + capture_output=True, |
| 56 | + text=True, |
| 57 | + ) |
| 58 | + return json.loads(result.stdout)["target"][target] |
| 59 | + |
| 60 | + |
| 61 | +def _tracked_files(path: str) -> list[Path]: |
| 62 | + """Every git-tracked file under `path` (a file or a directory), sorted.""" |
| 63 | + result = subprocess.run( |
| 64 | + ["git", "ls-files", path], |
| 65 | + cwd=REPO_ROOT, |
| 66 | + check=True, |
| 67 | + capture_output=True, |
| 68 | + text=True, |
| 69 | + ) |
| 70 | + return sorted(REPO_ROOT / line for line in result.stdout.splitlines() if line) |
| 71 | + |
| 72 | + |
| 73 | +def compute_hash(bake_config: dict, dependencies: list[str]) -> str: |
| 74 | + """Content hash for a base image target: bake config (minus tags) + Dockerfile |
| 75 | + content + content of every git-tracked file under `dependencies`. |
| 76 | + """ |
| 77 | + digest = hashlib.sha256() |
| 78 | + |
| 79 | + config_without_tags = {k: v for k, v in bake_config.items() if k != "tags"} |
| 80 | + digest.update(json.dumps(config_without_tags, sort_keys=True).encode()) |
| 81 | + |
| 82 | + dockerfile = REPO_ROOT / bake_config["context"] / bake_config["dockerfile"] |
| 83 | + digest.update(dockerfile.read_bytes()) |
| 84 | + |
| 85 | + files: list[Path] = [] |
| 86 | + for dep in dependencies: |
| 87 | + files.extend(_tracked_files(dep)) |
| 88 | + |
| 89 | + for file in sorted(set(files)): |
| 90 | + digest.update(str(file.relative_to(REPO_ROOT)).encode()) |
| 91 | + digest.update(file.read_bytes()) |
| 92 | + |
| 93 | + return digest.hexdigest()[:12] |
| 94 | + |
| 95 | + |
| 96 | +def image_exists(tag: str) -> bool: |
| 97 | + result = subprocess.run( |
| 98 | + ["docker", "manifest", "inspect", tag], |
| 99 | + cwd=REPO_ROOT, |
| 100 | + check=False, |
| 101 | + capture_output=True, |
| 102 | + ) |
| 103 | + return result.returncode == 0 |
| 104 | + |
| 105 | + |
| 106 | +def build_and_push(bake_file: Path, target: str, tag: str) -> None: |
| 107 | + print(f"Building and pushing {tag}") |
| 108 | + subprocess.run( |
| 109 | + [ |
| 110 | + "docker", |
| 111 | + "buildx", |
| 112 | + "bake", |
| 113 | + "--push", |
| 114 | + "--progress=plain", |
| 115 | + "--set", |
| 116 | + f"{target}.tags={tag}", |
| 117 | + "-f", |
| 118 | + str(bake_file), |
| 119 | + target, |
| 120 | + ], |
| 121 | + cwd=REPO_ROOT, |
| 122 | + check=True, |
| 123 | + ) |
| 124 | + |
| 125 | + |
| 126 | +def process_library(library: str, *, dry_run: bool) -> None: |
| 127 | + dependencies_by_target = WeblogMetaData.load_base_image_dependencies(library) |
| 128 | + if not dependencies_by_target: |
| 129 | + return |
| 130 | + |
| 131 | + bake_file = _bake_file(library) |
| 132 | + if not bake_file.exists(): |
| 133 | + print(f"Warning: {library} declares base_image_dependencies but has no docker-bake.hcl, skipping") |
| 134 | + return |
| 135 | + |
| 136 | + for target, dependencies in dependencies_by_target.items(): |
| 137 | + bake_config = _bake_config(bake_file, target) |
| 138 | + base_tag = bake_config["tags"][0] |
| 139 | + content_hash = compute_hash(bake_config, dependencies) |
| 140 | + tag = f"{base_tag}-{content_hash}" |
| 141 | + |
| 142 | + if dry_run: |
| 143 | + state = "exists" if image_exists(tag) else "missing" |
| 144 | + print(f"{library}/{target}: {tag} ({state})") |
| 145 | + continue |
| 146 | + |
| 147 | + if image_exists(tag): |
| 148 | + print(f"{tag} already exists, skipping") |
| 149 | + continue |
| 150 | + |
| 151 | + build_and_push(bake_file, target, tag) |
| 152 | + |
| 153 | + |
| 154 | +def main() -> None: |
| 155 | + parser = argparse.ArgumentParser(description="Rebuild and push weblog base images with a content-hash tag") |
| 156 | + parser.add_argument("--library", help="Only process this library (default: all libraries)") |
| 157 | + parser.add_argument( |
| 158 | + "--dry-run", |
| 159 | + action="store_true", |
| 160 | + help="Only print the computed tag and whether it exists on Docker Hub; never build or push", |
| 161 | + ) |
| 162 | + args = parser.parse_args() |
| 163 | + |
| 164 | + libraries = [args.library] if args.library else sorted(COMPONENT_GROUPS.all) |
| 165 | + for library in libraries: |
| 166 | + process_library(library, dry_run=args.dry_run) |
| 167 | + |
| 168 | + |
| 169 | +if __name__ == "__main__": |
| 170 | + main() |
0 commit comments