builder.py

# Copyright (c) Contributors to the aswf-docker Project. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""
CI Image and Package Builder
"""
import logging
import subprocess
import json
import os
import tempfile
import typing

from aswfdocker import constants, aswfinfo, utils, groupinfo, index

logger = logging.getLogger(__name__)


class Builder:
    """Builder generates a "docker buildx bake" json file to drive the parallel builds of Docker images."""

    def __init__(
        self,
        build_info: aswfinfo.ASWFInfo,
        group_info: groupinfo.GroupInfo,
        push: bool = False,
    ):
        self.push = push
        self.build_info = build_info
        self.group_info = group_info
        self.index = index.Index()

    def make_bake_dict(
        self,
        build_missing: bool,
        no_remote: bool,
    ) -> typing.Dict[str, dict]:
        # pylint: disable=too-many-locals
        root: typing.Dict[str, dict] = {}
        root["target"] = {}
        for image, version in self.group_info.iter_images_versions():
            versions_to_bake = set()
            major_version = utils.get_major_version(version)
            version_info = self.index.version_info(major_version)
            if self.group_info.type == constants.ImageType.PACKAGE:
                image_base = image.replace("ci-package-", "")
                self.index.get_group_from_image(self.group_info.type, image_base)
                if version in versions_to_bake:
                    # Only one version per image needed
                    continue
                # if version_info.ci_common_version != major_version:
                # Only bake conan images in ci_common container!
                # version = version_info.ci_common_version
                # major_version = utils.get_major_version(version)
                versions_to_bake.add(version)
                # tags may not make much sense for Conan packages
                tags = list(
                    map(
                        lambda tag: f"{constants.DOCKER_REGISTRY}/{self.build_info.docker_org}"
                        + f"/ci-baseos-gl-conan:{tag}",
                        [version, major_version],
                    )
                )
                docker_file = "packages/common/Dockerfile"
            else:
                tags = version_info.get_tags(version, self.build_info.docker_org, image)
                docker_file = f"{image}/Dockerfile"

            major_version_no_clang = major_version.split("-")[0]
            if version_info.ci_common_version == major_version_no_clang:
                channel = f"ci_common{major_version_no_clang}"
            else:
                channel = f"vfx{version_info.major_version}"
            args = {
                "ASWF_ORG": self.build_info.docker_org,
                "ASWF_PKG_ORG": self.build_info.package_org,
                "ASWF_VERSION": version,
                "CI_COMMON_VERSION": version_info.ci_common_version,
                "ASWF_CONAN_CHANNEL": channel,
            }
            if self.group_info.type == constants.ImageType.PACKAGE:
                # params as env var needed for conan build
                args.update(
                    {
                        "ASWF_PKG_NAME": image.replace("ci-package-", ""),
                        "ASWF_PKG_VERSION": version_info.package_versions.get(
                            "ASWF_"
                            # Handle dependencies with hyphens in their name.
                            + image.replace("ci-package-", "").upper().replace("-", "_")
                            + "_VERSION"
                        ),
                        "ASWF_CONAN_HOME": constants.ASWF_CONAN_HOME,
                        "ASWF_CONAN_BUILD_MISSING": (
                            "--build=missing" if build_missing else ""
                        ),
                        "ASWF_CONAN_NO_REMOTE": "--no-remote" if no_remote else "",
                        "ASWF_CONAN_PUSH": "TRUE" if self.push else "",
                    }
                )
            args.update(version_info.all_package_versions)
            target_dict = {
                "context": ".",
                "dockerfile": docker_file,
                "args": args,
                "labels": {
                    "org.opencontainers.image.created": self.build_info.build_date,
                    "org.opencontainers.image.revision": self.build_info.vcs_ref,
                },
                "tags": tags,
                "output": ["type=registry,push=true" if self.push else "type=docker"],
                "secret": [
                    "id=conan_login_username,env=CONAN_LOGIN_USERNAME",
                    "id=conan_password,env=CONAN_PASSWORD",
                ],
            }
            root["target"][f"{image}-{major_version}"] = target_dict

        root["group"] = {"default": {"targets": list(root["target"].keys())}}
        return root

    def make_bake_jsonfile(
        self,
        build_missing: bool,
        no_remote: bool,
    ) -> typing.Optional[str]:
        d = self.make_bake_dict(build_missing, no_remote)
        if not d["group"]["default"]["targets"]:
            return None
        groups = "-".join(self.group_info.names)
        versions = "-".join(self.group_info.versions)
        path = os.path.join(
            tempfile.gettempdir(),
            f"docker-bake-{self.group_info.type.name}-{groups}-{versions}.json",
        )
        with open(path, "w", encoding="utf-8") as f:
            json.dump(d, f, indent=4, sort_keys=True)
        return path

    def _run(self, cmd: str, dry_run: bool):
        if dry_run:
            logger.info("Would run: '%s'", cmd)
        else:
            logger.info("Building: '%s'", cmd)
            subprocess.run(cmd, shell=True, check=True, cwd=self.build_info.repo_root)

    def _run_in_docker(self, base_cmd, cmd, dry_run):
        self._run(
            " ".join(base_cmd + cmd),
            dry_run=dry_run,
        )

    def _get_conan_env_vars(self, version_info):
        envs = {
            "ASWF_CONAN_HOME": constants.ASWF_CONAN_HOME,
            "CONAN_HOME": os.path.join(constants.ASWF_CONAN_HOME, ".conan2"),
            "CCACHE_DIR": "/tmp/ccache",
            "CONAN_NON_INTERACTIVE": "1",
        }
        if "CONAN_LOGIN_USERNAME" in os.environ:
            envs["CONAN_LOGIN_USERNAME"] = os.environ["CONAN_LOGIN_USERNAME"]
        if "ARTIFACTORY_USER" in os.environ:
            envs["CONAN_LOGIN_USERNAME"] = os.environ["ARTIFACTORY_USER"]
        if "CONAN_PASSWORD" in os.environ:
            envs["CONAN_PASSWORD"] = os.environ["CONAN_PASSWORD"]
        if "ARTIFACTORY_TOKEN" in os.environ:
            envs["CONAN_PASSWORD"] = os.environ["ARTIFACTORY_TOKEN"]
        for name, value in version_info.all_package_versions.items():
            envs[name] = value
        return envs

    # We should leverage this to avoid repeating volume definitions in packages/common/Dockerfiles
    def _get_conan_vols(self):
        conan_base = os.path.join(utils.get_git_top_level(), "packages", "conan")
        vols = {
            os.path.join(conan_base, "settings"): os.path.join(
                constants.ASWF_CONAN_HOME, ".conan2"
            ),
            os.path.join(conan_base, "data"): os.path.join(
                constants.ASWF_CONAN_HOME, "d"
            ),
            os.path.join(conan_base, "recipes"): os.path.join(
                constants.ASWF_CONAN_HOME, "recipes"
            ),
            os.path.join(conan_base, "ccache"): "/tmp/ccache",
        }
        return vols

    def _get_conan_base_cmd(self, version_info):
        base_cmd = ["docker", "run"]
        for name, value in self._get_conan_env_vars(version_info).items():
            base_cmd.append("-e")
            base_cmd.append(f"{name}={value}")
        for name, value in self._get_conan_vols().items():
            base_cmd.append("-v")
            base_cmd.append(f"{name}:{value}")
        tag = (
            f"{constants.DOCKER_REGISTRY}/{self.build_info.docker_org}"
            + f"/ci-baseos-gl-conan:{version_info.ci_common_version}"
        )
        base_cmd.append(tag)
        return base_cmd

    def _build_conan_package(
        self,
        image,
        version,
        dry_run,
        progress,
        bake_jsonfile,
    ):
        major_version = utils.get_major_version(version)

        # buildx bake --set allows us to override settings in the bake file and avoid having
        #   to rewrite it.
        # output=type=cacheonly : no container is produced, we only want the cache containing
        #   the output of conan builds
        # target.target=ci-conan-package-builder : see packages/common/Dockerfile for the Conan
        #   build container which runs:
        #
        # - conan create
        # - conan upload main version (conditional)
        #
        runcmd = (
            f"docker buildx bake -f {bake_jsonfile} --set=*.output=type=cacheonly "
            f"--set=*.target.target=ci-conan-package-builder --progress {progress} "
            f"ci-package-{image}-{major_version}"
        )

        self._run(
            runcmd,
            dry_run=dry_run,
        )

    def build(
        self,
        dry_run: bool = False,
        progress: str = "",
        build_missing=False,
        no_remote=False,
    ) -> None:
        images_and_versions = []
        for image, version in self.group_info.iter_images_versions(get_image=True):
            images_and_versions.append((image, version))

        if not images_and_versions:
            return

        path = self.make_bake_jsonfile(build_missing, no_remote)
        if path:
            if self.group_info.type == constants.ImageType.IMAGE:
                self._run(
                    f"docker buildx bake -f {path} --progress {progress}",
                    dry_run=dry_run,
                )
            else:
                conan_base = os.path.join(
                    utils.get_git_top_level(), "packages", "conan"
                )
                # As of Conan 2.18, parallel builds are still not supported and can result in
                # database lock failures, so we have to build sequentially.
                for image, version in images_and_versions:
                    recipe_path = os.path.join(conan_base, "recipes", image)
                    if not os.path.exists(recipe_path):
                        logger.warning("Recipe for %s not found: skipping!", image)
                        continue
                    self._build_conan_package(
                        image,
                        version,
                        dry_run,
                        progress,
                        path,
                    )