diff --git a/CONSTRUCT.md b/CONSTRUCT.md index 17a71302f..c29d29c4e 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -235,6 +235,7 @@ The type of the installer being created. Possible values are: - `sh`: shell-based installer for Linux or macOS - `pkg`: macOS GUI installer built with Apple's `pkgbuild` - `exe`: Windows GUI installer built with NSIS +- `docker`: generates a `Dockerfile` or image of the installation environment. The default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well @@ -679,6 +680,29 @@ freeze_base: message: "This base environment is frozen and cannot be modified." ``` +### `docker_base_image` + +Required to use docker-related features. +Base image to use for docker builds and Dockerfiles. This can be any valid docker image reference, including a tag and/or digest. +For example: `debian:13.4-slim@sha256:abc123...`. + +### `docker_tag` + +Tag to use for the docker image. +If not provided, it will default to `-:-`. +Has no effect if not using the docker build output feature. + +### `docker_labels` + +Additional labels to add to the built docker image. +The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` +are set automatically from `name` and `version`. + +### `docker_image` + +If set, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball either uncompressed or compressed. +``----docker.tar`` will be created in the output docker directory. + ## Available selectors - `aarch64` diff --git a/constructor/_schema.py b/constructor/_schema.py index 87945ad03..943473f0e 100644 --- a/constructor/_schema.py +++ b/constructor/_schema.py @@ -42,6 +42,7 @@ class InstallerTypes(StrEnum): EXE = "exe" PKG = "pkg" SH = "sh" + DOCKER = "docker" class PkgDomains(StrEnum): @@ -403,6 +404,7 @@ class ConstructorConfiguration(BaseModel): - `sh`: shell-based installer for Linux or macOS - `pkg`: macOS GUI installer built with Apple's `pkgbuild` - `exe`: Windows GUI installer built with NSIS + - `docker`: generates a `Dockerfile` or image of the installation environment. The default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well @@ -853,6 +855,29 @@ class ConstructorConfiguration(BaseModel): message: "This base environment is frozen and cannot be modified." ``` """ + docker_base_image: Annotated[str, Field(min_length=1)] | None = None + """ + Required to use docker-related features. + Base image to use for docker builds and Dockerfiles. This can be any valid docker image reference, including a tag and/or digest. + For example: `debian:13.4-slim@sha256:abc123...`. + """ + docker_tag: NonEmptyStr | None = None + """ + Tag to use for the docker image. + If not provided, it will default to `-:-`. + Has no effect if not using the docker build output feature. + """ + docker_labels: dict[NonEmptyStr, NonEmptyStr] = {} + """ + Additional labels to add to the built docker image. + The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` + are set automatically from `name` and `version`. + """ + docker_image: Literal["tar", "gz", "zst"] = "tar" + """ + If set, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball either uncompressed or compressed. + ``----docker.tar`` will be created in the output docker directory. + """ def fix_descriptions(obj): diff --git a/constructor/data/construct.schema.json b/constructor/data/construct.schema.json index f0178d738..a5d10d7f4 100644 --- a/constructor/data/construct.schema.json +++ b/constructor/data/construct.schema.json @@ -245,7 +245,8 @@ "all", "exe", "pkg", - "sh" + "sh", + "docker" ], "title": "InstallerTypes", "type": "string" @@ -638,6 +639,58 @@ "description": "Set default installation prefix for domain users. If not provided, the installation prefix for domain users will be `%LOCALAPPDATA%\\`. By default, it is different from the `default_prefix` value to avoid installing the distribution into the roaming profile. Environment variables will be expanded at install time. Windows only.", "title": "Default Prefix Domain User" }, + "docker_base_image": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required to use docker-related features. Base image to use for docker builds and Dockerfiles. This can be any valid docker image reference, including a tag and/or digest. For example: `debian:13.4-slim@sha256:abc123...`.", + "title": "Docker Base Image" + }, + "docker_image": { + "default": "tar", + "description": "If set, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball either uncompressed or compressed. ``----docker.tar`` will be created in the output docker directory.", + "enum": [ + "tar", + "gz", + "zst" + ], + "title": "Docker Image", + "type": "string" + }, + "docker_labels": { + "additionalProperties": { + "minLength": 1, + "type": "string" + }, + "default": {}, + "description": "Additional labels to add to the built docker image. The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` are set automatically from `name` and `version`.", + "propertyNames": { + "minLength": 1 + }, + "title": "Docker Labels", + "type": "object" + }, + "docker_tag": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Tag to use for the docker image. If not provided, it will default to `-:-`. Has no effect if not using the docker build output feature.", + "title": "Docker Tag" + }, "environment": { "anyOf": [ { @@ -864,7 +917,7 @@ } ], "default": null, - "description": "The type of the installer being created. Possible values are:\n- `sh`: shell-based installer for Linux or macOS\n- `pkg`: macOS GUI installer built with Apple's `pkgbuild`\n- `exe`: Windows GUI installer built with NSIS\nThe default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well as `sh` on Linux and `exe` on Windows.", + "description": "The type of the installer being created. Possible values are:\n- `sh`: shell-based installer for Linux or macOS\n- `pkg`: macOS GUI installer built with Apple's `pkgbuild`\n- `exe`: Windows GUI installer built with NSIS\n- `docker`: generates a `Dockerfile` or image of the installation environment.\nThe default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well as `sh` on Linux and `exe` on Windows.", "title": "Installer Type" }, "keep_pkgs": { diff --git a/constructor/docker_build.py b/constructor/docker_build.py new file mode 100644 index 000000000..873b21f2e --- /dev/null +++ b/constructor/docker_build.py @@ -0,0 +1,156 @@ +"""Logic for creating a Dockerfile and/or building portable Docker images from Constructor installers.""" + +import logging +import shutil +import subprocess +import tempfile +from pathlib import Path + +from jinja2 import Template + +from . import __version__ + +logger = logging.getLogger(__name__) + +TEMPLATE_PATH = Path(__file__).parent / "dockerfile_template.tmpl" + +DOCKER_PLATFORM_MAP = { + "linux-64": "linux/amd64", + "linux-aarch64": "linux/arm64", + "linux-armv7l": "linux/arm/v7", + "linux-32": "linux/386", + "linux-ppc64le": "linux/ppc64le", + "linux-s390x": "linux/s390x", +} + + +def generate_dockerfile(info: dict, docker_dir: Path) -> Path: + """ + Render the Dockerfile template and write it to the Docker build directory. + + Parameters + ---------- + info: dict + Constructor installer info dict. + docker_dir: Path + Path to the Docker build directory returned by prepare_docker_context(). + + Returns + ------- + Path + Path to the generated Dockerfile. + """ + from .conda_interface import MatchSpec + + specs = {MatchSpec(spec).name for spec in info.get("specs", ())} + has_mamba = "mamba" in specs + + docker_template = Template(TEMPLATE_PATH.read_text()) + + rendered_dockerfile = docker_template.render( + constructor_version=__version__, + base_image=info.get("docker_base_image"), + default_prefix=info.get("default_prefix", f"/opt/{info['name'].lower()}"), + installer_filename=Path(info["_outpath"]).name, + name=info["name"], + version=info["version"], + labels=info.get("docker_labels", {}), + init_cmd="$PREFIX/bin/mamba shell" if has_mamba else "$PREFIX/bin/python -m conda", + register_envs=info.get("register_envs"), + keep_pkgs=info.get("keep_pkgs"), + ) + + logger.info("Writing Dockerfile...") + dockerfile_path = docker_dir / "Dockerfile" + dockerfile_path.write_text(rendered_dockerfile) + return dockerfile_path + + +def build_image(info: dict, docker_dir: Path) -> Path: + """Optionally build the docker image from the generated Dockerfile. + Currently supported on linux and macOS platforms. + + Parameters + ---------- + info: dict + Constructor installer info dict. + docker_dir: Path + Path to the Docker directory containing the Docker outputs. + + Returns + ------- + Path + Path to the saved Docker image tarball. + + """ + if not (docker_platform := DOCKER_PLATFORM_MAP.get(info["_platform"])): + raise RuntimeError( + f"Unsupported platform for Docker build: {info['_platform']}" + f"Supported platforms are: {', '.join(DOCKER_PLATFORM_MAP.keys())}" + ) + + if info.get("docker_tag") and ":" not in info.get("docker_tag"): + raise ValueError( + "Invalid docker_tag: '{info['docker_tag']}'. Must be in format 'name:tag'." + ) + + tag = info.get("docker_tag", f"{info['name'].lower()}:{info['version']}") + tarball_dest = docker_dir / f"{Path(info['_outpath']).stem}-docker.tar" + + cmd = [ + "docker", + "buildx", + "build", + str(docker_dir), + "--platform", + docker_platform, + "-t", + tag, + "--load", + ] + + logger.info("Building Docker image: '%s'", tag) + subprocess.run(cmd, check=True) + + logger.info("Saving Docker image to tarball: '%s'", tarball_dest) + subprocess.run(["docker", "save", tag, "-o", str(tarball_dest)], check=True) + return tarball_dest + + +def create(info: dict, verbose: bool = False) -> None: + """Build a Docker output + + Parameters + ---------- + info: dict + Constructor installer info dict. + verbose: bool, optional + If ``True``, enables verbose logging. + Defaults to ``False``. + + """ + with tempfile.TemporaryDirectory() as temp_dir: + docker_tmp_dir = Path(temp_dir) + + installer_path = Path(info["_outpath"]) + if not installer_path.exists(): + raise RuntimeError(f"Expected .sh installer not found: {installer_path}") + shutil.copy(installer_path, docker_tmp_dir / installer_path.name) + logger.info("Copied installer to build directory.") + + generate_dockerfile(info, docker_tmp_dir) + + if info.get("docker_image") == "tar": + tarball = build_image(info, docker_tmp_dir) + if tarball: + shutil.copy(tarball, Path(info["_output_dir"]) / tarball.name) + else: + output_dir = Path(info["_output_dir"]) / installer_path.stem + output_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(docker_tmp_dir / "Dockerfile", output_dir / "Dockerfile") + shutil.copy( + docker_tmp_dir / Path(info["_outpath"]).name, + output_dir / Path(info["_outpath"]).name, + ) + + logger.info("Docker output complete. Docker directory: '%s'", info["_output_dir"]) diff --git a/constructor/dockerfile_template.tmpl b/constructor/dockerfile_template.tmpl new file mode 100644 index 000000000..bba8e3a1d --- /dev/null +++ b/constructor/dockerfile_template.tmpl @@ -0,0 +1,49 @@ +# Dockerfile generated by constructor {{ constructor_version }} + +########################################################## +# Stage 1: Run .sh installer +########################################################## + +FROM {{ base_image }} AS builder + +ARG PREFIX={{ default_prefix }} + +COPY {{ installer_filename }} /tmp/installer.sh + +RUN sh /tmp/installer.sh -b -p "${PREFIX}" && \ + rm -f "${PREFIX}/uninstall.sh" && \ + rm -f "${PREFIX}/_conda" && \ + find "${PREFIX}" -follow -type f -name '*.js.map' -delete && \ + find "${PREFIX}" -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true && \ + {%- if not keep_pkgs %} + rm -rf "${PREFIX}/pkgs" && \ + {%- endif %} + find "${PREFIX}" -follow -type f -name '*.a' -delete + +########################################################## +# Stage 2: Final image +########################################################## + +FROM {{ base_image }} + +ARG PREFIX={{ default_prefix }} + +LABEL org.opencontainers.image.title="{{ name }}" +LABEL org.opencontainers.image.version="{{ version }}" +{%- for k, v in labels.items() %} +LABEL {{ k }}="{{ v }}" +{%- endfor %} + +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV PATH="${PREFIX}/bin:${PATH}" + +COPY --from=builder ${PREFIX} ${PREFIX} +{% if register_envs %} +COPY --from=builder /root/.conda /root/.conda +{% endif %} + +RUN echo 'export PATH=$(sed -e "s,:\?{{ default_prefix }}/bin:,," <<< "${PATH}")' >> ~/.bashrc && \ +/ {{ init_cmd }} init --all + +CMD [ "/bin/bash" ] diff --git a/constructor/main.py b/constructor/main.py index 5a8ce9af7..35893091c 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -15,6 +15,7 @@ import json import logging import os +import shutil import subprocess import sys from os.path import abspath, expanduser, isdir, join @@ -40,10 +41,27 @@ def get_installer_type(info: dict): osname, unused_arch = info["_platform"].split("-") - os_allowed = {"linux": ("sh",), "osx": ("sh", "pkg"), "win": ("exe",)} - all_allowed = set(sum(os_allowed.values(), ("all",))) + os_allowed = { + "linux": ("sh",), + "osx": ( + "sh", + "pkg", + ), + "win": ("exe",), + } + all_allowed = set(sum(os_allowed.values(), ("all", "docker"))) itype = info.get("installer_type") + + is_docker_image = info.get("docker_image") + if is_docker_image and shutil.which("docker") is None: + raise RuntimeError( + "Building a Docker image requires the 'docker' CLI tool to be installed and available in PATH. " + "Install Docker Desktop or Docker Engine to proceed, or " + "use `installer_type: docker` in construct.yaml to " + "generate the Dockerfile without building the image." + ) + if not itype: return os_allowed[osname][:1] elif itype == "all": @@ -51,6 +69,8 @@ def get_installer_type(info: dict): elif itype not in all_allowed: all_allowed = ", ".join(sorted(all_allowed)) sys.exit("Error: invalid installer type '%s'; allowed: %s" % (itype, all_allowed)) + elif itype == "docker" or is_docker_image: + return ("sh", "docker") elif itype not in os_allowed[osname]: os_allowed = ", ".join(sorted(os_allowed[osname])) sys.exit( @@ -202,6 +222,12 @@ def main_build( info["_debug"] = debug itypes = get_installer_type(info) + if "docker" in itypes and not info.get("docker_base_image"): + sys.exit( + "Error: docker_base_image is required when building Docker output. " + "Please specify a base image using the 'docker_base_image' key in construct.yaml." + ) + if platform != cc_platform and "pkg" in itypes and not cc_platform.startswith("osx-"): sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform) @@ -351,6 +377,11 @@ def main_build( "enable_currentUserHome": "true", } + if "docker" in itypes and not info.get("docker_base_image"): + sys.exit( + "Error: 'docker_base_image' not specified in construct.yaml. Skipping Dockerfile generation." + ) + if osname == "win": info["_win_install_needs_python_exe"] = _win_install_needs_python_exe( info["_conda_exe"], @@ -399,12 +430,27 @@ def main_build( from .winexe import create as winexe_create create = winexe_create + elif itype == "docker": + from .docker_build import create as docker_create + + create = docker_create info["installer_type"] = itype - info["_outpath"] = abspath(join(output_dir, get_output_filename(info))) + if itype != "docker": + info["_outpath"] = abspath(join(output_dir, get_output_filename(info))) + else: + info["_outpath"] = abspath(join(output_dir, get_output_filename(info))).replace( + ".docker", ".sh" + ) create(info, verbose=verbose) if len(itypes) > 1: info_dicts.append(info.copy()) - logger.info("Successfully created '%(_outpath)s'.", info) + if itype == "docker": + logger.info( + "Docker output complete. Docker directory: '%s'", + Path(info["_output_dir"]) / "docker", + ) + else: + logger.info("Successfully created '%(_outpath)s'.", info) # Merge info files for each installer type if len(itypes) > 1: diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index 17a71302f..c29d29c4e 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -235,6 +235,7 @@ The type of the installer being created. Possible values are: - `sh`: shell-based installer for Linux or macOS - `pkg`: macOS GUI installer built with Apple's `pkgbuild` - `exe`: Windows GUI installer built with NSIS +- `docker`: generates a `Dockerfile` or image of the installation environment. The default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well @@ -679,6 +680,29 @@ freeze_base: message: "This base environment is frozen and cannot be modified." ``` +### `docker_base_image` + +Required to use docker-related features. +Base image to use for docker builds and Dockerfiles. This can be any valid docker image reference, including a tag and/or digest. +For example: `debian:13.4-slim@sha256:abc123...`. + +### `docker_tag` + +Tag to use for the docker image. +If not provided, it will default to `-:-`. +Has no effect if not using the docker build output feature. + +### `docker_labels` + +Additional labels to add to the built docker image. +The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` +are set automatically from `name` and `version`. + +### `docker_image` + +If set, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball either uncompressed or compressed. +``----docker.tar`` will be created in the output docker directory. + ## Available selectors - `aarch64` diff --git a/examples/docker_image/construct.yaml b/examples/docker_image/construct.yaml new file mode 100644 index 000000000..551a763e4 --- /dev/null +++ b/examples/docker_image/construct.yaml @@ -0,0 +1,22 @@ +name: test_docker_image +version: 1.0.0 + +channels: + - conda-forge + +specs: + - python + - numpy + - conda + +docker_image: tar + +docker_base_image: "debian:13.4-slim@sha256:cedb1ef40439206b673ee8b33a46a03a0c9fa90bf3732f54704f99cb061d2c5a" + +keep_pkgs: true + +register_envs: true + +docker_labels: + maintainer: "jaidarice" + description: "Test Docker image built with constructor." diff --git a/examples/dockerfile/construct.yaml b/examples/dockerfile/construct.yaml new file mode 100644 index 000000000..3e205847f --- /dev/null +++ b/examples/dockerfile/construct.yaml @@ -0,0 +1,22 @@ +name: test_dockerfile +version: 1.0.0 + +channels: + - conda-forge + +specs: + - python + - numpy + - conda + +installer_type: docker + +docker_base_image: "debian:13.4-slim@sha256:cedb1ef40439206b673ee8b33a46a03a0c9fa90bf3732f54704f99cb061d2c5a" + +keep_pkgs: true + +register_envs: true + +docker_labels: + maintainer: "jaidarice" + description: "Test Dockerfile generated with constructor." diff --git a/news/1219-docker-implementation b/news/1219-docker-implementation new file mode 100644 index 000000000..6cab15110 --- /dev/null +++ b/news/1219-docker-implementation @@ -0,0 +1,20 @@ +### Enhancements + +* Add `installer_type: docker` support to generate a Dockerfile from a constructor build to be used as-is or modified. (#1219) +* Add `docker_image: tar` support to build a portable Docker image. (#1219) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 4ec5c9a64..33e7f184d 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1554,3 +1554,70 @@ def test_frozen_environment(tmp_path, request, has_conflict): s in c.value.stderr for s in ("RuntimeError", "freeze_base / freeze_env", "extra_files", "base") ) + + +def test_dockerfile_generation(tmp_path): + input_path = _example_path("dockerfile") + output_path = tmp_path / "output" + + yaml = YAML() + with open(input_path / "construct.yaml") as f: + config = yaml.load(f) + + for installer, info in create_installer(input_path, output_path): + if info.get("installer_type") == "docker": + docker_output_dir = output_path / "installer" / installer.stem + assert (docker_output_dir / "Dockerfile").exists() + assert (docker_output_dir / installer.name).exists() + + dockerfile_text = (docker_output_dir / "Dockerfile").read_text() + + assert f"FROM {config['docker_base_image']}" in dockerfile_text + + for key, value in config.get("docker_labels", {}).items(): + assert f'{key}="{value}"' in dockerfile_text + + +@pytest.mark.skipif(not shutil.which("docker"), reason="Docker not available") +def test_docker_image_build(tmp_path): + input_path = _example_path("docker_image") + output_path = tmp_path / "output" + + yaml = YAML() + with open(input_path / "construct.yaml") as f: + config = yaml.load(f) + image_name = f"{config['name'].lower()}:{config['version']}" + + for installer, _ in create_installer(input_path, output_path): + if installer.suffix == ".sh": + installer_stem = Path(installer).stem + tarball = output_path / "installer" / f"{installer_stem}-docker.tar" + assert tarball.exists(), f"Expected tarball not found: {tarball}" + + try: + subprocess.run(["docker", "load", "-i", str(tarball)], check=True) + + result = subprocess.run( + ["docker", "run", "--rm", image_name, "conda", "--version"], + capture_output=True, + text=True, + check=True, + ) + + assert "conda" in result.stdout + + inspect_result = subprocess.run( + ["docker", "inspect", "--format", "{{ json .Config.Labels }}", image_name], + capture_output=True, + text=True, + check=True, + ) + labels = json.loads(inspect_result.stdout) + + for key, value in config.get("docker_labels", {}).items(): + assert labels.get(key) == value, f"Label {key}: {value} not found in Docker image" + assert labels.get("org.opencontainers.image.title") == config["name"] + assert labels.get("org.opencontainers.image.version") == config["version"] + + finally: + subprocess.run(["docker", "rmi", image_name], check=False)