From 557eacaf95b96f73f937b021515c229282401c77 Mon Sep 17 00:00:00 2001 From: Vince Perri <5596945+vinceaperri@users.noreply.github.com> Date: Tue, 30 Jun 2026 21:11:41 +0000 Subject: [PATCH 1/2] Add Azure Linux 4.0 PXE boot support for LiveOS images Add PXE (network) boot support for Azure Linux 4.0 LiveOS images. Azure Linux 4.0 boots via BLS (Boot Loader Specification) entries under boot/loader/entries, which grub cannot enumerate over TFTP, so a PXE-booted AZL4 image reached an empty grub menu. This expands the BLS Type #1 entries into explicit grub menuentry blocks when the PXE artifacts are built, so netboot works without grub reading the entries directory over TFTP. It is wired per distro through a new DistroHandler.UpdateLiveOSGrubCfgForPxe hook plus a shared BLS renderer, so other BLS distros benefit too. Tests (in support of the above): test_pxe.py adds bootstrap and full-OS PXE boot tests for AZL3 and AZL4 over a transient libvirt NAT network (dnsmasq DHCP and TFTP) plus an HTTP server over the extracted artifacts (pxe_server.py). They are skipped on Azure Linux hosts, whose edk2-ovmf lacks the UEFI network stack (EDK II NetworkPkg) and cannot netboot a VM, and run on Ubuntu hosts. --- .../vmtests/imagecustomizer/test_pxe.py | 280 ++++++++++++++++++ test/vmtests/vmtests/utils/host_utils.py | 9 +- test/vmtests/vmtests/utils/imagecustomizer.py | 32 ++ test/vmtests/vmtests/utils/libvirt_utils.py | 97 ++++-- test/vmtests/vmtests/utils/libvirt_vm.py | 3 + test/vmtests/vmtests/utils/pxe_server.py | 161 ++++++++++ .../tools/pkg/imagecustomizerlib/blsutils.go | 96 ++++++ .../pkg/imagecustomizerlib/blsutils_test.go | 72 +++++ .../pkg/imagecustomizerlib/distrohandler.go | 9 + .../imagecustomizerlib/distrohandler_acl.go | 12 + .../distrohandler_azurelinux.go | 12 + .../distrohandler_azurelinux4.go | 20 +- .../distrohandler_fedora.go | 12 + .../distrohandler_ubuntu.go | 13 +- .../liveosisobuilder_test.go | 8 - .../pkg/imagecustomizerlib/liveosisogrub.go | 78 ++++- .../pkg/imagecustomizerlib/liveosisoimages.go | 47 ++- .../tools/pkg/imagecustomizerlib/liveospxe.go | 28 +- .../testdata/pxe-bootstrap-vm-azl3.yaml | 30 ++ .../testdata/pxe-bootstrap-vm-azl4.yaml | 24 ++ .../testdata/pxe-full-os-vm-azl3.yaml | 25 ++ .../testdata/pxe-full-os-vm-azl4.yaml | 26 ++ 22 files changed, 1023 insertions(+), 71 deletions(-) create mode 100644 test/vmtests/vmtests/imagecustomizer/test_pxe.py create mode 100644 test/vmtests/vmtests/utils/pxe_server.py create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl3.yaml create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl4.yaml create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl3.yaml create mode 100644 toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl4.yaml diff --git a/test/vmtests/vmtests/imagecustomizer/test_pxe.py b/test/vmtests/vmtests/imagecustomizer/test_pxe.py new file mode 100644 index 0000000000..9c4ba0d519 --- /dev/null +++ b/test/vmtests/vmtests/imagecustomizer/test_pxe.py @@ -0,0 +1,280 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import logging +import platform +import random +import string +import tarfile +from pathlib import Path +from typing import List, Tuple + +import libvirt # type: ignore +import pytest +from docker import DockerClient + +from ..conftest import TEST_CONFIGS_DIR +from ..utils.closeable import Closeable +from ..utils.host_utils import get_host_distro +from ..utils.imagecustomizer import ( + add_preview_features_to_config, + add_pxe_bootstrap_base_url_to_config, + add_ssh_to_config, + run_image_customizer, +) +from ..utils.libvirt_utils import VmSpec, create_libvirt_domain_xml +from ..utils.libvirt_vm import LibvirtVm +from ..utils.pxe_server import PXE_HTTP_PORT, PXE_NETWORK_GATEWAY_IP, PxeEnvironment +from ..utils.user_utils import get_username +from .test_min_change import run_basic_checks + +# The full-OS image is downloaded into RAM during PXE bootstrap, so the VM needs more memory than a disk/ISO boot. +PXE_VM_MEMORY_MIB = 8192 +PXE_VM_CORE_COUNT = 4 + +# PXE boot adds a firmware netboot phase plus an over-the-network bootstrap-image download before the OS requests its +# DHCP lease, so it needs additional time to boot. +PXE_BOOT_IP_WAIT_TIME_EXTRA_SECONDS = 600 + + +def run_pxe_test( + docker_client: DockerClient, + image_customizer_container_url: str, + input_image: Path, + input_image_azl_release: int, + initramfs_type: str, + config_path: Path, + ssh_key: Tuple[str, Path], + test_temp_dir: Path, + test_instance_name: str, + logs_dir: Path, + libvirt_conn: libvirt.virConnect, + close_list: List[Closeable], +) -> None: + + ssh_public_key, ssh_private_key_path = ssh_key + + if platform.machine() == "x86_64": + boot_loader_file = "bootx64.efi" + else: + boot_loader_file = "bootaa64.efi" + + username = get_username() + + modified_config_path = add_ssh_to_config(config_path, username, ssh_public_key, close_list) + + if initramfs_type == "bootstrap": + bootstrap_base_url = f"http://{PXE_NETWORK_GATEWAY_IP}:{PXE_HTTP_PORT}" + modified_config_path = add_pxe_bootstrap_base_url_to_config( + modified_config_path, bootstrap_base_url, close_list + ) + + pxe_tar_path = test_temp_dir.joinpath("pxe-artifacts.tar.gz") + run_image_customizer( + docker_client, + image_customizer_container_url, + "customize", + modified_config_path, + "pxe-tar", + pxe_tar_path, + image_file=input_image, + ) + + pxe_artifacts_dir = test_temp_dir.joinpath("pxe-artifacts") + pxe_artifacts_dir.mkdir() + with tarfile.open(pxe_tar_path, "r:gz") as tar: + tar.extractall(pxe_artifacts_dir) + + customized_name = ( + "pxe_" + + initramfs_type.replace("-", "_") + + "_" + + get_host_distro() + + "_efi_azl" + + str(input_image_azl_release) + + "_to_efi" + ) + customized_log_path = str(logs_dir) + "/" + customized_name + http_log_file_path = Path(customized_log_path + ".http.log") + vm_console_log_file_path = customized_log_path + ".console.log" + + suffix = "".join(random.choice(string.ascii_lowercase) for _ in range(5)) + network_name = test_instance_name + "-pxe" + bridge_name = "pxebr" + suffix + + pxe_env = PxeEnvironment( + libvirt_conn, + network_name, + bridge_name, + pxe_artifacts_dir, + boot_loader_file, + http_log_file_path, + ) + close_list.append(pxe_env) + + # Create the VM: no disk, boots from the PXE network. + vm_spec = VmSpec( + test_instance_name, + PXE_VM_MEMORY_MIB, + PXE_VM_CORE_COUNT, + None, + "efi", + secure_boot=False, + pxe_boot=True, + network_name=pxe_env.network_name, + ) + domain_xml = create_libvirt_domain_xml(libvirt_conn, vm_spec) + logging.debug(f"\n\ndomain_xml = {domain_xml}\n\n") + + vm = LibvirtVm(test_instance_name, domain_xml, vm_console_log_file_path, libvirt_conn) + close_list.append(vm) + + # Start the VM. + vm.start() + + # Connect to the VM and run the basic boot validation. + with vm.create_ssh_client( + ssh_private_key_path, + test_temp_dir, + username, + ip_wait_time_extra=PXE_BOOT_IP_WAIT_TIME_EXTRA_SECONDS, + ) as ssh_client: + run_basic_checks(ssh_client, input_image_azl_release, test_temp_dir) + + +@pytest.mark.skipif( + get_host_distro() == "azurelinux", + reason="PXE requires a network-enabled host UEFI firmware (EDK II NetworkPkg), which only Ubuntu hosts provide", +) +def test_pxe_bootstrap_efi_azl3( + docker_client: DockerClient, + image_customizer_container_url: str, + core_efi_azl3: Path, + ssh_key: Tuple[str, Path], + test_temp_dir: Path, + test_instance_name: str, + logs_dir: Path, + libvirt_conn: libvirt.virConnect, + close_list: List[Closeable], +) -> None: + azl_release = 3 + config_path = TEST_CONFIGS_DIR.joinpath("pxe-bootstrap-vm-azl3.yaml") + + run_pxe_test( + docker_client, + image_customizer_container_url, + core_efi_azl3, + azl_release, + "bootstrap", + config_path, + ssh_key, + test_temp_dir, + test_instance_name, + logs_dir, + libvirt_conn, + close_list, + ) + + +@pytest.mark.skipif( + get_host_distro() == "azurelinux", + reason="PXE requires a network-enabled host UEFI firmware (EDK II NetworkPkg), which only Ubuntu hosts provide", +) +def test_pxe_bootstrap_efi_azl4( + docker_client: DockerClient, + image_customizer_container_url: str, + core_efi_azl4: Path, + ssh_key: Tuple[str, Path], + test_temp_dir: Path, + test_instance_name: str, + logs_dir: Path, + libvirt_conn: libvirt.virConnect, + close_list: List[Closeable], +) -> None: + azl_release = 4 + config_path = TEST_CONFIGS_DIR.joinpath("pxe-bootstrap-vm-azl4.yaml") + config_path = add_preview_features_to_config(config_path, "preview-distro-version", close_list) + + run_pxe_test( + docker_client, + image_customizer_container_url, + core_efi_azl4, + azl_release, + "bootstrap", + config_path, + ssh_key, + test_temp_dir, + test_instance_name, + logs_dir, + libvirt_conn, + close_list, + ) + + +@pytest.mark.skipif( + get_host_distro() == "azurelinux", + reason="PXE requires a network-enabled host UEFI firmware (EDK II NetworkPkg), which only Ubuntu hosts provide", +) +def test_pxe_full_os_efi_azl3( + docker_client: DockerClient, + image_customizer_container_url: str, + core_efi_azl3: Path, + ssh_key: Tuple[str, Path], + test_temp_dir: Path, + test_instance_name: str, + logs_dir: Path, + libvirt_conn: libvirt.virConnect, + close_list: List[Closeable], +) -> None: + azl_release = 3 + config_path = TEST_CONFIGS_DIR.joinpath("pxe-full-os-vm-azl3.yaml") + + run_pxe_test( + docker_client, + image_customizer_container_url, + core_efi_azl3, + azl_release, + "full-os", + config_path, + ssh_key, + test_temp_dir, + test_instance_name, + logs_dir, + libvirt_conn, + close_list, + ) + + +@pytest.mark.skipif( + get_host_distro() == "azurelinux", + reason="PXE requires a network-enabled host UEFI firmware (EDK II NetworkPkg), which only Ubuntu hosts provide", +) +def test_pxe_full_os_efi_azl4( + docker_client: DockerClient, + image_customizer_container_url: str, + core_efi_azl4: Path, + ssh_key: Tuple[str, Path], + test_temp_dir: Path, + test_instance_name: str, + logs_dir: Path, + libvirt_conn: libvirt.virConnect, + close_list: List[Closeable], +) -> None: + azl_release = 4 + config_path = TEST_CONFIGS_DIR.joinpath("pxe-full-os-vm-azl4.yaml") + config_path = add_preview_features_to_config(config_path, "preview-distro-version", close_list) + + run_pxe_test( + docker_client, + image_customizer_container_url, + core_efi_azl4, + azl_release, + "full-os", + config_path, + ssh_key, + test_temp_dir, + test_instance_name, + logs_dir, + libvirt_conn, + close_list, + ) diff --git a/test/vmtests/vmtests/utils/host_utils.py b/test/vmtests/vmtests/utils/host_utils.py index c66e50a537..4fbb076623 100644 --- a/test/vmtests/vmtests/utils/host_utils.py +++ b/test/vmtests/vmtests/utils/host_utils.py @@ -4,13 +4,14 @@ def get_host_distro() -> str: file_path = "/etc/os-release" - name_value = "" + id_value = "" with open(file_path, "r") as file: for line in file: if line.startswith("ID="): - name_value = line.strip().split("=", 1)[1] # Get the value part + id_value = line.strip().split("=", 1)[1] break - if name_value == "": + + if id_value == "": raise Exception("ID field not found in os-release file") - return name_value + return id_value diff --git a/test/vmtests/vmtests/utils/imagecustomizer.py b/test/vmtests/vmtests/utils/imagecustomizer.py index b4c3528ee6..e5e5367c7e 100644 --- a/test/vmtests/vmtests/utils/imagecustomizer.py +++ b/test/vmtests/vmtests/utils/imagecustomizer.py @@ -209,6 +209,38 @@ def add_preview_features_to_config(config_path: Path, preview_feature: str, clos return path +def add_pxe_bootstrap_base_url_to_config( + config_path: Path, bootstrap_base_url: str, close_list: List[Closeable] +) -> Path: + """Modify an image customizer config file to set the PXE bootstrap base URL. + + This URL is baked into the generated PXE artifacts (grub.cfg's root=live:/image.iso), so it must match where + the bootstrap image is actually served at boot time. + + Args: + config_path: Path to the base config file + bootstrap_base_url: Base URL where the bootstrap image is served at boot time + close_list: List of resources to be cleaned up + + Returns: + Path to the modified config file + """ + config_str = config_path.read_text() + config = yaml.safe_load(config_str) + + pxe = dict_get_or_set(config, "pxe", {}) + pxe["bootstrapBaseUrl"] = bootstrap_base_url + + # Write out new config file to a temporary file. + fd, modified_config_path = tempfile.mkstemp(prefix=config_path.name + "~", suffix=".tmp", dir=config_path.parent) + with fdopen(fd, mode="w") as file: + yaml.safe_dump(config, file) + + path = Path(modified_config_path) + close_list.append(RemoveFileOnClose(path)) + return path + + def dict_get_or_set(dictionary: Dict[Any, Any], value_name: str, default: Any = None) -> Any: value = dictionary.get(value_name) if value is None: diff --git a/test/vmtests/vmtests/utils/libvirt_utils.py b/test/vmtests/vmtests/utils/libvirt_utils.py index 9855013e40..6d92db41c2 100644 --- a/test/vmtests/vmtests/utils/libvirt_utils.py +++ b/test/vmtests/vmtests/utils/libvirt_utils.py @@ -7,21 +7,51 @@ import platform import xml.etree.ElementTree as ET # noqa: N817 from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import libvirt # type: ignore class VmSpec: def __init__( - self, name: str, memory_mib: int, core_count: int, os_disk_path: Path, boot_type: str, secure_boot: bool + self, + name: str, + memory_mib: int, + core_count: int, + os_disk_path: Optional[Path], + boot_type: str, + secure_boot: bool, + pxe_boot: bool = False, + network_name: str = "default", ): + """Describe a libvirt VM for create_libvirt_domain_xml to render into a domain definition. + + Args: + name: libvirt domain name. + memory_mib: Guest RAM, in MiB. + core_count: Number of vCPUs. + os_disk_path: OS disk image to attach (a .iso boots as a CD-ROM, anything else as a virtio disk). Must be + None when pxe_boot is True, and must be set otherwise. + boot_type: "efi" for UEFI boot. + secure_boot: Whether to enable UEFI Secure Boot. + pxe_boot: When True, attach no disk and boot from the network instead: the firmware PXE-boots off the NIC + on network_name. + network_name: The libvirt network the VM's NIC attaches to. For PXE this is also the network that serves + the boot artifacts (DHCP/TFTP/HTTP). Defaults to libvirt's "default" network. + """ + if pxe_boot and os_disk_path is not None: + raise ValueError("os_disk_path must be None when pxe_boot is True") + if not pxe_boot and os_disk_path is None: + raise ValueError("os_disk_path is required when pxe_boot is False") + self.name: str = name self.memory_mib: int = memory_mib self.core_count: int = core_count - self.os_disk_path: Path = os_disk_path + self.os_disk_path: Optional[Path] = os_disk_path self.boot_type: str = boot_type self.secure_boot: bool = secure_boot + self.pxe_boot: bool = pxe_boot + self.network_name: str = network_name def _get_domain_caps( @@ -163,8 +193,6 @@ def create_libvirt_domain_xml(libvirt_conn: libvirt.virConnect, vm_spec: VmSpec) if vm_spec.boot_type == "efi": ET.SubElement(os_tag, "nvram") - os_boot = ET.SubElement(os_tag, "boot") - if vm_spec.boot_type == "efi": loader = ET.SubElement(os_tag, "loader") loader.attrib["readonly"] = "yes" @@ -238,38 +266,47 @@ def create_libvirt_domain_xml(libvirt_conn: libvirt.virConnect, vm_spec: VmSpec) network_interface.attrib["type"] = "network" network_interface_source = ET.SubElement(network_interface, "source") - network_interface_source.attrib["network"] = "default" + network_interface_source.attrib["network"] = vm_spec.network_name network_interface_model = ET.SubElement(network_interface, "model") network_interface_model.attrib["type"] = "virtio" next_disk_indexes: Dict[str, int] = {} - _, os_disk_ext = os.path.splitext(vm_spec.os_disk_path) - if os_disk_ext.lower() != ".iso": - os_boot.attrib["dev"] = "hd" - _add_disk_xml( - devices=devices, - file_path=str(vm_spec.os_disk_path), - device_type="disk", - image_type="qcow2", - bus_type="virtio", - device_prefix="vd", - read_only=False, - next_disk_indexes=next_disk_indexes, - ) + if vm_spec.pxe_boot: + # Do not attach a disk for PXE network boot, and do not set a global order since + # OVMF ignores that. Instead, set the boot order on the NIC itself, which is needed for OVMF (unlike SeaBIOS) so + # it is recognized as a boot device. + interface_boot = ET.SubElement(network_interface, "boot") + interface_boot.attrib["order"] = "1" else: - os_boot.attrib["dev"] = "cdrom" - _add_disk_xml( - devices=devices, - file_path=str(vm_spec.os_disk_path), - device_type="cdrom", - image_type="raw", - bus_type="scsi", - device_prefix="sd", - read_only=True, - next_disk_indexes=next_disk_indexes, - ) + os_boot = ET.SubElement(os_tag, "boot") + assert vm_spec.os_disk_path is not None # For type-checking + _, os_disk_ext = os.path.splitext(vm_spec.os_disk_path) + if os_disk_ext.lower() != ".iso": + os_boot.attrib["dev"] = "hd" + _add_disk_xml( + devices=devices, + file_path=str(vm_spec.os_disk_path), + device_type="disk", + image_type="qcow2", + bus_type="virtio", + device_prefix="vd", + read_only=False, + next_disk_indexes=next_disk_indexes, + ) + else: + os_boot.attrib["dev"] = "cdrom" + _add_disk_xml( + devices=devices, + file_path=str(vm_spec.os_disk_path), + device_type="cdrom", + image_type="raw", + bus_type="scsi", + device_prefix="sd", + read_only=True, + next_disk_indexes=next_disk_indexes, + ) xml = ET.tostring(domain, "unicode") return xml diff --git a/test/vmtests/vmtests/utils/libvirt_vm.py b/test/vmtests/vmtests/utils/libvirt_vm.py index 64be24cf11..bea6101f3c 100644 --- a/test/vmtests/vmtests/utils/libvirt_vm.py +++ b/test/vmtests/vmtests/utils/libvirt_vm.py @@ -107,6 +107,7 @@ def create_ssh_client( ssh_private_key_path: Path, test_temp_dir: Path, username: str, + ip_wait_time_extra: int = 0, ) -> SshClient: ssh_known_hosts_path = test_temp_dir.joinpath("known_hosts") @@ -118,6 +119,8 @@ def create_ssh_client( if platform.machine() == "aarch64": ip_wait_time = 300 + ip_wait_time += ip_wait_time_extra + # For arm64 runs, we are seeing a behavior where the first IP address that # gets assigned becomes unusable by the time we try to ssh into the machine # and then ssh fails to connect. diff --git a/test/vmtests/vmtests/utils/pxe_server.py b/test/vmtests/vmtests/utils/pxe_server.py new file mode 100644 index 0000000000..fd167a895d --- /dev/null +++ b/test/vmtests/vmtests/utils/pxe_server.py @@ -0,0 +1,161 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import logging +import os +import subprocess +import sys +import xml.etree.ElementTree as ET # noqa: N817 +from pathlib import Path +from typing import IO, Optional + +import libvirt # type: ignore + +# Fixed addressing for the dedicated PXE network. These must not collide with libvirt's default network +# (192.168.122.0/24). +PXE_NETWORK_GATEWAY_IP = "192.168.123.1" +PXE_NETWORK_NETMASK = "255.255.255.0" +PXE_NETWORK_DHCP_START = "192.168.123.2" +PXE_NETWORK_DHCP_END = "192.168.123.254" +PXE_HTTP_PORT = 8080 + + +def _build_pxe_network_xml(network_name: str, bridge_name: str, tftp_root: str, boot_loader_file: str) -> str: + network = ET.Element("network") + + name = ET.SubElement(network, "name") + name.text = network_name + + forward = ET.SubElement(network, "forward") + forward.attrib["mode"] = "nat" + + bridge = ET.SubElement(network, "bridge") + bridge.attrib["name"] = bridge_name + bridge.attrib["stp"] = "on" + bridge.attrib["delay"] = "0" + + ip = ET.SubElement(network, "ip") + ip.attrib["address"] = PXE_NETWORK_GATEWAY_IP + ip.attrib["netmask"] = PXE_NETWORK_NETMASK + + tftp = ET.SubElement(ip, "tftp") + tftp.attrib["root"] = tftp_root + + dhcp = ET.SubElement(ip, "dhcp") + + dhcp_range = ET.SubElement(dhcp, "range") + dhcp_range.attrib["start"] = PXE_NETWORK_DHCP_START + dhcp_range.attrib["end"] = PXE_NETWORK_DHCP_END + + bootp = ET.SubElement(dhcp, "bootp") + bootp.attrib["file"] = boot_loader_file + + return ET.tostring(network, "unicode") + + +def _make_world_readable(root: Path) -> None: + os.chmod(root, 0o755) + for dir_path, dir_names, file_names in os.walk(root): + for dir_name in dir_names: + os.chmod(os.path.join(dir_path, dir_name), 0o755) + for file_name in file_names: + os.chmod(os.path.join(dir_path, file_name), 0o644) + + +def _make_ancestors_traversable(leaf: Path) -> None: + # Add the world-execute bit (traverse only, not read/list) to each ancestor that lacks it, all the way up to the + # filesystem root, so the artifacts stay reachable without widening read access to any directory's contents. + current = leaf.parent + while True: + mode = current.stat().st_mode + if not mode & 0o001: + os.chmod(current, mode | 0o001) + if current == current.parent: + break + current = current.parent + + +# Stands up the network-boot environment a PXE VM test needs: +# +# - A dedicated, transient libvirt NAT network whose embedded dnsmasq serves DHCP + TFTP. The TFTP root points at the +# extracted PXE artifacts directory. +# - A plain HTTP server over the same artifacts directory. The bootstrap initramfs downloads the full-OS image from +# here. (HTTP is used for that transfer because it is far faster than TFTP for a large file.) +# +# It is Closeable so can be added to a test's close list. +class PxeEnvironment: + def __init__( + self, + libvirt_conn: libvirt.virConnect, + network_name: str, + bridge_name: str, + artifacts_dir: Path, + boot_loader_file: str, + http_log_file_path: Path, + ): + self.network_name: str = network_name + + self._network: Optional[libvirt.virNetwork] = None + self._http_process: Optional[subprocess.Popen[bytes]] = None + self._http_log: Optional[IO[str]] = None + + # pytest runs as root (via sudo) and creates the intermediate temp directories with mkdtemp (mode 0700). The + # workspace root itself may be group-private (e.g. 0750). Since dnsmasq (TFTP) and qemu run as their own users, + # ensure the artifacts are readable by others and can traverse into the artifacts directory. + _make_world_readable(artifacts_dir) + _make_ancestors_traversable(artifacts_dir) + + network_xml = _build_pxe_network_xml(network_name, bridge_name, str(artifacts_dir), boot_loader_file) + logging.debug(f"Creating PXE libvirt network:\n{network_xml}") + + try: + self._network = libvirt_conn.networkCreateXML(network_xml) + + logging.debug(f"Starting PXE HTTP server on port {PXE_HTTP_PORT} serving ({artifacts_dir})") + self._http_log = open(http_log_file_path, "w", encoding="utf-8") + self._http_process = subprocess.Popen( + [ + sys.executable, + "-m", + "http.server", + str(PXE_HTTP_PORT), + "--directory", + str(artifacts_dir), + # Bind only to the PXE NAT network's gateway IP, never 0.0.0.0. The artifacts are served + # unauthenticated over plain HTTP and are world-readable, so the listener must not be reachable from + # the build host's physical LAN. Binding to the gateway IP keeps it on the libvirt bridge interface + # only, so the guest VM under test can still reach it while other physical machines on the LAN + # cannot. The bridge interface already holds this IP because the libvirt network was created above. + "--bind", + PXE_NETWORK_GATEWAY_IP, + ], + stdout=self._http_log, + stderr=subprocess.STDOUT, + ) + except BaseException: + self.close() + raise + + def close(self) -> None: + if self._http_process is not None: + logging.debug("Stopping PXE HTTP server") + self._http_process.terminate() + try: + self._http_process.wait(timeout=10) + except subprocess.TimeoutExpired: + self._http_process.kill() + + self._http_process = None + + if self._http_log is not None: + self._http_log.close() + self._http_log = None + + if self._network is not None: + logging.debug(f"Destroying PXE libvirt network: {self.network_name}") + try: + self._network.destroy() + except libvirt.libvirtError as ex: + logging.warning(f"PXE network destroy failed. {ex}") + + self._network = None diff --git a/toolkit/tools/pkg/imagecustomizerlib/blsutils.go b/toolkit/tools/pkg/imagecustomizerlib/blsutils.go index a3d3a37c45..64eae53ca1 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/blsutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/blsutils.go @@ -135,6 +135,102 @@ func readKernelCmdlinesFromBLSEntries(bootDir string) (map[string][]grubConfigLi return kernelToArgs, nil } +// renderGrubMenuEntriesFromBLS reads the Boot Loader Specification (BLS) entries in {bootDir}/loader/entries/*.conf and +// renders an equivalent grub `menuentry` block for each non-recovery entry, in directory order, mirroring what grub's +// `blscfg` command generates at boot. It is used where grub cannot run `blscfg` itself: over PXE the boot transport +// (TFTP) cannot enumerate the entries directory, so the menu must be spelled out ahead of time. +func renderGrubMenuEntriesFromBLS(bootDir string) (string, error) { + entriesDir := filepath.Join(bootDir, "loader", "entries") + entries, err := os.ReadDir(entriesDir) + if err != nil { + return "", fmt.Errorf("failed to read BLS entries directory (%s):\n%w", entriesDir, err) + } + + var builder strings.Builder + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".conf") { + logger.Log.Debugf("Skipping non-.conf BLS entry file (%s) in directory (%s)", entry.Name(), entriesDir) + continue + } + + absPath := filepath.Join(entriesDir, entry.Name()) + content, err := os.ReadFile(absPath) + if err != nil { + return "", fmt.Errorf("failed to read BLS entry file (%s):\n%w", absPath, err) + } + + var title string + var titleSeen bool + var linux string + var initrd string + var options string + var optionsSeen bool + + for _, field := range bls.ParseFields(string(content)) { + switch field.Key { + case "linux": + if linux != "" { + return "", fmt.Errorf("duplicate key (%s) in BLS entry (%s)", field.Key, absPath) + } + if field.Value == "" { + return "", fmt.Errorf("BLS entry (%s) 'linux' key has empty value", absPath) + } + linux = field.Value + case "initrd": + if initrd != "" { + return "", fmt.Errorf("duplicate key (%s) in BLS entry (%s)", field.Key, absPath) + } + initrd = field.Value + case "title": + // Per BLS spec each non-options key may appear at most once. An empty title value is still a valid + // (normal) entry, so track its presence explicitly rather than inferring from title != "". + if titleSeen { + return "", fmt.Errorf("duplicate key (%s) in BLS entry (%s)", field.Key, absPath) + } + titleSeen = true + title = field.Value + case "efi", "uki", "uki-url": + return "", fmt.Errorf("BLS entry (%s) uses '%s' key, which is not supported", absPath, field.Key) + case "options": + // "options" may appear multiple times per BLS spec; concatenate in source order so the kernel command + // line is preserved verbatim. + if optionsSeen { + options += " " + field.Value + } else { + options = field.Value + optionsSeen = true + } + } + } + + if linux == "" { + return "", fmt.Errorf("BLS entry (%s) is missing 'linux' key", absPath) + } + + // Entries without titles are treated as normal entries. + if bls.IsRescueEntryTitle(title) { + logger.Log.Debugf("Skipping recovery/rescue BLS entry with title (%s) in file (%s)", title, absPath) + continue + } + + // A grub single-quoted string cannot contain a literal single quote; use the '\'' idiom to embed one. + quotedTitle := strings.ReplaceAll(title, "'", `'\''`) + + builder.WriteString(fmt.Sprintf("menuentry '%s' {\n", quotedTitle)) + if options != "" { + builder.WriteString(fmt.Sprintf("\tlinux %s %s\n", linux, options)) + } else { + builder.WriteString(fmt.Sprintf("\tlinux %s\n", linux)) + } + if initrd != "" { + builder.WriteString(fmt.Sprintf("\tinitrd %s\n", initrd)) + } + builder.WriteString("}\n") + } + + return builder.String(), nil +} + // readNonRecoveryKernelCmdlinesFromBLS reads the first non-recovery kernel's command-line // arguments from BLS entry files. func readNonRecoveryKernelCmdlinesFromBLS(bootDir string, argNames []string) (map[string]string, error) { diff --git a/toolkit/tools/pkg/imagecustomizerlib/blsutils_test.go b/toolkit/tools/pkg/imagecustomizerlib/blsutils_test.go index 0dffa0d60e..b6ae4da0ba 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/blsutils_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/blsutils_test.go @@ -594,6 +594,24 @@ func writeTestBLSEntry(t *testing.T, bootDir string) string { return entryPath } +// writeTestPxeGrubCfg creates a minimal AZL4-style grub.cfg with a `blscfg` invocation under {bootDir}/grub2 and +// returns the grub.cfg's absolute path. It mirrors the relevant lines that finalizeLiveOSPxeBLSEntries rewrites. +func writeTestPxeGrubCfg(t *testing.T, bootDir string) string { + grubDir := filepath.Join(bootDir, "grub2") + err := os.MkdirAll(grubDir, 0o755) + assert.NoError(t, err) + + content := "### BEGIN /etc/grub.d/10_linux ###\n" + + "insmod blscfg\n" + + "blscfg\n" + + "### END /etc/grub.d/10_linux ###\n" + + grubCfgPath := filepath.Join(grubDir, "grub.cfg") + err = os.WriteFile(grubCfgPath, []byte(content), 0o644) + assert.NoError(t, err) + return grubCfgPath +} + func TestUpdateLiveOSBLSEntriesFullOS(t *testing.T) { bootDir := t.TempDir() entryPath := writeTestBLSEntry(t, bootDir) @@ -661,6 +679,60 @@ func TestSetLiveOSBLSEntriesRootForIso(t *testing.T) { assert.NotContains(t, gotStr, "root=UUID=") } +func TestFinalizeLiveOSPxeBLSEntries(t *testing.T) { + bootDir := t.TempDir() + entryPath := writeTestBLSEntry(t, bootDir) + grubCfgPath := writeTestPxeGrubCfg(t, bootDir) + + err := finalizeLiveOSPxeBLSEntries(bootDir, imagecustomizerapi.InitramfsImageTypeBootstrap, + "" /*bootstrapBaseUrl*/, "http://pxe-server/my.iso" /*bootstrapFileUrl*/) + assert.NoError(t, err) + + got, err := os.ReadFile(entryPath) + assert.NoError(t, err) + gotStr := string(got) + + assert.Contains(t, gotStr, "root=live:http://pxe-server/my.iso") + assert.NotContains(t, gotStr, "root=UUID=") + assert.Contains(t, gotStr, "rd.live.azldownloader=enable") + assert.Regexp(t, `(?m)^options .*ip=dhcp`, gotStr) + + // The BLS entry is expanded into an explicit grub menuentry. + grubCfg, err := os.ReadFile(grubCfgPath) + assert.NoError(t, err) + grubCfgStr := string(grubCfg) + assert.Regexp(t, `(?m)^menuentry 'Azure Linux \(6\.18\.31-1\.5\.azl4\.x86_64\) 4\.0' \{$`, grubCfgStr) + assert.Regexp(t, `(?m)^\tlinux /boot/vmlinuz-6\.18\.31-1\.5\.azl4\.x86_64 .*ip=dhcp`, grubCfgStr) + assert.Regexp(t, `(?m)^\tinitrd /boot/initramfs-6\.18\.31-1\.5\.azl4\.x86_64\.img$`, grubCfgStr) + assert.Contains(t, grubCfgStr, "set default=0") + assert.NotRegexp(t, `(?m)^[ \t]*blscfg[ \t]*$`, grubCfgStr) + assert.Contains(t, grubCfgStr, "insmod blscfg") +} + +func TestFinalizeLiveOSPxeBLSEntriesFullOSNoOp(t *testing.T) { + bootDir := t.TempDir() + entryPath := writeTestBLSEntry(t, bootDir) + grubCfgPath := writeTestPxeGrubCfg(t, bootDir) + + before, err := os.ReadFile(entryPath) + assert.NoError(t, err) + + // Full-OS pxe has no bootstrap url to inject, so the BLS entry's kernel args must be left untouched. + err = finalizeLiveOSPxeBLSEntries(bootDir, imagecustomizerapi.InitramfsImageTypeFullOS, "", "") + assert.NoError(t, err) + + after, err := os.ReadFile(entryPath) + assert.NoError(t, err) + assert.Equal(t, string(before), string(after)) + + // The grub.cfg is still expanded into menuentries. + grubCfg, err := os.ReadFile(grubCfgPath) + assert.NoError(t, err) + grubCfgStr := string(grubCfg) + assert.Regexp(t, `(?m)^menuentry 'Azure Linux \(6\.18\.31-1\.5\.azl4\.x86_64\) 4\.0' \{$`, grubCfgStr) + assert.NotRegexp(t, `(?m)^[ \t]*blscfg[ \t]*$`, grubCfgStr) +} + func TestSetBLSEntryField(t *testing.T) { content := "title Foo\n" + "linux /boot/vmlinuz-6.6\n" + diff --git a/toolkit/tools/pkg/imagecustomizerlib/distrohandler.go b/toolkit/tools/pkg/imagecustomizerlib/distrohandler.go index 61c22ecdb2..a98f4a48d9 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/distrohandler.go +++ b/toolkit/tools/pkg/imagecustomizerlib/distrohandler.go @@ -141,6 +141,15 @@ type DistroHandler interface { UpdateLiveOSGrubCfgForIso(grubCfgContent string, bootDir string, initramfsType imagecustomizerapi.InitramfsImageType) (string, error) + // UpdateLiveOSGrubCfgForPxe applies the pxe-specific edits on top of the LiveOS edits. + UpdateLiveOSGrubCfgForPxe(grubCfgContent string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string) (string, error) + + // FinalizeLiveOSPxeBootConfig writes the pxe-specific root=live: kernel arg into the staged PXE artifacts at + // pxeBootDir. + FinalizeLiveOSPxeBootConfig(pxeBootDir string, initramfsType imagecustomizerapi.InitramfsImageType, + bootstrapBaseUrl string, bootstrapFileUrl string) error + // ShimPackage returns the package that provides the shim EFI binary for this distro on the current architecture. ShimPackage() string diff --git a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_acl.go b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_acl.go index 49e2037ca0..690570a1b9 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_acl.go +++ b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_acl.go @@ -308,6 +308,18 @@ func (d *aclDistroHandler) UpdateLiveOSGrubCfgForIso(grubCfgContent string, boot return updateGrubCfgForIso(grubCfgContent, initramfsType) } +func (d *aclDistroHandler) UpdateLiveOSGrubCfgForPxe(grubCfgContent string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) (string, error) { + return updateGrubCfgForPxe(grubCfgContent, initramfsType, bootstrapBaseUrl, bootstrapFileUrl) +} + +func (d *aclDistroHandler) FinalizeLiveOSPxeBootConfig(pxeBootDir string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) error { + return nil +} + func (d *aclDistroHandler) ShimPackage() string { // ACL uses systemd-boot + UKI (no shim/grub from a package). return "" diff --git a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux.go b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux.go index 8e2311c50d..1c4e6c9664 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux.go +++ b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux.go @@ -234,6 +234,18 @@ func (d *azureLinuxDistroHandler) UpdateLiveOSGrubCfgForIso(grubCfgContent strin return updateGrubCfgForIso(grubCfgContent, initramfsType) } +func (d *azureLinuxDistroHandler) UpdateLiveOSGrubCfgForPxe(grubCfgContent string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) (string, error) { + return updateGrubCfgForPxe(grubCfgContent, initramfsType, bootstrapBaseUrl, bootstrapFileUrl) +} + +func (d *azureLinuxDistroHandler) FinalizeLiveOSPxeBootConfig(pxeBootDir string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) error { + return nil +} + func (d *azureLinuxDistroHandler) ShimPackage() string { return shimPackageAzl3 } diff --git a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux4.go b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux4.go index 5da08fd2d3..419bbfa913 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux4.go +++ b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_azurelinux4.go @@ -114,19 +114,11 @@ func (d *azureLinux4DistroHandler) ValidateConfig(rc *ResolvedConfig) error { return nil } -var ErrAzureLinux4PxeUnsupported = NewImageCustomizerError("Validation:AzureLinux4PxeUnsupported", - "PXE output format is not supported yet for Azure Linux 4.0 images") - func (d *azureLinux4DistroHandler) checkForUnsupportedApis(rc *ResolvedConfig) error { if rc.HasPackageSnapshotTime() { return ErrUnsupportedPackageSnapshotTime } - switch rc.OutputImageFormat { - case imagecustomizerapi.ImageFormatTypePxeDir, imagecustomizerapi.ImageFormatTypePxeTar: - return ErrAzureLinux4PxeUnsupported - } - return nil } @@ -312,6 +304,18 @@ func (d *azureLinux4DistroHandler) UpdateLiveOSGrubCfgForIso(grubCfgContent stri return updateLiveOSGrubCfgBLSForIso(grubCfgContent, bootDir, initramfsType) } +func (d *azureLinux4DistroHandler) UpdateLiveOSGrubCfgForPxe(grubCfgContent string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) (string, error) { + return updateLiveOSGrubCfgBLSForPxe(grubCfgContent) +} + +func (d *azureLinux4DistroHandler) FinalizeLiveOSPxeBootConfig(pxeBootDir string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) error { + return finalizeLiveOSPxeBLSEntries(pxeBootDir, initramfsType, bootstrapBaseUrl, bootstrapFileUrl) +} + func (d *azureLinux4DistroHandler) warnIfUnsignedSystemdBootPackage(detectedPackage string) { if detectedPackage == systemdBootUnsignedPackageAzl4 { logger.Log.Warnf("Detected package (%s): Customized image will fail Secure Boot verification", detectedPackage) diff --git a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_fedora.go b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_fedora.go index f724416007..09edecca49 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_fedora.go +++ b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_fedora.go @@ -299,6 +299,18 @@ func (d *fedoraDistroHandler) UpdateLiveOSGrubCfgForIso(grubCfgContent string, b return updateLiveOSGrubCfgBLSForIso(grubCfgContent, bootDir, initramfsType) } +func (d *fedoraDistroHandler) UpdateLiveOSGrubCfgForPxe(grubCfgContent string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) (string, error) { + return updateLiveOSGrubCfgBLSForPxe(grubCfgContent) +} + +func (d *fedoraDistroHandler) FinalizeLiveOSPxeBootConfig(pxeBootDir string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) error { + return finalizeLiveOSPxeBLSEntries(pxeBootDir, initramfsType, bootstrapBaseUrl, bootstrapFileUrl) +} + func (d *fedoraDistroHandler) ShimPackage() string { switch runtime.GOARCH { case "amd64": diff --git a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_ubuntu.go b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_ubuntu.go index 13b98c331a..017b46fb08 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/distrohandler_ubuntu.go +++ b/toolkit/tools/pkg/imagecustomizerlib/distrohandler_ubuntu.go @@ -248,6 +248,18 @@ func (d *ubuntuDistroHandler) UpdateLiveOSGrubCfgForIso(grubCfgContent string, b return updateGrubCfgForIso(grubCfgContent, initramfsType) } +func (d *ubuntuDistroHandler) UpdateLiveOSGrubCfgForPxe(grubCfgContent string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) (string, error) { + return updateGrubCfgForPxe(grubCfgContent, initramfsType, bootstrapBaseUrl, bootstrapFileUrl) +} + +func (d *ubuntuDistroHandler) FinalizeLiveOSPxeBootConfig(pxeBootDir string, + initramfsType imagecustomizerapi.InitramfsImageType, bootstrapBaseUrl string, bootstrapFileUrl string, +) error { + return nil +} + func (d *ubuntuDistroHandler) ShimPackage() string { return "shim" } @@ -270,7 +282,6 @@ func (d *ubuntuDistroHandler) LiveOSGrubEfiPrefixDir() string { } func (d *ubuntuDistroHandler) LiveOSInitrdDracutModules() []string { - // Ubuntu LiveOS is not a validated path; default to the inline distros' module set. return liveOSInitrdDracutModulesAzl3 } diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go index b7d8e32cf7..1397bcbf07 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go @@ -862,10 +862,6 @@ func TestCustomizeImageLiveOSPxe1(t *testing.T) { err := basicCustomizeImage(t.Context(), buildDir, testDir, config, baseImage, outImageFilePath, string(imagecustomizerapi.ImageFormatTypePxeTar), baseImageInfo.PreviewFeatures) - if baseImageInfo.Version == baseImageVersionAzl4 { - assert.ErrorIs(t, err, ErrAzureLinux4PxeUnsupported) - return - } if !assert.NoError(t, err) { return } @@ -890,10 +886,6 @@ func TestCustomizeImageLiveOSPxe2(t *testing.T) { err := basicCustomizeImage(t.Context(), buildDir, testDir, config, baseImage, outImageFilePath, string(imagecustomizerapi.ImageFormatTypePxeTar), baseImageInfo.PreviewFeatures) - if baseImageInfo.Version == baseImageVersionAzl4 { - assert.ErrorIs(t, err, ErrAzureLinux4PxeUnsupported) - return - } if !assert.NoError(t, err) { return } diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisogrub.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisogrub.go index c1d17f7232..f2b4e51e02 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisogrub.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisogrub.go @@ -6,6 +6,8 @@ package imagecustomizerlib import ( "fmt" "path/filepath" + "regexp" + "strings" "github.com/microsoft/azure-linux-image-tools/toolkit/tools/imagecustomizerapi" "github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/file" @@ -31,6 +33,10 @@ const ( initrdPathAzl2Template = "/boot/initrd.img-%s" ) +// blscfgCommandRegex matches the bare `blscfg` command invocation (the line that, at boot, enumerates the BLS entries +// under /boot/loader/entries) on its own line. +var blscfgCommandRegex = regexp.MustCompile(`(?m)^[ \t]*blscfg[ \t]*$`) + // updateLiveOSGrubCfgBLSForLiveOS applies the common LiveOS-compatibility edits for distros that use Boot Loader // Specification entries. func updateLiveOSGrubCfgBLSForLiveOS(grubCfgContent string, bootDir string, @@ -63,6 +69,74 @@ func updateLiveOSGrubCfgBLSForIso(grubCfgContent string, bootDir string, return grubCfgContent, nil } +// updateLiveOSGrubCfgBLSForPxe applies the pxe-specific grub.cfg edits (removing 'search') for BLS distros. +func updateLiveOSGrubCfgBLSForPxe(grubCfgContent string) (string, error) { + grubCfgContent, err := removeCommandAll(grubCfgContent, "search") + if err != nil { + return "", fmt.Errorf("failed to remove the 'search' commands from PXE grub.cfg:\n%w", err) + } + return grubCfgContent, nil +} + +// finalizeLiveOSPxeBLSEntries writes the pxe-specific root=live: arg (plus the dracut pxe args) to the BLS +// entries under bootDir. +func finalizeLiveOSPxeBLSEntries(bootDir string, initramfsImageType imagecustomizerapi.InitramfsImageType, + bootstrapBaseUrl string, bootstrapFileUrl string, +) error { + if initramfsImageType == imagecustomizerapi.InitramfsImageTypeBootstrap { + fileUrl, err := getPxeBootstrapFileUrl(bootstrapBaseUrl, bootstrapFileUrl) + if err != nil { + return err + } + + rootValue := fmt.Sprintf(rootValuePxeTemplate, fileUrl) + err = setLiveOSBLSEntriesRoot(bootDir, rootValue, strings.Fields(pxeBootstrapKernelsArgs)) + if err != nil { + return err + } + } + + // The BLS entries now carry their final kernel command line, so expand them into explicit grub menuentries. + return expandBLSEntriesToPxeGrubMenu(bootDir) +} + +// expandBLSEntriesToPxeGrubMenu rewrites the staged PXE grub.cfg so it carries explicit menuentries instead of relying +// on the `blscfg` command. +func expandBLSEntriesToPxeGrubMenu(bootDir string) error { + menuEntries, err := renderGrubMenuEntriesFromBLS(bootDir) + if err != nil { + return err + } + if menuEntries == "" { + return fmt.Errorf("found no BLS entries under (%s) to build the PXE grub menu", + filepath.Join(bootDir, "loader", "entries")) + } + + grubCfgPath := filepath.Join(bootDir, "grub2", isoGrubCfg) + content, err := file.Read(grubCfgPath) + if err != nil { + return fmt.Errorf("failed to read PXE grub.cfg (%s):\n%w", grubCfgPath, err) + } + + if !blscfgCommandRegex.MatchString(content) { + return fmt.Errorf("expected a 'blscfg' command in PXE grub.cfg (%s) but found none", grubCfgPath) + } + + // Boot the first entry deterministically: over PXE the grubenv is not writable, so the header's + // 'set default="${saved_entry}"' resolves to an empty value. + replacement := "set default=0\n" + strings.TrimRight(menuEntries, "\n") + + // Use the literal replacement form so '$' in a kernel command line is not treated as a regexp group reference. + newContent := blscfgCommandRegex.ReplaceAllLiteralString(content, replacement) + + err = file.Write(newContent, grubCfgPath) + if err != nil { + return fmt.Errorf("failed to write PXE grub.cfg (%s):\n%w", grubCfgPath, err) + } + + return nil +} + func updateGrubCfgForLiveOS(inputContentString string, initramfsImageType imagecustomizerapi.InitramfsImageType, disableSELinux bool, savedConfigs *SavedConfigs, kernelVersions []string, ) (string, error) { @@ -263,8 +337,8 @@ func updateGrubCfg(inputGrubCfgPath string, outputFormat imagecustomizerapi.Imag return fmt.Errorf("cannot generate grub.cfg for PXE booting.\n%v", err) } } - pxeContentString, err := updateGrubCfgForPxe(liveosContentString, initramfsImageType, savedConfigs.Pxe.bootstrapBaseUrl, - savedConfigs.Pxe.bootstrapFileUrl) + pxeContentString, err := distroHandler.UpdateLiveOSGrubCfgForPxe(liveosContentString, initramfsImageType, + savedConfigs.Pxe.bootstrapBaseUrl, savedConfigs.Pxe.bootstrapFileUrl) if err != nil { return fmt.Errorf("failed to create grub configuration for PXE booting.\n%w", err) } diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisoimages.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisoimages.go index 97754b0bcd..0f979688b4 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisoimages.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisoimages.go @@ -242,10 +242,19 @@ func stageLiveOSFile(stageDirPath string, stageFile StageFile) error { return fmt.Errorf("failed to create destination directory (%s):\n%w", targetDir, err) } - // Preserve symlinks rather than dereferencing them. Some kernel boot files (e.g. Azure Linux 4.0's - // symvers-.xz) are symlinks into the rootfs that dangle in the flattened artifacts directory. The - // artifacts were copied in with --no-dereference, so stage them the same way; mkisofs records the symlinks - // via Rock Ridge. Regular files fall through to a normal copy. + // Skip dangling symlinks. Some kernel boot files (e.g. Azure Linux 4.0's symvers-.xz, a symlink into + // /lib/modules//) dangle in the flattened artifacts directory. They are kernel module-build artifacts that + // are not needed to boot, and a dangling symlink on the boot media serves no purpose while potentially breaking + // consumers that extract the artifacts onto a real filesystem. + if info, lerr := os.Lstat(stageFile.sourcePath); lerr == nil && info.Mode()&os.ModeSymlink != 0 { + if _, serr := os.Stat(stageFile.sourcePath); serr != nil { + logger.Log.Debugf("Skipping dangling symlink while staging Live OS file (%s)", stageFile.sourcePath) + return nil + } + } + + // Preserve symlinks rather than dereferencing them. The artifacts were copied in with --no-dereference, so stage + // them the same way; mkisofs records the symlinks via Rock Ridge. Regular files fall through to a normal copy. err = file.NewFileCopyBuilder(stageFile.sourcePath, targetPath). SetNoDereference(). Run() @@ -437,7 +446,7 @@ func createIsoImage(buildDir string, initramfsType imagecustomizerapi.InitramfsI } if grubPrefixDir := distroHandler.LiveOSGrubEfiPrefixDir(); grubPrefixDir != "" { - err = writeIsoGrubPrefixRedirector(stagingDir, grubPrefixDir) + err = writeGrubPrefixRedirector(stagingDir, grubPrefixDir, isogenerator.DefaultVolumeId) if err != nil { return fmt.Errorf("failed to stage grub prefix redirector:\n%w", err) } @@ -453,17 +462,23 @@ func createIsoImage(buildDir string, initramfsType imagecustomizerapi.InitramfsI return nil } -// writeIsoGrubPrefixRedirector writes a minimal grub.cfg into grubPrefixDir (the directory the ISO's grub EFI binary -// uses as its baked-in 'prefix', e.g. EFI/azurelinux) within stagingPath. BLS-style distros (Azure Linux 4.0, Fedora) -// ship a grub binary whose prefix points at EFI/ rather than the boot media root, so without this file grub -// would look for /grub.cfg, find nothing, and drop to the 'grub>' rescue prompt. The redirector locates the -// LiveOS volume by label and chains to the real configuration at isoGrubCfgPath. -func writeIsoGrubPrefixRedirector(stagingPath string, grubPrefixDir string) error { - redirector := fmt.Sprintf("search --label %s --set root\n"+ - "set prefix=($root)%s\n"+ - "configfile ($root)%s\n", isogenerator.DefaultVolumeId, grubCfgDir, isoGrubCfgPath) - - targetPath := filepath.Join(stagingPath, grubPrefixDir, isoGrubCfg) +// writeGrubPrefixRedirector writes a minimal grub.cfg into grubPrefixDir (the directory the grub EFI binary uses as +// its baked-in 'prefix', e.g. EFI/azurelinux) within rootDir, and chains to the real configuration at isoGrubCfgPath. +// BLS-style distros (Azure Linux 4.0, Fedora) ship a grub binary whose prefix points at EFI/ rather than the +// boot media root, so without this file grub would look for /grub.cfg, find nothing, and drop to the 'grub>' +// rescue prompt. +// +// How $root is established before chaining depends on the boot media: for media grub must locate by filesystem label +// (the ISO), pass its volume id in searchVolumeId so the redirector issues a 'search --label'; for network boot (PXE) +// pass "" because $root is already the TFTP device grub was network-loaded from. +func writeGrubPrefixRedirector(rootDir, grubPrefixDir, searchVolumeId string) error { + redirector := "" + if searchVolumeId != "" { + redirector = fmt.Sprintf("search --label %s --set root\n", searchVolumeId) + } + redirector += fmt.Sprintf("set prefix=($root)%s\nconfigfile ($root)%s\n", grubCfgDir, isoGrubCfgPath) + + targetPath := filepath.Join(rootDir, grubPrefixDir, isoGrubCfg) err := os.MkdirAll(filepath.Dir(targetPath), os.ModePerm) if err != nil { return fmt.Errorf("failed to create grub prefix directory (%s):\n%w", filepath.Dir(targetPath), err) diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveospxe.go b/toolkit/tools/pkg/imagecustomizerlib/liveospxe.go index 710a882762..558d0da437 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveospxe.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveospxe.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "github.com/microsoft/azure-linux-image-tools/toolkit/tools/imagecustomizerapi" "github.com/microsoft/azure-linux-image-tools/toolkit/tools/internal/file" @@ -45,7 +46,8 @@ func getPxeBootstrapFileName(bootstrapBaseUrl, bootstrapFileUrl string) (string, func createPXEArtifacts(buildDir string, outputFormat imagecustomizerapi.ImageFormatType, initramfsType imagecustomizerapi.InitramfsImageType, artifactsStore *IsoArtifactsStore, kdumpBootFiles *imagecustomizerapi.KdumpBootFilesType, additionalIsoFiles imagecustomizerapi.AdditionalFileList, - bootstrapBaseUrl, bootstrapFileUrl, outputPath string, distroHandler DistroHandler) (err error) { + bootstrapBaseUrl, bootstrapFileUrl, outputPath string, distroHandler DistroHandler, +) (err error) { logger.Log.Infof("Creating PXE output at (%s)", outputPath) outputPXEArtifactsDir := "" @@ -112,6 +114,15 @@ func createPXEArtifacts(buildDir string, outputFormat imagecustomizerapi.ImageFo } } + // Finalize the pxe boot configuration. It must run after the embedded ISO above is sealed, since that ISO is built + // from the iso-flavored (root=live:LABEL) entries. For inline-grub distros this is a no-op + // (the pxe grub.cfg already carries the arg). + err = distroHandler.FinalizeLiveOSPxeBootConfig(filepath.Join(outputPXEArtifactsDir, "boot"), initramfsType, + bootstrapBaseUrl, bootstrapFileUrl) + if err != nil { + return fmt.Errorf("failed to finalize the PXE boot configuration:\n%w", err) + } + // Note that the moves/removes must take place afer the bootstrapped ISO is // created because some of these files are needed by the ISO so it is // bootable. @@ -127,7 +138,10 @@ func createPXEArtifacts(buildDir string, outputFormat imagecustomizerapi.ImageFo for _, bootloaderFile := range bootloaderFiles { sourcePath := filepath.Join(bootloaderSrcDir, bootloaderFile) - targetPath := filepath.Join(outputPXEArtifactsDir, bootloaderFile) + // The PXE TFTP root uses the conventional lowercase boot loader names (bootx64.efi, grubx64.efi) documented in + // the PXE layout. Some distros name the ESP shim in uppercase (Fedora and Azure Linux 4 ship BOOTX64.EFI), so + // normalize the name at the PXE root. + targetPath := filepath.Join(outputPXEArtifactsDir, strings.ToLower(bootloaderFile)) err = file.Move(sourcePath, targetPath) if err != nil { return fmt.Errorf("failed to move boot loader file from (%s) to (%s) while generated the PXE artifacts folder:\n%w", sourcePath, targetPath, err) @@ -141,6 +155,16 @@ func createPXEArtifacts(buildDir string, outputFormat imagecustomizerapi.ImageFo return fmt.Errorf("failed to remove folder (%s):\n%w", isoEFIDir, err) } + // Some distros (Azure Linux 4.0, Fedora) ship a grub EFI binary whose baked-in prefix points at + // EFI/ rather than the TFTP root, so over PXE grub looks for /grub.cfg initially. Write a + // redirector here that chains to /boot/grub2/grub.cfg. + if grubPrefixDir := distroHandler.LiveOSGrubEfiPrefixDir(); grubPrefixDir != "" { + err = writeGrubPrefixRedirector(outputPXEArtifactsDir, grubPrefixDir, "" /*searchVolumeId*/) + if err != nil { + return err + } + } + // If a tar.gz is requested, create the archive if outputFormat == imagecustomizerapi.ImageFormatTypePxeTar { err = tarutils.CreateTarGzArchive(outputPXEArtifactsDir, outputPXEImage) diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl3.yaml b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl3.yaml new file mode 100644 index 0000000000..e1b7fdf512 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl3.yaml @@ -0,0 +1,30 @@ +# Azure Linux 3.0 PXE (bootstrap) VM test config. +# +# The bootstrap initramfs downloads the full-OS image (image.iso) over the network at boot time. The download URL is +# derived from pxe.bootstrapBaseUrl, which is injected by the test at run time (it must match the HTTP server the test +# stands up), so it is intentionally not set here. +# +# Differs from pxe-bootstrap-vm-azl4 only in the SELinux package set: Azure Linux 3.0 additionally requires +# selinux-policy-modules. +pxe: + initramfsType: bootstrap + + additionalFiles: + # Enable DHCP client on all of the physical NICs so the booted OS can be reached over SSH. + - source: files/89-ethernet.network + destination: /etc/systemd/network/89-ethernet.network + +os: + selinux: + mode: enforcing + + packages: + install: + # iso/pxe required packages + - squashfs-tools + - tar + - device-mapper + - curl + # Required packages for SELinux. + - selinux-policy + - selinux-policy-modules diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl4.yaml b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl4.yaml new file mode 100644 index 0000000000..36e912f796 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-bootstrap-vm-azl4.yaml @@ -0,0 +1,24 @@ +# Differs from pxe-bootstrap-vm-azl3 in that it installs dracut-live and doesn't install selinux-policy-modules. +pxe: + initramfsType: bootstrap + + additionalFiles: + # Enable DHCP client on all of the physical NICs so the booted OS can be reached over SSH. + - source: files/89-ethernet.network + destination: /etc/systemd/network/89-ethernet.network + +os: + selinux: + mode: enforcing + + packages: + install: + # iso/pxe required packages + - squashfs-tools + - tar + - device-mapper + - curl + # Azure Linux 4.0 ships the dmsquash-live dracut module in a separate dracut-live package. + - dracut-live + # Required package for SELinux. + - selinux-policy diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl3.yaml b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl3.yaml new file mode 100644 index 0000000000..f08b57c200 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl3.yaml @@ -0,0 +1,25 @@ +pxe: + initramfsType: full-os + +os: + selinux: + mode: disabled + + kernelCommandLine: + extraCommandLine: + - "selinux=0" + + packages: + install: + # multi-kernel test + - kernel-6.6.57.1-6.azl3 + # iso/pxe required packages + - squashfs-tools + - tar + - device-mapper + - curl + + additionalFiles: + # Enable DHCP client on all of the physical NICs so the booted OS can be reached over SSH. + - source: files/89-ethernet.network + destination: /etc/systemd/network/89-ethernet.network diff --git a/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl4.yaml b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl4.yaml new file mode 100644 index 0000000000..8c77a24d56 --- /dev/null +++ b/toolkit/tools/pkg/imagecustomizerlib/testdata/pxe-full-os-vm-azl4.yaml @@ -0,0 +1,26 @@ +# Differs from pxe-full-os-vm-azl3 in the kernel version. +pxe: + initramfsType: full-os + +os: + selinux: + mode: disabled + + kernelCommandLine: + extraCommandLine: + - "selinux=0" + + packages: + install: + # multi-kernel test + - kernel-6.18.5-1.8.azl4 + # iso/pxe required packages + - squashfs-tools + - tar + - device-mapper + - curl + + additionalFiles: + # Enable DHCP client on all of the physical NICs so the booted OS can be reached over SSH. + - source: files/89-ethernet.network + destination: /etc/systemd/network/89-ethernet.network From 629c1708afd7013fcae000b6838823ec1c1bd681 Mon Sep 17 00:00:00 2001 From: Vince Perri <5596945+vinceaperri@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:43:29 +0000 Subject: [PATCH 2/2] Address review feedback - Run the PXE HTTP server via multiprocessing.Process instead of a python -m http.server subprocess. - Extract PXE artifacts into a world-traversable /var/tmp directory and drop the chmod of ancestor directories. --- .../vmtests/imagecustomizer/test_pxe.py | 8 +- test/vmtests/vmtests/utils/pxe_server.py | 101 +++++++++--------- 2 files changed, 51 insertions(+), 58 deletions(-) diff --git a/test/vmtests/vmtests/imagecustomizer/test_pxe.py b/test/vmtests/vmtests/imagecustomizer/test_pxe.py index 9c4ba0d519..073f5ad7eb 100644 --- a/test/vmtests/vmtests/imagecustomizer/test_pxe.py +++ b/test/vmtests/vmtests/imagecustomizer/test_pxe.py @@ -5,7 +5,6 @@ import platform import random import string -import tarfile from pathlib import Path from typing import List, Tuple @@ -80,11 +79,6 @@ def run_pxe_test( image_file=input_image, ) - pxe_artifacts_dir = test_temp_dir.joinpath("pxe-artifacts") - pxe_artifacts_dir.mkdir() - with tarfile.open(pxe_tar_path, "r:gz") as tar: - tar.extractall(pxe_artifacts_dir) - customized_name = ( "pxe_" + initramfs_type.replace("-", "_") @@ -106,7 +100,7 @@ def run_pxe_test( libvirt_conn, network_name, bridge_name, - pxe_artifacts_dir, + pxe_tar_path, boot_loader_file, http_log_file_path, ) diff --git a/test/vmtests/vmtests/utils/pxe_server.py b/test/vmtests/vmtests/utils/pxe_server.py index fd167a895d..0c37774980 100644 --- a/test/vmtests/vmtests/utils/pxe_server.py +++ b/test/vmtests/vmtests/utils/pxe_server.py @@ -2,12 +2,17 @@ # Licensed under the MIT License. import logging +import multiprocessing import os -import subprocess +import shutil import sys +import tarfile +import tempfile import xml.etree.ElementTree as ET # noqa: N817 +from functools import partial +from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path -from typing import IO, Optional +from typing import Optional import libvirt # type: ignore @@ -62,17 +67,13 @@ def _make_world_readable(root: Path) -> None: os.chmod(os.path.join(dir_path, file_name), 0o644) -def _make_ancestors_traversable(leaf: Path) -> None: - # Add the world-execute bit (traverse only, not read/list) to each ancestor that lacks it, all the way up to the - # filesystem root, so the artifacts stay reachable without widening read access to any directory's contents. - current = leaf.parent - while True: - mode = current.stat().st_mode - if not mode & 0o001: - os.chmod(current, mode | 0o001) - if current == current.parent: - break - current = current.parent +def _serve_http(directory: Path, bind_ip: str, port: int, log_file_path: Path) -> None: + with open(log_file_path, "w", encoding="utf-8", buffering=1) as log_file: + sys.stdout = log_file + sys.stderr = log_file + handler = partial(SimpleHTTPRequestHandler, directory=str(directory)) + with HTTPServer((bind_ip, port), handler) as httpd: + httpd.serve_forever() # Stands up the network-boot environment a PXE VM test needs: @@ -89,49 +90,46 @@ def __init__( libvirt_conn: libvirt.virConnect, network_name: str, bridge_name: str, - artifacts_dir: Path, + pxe_tar_path: Path, boot_loader_file: str, http_log_file_path: Path, ): self.network_name: str = network_name self._network: Optional[libvirt.virNetwork] = None - self._http_process: Optional[subprocess.Popen[bytes]] = None - self._http_log: Optional[IO[str]] = None - - # pytest runs as root (via sudo) and creates the intermediate temp directories with mkdtemp (mode 0700). The - # workspace root itself may be group-private (e.g. 0750). Since dnsmasq (TFTP) and qemu run as their own users, - # ensure the artifacts are readable by others and can traverse into the artifacts directory. - _make_world_readable(artifacts_dir) - _make_ancestors_traversable(artifacts_dir) - - network_xml = _build_pxe_network_xml(network_name, bridge_name, str(artifacts_dir), boot_loader_file) - logging.debug(f"Creating PXE libvirt network:\n{network_xml}") + self._http_process: Optional[multiprocessing.Process] = None + self._artifacts_dir: Optional[Path] = None try: + # Extract the artifacts under /var/tmp. dnsmasq (TFTP) and qemu run as their own unprivileged users, so + # every directory from the filesystem root down to the artifacts must be traversable by others. /var/tmp is + # world-traversable (1777) and on disk, unlike the tmpfs-backed /tmp (the artifacts include the full-OS + # image, which is too large for RAM). + artifacts_dir = Path(tempfile.mkdtemp(prefix="pxe-artifacts-", dir="/var/tmp")) + self._artifacts_dir = artifacts_dir + with tarfile.open(pxe_tar_path, "r:gz") as tar: + tar.extractall(artifacts_dir) + + # mkdtemp created the directory 0700 and the tarball preserves its own modes, so widen the extracted tree + # to be readable and traversable by others. + _make_world_readable(artifacts_dir) + + network_xml = _build_pxe_network_xml(network_name, bridge_name, str(artifacts_dir), boot_loader_file) + logging.debug(f"Creating PXE libvirt network:\n{network_xml}") self._network = libvirt_conn.networkCreateXML(network_xml) logging.debug(f"Starting PXE HTTP server on port {PXE_HTTP_PORT} serving ({artifacts_dir})") - self._http_log = open(http_log_file_path, "w", encoding="utf-8") - self._http_process = subprocess.Popen( - [ - sys.executable, - "-m", - "http.server", - str(PXE_HTTP_PORT), - "--directory", - str(artifacts_dir), - # Bind only to the PXE NAT network's gateway IP, never 0.0.0.0. The artifacts are served - # unauthenticated over plain HTTP and are world-readable, so the listener must not be reachable from - # the build host's physical LAN. Binding to the gateway IP keeps it on the libvirt bridge interface - # only, so the guest VM under test can still reach it while other physical machines on the LAN - # cannot. The bridge interface already holds this IP because the libvirt network was created above. - "--bind", - PXE_NETWORK_GATEWAY_IP, - ], - stdout=self._http_log, - stderr=subprocess.STDOUT, + # Bind only to the PXE NAT network's gateway IP, never 0.0.0.0. The artifacts are served + # unauthenticated over plain HTTP and are world-readable, so the listener must not be reachable from + # the build host's physical LAN. Binding to the gateway IP keeps it on the libvirt bridge interface + # only, so the guest VM under test can still reach it while other physical machines on the LAN + # cannot. The bridge interface already holds this IP because the libvirt network was created above. + self._http_process = multiprocessing.Process( + target=_serve_http, + args=(artifacts_dir, PXE_NETWORK_GATEWAY_IP, PXE_HTTP_PORT, http_log_file_path), + daemon=True, ) + self._http_process.start() except BaseException: self.close() raise @@ -140,17 +138,13 @@ def close(self) -> None: if self._http_process is not None: logging.debug("Stopping PXE HTTP server") self._http_process.terminate() - try: - self._http_process.wait(timeout=10) - except subprocess.TimeoutExpired: + self._http_process.join(timeout=10) + if self._http_process.is_alive(): self._http_process.kill() + self._http_process.join() self._http_process = None - if self._http_log is not None: - self._http_log.close() - self._http_log = None - if self._network is not None: logging.debug(f"Destroying PXE libvirt network: {self.network_name}") try: @@ -159,3 +153,8 @@ def close(self) -> None: logging.warning(f"PXE network destroy failed. {ex}") self._network = None + + if self._artifacts_dir is not None: + logging.debug(f"Removing PXE artifacts directory: {self._artifacts_dir}") + shutil.rmtree(self._artifacts_dir, ignore_errors=True) + self._artifacts_dir = None