From ac9ac2c16cb10f658cd28c7acba593134704d141 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 20:55:38 -0400 Subject: [PATCH 01/40] feat(py_image_layer): support multiple binaries Union concrete wheel installs, grouped first-party inputs, and interpreter layers across all binaries so separate dependency contexts remain runnable in one image. Relocate each launcher into an explicit image directory, reject ambiguous runfiles and duplicate launcher names, and preserve the single-binary API and output groups. Add OCI and analysis coverage for two interpreters, separate lock universes, and invalid launcher setup. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 73 +++- .../image_layer_analysis_tests.bzl | 55 +++ .../launchers_image_command_test.yaml | 12 + e2e/cases/oci/py_image_layer/worker.py | 15 + .../oci/py_image_layer/worker_support.py | 2 + py/private/py_image_layer.bzl | 315 +++++++++--------- 6 files changed, 304 insertions(+), 168 deletions(-) create mode 100644 e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl create mode 100644 e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml create mode 100644 e2e/cases/oci/py_image_layer/worker.py create mode 100644 e2e/cases/oci/py_image_layer/worker_support.py diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index a17d89634..8c18f4a1e 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -1,9 +1,10 @@ -load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_test") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library", "py_test") load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@container_structure_test//:defs.bzl", "container_structure_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") load("//tools:asserts.bzl", "assert_tar_listing") +load(":image_layer_analysis_tests.bzl", "image_layer_analysis_test_suite") platform( name = "aarch64_linux", @@ -98,9 +99,9 @@ py_image_layer( ) # Multi-member group coverage: two pip packages mapped to the same group name -# go through the _merge_aspect path (no per-package tars; single merged tar -# from both install_dirs). Binary also adds pyproject_hooks to make the -# closure non-trivial. Zstd-compressed to exercise _compression_ext. +# produce a single merged tar from both install_dirs. Binary also adds +# pyproject_hooks to make the closure non-trivial. Zstd-compressed to exercise +# _compression_ext. py_binary( name = "my_app_multi_bin", srcs = ["__main__.py"], @@ -131,6 +132,70 @@ py_image_layer( layer_tier = ":my_app_multi_tier", ) +# Exercise the actual multi-launcher flow with separate lock universes and two +# interpreter versions. colorama has the same normalized label in both +# closures, but distinct 3.11/3.12 install trees; both launchers must resolve +# their own venv and the unioned dependency layers from the shared runfiles tree. +py_library( + name = "worker_support", + srcs = ["worker_support.py"], + imports = ["."], +) + +py_binary( + name = "my_app_worker_bin", + srcs = ["worker.py"], + dep_group = "venv_images", + python_version = "3.12", + deps = [ + ":worker_support", + "@pypi_oci_py_venv_image_layer//colorama", + "@pypi_oci_py_venv_image_layer//pyproject_hooks", + ], +) + +py_layer_tier( + name = "my_app_launchers_tier", + groups = { + "//oci/py_image_layer:worker_support": "worker_support", + "@pip//colorama": "third_party", + "@pip//pyproject_hooks": "third_party", + }, + interpreter_group = "interpreter", +) + +py_image_layer( + name = "my_app_launchers_layers", + additional_binaries = [":my_app_worker_bin"], + binary = ":my_app_bin", + launcher_dir = "/app/bin", + layer_tier = ":my_app_launchers_tier", +) + +oci_image( + name = "launchers_image", + base = "@ubuntu", + env = {"RUNFILES_DIR": "/app.runfiles"}, + tars = [":my_app_launchers_layers"], +) + +platform_transition_filegroup( + name = "amd64_launchers_image", + srcs = [":launchers_image"], + target_platform = ":x86_64_linux", +) + +container_structure_test( + name = "launchers_image_command_test", + args = ["--verbosity=debug"], + configs = ["launchers_image_command_test.yaml"], + image = ":amd64_launchers_image", + platform = "linux/amd64", + tags = ["requires-docker"], +) + +image_layer_analysis_test_suite() + platform_transition_filegroup( name = "platform_layers", srcs = [":my_app_layers"], diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl new file mode 100644 index 000000000..b83deb713 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -0,0 +1,55 @@ +"""Analysis coverage for invalid multi-launcher image-layer configurations.""" + +load("@aspect_rules_py//py:defs.bzl", "py_image_layer") +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") + +def _expected_failure_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, ctx.attr.expected_error) + return analysistest.end(env) + +_expected_failure_test = analysistest.make( + _expected_failure_impl, + attrs = {"expected_error": attr.string(mandatory = True)}, + expect_failure = True, +) + +def image_layer_analysis_test_suite(): + py_image_layer( + name = "_missing_launcher_dir_layers", + binary = ":my_app_bin", + additional_binaries = [":my_app_worker_bin"], + ) + _expected_failure_test( + name = "missing_launcher_dir_test", + expected_error = "py_image_layer with multiple binaries requires launcher_dir", + target_under_test = ":_missing_launcher_dir_layers", + ) + + py_image_layer( + name = "_relative_launcher_dir_layers", + binary = ":my_app_bin", + additional_binaries = [":my_app_worker_bin"], + launcher_dir = "app/bin", + ) + _expected_failure_test( + name = "relative_launcher_dir_test", + expected_error = "py_image_layer.launcher_dir must be an absolute image path", + target_under_test = ":_relative_launcher_dir_layers", + ) + + native.alias( + name = "_my_app_bin_alias", + actual = ":my_app_bin", + ) + py_image_layer( + name = "_duplicate_launcher_layers", + binary = ":my_app_bin", + additional_binaries = [":_my_app_bin_alias"], + launcher_dir = "/app/bin", + ) + _expected_failure_test( + name = "duplicate_launcher_basename_test", + expected_error = "duplicate py_image_layer launcher basename: my_app_bin", + target_under_test = ":_duplicate_launcher_layers", + ) diff --git a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml new file mode 100644 index 000000000..052378d34 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml @@ -0,0 +1,12 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: primary launcher runs from launcher_dir + exitCode: 0 + command: /app/bin/my_app_bin + expectedOutput: ["Hello rules_py - 3.14"] + + - name: additional launcher uses its own interpreter and pip closure + exitCode: 0 + command: /app/bin/my_app_worker_bin + expectedOutput: ["worker ok 12 0.4.6 1.2.0 grouped"] diff --git a/e2e/cases/oci/py_image_layer/worker.py b/e2e/cases/oci/py_image_layer/worker.py new file mode 100644 index 000000000..19065ab37 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/worker.py @@ -0,0 +1,15 @@ +import sys + +import colorama +import pyproject_hooks +from worker_support import support + +if __name__ == "__main__": + print( + "worker ok {} {} {} {}".format( + sys.version_info.minor, + colorama.__version__, + pyproject_hooks.__version__, + support(), + ) + ) diff --git a/e2e/cases/oci/py_image_layer/worker_support.py b/e2e/cases/oci/py_image_layer/worker_support.py new file mode 100644 index 000000000..0a1b5bc05 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/worker_support.py @@ -0,0 +1,2 @@ +def support(): + return "grouped" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 0ed063c34..6f939820c 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -1,21 +1,15 @@ """py_image_layer — analysis-time grouped OCI layers with globally shared pip tars. -Two rule-propagated aspects wire onto `py_image_layer.binary`: - - 1. `_layer_aspect` — propagates through `deps`/`data`/`actual`. For pip packages it - creates aspect-owned per-package tars at the pip target's namespace (globally - shared across every rule using that package) for solo whole-groups and subpath - splits. Members of multi-member whole groups get NO per-package tar — intermediate - elided; they just flag `merge_group` on their _LayerInfo struct. First-party - PyInfo targets matched by `py_layer_tier.groups` are captured as `first_party_layers` - entries (label, files, group) to be tarred per-group at the binary. Produces - `_LayerInfo`. - - 2. `_merge_aspect` — runs only on the binary (`attr_aspects = []`), after `_layer_aspect` - via `required_aspect_providers`. Reads `_LayerInfo.pip_packages` (full closure), - buckets install_dirs by `merge_group`, and emits one bsdtar action per group from - raw install_dirs — single pass, no intermediates, content exactly matches closure - (no dep leak). Produces `_MergedLayerInfo` declared at the binary's output namespace. +One rule-propagated aspect wires onto `py_image_layer.binaries`: + +`_layer_aspect` propagates through `deps`/`data`/`actual`. For pip packages it +creates aspect-owned per-package tars at the pip target's namespace (globally +shared across every rule using that package) for solo whole-groups and subpath +splits. Members of multi-member whole groups get NO per-package tar — intermediate +elided; they just flag `merge_group` on their _LayerInfo struct. First-party +PyInfo targets matched by `py_layer_tier.groups` are captured as `first_party_layers` +entries (label, files, group) to be tarred per-group at the binary. Produces +`_LayerInfo`. Layer tier (groups + compression) is carried by the `py_layer_tier` rule which produces `PyLayerTierInfo`. Aspects read it via a private `_layer_tier` attr whose default is @@ -25,10 +19,10 @@ Layer tier (groups + compression) is carried by the `py_layer_tier` rule which p Sharing model: - Solo whole-group + subpath-split pip tars: action-shared across every rule using that package (declared at the pip target's namespace). - - Multi-member merged tars: per-binary action, but deterministic content (canonical + - Multi-member merged tars: per-rule action, but deterministic content (canonical mtree, fixed bsdtar options) → remote CAS / OCI registry dedupe by digest. - - First-party grouped tars: per-binary action, one tar per group, collected from - matched py_library targets in the binary's dep closure. + - First-party grouped tars: per-rule action, one tar per group, collected from + matched py_library targets in the binaries' dep closures. - Ungrouped pip packages: squashed by the rule into one per-rule tar. """ @@ -224,7 +218,7 @@ def _build_pip_layers(ctx, plan, label, install_dir): Returns (layers, merge_group): layers is a tuple of struct(tar, group) entries for this package; merge_group is set (and layers is empty) iff - the package is deferred to `_merge_aspect`. + the package is deferred to the image-layer rule. """ subpath_for_this = plan.subpath_groups.get(label, {}) whole_group = plan.whole_groups.get(label, None) @@ -506,76 +500,6 @@ _layer_aspect = aspect( provides = [_LayerInfo], ) -_MergedLayerInfo = provider( - doc = "Private: closure-filtered merged tars for multi-member groups, produced by _merge_aspect.", - fields = { - "merged_tars": "dict[group_name, File] — one merged tar per multi-member group.", - }, -) - -def _merge_aspect_impl(target, ctx): - info = target[_LayerInfo] - - bucket = {} - seen = {} - for pkg in info.pip_packages.to_list(): - if pkg.label in seen: - continue - seen[pkg.label] = True - if pkg.merge_group != None: - bucket.setdefault(pkg.merge_group, []).append(pkg.files) - - if not bucket: - return [_MergedLayerInfo(merged_tars = {})] - - bsdtar, bsdtar_files = _tar_toolchain(ctx) - plan = ctx.attr._layer_tier[PyLayerTierInfo] - - merged_tars = {} - for group_name in sorted(bucket): - install_dirs = bucket[group_name] - algorithm, level, ext = _compression_for(plan, group_name) - tar_out = ctx.actions.declare_file("_merged_pip_layer_{}{}".format(group_name, ext)) - _run_tar_action( - ctx, - bsdtar, - bsdtar_files, - tar_out, - depset(transitive = install_dirs), - _pkg_file_to_mtree, - algorithm, - level, - {}, - "PyImageMergedLayer", - "Merging %d pip packages into %s[%s]" % (len(install_dirs), target.label, group_name), - ) - merged_tars[group_name] = tar_out - - return [_MergedLayerInfo(merged_tars = merged_tars)] - -_merge_aspect = aspect( - implementation = _merge_aspect_impl, - attr_aspects = [], - attrs = { - "_layer_tier": attr.label( - default = "//py:layer_tier", - providers = [PyLayerTierInfo], - ), - "_awk_script": attr.label( - default = "//py/private:modify_mtree.awk", - allow_single_file = True, - ), - "_awk": attr.label( - default = "@gawk", - cfg = "exec", - executable = True, - ), - }, - toolchains = [_TAR_TOOLCHAIN], - required_aspect_providers = [[_LayerInfo]], - provides = [_MergedLayerInfo], -) - def _apply_strip_prefix(sp, strip_prefix, root): prefix = strip_prefix.replace("\\/", "/") if sp == prefix: @@ -585,9 +509,11 @@ def _apply_strip_prefix(sp, strip_prefix, root): return "." + root + sp[len(prefix):] return "./app.runfiles/_main/" + sp -def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False): +def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False, executable_short_path = "", executable_dst = ""): sp = f.short_path - if sp == "_repo_mapping": + if executable_dst and sp == executable_short_path: + dst = executable_dst + elif sp == "_repo_mapping": # Bazel synthesizes a top-level `_repo_mapping` runfile (no `_main/` # prefix); replicate that placement so runfiles.bash can find it. dst = "./app.runfiles/_repo_mapping" @@ -615,15 +541,23 @@ def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_ f.path.replace(" ", "\\040"), ) -def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink): +def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, executable_short_path = "", executable_dst = ""): # 0755 throughout: keeps launcher/interpreter/venv shims executable; Bazel # doesn't expose per-input source mode for us to propagate. if f.is_directory: return [ - _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink) + _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink, executable_short_path, executable_dst) for child in dir_expander.expand(f) ] - return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink) + return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_short_path, executable_dst) + +def _make_source_map(binary_short_path, strip_prefix, root, maybe_symlink, executable_dst = ""): + effective_strip_prefix = strip_prefix or binary_short_path + + def _source_map(f, d): + return _source_file_to_mtree(f, d, effective_strip_prefix, root, maybe_symlink, binary_short_path, executable_dst) + + return _source_map def _user_file_to_mtree(f, dir_expander): if f.is_directory: @@ -802,21 +736,33 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m return tar_out def _py_image_layer_impl(ctx): - info = ctx.attr.binary[_LayerInfo] - merged = ctx.attr.binary[_MergedLayerInfo] + binaries = ctx.attr.binaries + infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) - pkg_by_label = {} - for pkg in info.pip_packages.to_list(): - if pkg.label not in pkg_by_label: - pkg_by_label[pkg.label] = pkg - all_pkgs = pkg_by_label.values() + # Labels are normalized for tier lookup and can collide across lock universes. + # Concrete install-tree paths identify the configured wheel artifacts. + pkg_by_files = {} + for info in infos: + for pkg in info.pip_packages.to_list(): + key = tuple(sorted([f.path for f in pkg.files.to_list()])) + if key not in pkg_by_files: + pkg_by_files[key] = pkg + all_pkgs = pkg_by_files.values() + pip_labels = {pkg.label: True for pkg in all_pkgs} # `_platform_cfg` rewrites the `//py:layer_tier` flag from `attr.layer_tier`, # so `_layer_tier` always resolves to the effective tier. plan = ctx.attr._layer_tier[PyLayerTierInfo] root = plan.root strip_prefix = plan.strip_prefix + launcher_dir = ctx.attr.launcher_dir + if len(binaries) > 1 and not launcher_dir: + fail("py_image_layer with multiple binaries requires launcher_dir") + if launcher_dir and not launcher_dir.startswith("/"): + fail("py_image_layer.launcher_dir must be an absolute image path") + if launcher_dir != "/": + launcher_dir = launcher_dir.rstrip("/") # 3p pip layers are action-shared across the graph and hard-code their # destination under `./app.runfiles//...`, so the consumer's source @@ -824,24 +770,12 @@ def _py_image_layer_impl(ctx): # to find them. Default the binary's `short_path` to `strip_prefix` so the # natural runfile layout maps onto `./app.runfiles/_main/...` without each # caller wiring it up. - binary_short_path = ctx.attr.binary[DefaultInfo].files_to_run.executable.short_path - if not strip_prefix: - strip_prefix = binary_short_path - all_tars = [] - # Without a dedicated interpreter tar, repo-rule-staged symlinks like - # rules_python's `bin/python -> python3.11` end up in the source layer - # and need awk's readlink scan to be preserved. - source_maybe_symlink = info.interpreter_layer == None - - def _source_map(f, d): - return _source_file_to_mtree(f, d, strip_prefix, root, source_maybe_symlink) - rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) - if dep_label in pkg_by_label: + if dep_label in pip_labels: continue tar_out = _declare_group_tar( ctx, @@ -856,20 +790,17 @@ def _py_image_layer_impl(ctx): all_tars.append(tar_out) fp_by_group = {} - seen_fp_labels = {} - for entry in info.first_party_layers.to_list(): - if entry.label in seen_fp_labels: - continue - seen_fp_labels[entry.label] = True - fp_by_group.setdefault(entry.group, []).append(entry.files) - - # Slot the pre-built, action-shared interpreter tar into the fp ordering - # under its group name so users see a stable per-group sort regardless of - # where the tar was declared. - prebuilt_group_tars = {} - if info.interpreter_layer != None: - prebuilt_group_tars[info.interpreter_layer.group] = info.interpreter_layer.tar - fp_by_group.setdefault(info.interpreter_layer.group, []) + for info in infos: + for entry in info.first_party_layers.to_list(): + fp_by_group.setdefault(entry.group, []).append(entry.files) + + # Interpreter tars are declared at the configured toolchain, so identical + # runtimes action-share while distinct interpreter artifacts are retained. + interpreter_tars = {} + for info in infos: + if info.interpreter_layer != None: + tar = info.interpreter_layer.tar + interpreter_tars[tar.path] = tar for group_name in sorted(fp_by_group): if group_name in rule_group_names: @@ -879,27 +810,47 @@ def _py_image_layer_impl(ctx): "deps with a different file layout than first-party sources, so they " + "cannot share a tar.") % group_name, ) - if group_name in prebuilt_group_tars: - tar_out = prebuilt_group_tars[group_name] - else: - tar_out = _declare_group_tar( - ctx, - bsdtar, - bsdtar_files, - "{}_{}.tar.gz".format(ctx.attr.name, group_name), - group_name, - depset(transitive = fp_by_group[group_name]), - _source_map, - "Creating first-party layer %s[%s]" % (ctx.label, group_name), - ) + tar_out = _declare_group_tar( + ctx, + bsdtar, + bsdtar_files, + "{}_{}.tar.gz".format(ctx.attr.name, group_name), + group_name, + depset(transitive = fp_by_group[group_name]), + _make_source_map(binaries[0][DefaultInfo].files_to_run.executable.short_path, strip_prefix, root, any([info.interpreter_layer == None for info in infos])), + "Creating first-party layer %s[%s]" % (ctx.label, group_name), + ) all_tars.append(tar_out) + all_tars.extend(interpreter_tars.values()) + + pip_tars = {} + merged = {} for pkg in all_pkgs: for layer in pkg.layers: - all_tars.append(layer.tar) + pip_tars[layer.tar.path] = layer.tar + if pkg.merge_group != None: + merged.setdefault(pkg.merge_group, []).append(pkg.files) + + all_tars.extend(pip_tars.values()) - for _group_name, merged_tar in sorted(merged.merged_tars.items()): - all_tars.append(merged_tar) + for group_name in sorted(merged): + algorithm, level, ext = _compression_for(plan, group_name) + tar_out = ctx.actions.declare_file("{}_merged_pip_layer_{}{}".format(ctx.attr.name, group_name, ext)) + _run_tar_action( + ctx, + bsdtar, + bsdtar_files, + tar_out, + depset(transitive = merged[group_name]), + _pkg_file_to_mtree, + algorithm, + level, + {}, + "PyImageMergedLayer", + "Merging %d pip packages into %s[%s]" % (len(merged[group_name]), ctx.label, group_name), + ) + all_tars.append(tar_out) ungrouped_pkgs = [p for p in all_pkgs if len(p.layers) == 0 and p.merge_group == None] if ungrouped_pkgs: @@ -919,17 +870,41 @@ def _py_image_layer_impl(ctx): # snapshot here to avoid double-bookkeeping during construction. dep_tars = list(all_tars) - source_tar = _declare_group_tar( - ctx, - bsdtar, - bsdtar_files, - "{}_default.tar.gz".format(ctx.attr.name), - "default", - info.source_files, - _source_map, - "Creating source layer for %s" % ctx.label, - ) - all_tars.append(source_tar) + source_tars = [] + launcher_names = {} + runfile_paths = {} + for binary, info in zip(binaries, infos): + executable = binary[DefaultInfo].files_to_run.executable + launcher_name = executable.basename + if launcher_dir: + if launcher_name in launcher_names: + fail("duplicate py_image_layer launcher basename: {}".format(launcher_name)) + launcher_names[launcher_name] = True + + # Source tars share one runfiles root. Two different artifacts mapped to + # the same runfile (especially `_repo_mapping`) would be order-dependent. + for f in info.source_files.to_list(): + previous = runfile_paths.get(f.short_path, None) + if previous != None and previous != f.path: + fail("py_image_layer runfile collision at {}: {} and {}".format(f.short_path, previous, f.path)) + runfile_paths[f.short_path] = f.path + + executable_dst = "." + launcher_dir.rstrip("/") + "/" + launcher_name if launcher_dir else "" + source_name = "{}_default.tar.gz".format(ctx.attr.name) + if len(binaries) > 1: + source_name = "{}_{}_default.tar.gz".format(ctx.attr.name, launcher_name) + source_tar = _declare_group_tar( + ctx, + bsdtar, + bsdtar_files, + source_name, + "default", + info.source_files, + _make_source_map(executable.short_path, strip_prefix, root, info.interpreter_layer == None, executable_dst), + "Creating source layer for %s[%s]" % (ctx.label, binary.label), + ) + all_tars.append(source_tar) + source_tars.append(source_tar) validation = ctx.actions.declare_file(ctx.attr.name + "_validation.log") validation_args = ctx.actions.args() @@ -952,7 +927,7 @@ def _py_image_layer_impl(ctx): DefaultInfo(files = depset(all_tars)), OutputGroupInfo( deps = depset(dep_tars), - sources = depset([source_tar]), + sources = depset(source_tars), _validation = depset([validation]), ), ] @@ -960,9 +935,14 @@ def _py_image_layer_impl(ctx): _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { - "binary": attr.label( + "binaries": attr.label_list( mandatory = True, - aspects = [_layer_aspect, _merge_aspect], + allow_empty = False, + aspects = [_layer_aspect], + ), + "launcher_dir": attr.string( + default = "", + doc = "Absolute image directory for binary launchers. Required with multiple binaries.", ), "groups": attr.label_keyed_string_dict(default = {}), "group_execution_requirements": attr.string_list_dict(default = {}), @@ -1005,8 +985,10 @@ def py_image_layer( warn_layer_count = 90, platform = None, layer_tier = None, + additional_binaries = [], + launcher_dir = "", **kwargs): - """Create OCI-compatible tars from a py_binary or py_venv target. + """Create OCI-compatible tars from one or more py_binary targets. Pip-package grouping + compression is resolved from the `//py:layer_tier` label_flag. Override globally with `--//py:layer_tier=//path:custom_tier`, @@ -1020,14 +1002,14 @@ def py_image_layer( binary's dep closure). 3. Solo-group and subpath-split pip tars — built by `_layer_aspect` at each pip target's own namespace; globally shared across every rule using that package. - 4. Multi-member merged tars — one per group, built by `_merge_aspect` at the - binary's namespace from the closure-filtered union of member install_dirs. + 4. Multi-member merged tars — one per group from the closure-filtered union + of member install_dirs across all binaries. 5. Ungrouped pip packages → one squashed rule-created tar. 6. Remaining first-party Python source files → the "default" layer. Args: name: Name of the generated target. - binary: A py_venv or py_binary target. + binary: A py_binary target. groups: Maps a NON-PIP dep label to a group name. Each gets its own rule-created tar. All pip-package grouping (whole-package, subpath, multi-member) belongs in py_layer_tier — subpath glob keys passed here fail loudly. @@ -1042,6 +1024,10 @@ def py_image_layer( layer_tier: Optional py_layer_tier target pinned for this rule. Sets the `@aspect_rules_py//py:layer_tier` label_flag via the rule transition, overriding any command-line value for this rule's subgraph. + additional_binaries: Additional py_binary targets whose dependency and + source layers are included in the image. + launcher_dir: Absolute image directory for the binary launchers. Required + with additional_binaries. Set RUNFILES_DIR=/app.runfiles in the image. **kwargs: Forwarded to inner rule. """ tags = kwargs.pop("tags", []) + ["manual"] @@ -1055,7 +1041,8 @@ def py_image_layer( _py_image_layer( name = name, - binary = binary, + binaries = [binary] + additional_binaries, + launcher_dir = launcher_dir, groups = groups, group_execution_requirements = group_execution_requirements, group_compress_levels = group_compress_levels, From c1443937d269ea72254ba1d61dcf80f0ec7abff1 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 22:15:47 -0400 Subject: [PATCH 02/40] fix(py_image_layer): deduplicate shared sources Union all binary source closures into one layer so shared interpreter, first-party, and data runfiles are emitted once and Bazel-tree symlinks retain their in-layer targets. Validate grouped and ungrouped files against canonical image destinations, including expanded TreeArtifacts and rule-level groups. Preserve root launcher paths and allow identical configured inputs across layer modes. Add focused collision, symlink, and byte-dedup coverage. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 39 ++++ .../oci/py_image_layer/assert_source_dedup.py | 23 +++ .../image_layer_analysis_tests.bzl | 149 ++++++++++++++- py/private/BUILD.bazel | 14 ++ py/private/modify_mtree.awk | 62 +++++- py/private/modify_mtree_test.sh | 50 +++++ py/private/py_image_layer.bzl | 180 ++++++++++++------ 7 files changed, 454 insertions(+), 63 deletions(-) create mode 100644 e2e/cases/oci/py_image_layer/assert_source_dedup.py create mode 100755 py/private/modify_mtree_test.sh diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 8c18f4a1e..ac616031e 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -172,6 +172,45 @@ py_image_layer( layer_tier = ":my_app_launchers_tier", ) +# Default-tier source closures share the interpreter, branding source, and +# data. The listing test proves the unioned source tar ships each byte once. +py_binary( + name = "my_app_peer_bin", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.11", + deps = ["//oci/py_image_layer/branding"], +) + +py_image_layer( + name = "my_app_shared_layers", + additional_binaries = [":my_app_peer_bin"], + binary = ":my_app_bin", + launcher_dir = "/app/bin", +) + +genrule( + name = "my_app_shared_sources_listing", + srcs = [":my_app_shared_layers_only_src"], + outs = ["_my_app_shared_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + +py_test( + name = "my_app_shared_sources_test", + srcs = ["assert_source_dedup.py"], + args = [ + "$(rootpath :my_app_shared_sources_listing)", + "/branding/__init__.py", + "/branding/palette.txt", + "/lib/python3.11/os.py", + "/app/bin/my_app_bin", + "/app/bin/my_app_peer_bin", + ], + data = [":my_app_shared_sources_listing"], +) + oci_image( name = "launchers_image", base = "@ubuntu", diff --git a/e2e/cases/oci/py_image_layer/assert_source_dedup.py b/e2e/cases/oci/py_image_layer/assert_source_dedup.py new file mode 100644 index 000000000..1cf2b17a9 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/assert_source_dedup.py @@ -0,0 +1,23 @@ +"""Assert that each shared source-layer sentinel is present exactly once.""" + +import sys + + +def main(argv): + listing, sentinels = argv[1], argv[2:] + with open(listing, encoding="utf-8") as f: + entries = f.read().splitlines() + + failures = [] + for sentinel in sentinels: + count = sum(entry.endswith(sentinel) for entry in entries) + if count != 1: + failures.append("{}: expected once, found {}".format(sentinel, count)) + if failures: + sys.exit("\n".join(failures)) + + print("assert_source_dedup: ok ({})".format(", ".join(sentinels))) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index b83deb713..66fd1b541 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -1,6 +1,6 @@ """Analysis coverage for invalid multi-launcher image-layer configurations.""" -load("@aspect_rules_py//py:defs.bzl", "py_image_layer") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): @@ -14,6 +14,24 @@ _expected_failure_test = analysistest.make( expect_failure = True, ) +def _configured_tree_or_file_impl(ctx): + if ctx.attr.tree: + out = ctx.actions.declare_directory("generated_tree") + ctx.actions.run_shell( + outputs = [out], + command = "mkdir -p \"$1\" && echo tree > \"$1/support.py\"", + arguments = [out.path], + ) + else: + out = ctx.actions.declare_file("generated_tree/support.py") + ctx.actions.write(out, "file\n") + return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] + +_configured_tree_or_file = rule( + implementation = _configured_tree_or_file_impl, + attrs = {"tree": attr.bool()}, +) + def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", @@ -46,10 +64,137 @@ def image_layer_analysis_test_suite(): name = "_duplicate_launcher_layers", binary = ":my_app_bin", additional_binaries = [":_my_app_bin_alias"], - launcher_dir = "/app/bin", + launcher_dir = "////", ) _expected_failure_test( name = "duplicate_launcher_basename_test", expected_error = "duplicate py_image_layer launcher basename: my_app_bin", target_under_test = ":_duplicate_launcher_layers", ) + + native.config_setting( + name = "_python_3_11", + flag_values = {"@aspect_rules_py//py/private/interpreter:python_version": "3.11"}, + ) + native.genrule( + name = "_generated_support", + outs = ["generated_support.py"], + cmd = select({ + ":_python_3_11": "echo 'VALUE = 11' > $@", + "//conditions:default": "echo 'VALUE = 12' > $@", + }), + ) + py_library( + name = "_generated_support_lib", + srcs = [":_generated_support"], + imports = ["."], + ) + py_binary( + name = "_configured_group_311", + srcs = ["server.py"], + python_version = "3.11", + deps = [":_generated_support_lib"], + ) + py_binary( + name = "_configured_group_312", + srcs = ["server.py"], + python_version = "3.12", + deps = [":_generated_support_lib"], + ) + py_binary( + name = "_configured_source_312", + srcs = ["server.py"], + data = [":_generated_support"], + python_version = "3.12", + ) + py_layer_tier( + name = "_generated_support_tier", + groups = {"//oci/py_image_layer:_generated_support_lib": "generated_support"}, + ) + + py_image_layer( + name = "_configured_group_collision_layers", + binary = ":_configured_group_311", + additional_binaries = [":_configured_group_312"], + launcher_dir = "/app/bin", + layer_tier = ":_generated_support_tier", + ) + _expected_failure_test( + name = "configured_group_collision_test", + expected_error = "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_support.py", + target_under_test = ":_configured_group_collision_layers", + ) + + py_image_layer( + name = "_configured_group_source_collision_layers", + binary = ":_configured_group_311", + additional_binaries = [":_configured_source_312"], + launcher_dir = "/app/bin", + layer_tier = ":_generated_support_tier", + ) + _expected_failure_test( + name = "configured_group_source_collision_test", + expected_error = "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_support.py", + target_under_test = ":_configured_group_source_collision_layers", + ) + + # Manual action-failure fixture: the 3.11 tree expands in the grouped tar, + # while the 3.12 file lands in the default tar. The global mtree validator + # must reject their shared destination. + _configured_tree_or_file( + name = "_configured_tree_or_file", + tree = select({ + ":_python_3_11": True, + "//conditions:default": False, + }), + ) + py_library( + name = "_configured_tree_lib", + srcs = [":_configured_tree_or_file"], + imports = ["."], + ) + py_binary( + name = "_configured_tree_311", + srcs = ["server.py"], + python_version = "3.11", + deps = [":_configured_tree_lib"], + ) + py_binary( + name = "_configured_file_312", + srcs = [ + "server.py", + ":_configured_tree_or_file", + ], + main = "server.py", + python_version = "3.12", + ) + py_layer_tier( + name = "_configured_tree_tier", + groups = {"//oci/py_image_layer:_configured_tree_lib": "generated_tree"}, + ) + py_image_layer( + name = "_expanded_tree_collision_layers", + binary = ":_configured_tree_311", + additional_binaries = [":_configured_file_312"], + launcher_dir = "/app/bin", + layer_tier = ":_configured_tree_tier", + ) + + # Manual action-failure fixture for a rule-level group built in the image + # configuration colliding with a transitioned binary source file. + py_image_layer( + name = "_rule_group_collision_layers", + binary = ":_configured_group_311", + groups = {":_generated_support": "generated_support"}, + ) + + py_binary( + name = "_same_group_source_bin", + srcs = ["server.py"], + data = [":_generated_support"], + ) + py_image_layer( + name = "_same_rule_group_source_layers", + binary = ":_same_group_source_bin", + groups = {":_generated_support": "generated_support"}, + ) diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 1fd5eb05e..7bc96203f 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -1,4 +1,5 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//py:defs.bzl", _py_binary_rule = "py_binary", _py_test_rule = "py_test") load(":py_image_layer.bzl", "py_layer_tier") load(":py_library.bzl", "py_library") @@ -40,6 +41,19 @@ _py_test_rule( main = "py_image_layer_validator_test.py", ) +sh_test( + name = "modify_mtree_test", + srcs = ["modify_mtree_test.sh"], + args = [ + "$(location :modify_mtree.awk)", + "$(location @gawk)", + ], + data = [ + ":modify_mtree.awk", + "@gawk", + ], +) + py_layer_tier( name = "default_layer_tier", visibility = ["//visibility:public"], diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index 67366fbf7..4dd064dca 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -58,7 +58,56 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back return back_steps target } +function normalize_destination(path, count, i, n, part, parts, normalized) { + count = split(path, parts, "/") + n = 0 + for (i = 1; i <= count; i++) { + part = parts[i] + if (part == "" || part == ".") { + continue + } + if (part == "..") { + if (n == 0) { + print "py_image_layer image destination escapes its root: " path > "/dev/stderr" + failed = 1 + exit 1 + } + delete normalized[n--] + continue + } + normalized[++n] = part + } + path = "." + for (i = 1; i <= n; i++) { + path = path "/" normalized[i] + } + return path +} + { + if ($0 !~ /^#/) { + destination = normalize_destination($1) + sub(/^[^ ]+/, destination) + match($0, /(contents|content|link)=[^ ]+/) + current_source = substr($0, RSTART, RLENGTH) + current_source_path = substr(current_source, index(current_source, "=") + 1) + if (destination in destination_rows) { + if (destination_rows[destination] == $0 || (validate_only && destination_source_paths[destination] == current_source_path)) { + next + } + print "py_image_layer runfile collision at " destination ": " destination_sources[destination] " and " current_source > "/dev/stderr" + failed = 1 + exit 1 + } + destination_rows[destination] = $0 + destination_sources[destination] = current_source + destination_source_paths[destination] = current_source_path + } + + if (validate_only) { + next + } + symlink = "" symlink_content = "" # Two markers Starlark emits for paths that could be symlinks: @@ -151,17 +200,24 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back } } if (symlink != "") { - line_array[NR] = $0 SUBSEP $1 SUBSEP resolved_path + line_array[++row_count] = $0 SUBSEP $1 SUBSEP resolved_path } else { - line_array[NR] = $0 + line_array[++row_count] = $0 } } END { + if (failed) { + exit 1 + } + if (validate_only) { + print "#mtree" > outfile + exit + } # Buffer rewritten rows, sort byte-wise (asort under LC_ALL=C, set by # the action env), and write to `outfile`. n = 0 - for (i = 1; i <= NR; i++) { + for (i = 1; i <= row_count; i++) { line = line_array[i] if (index(line, SUBSEP) > 0) { split(line, fields, SUBSEP) diff --git a/py/private/modify_mtree_test.sh b/py/private/modify_mtree_test.sh new file mode 100755 index 000000000..2aa80b8b5 --- /dev/null +++ b/py/private/modify_mtree_test.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +awk_script="$1" +gawk="$2" + +run_case() { + local name="$1" + local expected="$2" + local input="$3" + local diagnostic="${4:-py_image_layer runfile collision at ./generated_tree/support.py:}" + local output="$TEST_TMPDIR/$name.out" + local error="$TEST_TMPDIR/$name.err" + + if "$gawk" -v "outfile=$output" -f "$awk_script" <<<"$input" 2>"$error"; then + if [[ "$expected" != pass ]]; then + echo "$name: expected collision failure" >&2 + exit 1 + fi + elif [[ "$expected" != fail ]]; then + echo "$name: unexpected failure" >&2 + sed -n '1,80p' "$error" >&2 + exit 1 + fi + + if [[ "$expected" == fail ]] && ! grep -Fq "$diagnostic" "$error"; then + echo "$name: missing collision diagnostic" >&2 + sed -n '1,80p' "$error" >&2 + exit 1 + fi +} + +run_case disjoint pass $'#mtree\n./generated_tree/first.py type=file contents=first\n./generated_tree/second.py type=file contents=second' +run_case identical pass $'#mtree\n./generated_tree/support.py type=file contents=same\n./generated_tree/support.py type=file contents=same' +[[ $(grep -Fc './generated_tree/support.py ' "$TEST_TMPDIR/identical.out") == 1 ]] +run_case conflicting fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' +run_case dot_alias fail $'#mtree\n./generated_tree/./support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' +run_case parent_alias fail $'#mtree\n./generated_tree/nested/../support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' +run_case above_root fail $'#mtree\n../generated_tree/support.py type=file contents=first' 'py_image_layer image destination escapes its root:' + +if "$gawk" -v "outfile=$TEST_TMPDIR/validate_only.out" -v validate_only=1 -f "$awk_script" \ + <<< $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' \ + 2>"$TEST_TMPDIR/validate_only.err"; then + echo 'validate_only: expected collision failure' >&2 + exit 1 +fi +grep -Fq 'py_image_layer runfile collision at ./generated_tree/support.py:' "$TEST_TMPDIR/validate_only.err" + +"$gawk" -v "outfile=$TEST_TMPDIR/validate_same_source.out" -v validate_only=1 -f "$awk_script" \ + <<< $'#mtree\n./generated_tree/support.py type=file mode=0755 content=same\n./generated_tree/support.py type=file mode=0644 contents=same' diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 6f939820c..06c8d460e 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -509,20 +509,38 @@ def _apply_strip_prefix(sp, strip_prefix, root): return "." + root + sp[len(prefix):] return "./app.runfiles/_main/" + sp -def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False, executable_short_path = "", executable_dst = ""): - sp = f.short_path - if executable_dst and sp == executable_short_path: - dst = executable_dst - elif sp == "_repo_mapping": +def _normalize_destination(path): + parts = [] + for part in path.split("/"): + if not part or part == ".": + continue + if part == "..": + if not parts: + fail("py_image_layer image destination escapes its root: {}".format(path)) + parts.pop() + continue + parts.append(part) + return "./" + "/".join(parts) + +def _source_destination(sp, strip_prefix, root, executable_dsts): + executable_dst = executable_dsts.get(sp, None) + if executable_dst: + return _normalize_destination(executable_dst) + if sp == "_repo_mapping": # Bazel synthesizes a top-level `_repo_mapping` runfile (no `_main/` # prefix); replicate that placement so runfiles.bash can find it. - dst = "./app.runfiles/_repo_mapping" - elif sp.startswith("../"): - dst = "./app.runfiles/" + sp[3:] - elif strip_prefix: - dst = _apply_strip_prefix(sp, strip_prefix, root) - else: - dst = "./app.runfiles/_main/" + sp + return "./app.runfiles/_repo_mapping" + if sp.startswith("../"): + return _normalize_destination("./app.runfiles/" + sp[3:]) + if strip_prefix: + return _normalize_destination(_apply_strip_prefix(sp, strip_prefix, root)) + for executable_short_path in executable_dsts: + if sp == executable_short_path or sp.startswith(executable_short_path + ".runfiles/") or sp.startswith(executable_short_path + "/"): + return _normalize_destination(_apply_strip_prefix(sp, executable_short_path, root)) + return _normalize_destination("./app.runfiles/_main/" + sp) + +def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False, executable_dsts = {}): + dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) # `f.is_symlink` emits `type=link` (awk readlinks once); `maybe_symlink=True` # emits `type=file content=` (awk readlinks to detect repo-rule-staged @@ -541,24 +559,30 @@ def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_ f.path.replace(" ", "\\040"), ) -def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, executable_short_path = "", executable_dst = ""): +def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, executable_dsts): # 0755 throughout: keeps launcher/interpreter/venv shims executable; Bazel # doesn't expose per-input source mode for us to propagate. if f.is_directory: return [ - _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink, executable_short_path, executable_dst) + _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink, executable_dsts) for child in dir_expander.expand(f) ] - return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_short_path, executable_dst) - -def _make_source_map(binary_short_path, strip_prefix, root, maybe_symlink, executable_dst = ""): - effective_strip_prefix = strip_prefix or binary_short_path + return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_dsts) +def _make_source_map(strip_prefix, root, maybe_symlink, executable_dsts): def _source_map(f, d): - return _source_file_to_mtree(f, d, effective_strip_prefix, root, maybe_symlink, binary_short_path, executable_dst) + return _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) return _source_map +def _check_runfile_collision(f, dst, runfile_paths): + if f.is_directory: + return + previous = runfile_paths.get(dst, None) + if previous != None and previous != f.path: + fail("py_image_layer runfile collision at {}: {} and {}".format(dst, previous, f.path)) + runfile_paths[dst] = f.path + def _user_file_to_mtree(f, dir_expander): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] @@ -716,6 +740,31 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, use_default_shell_env = False, ) +def _validate_source_mtree(ctx, source_files, source_map, rule_group_files): + mtree_args = ctx.actions.args() + mtree_args.set_param_file_format("multiline") + mtree_args.use_param_file("%s", use_always = True) + mtree_args.add("#mtree") + mtree_args.add_all(source_files, map_each = source_map, expand_directories = False, allow_closure = True) + for files in rule_group_files: + mtree_args.add_all(files, map_each = _user_file_to_mtree, expand_directories = False) + + output = ctx.actions.declare_file(ctx.attr.name + "_source_validation.mtree") + gawk_args = ctx.actions.args() + gawk_args.add("-v", output, format = "outfile=%s") + gawk_args.add("-v", "validate_only=1") + gawk_args.add("-f", ctx.file._awk_script) + ctx.actions.run( + executable = ctx.executable._awk, + inputs = depset(direct = [ctx.file._awk_script], transitive = [source_files] + rule_group_files), + outputs = [output], + arguments = [gawk_args, mtree_args], + env = {"LC_ALL": "C"}, + mnemonic = "PyImageLayerValidateMtree", + progress_message = "Validating source runfiles for %{output}", + ) + return output + def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, map_each, progress): tar_out = ctx.actions.declare_file(out_name) level = ctx.attr.group_compress_levels.get(group_name, "6") @@ -757,12 +806,37 @@ def _py_image_layer_impl(ctx): root = plan.root strip_prefix = plan.strip_prefix launcher_dir = ctx.attr.launcher_dir + if launcher_dir: + launcher_dir = launcher_dir.rstrip("/") or "/" if len(binaries) > 1 and not launcher_dir: fail("py_image_layer with multiple binaries requires launcher_dir") if launcher_dir and not launcher_dir.startswith("/"): fail("py_image_layer.launcher_dir must be an absolute image path") - if launcher_dir != "/": - launcher_dir = launcher_dir.rstrip("/") + + launcher_names = {} + executable_dsts = {} + for binary in binaries: + executable = binary[DefaultInfo].files_to_run.executable + launcher_name = executable.basename + if launcher_dir: + if launcher_name in launcher_names: + fail("duplicate py_image_layer launcher basename: {}".format(launcher_name)) + launcher_names[launcher_name] = True + executable_dsts[executable.short_path] = "." + launcher_dir.rstrip("/") + "/" + launcher_name + else: + executable_dsts[executable.short_path] = "" + + # Grouped and ungrouped sources share one runfiles tree. Reject different + # configured artifacts that map to the same actual image destination. + runfile_paths = {} + for info in infos: + for entry in info.first_party_layers.to_list(): + for f in entry.files.to_list(): + dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) + _check_runfile_collision(f, dst, runfile_paths) + for f in info.source_files.to_list(): + dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) + _check_runfile_collision(f, dst, runfile_paths) # 3p pip layers are action-shared across the graph and hard-code their # destination under `./app.runfiles//...`, so the consumer's source @@ -771,12 +845,14 @@ def _py_image_layer_impl(ctx): # natural runfile layout maps onto `./app.runfiles/_main/...` without each # caller wiring it up. all_tars = [] + rule_group_files = [] rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) if dep_label in pip_labels: continue + rule_group_files.append(dep[DefaultInfo].files) tar_out = _declare_group_tar( ctx, bsdtar, @@ -817,7 +893,7 @@ def _py_image_layer_impl(ctx): "{}_{}.tar.gz".format(ctx.attr.name, group_name), group_name, depset(transitive = fp_by_group[group_name]), - _make_source_map(binaries[0][DefaultInfo].files_to_run.executable.short_path, strip_prefix, root, any([info.interpreter_layer == None for info in infos])), + _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), "Creating first-party layer %s[%s]" % (ctx.label, group_name), ) all_tars.append(tar_out) @@ -870,41 +946,29 @@ def _py_image_layer_impl(ctx): # snapshot here to avoid double-bookkeeping during construction. dep_tars = list(all_tars) - source_tars = [] - launcher_names = {} - runfile_paths = {} - for binary, info in zip(binaries, infos): - executable = binary[DefaultInfo].files_to_run.executable - launcher_name = executable.basename - if launcher_dir: - if launcher_name in launcher_names: - fail("duplicate py_image_layer launcher basename: {}".format(launcher_name)) - launcher_names[launcher_name] = True + # Keep the complete source closure in one tar. `modify_mtree.awk` can only + # rewrite a Bazel-tree symlink when its target row is in the same mtree. + source_tar = _declare_group_tar( + ctx, + bsdtar, + bsdtar_files, + "{}_default.tar.gz".format(ctx.attr.name), + "default", + depset(transitive = [info.source_files for info in infos]), + _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), + "Creating source layer for %s" % ctx.label, + ) + all_tars.append(source_tar) - # Source tars share one runfiles root. Two different artifacts mapped to - # the same runfile (especially `_repo_mapping`) would be order-dependent. - for f in info.source_files.to_list(): - previous = runfile_paths.get(f.short_path, None) - if previous != None and previous != f.path: - fail("py_image_layer runfile collision at {}: {} and {}".format(f.short_path, previous, f.path)) - runfile_paths[f.short_path] = f.path - - executable_dst = "." + launcher_dir.rstrip("/") + "/" + launcher_name if launcher_dir else "" - source_name = "{}_default.tar.gz".format(ctx.attr.name) - if len(binaries) > 1: - source_name = "{}_{}_default.tar.gz".format(ctx.attr.name, launcher_name) - source_tar = _declare_group_tar( - ctx, - bsdtar, - bsdtar_files, - source_name, - "default", - info.source_files, - _make_source_map(executable.short_path, strip_prefix, root, info.interpreter_layer == None, executable_dst), - "Creating source layer for %s[%s]" % (ctx.label, binary.label), - ) - all_tars.append(source_tar) - source_tars.append(source_tar) + # Validate one expanded mtree spanning grouped and default sources. Each + # tar sees only its own rows, so this catches TreeArtifact child collisions + # that would otherwise land in different OCI layers. + source_validation = _validate_source_mtree( + ctx, + depset(transitive = [info.source_files for info in infos] + [entry.files for info in infos for entry in info.first_party_layers.to_list()]), + _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), + rule_group_files, + ) validation = ctx.actions.declare_file(ctx.attr.name + "_validation.log") validation_args = ctx.actions.args() @@ -927,8 +991,8 @@ def _py_image_layer_impl(ctx): DefaultInfo(files = depset(all_tars)), OutputGroupInfo( deps = depset(dep_tars), - sources = depset(source_tars), - _validation = depset([validation]), + sources = depset([source_tar]), + _validation = depset([validation, source_validation]), ), ] From 23a9fd932e2627a6a9233c6b14faea5f7f0f75e3 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 22:34:20 -0400 Subject: [PATCH 03/40] fix(py_image_layer): validate all runfile layers Validate configured wheel, grouped, and source artifacts against one canonical destination registry, including terminal-path collisions and unversioned wheel script or data paths. Keep unrelated launcher-prefix runfiles in their original location. Preserve configurable primary binaries with a separate label attribute, accept a top-level binaries selector, and require exactly one public binary form. Add focused runfile, wheel, and configurable-API coverage. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 34 +++++++++++++++++++ .../image_layer_analysis_tests.bzl | 28 +++++++++++++++ .../my_app_peer_bin/config.json | 1 + py/private/modify_mtree.awk | 24 +++++++++++++ py/private/modify_mtree_test.sh | 2 ++ py/private/py_image_layer.bzl | 34 ++++++++++++++----- 6 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index ac616031e..4524a83a1 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -177,6 +177,7 @@ py_image_layer( py_binary( name = "my_app_peer_bin", srcs = ["server.py"], + data = ["my_app_peer_bin/config.json"], dep_group = "images", python_version = "3.11", deps = ["//oci/py_image_layer/branding"], @@ -207,10 +208,43 @@ py_test( "/lib/python3.11/os.py", "/app/bin/my_app_bin", "/app/bin/my_app_peer_bin", + "/oci/py_image_layer/my_app_peer_bin/config.json", ], data = [":my_app_shared_sources_listing"], ) +py_image_layer( + name = "my_app_select_layers", + binary = select({ + ":_python_3_11": ":my_app_bin", + "//conditions:default": ":my_app_peer_bin", + }), + layer_tier = ":my_app_tier", +) + +py_image_layer( + name = "my_app_binaries_layers", + binaries = select({ + ":_python_3_11": [ + ":my_app_bin", + ":my_app_peer_bin", + ], + "//conditions:default": [ + ":my_app_peer_bin", + ":my_app_bin", + ], + }), + launcher_dir = "/app/bin", +) + +build_test( + name = "my_app_binary_api_test", + targets = [ + ":my_app_binaries_layers", + ":my_app_select_layers", + ], +) + oci_image( name = "launchers_image", base = "@ubuntu", diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 66fd1b541..2e3aadeb8 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -198,3 +198,31 @@ def image_layer_analysis_test_suite(): binary = ":_same_group_source_bin", groups = {":_generated_support": "generated_support"}, ) + + # Manual action-failure fixture for unversioned wheel script/data paths + # shared by separately configured whl_install trees. + py_binary( + name = "_wheel_scripts_311", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.11", + deps = ["@pypi_oci_py_image_layer//build"], + ) + py_binary( + name = "_wheel_scripts_312", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.12", + deps = ["@pypi_oci_py_image_layer//build"], + ) + py_layer_tier( + name = "_wheel_scripts_tier", + groups = {"@pip//build": "wheel_scripts"}, + ) + py_image_layer( + name = "_configured_wheel_collision_layers", + binary = ":_wheel_scripts_311", + additional_binaries = [":_wheel_scripts_312"], + launcher_dir = "/app/bin", + layer_tier = ":_wheel_scripts_tier", + ) diff --git a/e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json b/e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json new file mode 100644 index 000000000..7c111356f --- /dev/null +++ b/e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json @@ -0,0 +1 @@ +{"sentinel": "prefix-data"} diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index 4dd064dca..996ca3474 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -91,6 +91,22 @@ function normalize_destination(path, count, i, n, part, parts, normalized) { match($0, /(contents|content|link)=[^ ]+/) current_source = substr($0, RSTART, RLENGTH) current_source_path = substr(current_source, index(current_source, "=") + 1) + + count = split(destination, destination_parts, "/") + parent = destination_parts[1] + for (j = 2; j < count; j++) { + parent = parent "/" destination_parts[j] + if (parent in destination_rows) { + print "py_image_layer runfile collision at " destination ": " destination_sources[parent] " and " current_source > "/dev/stderr" + failed = 1 + exit 1 + } + } + if (destination in destination_descendants) { + print "py_image_layer runfile collision at " destination_descendants[destination] ": " destination_descendant_sources[destination] " and " current_source > "/dev/stderr" + failed = 1 + exit 1 + } if (destination in destination_rows) { if (destination_rows[destination] == $0 || (validate_only && destination_source_paths[destination] == current_source_path)) { next @@ -102,6 +118,14 @@ function normalize_destination(path, count, i, n, part, parts, normalized) { destination_rows[destination] = $0 destination_sources[destination] = current_source destination_source_paths[destination] = current_source_path + parent = destination_parts[1] + for (j = 2; j < count; j++) { + parent = parent "/" destination_parts[j] + if (!(parent in destination_descendants)) { + destination_descendants[parent] = destination + destination_descendant_sources[parent] = current_source + } + } } if (validate_only) { diff --git a/py/private/modify_mtree_test.sh b/py/private/modify_mtree_test.sh index 2aa80b8b5..37b25fb64 100755 --- a/py/private/modify_mtree_test.sh +++ b/py/private/modify_mtree_test.sh @@ -36,6 +36,8 @@ run_case identical pass $'#mtree\n./generated_tree/support.py type=file contents run_case conflicting fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' run_case dot_alias fail $'#mtree\n./generated_tree/./support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' run_case parent_alias fail $'#mtree\n./generated_tree/nested/../support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' +run_case ancestor_first fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py/data type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' +run_case descendant_first fail $'#mtree\n./generated_tree/support.py/data type=file contents=first\n./generated_tree/support.py type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' run_case above_root fail $'#mtree\n../generated_tree/support.py type=file contents=first' 'py_image_layer image destination escapes its root:' if "$gawk" -v "outfile=$TEST_TMPDIR/validate_only.out" -v validate_only=1 -f "$awk_script" \ diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 06c8d460e..9bfde2b4f 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -535,7 +535,7 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): if strip_prefix: return _normalize_destination(_apply_strip_prefix(sp, strip_prefix, root)) for executable_short_path in executable_dsts: - if sp == executable_short_path or sp.startswith(executable_short_path + ".runfiles/") or sp.startswith(executable_short_path + "/"): + if sp == executable_short_path or sp.startswith(executable_short_path + ".runfiles/"): return _normalize_destination(_apply_strip_prefix(sp, executable_short_path, root)) return _normalize_destination("./app.runfiles/_main/" + sp) @@ -740,7 +740,7 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, use_default_shell_env = False, ) -def _validate_source_mtree(ctx, source_files, source_map, rule_group_files): +def _validate_source_mtree(ctx, source_files, source_map, rule_group_files, wheel_files): mtree_args = ctx.actions.args() mtree_args.set_param_file_format("multiline") mtree_args.use_param_file("%s", use_always = True) @@ -748,6 +748,8 @@ def _validate_source_mtree(ctx, source_files, source_map, rule_group_files): mtree_args.add_all(source_files, map_each = source_map, expand_directories = False, allow_closure = True) for files in rule_group_files: mtree_args.add_all(files, map_each = _user_file_to_mtree, expand_directories = False) + for files in wheel_files: + mtree_args.add_all(files, map_each = _pkg_file_to_mtree, expand_directories = False) output = ctx.actions.declare_file(ctx.attr.name + "_source_validation.mtree") gawk_args = ctx.actions.args() @@ -756,7 +758,7 @@ def _validate_source_mtree(ctx, source_files, source_map, rule_group_files): gawk_args.add("-f", ctx.file._awk_script) ctx.actions.run( executable = ctx.executable._awk, - inputs = depset(direct = [ctx.file._awk_script], transitive = [source_files] + rule_group_files), + inputs = depset(direct = [ctx.file._awk_script], transitive = [source_files] + rule_group_files + wheel_files), outputs = [output], arguments = [gawk_args, mtree_args], env = {"LC_ALL": "C"}, @@ -785,7 +787,9 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m return tar_out def _py_image_layer_impl(ctx): - binaries = ctx.attr.binaries + binaries = ([ctx.attr.binary] if ctx.attr.binary != None else []) + ctx.attr.binaries + if not binaries: + fail("py_image_layer requires at least one binary") infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) @@ -968,6 +972,7 @@ def _py_image_layer_impl(ctx): depset(transitive = [info.source_files for info in infos] + [entry.files for info in infos for entry in info.first_party_layers.to_list()]), _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), rule_group_files, + [pkg.files for pkg in all_pkgs], ) validation = ctx.actions.declare_file(ctx.attr.name + "_validation.log") @@ -999,9 +1004,10 @@ def _py_image_layer_impl(ctx): _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { + "binary": attr.label( + aspects = [_layer_aspect], + ), "binaries": attr.label_list( - mandatory = True, - allow_empty = False, aspects = [_layer_aspect], ), "launcher_dir": attr.string( @@ -1041,7 +1047,7 @@ _py_image_layer = rule( def py_image_layer( name, - binary, + binary = None, groups = {}, group_execution_requirements = {}, group_compress_levels = {}, @@ -1051,6 +1057,7 @@ def py_image_layer( layer_tier = None, additional_binaries = [], launcher_dir = "", + binaries = None, **kwargs): """Create OCI-compatible tars from one or more py_binary targets. @@ -1092,10 +1099,20 @@ def py_image_layer( source layers are included in the image. launcher_dir: Absolute image directory for the binary launchers. Required with additional_binaries. Set RUNFILES_DIR=/app.runfiles in the image. + binaries: Alternative to binary/additional_binaries. A nonempty list of + py_binary targets to include in the image. **kwargs: Forwarded to inner rule. """ tags = kwargs.pop("tags", []) + ["manual"] + if binaries != None: + if binary != None or additional_binaries: + fail("py_image_layer accepts either binaries or binary/additional_binaries, not both") + else: + if binary == None: + fail("py_image_layer requires binary or binaries") + binaries = additional_binaries + for key in groups: if _split_glob_key(key)[1] != None: fail( @@ -1105,7 +1122,8 @@ def py_image_layer( _py_image_layer( name = name, - binaries = [binary] + additional_binaries, + binary = binary, + binaries = binaries, launcher_dir = launcher_dir, groups = groups, group_execution_requirements = group_execution_requirements, From eccd864f0593ac57fd16770b25ad1556267e2a2a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 22:45:14 -0400 Subject: [PATCH 04/40] fix(py_image_layer): validate interpreter layers Include each configured runtime's original interpreter files in the global mtree validator so distinct toolchains cannot overwrite a shared runfile destination during OCI extraction. Retain layer structs while deduplicating shared runtimes and add focused configured-runtime collision and shared-runtime coverage. --- .../image_layer_analysis_tests.bzl | 65 +++++++++++++++++++ py/private/modify_mtree_test.sh | 3 + py/private/py_image_layer.bzl | 15 +++-- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 2e3aadeb8..2eebae36f 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -32,6 +32,22 @@ _configured_tree_or_file = rule( attrs = {"tree": attr.bool()}, ) +def _configured_runtime_impl(ctx): + interpreter = ctx.actions.declare_file("shared_runtime/bin/python") + ctx.actions.write(interpreter, "#!/bin/sh\nexec /usr/bin/python3 \"$@\"\n", is_executable = True) + runtime = struct( + interpreter = interpreter, + interpreter_path = None, + interpreter_version_info = struct(major = 3, minor = ctx.attr.minor, micro = 0), + files = depset([interpreter]), + ) + return [platform_common.ToolchainInfo(py2_runtime = None, py3_runtime = runtime)] + +_configured_runtime = rule( + implementation = _configured_runtime_impl, + attrs = {"minor": attr.int(mandatory = True)}, +) + def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", @@ -226,3 +242,52 @@ def image_layer_analysis_test_suite(): launcher_dir = "/app/bin", layer_tier = ":_wheel_scripts_tier", ) + + # Manual action-failure fixture: one custom toolchain resolves under both + # Python configurations and emits distinct artifacts at the same runtime + # short_path. The interpreter mapper must participate in global validation. + _configured_runtime( + name = "_configured_runtime", + minor = select({ + ":_python_3_11": 11, + "//conditions:default": 12, + }), + ) + native.toolchain( + name = "_configured_python_toolchain", + toolchain = ":_configured_runtime", + toolchain_type = "@bazel_tools//tools/python:toolchain_type", + ) + py_binary( + name = "_configured_interpreter_311", + srcs = ["server.py"], + python_version = "3.11", + ) + py_binary( + name = "_configured_interpreter_312", + srcs = ["server.py"], + python_version = "3.12", + ) + py_binary( + name = "_configured_interpreter_peer_311", + srcs = ["server.py"], + python_version = "3.11", + ) + py_layer_tier( + name = "_configured_interpreter_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_configured_interpreter_collision_layers", + binary = ":_configured_interpreter_311", + additional_binaries = [":_configured_interpreter_312"], + launcher_dir = "/app/bin", + layer_tier = ":_configured_interpreter_tier", + ) + py_image_layer( + name = "_configured_interpreter_shared_layers", + binary = ":_configured_interpreter_311", + additional_binaries = [":_configured_interpreter_peer_311"], + launcher_dir = "/app/bin", + layer_tier = ":_configured_interpreter_tier", + ) diff --git a/py/private/modify_mtree_test.sh b/py/private/modify_mtree_test.sh index 37b25fb64..8a5066ee7 100755 --- a/py/private/modify_mtree_test.sh +++ b/py/private/modify_mtree_test.sh @@ -34,6 +34,9 @@ run_case disjoint pass $'#mtree\n./generated_tree/first.py type=file contents=fi run_case identical pass $'#mtree\n./generated_tree/support.py type=file contents=same\n./generated_tree/support.py type=file contents=same' [[ $(grep -Fc './generated_tree/support.py ' "$TEST_TMPDIR/identical.out") == 1 ]] run_case conflicting fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' +run_case interpreter_identical pass $'#mtree\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/cfg/bin/shared_runtime/bin/python\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/cfg/bin/shared_runtime/bin/python' 'py_image_layer runfile collision at ./app.runfiles/_main/shared_runtime/bin/python:' +[[ $(grep -Fc './app.runfiles/_main/shared_runtime/bin/python ' "$TEST_TMPDIR/interpreter_identical.out") == 1 ]] +run_case interpreter_conflicting fail $'#mtree\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/first/bin/shared_runtime/bin/python\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/second/bin/shared_runtime/bin/python' 'py_image_layer runfile collision at ./app.runfiles/_main/shared_runtime/bin/python:' run_case dot_alias fail $'#mtree\n./generated_tree/./support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' run_case parent_alias fail $'#mtree\n./generated_tree/nested/../support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' run_case ancestor_first fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py/data type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 9bfde2b4f..b37f2f88c 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -740,7 +740,7 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, use_default_shell_env = False, ) -def _validate_source_mtree(ctx, source_files, source_map, rule_group_files, wheel_files): +def _validate_source_mtree(ctx, source_files, source_map, rule_group_files, wheel_files, interpreter_files): mtree_args = ctx.actions.args() mtree_args.set_param_file_format("multiline") mtree_args.use_param_file("%s", use_always = True) @@ -750,6 +750,8 @@ def _validate_source_mtree(ctx, source_files, source_map, rule_group_files, whee mtree_args.add_all(files, map_each = _user_file_to_mtree, expand_directories = False) for files in wheel_files: mtree_args.add_all(files, map_each = _pkg_file_to_mtree, expand_directories = False) + for files in interpreter_files: + mtree_args.add_all(files, map_each = _interpreter_file_to_mtree, expand_directories = False) output = ctx.actions.declare_file(ctx.attr.name + "_source_validation.mtree") gawk_args = ctx.actions.args() @@ -758,7 +760,7 @@ def _validate_source_mtree(ctx, source_files, source_map, rule_group_files, whee gawk_args.add("-f", ctx.file._awk_script) ctx.actions.run( executable = ctx.executable._awk, - inputs = depset(direct = [ctx.file._awk_script], transitive = [source_files] + rule_group_files + wheel_files), + inputs = depset(direct = [ctx.file._awk_script], transitive = [source_files] + rule_group_files + wheel_files + interpreter_files), outputs = [output], arguments = [gawk_args, mtree_args], env = {"LC_ALL": "C"}, @@ -876,11 +878,11 @@ def _py_image_layer_impl(ctx): # Interpreter tars are declared at the configured toolchain, so identical # runtimes action-share while distinct interpreter artifacts are retained. - interpreter_tars = {} + interpreter_layers = {} for info in infos: if info.interpreter_layer != None: - tar = info.interpreter_layer.tar - interpreter_tars[tar.path] = tar + layer = info.interpreter_layer + interpreter_layers[layer.tar.path] = layer for group_name in sorted(fp_by_group): if group_name in rule_group_names: @@ -902,7 +904,7 @@ def _py_image_layer_impl(ctx): ) all_tars.append(tar_out) - all_tars.extend(interpreter_tars.values()) + all_tars.extend([layer.tar for layer in interpreter_layers.values()]) pip_tars = {} merged = {} @@ -973,6 +975,7 @@ def _py_image_layer_impl(ctx): _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), rule_group_files, [pkg.files for pkg in all_pkgs], + [layer.interpreter_files for layer in interpreter_layers.values()], ) validation = ctx.actions.declare_file(ctx.attr.name + "_validation.log") From de60b14fdd6fde97e1ff49702bfcc3e8f9305030 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 22:56:13 -0400 Subject: [PATCH 05/40] fix(py_image_layer): preserve aliased interpreters Forward interpreter layers and files through binary aliases so an aliased launcher keeps the runtime that built its venv and participates in global collision validation. Normalize the public scalar-binary API through a private alias, leaving the inner rule with one nonempty binaries attribute while preserving select, positional, additional-binary, and testonly compatibility. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 20 +++++++++++++++++++- py/private/py_image_layer.bzl | 22 ++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 4524a83a1..b62d430f6 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -154,6 +154,11 @@ py_binary( ], ) +alias( + name = "my_app_worker_alias", + actual = ":my_app_worker_bin", +) + py_layer_tier( name = "my_app_launchers_tier", groups = { @@ -166,7 +171,7 @@ py_layer_tier( py_image_layer( name = "my_app_launchers_layers", - additional_binaries = [":my_app_worker_bin"], + additional_binaries = [":my_app_worker_alias"], binary = ":my_app_bin", launcher_dir = "/app/bin", layer_tier = ":my_app_launchers_tier", @@ -237,11 +242,24 @@ py_image_layer( launcher_dir = "/app/bin", ) +py_binary( + name = "my_app_testonly_bin", + testonly = True, + srcs = ["server.py"], +) + +py_image_layer( + name = "my_app_testonly_layers", + testonly = True, + binary = ":my_app_testonly_bin", +) + build_test( name = "my_app_binary_api_test", targets = [ ":my_app_binaries_layers", ":my_app_select_layers", + ":my_app_testonly_layers", ], ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index b37f2f88c..d454c2e98 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -383,12 +383,13 @@ def _layer_aspect_impl(target, ctx): # it is already captured upstream as a pip package via the wheel-leaf # branch. Just propagate transitively for these targets. if kind == "alias": + interp = transitive_interp[0] if transitive_interp else None return [_LayerInfo( source_files = depset(transitive = transitive_source), pip_packages = depset(transitive = transitive_pkgs), first_party_layers = depset(transitive = transitive_fp), - interpreter_files = depset(), - interpreter_layer = None, + interpreter_files = interp.interpreter_files if interp != None else depset(), + interpreter_layer = interp, )] # Skip PyInfo deps (including wheel-leaf targets, which also emit PyInfo) — @@ -789,9 +790,7 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m return tar_out def _py_image_layer_impl(ctx): - binaries = ([ctx.attr.binary] if ctx.attr.binary != None else []) + ctx.attr.binaries - if not binaries: - fail("py_image_layer requires at least one binary") + binaries = ctx.attr.binaries infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) @@ -1007,11 +1006,9 @@ def _py_image_layer_impl(ctx): _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { - "binary": attr.label( - aspects = [_layer_aspect], - ), "binaries": attr.label_list( aspects = [_layer_aspect], + allow_empty = False, ), "launcher_dir": attr.string( default = "", @@ -1114,7 +1111,13 @@ def py_image_layer( else: if binary == None: fail("py_image_layer requires binary or binaries") - binaries = additional_binaries + native.alias( + name = name + "_binary", + actual = binary, + tags = tags, + testonly = kwargs.get("testonly", False), + ) + binaries = [":" + name + "_binary"] + additional_binaries for key in groups: if _split_glob_key(key)[1] != None: @@ -1125,7 +1128,6 @@ def py_image_layer( _py_image_layer( name = name, - binary = binary, binaries = binaries, launcher_dir = launcher_dir, groups = groups, From 67495e9d71d66cee72bbeea57d122542dead39dd Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 23:05:15 -0400 Subject: [PATCH 06/40] fix(py_image_layer): namespace merged pip layers Place rule-owned merged pip tars below the image target directory so a valid user group named merged_pip_layer_ cannot declare the same output artifact. Keep existing group, source, and squashed filenames stable and add a default-gzip two-wheel regression that builds both colliding names. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 16 ++++++++++++++++ py/private/py_image_layer.bzl | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index b62d430f6..2df0d91e5 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -132,6 +132,21 @@ py_image_layer( layer_tier = ":my_app_multi_tier", ) +py_layer_tier( + name = "my_app_merged_name_tier", + groups = { + "@pip//colorama": "third_party", + "@pip//pyproject_hooks": "third_party", + }, +) + +py_image_layer( + name = "my_app_merged_name_layers", + binary = ":my_app_multi_bin", + groups = {":worker_support": "merged_pip_layer_third_party"}, + layer_tier = ":my_app_merged_name_tier", +) + # Exercise the actual multi-launcher flow with separate lock universes and two # interpreter versions. colorama has the same normalized label in both # closures, but distinct 3.11/3.12 install trees; both launchers must resolve @@ -258,6 +273,7 @@ build_test( name = "my_app_binary_api_test", targets = [ ":my_app_binaries_layers", + ":my_app_merged_name_layers", ":my_app_select_layers", ":my_app_testonly_layers", ], diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index d454c2e98..23748bb49 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -917,7 +917,7 @@ def _py_image_layer_impl(ctx): for group_name in sorted(merged): algorithm, level, ext = _compression_for(plan, group_name) - tar_out = ctx.actions.declare_file("{}_merged_pip_layer_{}{}".format(ctx.attr.name, group_name, ext)) + tar_out = ctx.actions.declare_file("{}/merged_pip_layers/{}{}".format(ctx.attr.name, group_name, ext)) _run_tar_action( ctx, bsdtar, From 4a71d18f6a57437e9068145f9ca914c5d594401b Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 23:13:29 -0400 Subject: [PATCH 07/40] fix(py_image_layer): avoid helper collisions Forward scalar and list-valued binaries through separate inner attrs so a valid _binary target no longer collides with a generated alias. Combine their resolved values after the transition and reject an empty binary set. Keep scalar select, list select, positional, and additional-binary compatibility without reserving another target name, and replace the obsolete testonly-alias case with the colliding-name regression. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 10 ++++------ py/private/py_image_layer.bzl | 17 ++++++++--------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 2df0d91e5..ab33b9522 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -258,24 +258,22 @@ py_image_layer( ) py_binary( - name = "my_app_testonly_bin", - testonly = True, + name = "my_app_compat_layers_binary", srcs = ["server.py"], ) py_image_layer( - name = "my_app_testonly_layers", - testonly = True, - binary = ":my_app_testonly_bin", + name = "my_app_compat_layers", + binary = ":my_app_compat_layers_binary", ) build_test( name = "my_app_binary_api_test", targets = [ ":my_app_binaries_layers", + ":my_app_compat_layers", ":my_app_merged_name_layers", ":my_app_select_layers", - ":my_app_testonly_layers", ], ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 23748bb49..3d554c0c1 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -790,7 +790,9 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m return tar_out def _py_image_layer_impl(ctx): - binaries = ctx.attr.binaries + binaries = ([ctx.attr.binary] if ctx.attr.binary != None else []) + ctx.attr.binaries + if not binaries: + fail("py_image_layer requires at least one binary") infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) @@ -1006,9 +1008,11 @@ def _py_image_layer_impl(ctx): _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { + "binary": attr.label( + aspects = [_layer_aspect], + ), "binaries": attr.label_list( aspects = [_layer_aspect], - allow_empty = False, ), "launcher_dir": attr.string( default = "", @@ -1111,13 +1115,7 @@ def py_image_layer( else: if binary == None: fail("py_image_layer requires binary or binaries") - native.alias( - name = name + "_binary", - actual = binary, - tags = tags, - testonly = kwargs.get("testonly", False), - ) - binaries = [":" + name + "_binary"] + additional_binaries + binaries = additional_binaries for key in groups: if _split_glob_key(key)[1] != None: @@ -1128,6 +1126,7 @@ def py_image_layer( _py_image_layer( name = name, + binary = binary, binaries = binaries, launcher_dir = launcher_dir, groups = groups, From 9afa9b70848d8fc59f7d3a490b29bf8255771d72 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 23:38:56 -0400 Subject: [PATCH 08/40] fix(py_image_layer): handle ampersand paths Rebuild normalized mtree rows literally instead of passing destinations through awk's replacement-string expansion, which corrupts valid paths containing an ampersand. Add a focused normalized-path regression that preserves the destination, separator spacing, and attributes exactly. --- py/private/modify_mtree.awk | 3 ++- py/private/modify_mtree_test.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index 996ca3474..fb12c81e2 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -86,8 +86,9 @@ function normalize_destination(path, count, i, n, part, parts, normalized) { { if ($0 !~ /^#/) { + original_path = $1 destination = normalize_destination($1) - sub(/^[^ ]+/, destination) + $0 = destination substr($0, length(original_path) + 1) match($0, /(contents|content|link)=[^ ]+/) current_source = substr($0, RSTART, RLENGTH) current_source_path = substr(current_source, index(current_source, "=") + 1) diff --git a/py/private/modify_mtree_test.sh b/py/private/modify_mtree_test.sh index 8a5066ee7..dfbf4d4ed 100755 --- a/py/private/modify_mtree_test.sh +++ b/py/private/modify_mtree_test.sh @@ -31,6 +31,8 @@ run_case() { } run_case disjoint pass $'#mtree\n./generated_tree/first.py type=file contents=first\n./generated_tree/second.py type=file contents=second' +run_case ampersand_destination pass $'#mtree\n./assets/nested/../a&b.txt type=file mode=0644 contents=asset' +grep -Fxq './assets/a&b.txt type=file mode=0644 contents=asset' "$TEST_TMPDIR/ampersand_destination.out" run_case identical pass $'#mtree\n./generated_tree/support.py type=file contents=same\n./generated_tree/support.py type=file contents=same' [[ $(grep -Fc './generated_tree/support.py ' "$TEST_TMPDIR/identical.out") == 1 ]] run_case conflicting fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' From 23e16fbd296c5531895caabc298fa17fbc67089b Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Sun, 19 Jul 2026 23:52:10 -0400 Subject: [PATCH 09/40] fix(py_image_layer): parse mtree metadata fields Read source markers only from anchored metadata fields so legal content=, contents=, or link= text in a runfile destination cannot hide a collision. Reject malformed rows without a source marker. Reuse the parsed source for symlink resolution and rewrite metadata literally, preserving destination text and attributes during normal tar generation. Add focused validator and symlink destination regressions. --- py/private/modify_mtree.awk | 42 +++++++++++++++++++++------------ py/private/modify_mtree_test.sh | 14 +++++++++++ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index fb12c81e2..7c61e174f 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -84,13 +84,30 @@ function normalize_destination(path, count, i, n, part, parts, normalized) { return path } +function replace_metadata_field(row, pattern, replacement) { + if (!match(row, pattern)) { + return row + } + return substr(row, 1, RSTART) replacement substr(row, RSTART + RLENGTH) +} + { if ($0 !~ /^#/) { original_path = $1 destination = normalize_destination($1) $0 = destination substr($0, length(original_path) + 1) - match($0, /(contents|content|link)=[^ ]+/) - current_source = substr($0, RSTART, RLENGTH) + current_source = "" + for (field = 2; field <= NF; field++) { + if ($field ~ /^(contents|content|link)=[^ ]+$/) { + current_source = $field + break + } + } + if (current_source == "") { + print "invalid py_image_layer mtree row (missing source): " $0 > "/dev/stderr" + failed = 1 + exit 1 + } current_source_path = substr(current_source, index(current_source, "=") + 1) count = split(destination, destination_parts, "/") @@ -142,17 +159,10 @@ function normalize_destination(path, count, i, n, part, parts, normalized) { # might be symlinks Bazel didn't flag (repo-rule-staged ones like # rules_python's `bin/python -> python3.11`). Empty `readlink` means # it's a regular file and the row passes through unchanged. - is_hot_path = ($0 ~ /type=link/) && ($0 ~ /link=/) - is_slow_path = ($0 ~ /type=file/) && ($0 ~ /content=/) + is_hot_path = ($0 ~ /[[:space:]]type=link([[:space:]]|$)/) && (current_source ~ /^link=/) + is_slow_path = ($0 ~ /[[:space:]]type=file([[:space:]]|$)/) && (current_source ~ /^content=/) if (is_hot_path || is_slow_path) { - if (is_hot_path) { - match($0, /link=[^ ]+/) - } else { - match($0, /content=[^ ]+/) - } - content_field = substr($0, RSTART, RLENGTH) - split(content_field, parts, "=") - path = parts[2] + path = current_source_path symlink_map[path] = $1 # Plain `readlink` first: keep its result if relative @@ -261,9 +271,11 @@ END { # Already a relative path linked_to = resolved_path } - sub(/type=[^ ]+/, "type=link", original_line) - if (!sub(/content=[^ ]+/, "link=" linked_to, original_line)) { - sub(/link=[^ ]+/, "link=" linked_to, original_line) + original_line = replace_metadata_field(original_line, "[[:space:]]type=[^ ]+", "type=link") + if (original_line ~ /[[:space:]]content=[^ ]+/) { + original_line = replace_metadata_field(original_line, "[[:space:]]content=[^ ]+", "link=" linked_to) + } else { + original_line = replace_metadata_field(original_line, "[[:space:]]link=[^ ]+", "link=" linked_to) } out_lines[++n] = original_line } else { diff --git a/py/private/modify_mtree_test.sh b/py/private/modify_mtree_test.sh index dfbf4d4ed..f9c5c8d46 100755 --- a/py/private/modify_mtree_test.sh +++ b/py/private/modify_mtree_test.sh @@ -44,6 +44,7 @@ run_case parent_alias fail $'#mtree\n./generated_tree/nested/../support.py type= run_case ancestor_first fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py/data type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' run_case descendant_first fail $'#mtree\n./generated_tree/support.py/data type=file contents=first\n./generated_tree/support.py type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' run_case above_root fail $'#mtree\n../generated_tree/support.py type=file contents=first' 'py_image_layer image destination escapes its root:' +run_case missing_source fail $'#mtree\n./generated_tree/support.py type=file mode=0644' 'invalid py_image_layer mtree row (missing source):' if "$gawk" -v "outfile=$TEST_TMPDIR/validate_only.out" -v validate_only=1 -f "$awk_script" \ <<< $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' \ @@ -53,5 +54,18 @@ if "$gawk" -v "outfile=$TEST_TMPDIR/validate_only.out" -v validate_only=1 -f "$a fi grep -Fq 'py_image_layer runfile collision at ./generated_tree/support.py:' "$TEST_TMPDIR/validate_only.err" +if "$gawk" -v "outfile=$TEST_TMPDIR/validate_destination_decoy.out" -v validate_only=1 -f "$awk_script" \ + <<< $'#mtree\n./generated_tree/content=decoy/contents=also/link=x type=file mode=0755 contents=bazel-out/first/bin/file\n./generated_tree/content=decoy/contents=also/link=x type=file mode=0644 content=bazel-out/second/bin/file' \ + 2>"$TEST_TMPDIR/validate_destination_decoy.err"; then + echo 'validate_destination_decoy: expected collision failure' >&2 + exit 1 +fi +grep -Fq 'py_image_layer runfile collision at ./generated_tree/content=decoy/contents=also/link=x:' "$TEST_TMPDIR/validate_destination_decoy.err" + +ln -s target.py "$TEST_TMPDIR/destination_decoy_link" +"$gawk" -v "outfile=$TEST_TMPDIR/destination_decoy_link.out" -f "$awk_script" \ + <<< "#mtree"$'\n'"./generated_tree/content=decoy/contents=also/link=x type=link mode=0755 link=$TEST_TMPDIR/destination_decoy_link" +grep -Fxq './generated_tree/content=decoy/contents=also/link=x type=link mode=0755 link=target.py' "$TEST_TMPDIR/destination_decoy_link.out" + "$gawk" -v "outfile=$TEST_TMPDIR/validate_same_source.out" -v validate_only=1 -f "$awk_script" \ <<< $'#mtree\n./generated_tree/support.py type=file mode=0755 content=same\n./generated_tree/support.py type=file mode=0644 contents=same' From 372881dfbeacfed4f13e9a31b72ab6796a6b17ef Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 14:30:22 -0400 Subject: [PATCH 10/40] refactor(py): simplify multi-binary image API --- e2e/cases/oci/py_image_layer/BUILD.bazel | 12 ++++++--- .../image_layer_analysis_tests.bzl | 27 +++++++------------ py/private/py_image_layer.bzl | 20 +++++--------- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index ab33b9522..e3b7589a5 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -186,8 +186,10 @@ py_layer_tier( py_image_layer( name = "my_app_launchers_layers", - additional_binaries = [":my_app_worker_alias"], - binary = ":my_app_bin", + binaries = [ + ":my_app_bin", + ":my_app_worker_alias", + ], launcher_dir = "/app/bin", layer_tier = ":my_app_launchers_tier", ) @@ -205,8 +207,10 @@ py_binary( py_image_layer( name = "my_app_shared_layers", - additional_binaries = [":my_app_peer_bin"], - binary = ":my_app_bin", + binaries = [ + ":my_app_bin", + ":my_app_peer_bin", + ], launcher_dir = "/app/bin", ) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 2eebae36f..83392e4ce 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -51,8 +51,7 @@ _configured_runtime = rule( def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", - binary = ":my_app_bin", - additional_binaries = [":my_app_worker_bin"], + binaries = [":my_app_bin", ":my_app_worker_bin"], ) _expected_failure_test( name = "missing_launcher_dir_test", @@ -62,8 +61,7 @@ def image_layer_analysis_test_suite(): py_image_layer( name = "_relative_launcher_dir_layers", - binary = ":my_app_bin", - additional_binaries = [":my_app_worker_bin"], + binaries = [":my_app_bin", ":my_app_worker_bin"], launcher_dir = "app/bin", ) _expected_failure_test( @@ -78,8 +76,7 @@ def image_layer_analysis_test_suite(): ) py_image_layer( name = "_duplicate_launcher_layers", - binary = ":my_app_bin", - additional_binaries = [":_my_app_bin_alias"], + binaries = [":my_app_bin", ":_my_app_bin_alias"], launcher_dir = "////", ) _expected_failure_test( @@ -130,8 +127,7 @@ def image_layer_analysis_test_suite(): py_image_layer( name = "_configured_group_collision_layers", - binary = ":_configured_group_311", - additional_binaries = [":_configured_group_312"], + binaries = [":_configured_group_311", ":_configured_group_312"], launcher_dir = "/app/bin", layer_tier = ":_generated_support_tier", ) @@ -143,8 +139,7 @@ def image_layer_analysis_test_suite(): py_image_layer( name = "_configured_group_source_collision_layers", - binary = ":_configured_group_311", - additional_binaries = [":_configured_source_312"], + binaries = [":_configured_group_311", ":_configured_source_312"], launcher_dir = "/app/bin", layer_tier = ":_generated_support_tier", ) @@ -190,8 +185,7 @@ def image_layer_analysis_test_suite(): ) py_image_layer( name = "_expanded_tree_collision_layers", - binary = ":_configured_tree_311", - additional_binaries = [":_configured_file_312"], + binaries = [":_configured_tree_311", ":_configured_file_312"], launcher_dir = "/app/bin", layer_tier = ":_configured_tree_tier", ) @@ -237,8 +231,7 @@ def image_layer_analysis_test_suite(): ) py_image_layer( name = "_configured_wheel_collision_layers", - binary = ":_wheel_scripts_311", - additional_binaries = [":_wheel_scripts_312"], + binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], launcher_dir = "/app/bin", layer_tier = ":_wheel_scripts_tier", ) @@ -279,15 +272,13 @@ def image_layer_analysis_test_suite(): ) py_image_layer( name = "_configured_interpreter_collision_layers", - binary = ":_configured_interpreter_311", - additional_binaries = [":_configured_interpreter_312"], + binaries = [":_configured_interpreter_311", ":_configured_interpreter_312"], launcher_dir = "/app/bin", layer_tier = ":_configured_interpreter_tier", ) py_image_layer( name = "_configured_interpreter_shared_layers", - binary = ":_configured_interpreter_311", - additional_binaries = [":_configured_interpreter_peer_311"], + binaries = [":_configured_interpreter_311", ":_configured_interpreter_peer_311"], launcher_dir = "/app/bin", layer_tier = ":_configured_interpreter_tier", ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 3d554c0c1..02a4c9499 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -571,10 +571,7 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_dsts) def _make_source_map(strip_prefix, root, maybe_symlink, executable_dsts): - def _source_map(f, d): - return _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) - - return _source_map + return lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) def _check_runfile_collision(f, dst, runfile_paths): if f.is_directory: @@ -1059,7 +1056,6 @@ def py_image_layer( warn_layer_count = 90, platform = None, layer_tier = None, - additional_binaries = [], launcher_dir = "", binaries = None, **kwargs): @@ -1099,23 +1095,21 @@ def py_image_layer( layer_tier: Optional py_layer_tier target pinned for this rule. Sets the `@aspect_rules_py//py:layer_tier` label_flag via the rule transition, overriding any command-line value for this rule's subgraph. - additional_binaries: Additional py_binary targets whose dependency and - source layers are included in the image. launcher_dir: Absolute image directory for the binary launchers. Required - with additional_binaries. Set RUNFILES_DIR=/app.runfiles in the image. - binaries: Alternative to binary/additional_binaries. A nonempty list of - py_binary targets to include in the image. + with multiple binaries. Set RUNFILES_DIR=/app.runfiles in the image. + binaries: Alternative to binary. A nonempty list of py_binary targets to + include in the image. **kwargs: Forwarded to inner rule. """ tags = kwargs.pop("tags", []) + ["manual"] if binaries != None: - if binary != None or additional_binaries: - fail("py_image_layer accepts either binaries or binary/additional_binaries, not both") + if binary != None: + fail("py_image_layer accepts either binary or binaries, not both") else: if binary == None: fail("py_image_layer requires binary or binaries") - binaries = additional_binaries + binaries = [] for key in groups: if _split_glob_key(key)[1] != None: From f1c0de04ff3cffbb82ed5cbd3d6326dd33f4a93a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 14:39:59 -0400 Subject: [PATCH 11/40] test(py): exercise image-layer validation fixtures --- e2e/cases/oci/test.sh | 60 +++++++++++++++++++++++++++++++++++ py/private/py_image_layer.bzl | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100755 e2e/cases/oci/test.sh diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh new file mode 100755 index 000000000..c1b41b661 --- /dev/null +++ b/e2e/cases/oci/test.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# The py_image_layer collision fixtures fail while their validation actions run, +# so they cannot be sh_tests under //...; CI runs this script directly. +set -uo pipefail + +cd "$(dirname "$0")/.." || exit 1 + +BAZEL="${BAZEL:-bazel}" +PKG="//oci/py_image_layer" +output_log="$(mktemp)" +trap 'rm -f "$output_log"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +expect_diagnostic() { + local expected="$1" + + if ! grep -Fq "$expected" "$output_log"; then + cat "$output_log" >&2 + fail "expected validation diagnostic: $expected" + fi +} + +echo "== cross-layer collisions must fail validation ==" +if "$BAZEL" build --keep_going --output_groups=_validation -- \ + "${PKG}:_expanded_tree_collision_layers" \ + "${PKG}:_rule_group_collision_layers" \ + "${PKG}:_configured_wheel_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected cross-layer collision fixtures to fail validation" +fi +expect_diagnostic "generated_tree/support.py:" +expect_diagnostic "generated_support.py:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/" +expect_diagnostic "/bin/pyproject-build:" + +echo "== distinct interpreters at the same runfile path must fail validation ==" +if "$BAZEL" build \ + --extra_toolchains="${PKG}:_configured_python_toolchain" \ + --output_groups=_validation -- \ + "${PKG}:_configured_interpreter_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected interpreter collision fixture to fail validation" +fi +expect_diagnostic "shared_runtime/bin/python:" + +echo "== identical interpreters at the same runfile path must pass validation ==" +if ! "$BAZEL" build \ + --extra_toolchains="${PKG}:_configured_python_toolchain" \ + --output_groups=_validation -- \ + "${PKG}:_configured_interpreter_shared_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected shared interpreter fixture to pass validation" +fi + +echo "PASS: py_image_layer validates cross-layer and interpreter collisions" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 02a4c9499..c2b43f3a5 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -1,6 +1,6 @@ """py_image_layer — analysis-time grouped OCI layers with globally shared pip tars. -One rule-propagated aspect wires onto `py_image_layer.binaries`: +One rule-propagated aspect wires onto the `py_image_layer` binary inputs: `_layer_aspect` propagates through `deps`/`data`/`actual`. For pip packages it creates aspect-owned per-package tars at the pip target's namespace (globally From 40b1d69b8818bbcbf5e1c3d8061f96b91244a83d Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 14:44:16 -0400 Subject: [PATCH 12/40] test(py): tighten image-layer validation coverage --- e2e/cases/oci/test.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index c1b41b661..76caa30b3 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -33,10 +33,12 @@ if "$BAZEL" build --keep_going --output_groups=_validation -- \ cat "$output_log" >&2 fail "expected cross-layer collision fixtures to fail validation" fi -expect_diagnostic "generated_tree/support.py:" -expect_diagnostic "generated_support.py:" -expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/" -expect_diagnostic "/bin/pyproject-build:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_tree/support.py:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_support.py:" +if ! grep -F "py_image_layer runfile collision at ./app.runfiles/" "$output_log" | grep -Fq "/bin/pyproject-build:"; then + cat "$output_log" >&2 + fail "expected wheel-script collision validation diagnostic" +fi echo "== distinct interpreters at the same runfile path must fail validation ==" if "$BAZEL" build \ @@ -46,15 +48,16 @@ if "$BAZEL" build \ cat "$output_log" >&2 fail "expected interpreter collision fixture to fail validation" fi -expect_diagnostic "shared_runtime/bin/python:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/shared_runtime/bin/python:" -echo "== identical interpreters at the same runfile path must pass validation ==" +echo "== identical runfile artifacts must pass validation ==" if ! "$BAZEL" build \ --extra_toolchains="${PKG}:_configured_python_toolchain" \ --output_groups=_validation -- \ - "${PKG}:_configured_interpreter_shared_layers" >"$output_log" 2>&1; then + "${PKG}:_configured_interpreter_shared_layers" \ + "${PKG}:_same_rule_group_source_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 - fail "expected shared interpreter fixture to pass validation" + fail "expected shared-artifact fixtures to pass validation" fi echo "PASS: py_image_layer validates cross-layer and interpreter collisions" From f87e6da37cff657bc3be91f4883306496edb1c61 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 15:04:59 -0400 Subject: [PATCH 13/40] refactor(py): narrow multi-binary image layering --- e2e/cases/oci/py_image_layer/BUILD.bazel | 3 +- .../image_layer_analysis_tests.bzl | 229 +----------------- .../launchers_image_command_test.yaml | 2 +- e2e/cases/oci/test.sh | 63 ----- py/private/BUILD.bazel | 14 -- py/private/modify_mtree.awk | 125 ++-------- py/private/modify_mtree_test.sh | 71 ------ py/private/py_image_layer.bzl | 108 ++------- 8 files changed, 38 insertions(+), 577 deletions(-) delete mode 100755 e2e/cases/oci/test.sh delete mode 100755 py/private/modify_mtree_test.sh diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index e3b7589a5..d439da8e3 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -149,7 +149,7 @@ py_image_layer( # Exercise the actual multi-launcher flow with separate lock universes and two # interpreter versions. colorama has the same normalized label in both -# closures, but distinct 3.11/3.12 install trees; both launchers must resolve +# closures, but distinct 3.14/3.12 install trees; both launchers must resolve # their own venv and the unioned dependency layers from the shared runfiles tree. py_library( name = "worker_support", @@ -182,6 +182,7 @@ py_layer_tier( "@pip//pyproject_hooks": "third_party", }, interpreter_group = "interpreter", + strip_prefix = "oci/py_image_layer/my_app_bin", ) py_image_layer( diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 83392e4ce..d839687bd 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -1,6 +1,6 @@ """Analysis coverage for invalid multi-launcher image-layer configurations.""" -load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library") +load("@aspect_rules_py//py:defs.bzl", "py_image_layer") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): @@ -14,40 +14,6 @@ _expected_failure_test = analysistest.make( expect_failure = True, ) -def _configured_tree_or_file_impl(ctx): - if ctx.attr.tree: - out = ctx.actions.declare_directory("generated_tree") - ctx.actions.run_shell( - outputs = [out], - command = "mkdir -p \"$1\" && echo tree > \"$1/support.py\"", - arguments = [out.path], - ) - else: - out = ctx.actions.declare_file("generated_tree/support.py") - ctx.actions.write(out, "file\n") - return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] - -_configured_tree_or_file = rule( - implementation = _configured_tree_or_file_impl, - attrs = {"tree": attr.bool()}, -) - -def _configured_runtime_impl(ctx): - interpreter = ctx.actions.declare_file("shared_runtime/bin/python") - ctx.actions.write(interpreter, "#!/bin/sh\nexec /usr/bin/python3 \"$@\"\n", is_executable = True) - runtime = struct( - interpreter = interpreter, - interpreter_path = None, - interpreter_version_info = struct(major = 3, minor = ctx.attr.minor, micro = 0), - files = depset([interpreter]), - ) - return [platform_common.ToolchainInfo(py2_runtime = None, py3_runtime = runtime)] - -_configured_runtime = rule( - implementation = _configured_runtime_impl, - attrs = {"minor": attr.int(mandatory = True)}, -) - def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", @@ -89,196 +55,3 @@ def image_layer_analysis_test_suite(): name = "_python_3_11", flag_values = {"@aspect_rules_py//py/private/interpreter:python_version": "3.11"}, ) - native.genrule( - name = "_generated_support", - outs = ["generated_support.py"], - cmd = select({ - ":_python_3_11": "echo 'VALUE = 11' > $@", - "//conditions:default": "echo 'VALUE = 12' > $@", - }), - ) - py_library( - name = "_generated_support_lib", - srcs = [":_generated_support"], - imports = ["."], - ) - py_binary( - name = "_configured_group_311", - srcs = ["server.py"], - python_version = "3.11", - deps = [":_generated_support_lib"], - ) - py_binary( - name = "_configured_group_312", - srcs = ["server.py"], - python_version = "3.12", - deps = [":_generated_support_lib"], - ) - py_binary( - name = "_configured_source_312", - srcs = ["server.py"], - data = [":_generated_support"], - python_version = "3.12", - ) - py_layer_tier( - name = "_generated_support_tier", - groups = {"//oci/py_image_layer:_generated_support_lib": "generated_support"}, - ) - - py_image_layer( - name = "_configured_group_collision_layers", - binaries = [":_configured_group_311", ":_configured_group_312"], - launcher_dir = "/app/bin", - layer_tier = ":_generated_support_tier", - ) - _expected_failure_test( - name = "configured_group_collision_test", - expected_error = "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_support.py", - target_under_test = ":_configured_group_collision_layers", - ) - - py_image_layer( - name = "_configured_group_source_collision_layers", - binaries = [":_configured_group_311", ":_configured_source_312"], - launcher_dir = "/app/bin", - layer_tier = ":_generated_support_tier", - ) - _expected_failure_test( - name = "configured_group_source_collision_test", - expected_error = "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_support.py", - target_under_test = ":_configured_group_source_collision_layers", - ) - - # Manual action-failure fixture: the 3.11 tree expands in the grouped tar, - # while the 3.12 file lands in the default tar. The global mtree validator - # must reject their shared destination. - _configured_tree_or_file( - name = "_configured_tree_or_file", - tree = select({ - ":_python_3_11": True, - "//conditions:default": False, - }), - ) - py_library( - name = "_configured_tree_lib", - srcs = [":_configured_tree_or_file"], - imports = ["."], - ) - py_binary( - name = "_configured_tree_311", - srcs = ["server.py"], - python_version = "3.11", - deps = [":_configured_tree_lib"], - ) - py_binary( - name = "_configured_file_312", - srcs = [ - "server.py", - ":_configured_tree_or_file", - ], - main = "server.py", - python_version = "3.12", - ) - py_layer_tier( - name = "_configured_tree_tier", - groups = {"//oci/py_image_layer:_configured_tree_lib": "generated_tree"}, - ) - py_image_layer( - name = "_expanded_tree_collision_layers", - binaries = [":_configured_tree_311", ":_configured_file_312"], - launcher_dir = "/app/bin", - layer_tier = ":_configured_tree_tier", - ) - - # Manual action-failure fixture for a rule-level group built in the image - # configuration colliding with a transitioned binary source file. - py_image_layer( - name = "_rule_group_collision_layers", - binary = ":_configured_group_311", - groups = {":_generated_support": "generated_support"}, - ) - - py_binary( - name = "_same_group_source_bin", - srcs = ["server.py"], - data = [":_generated_support"], - ) - py_image_layer( - name = "_same_rule_group_source_layers", - binary = ":_same_group_source_bin", - groups = {":_generated_support": "generated_support"}, - ) - - # Manual action-failure fixture for unversioned wheel script/data paths - # shared by separately configured whl_install trees. - py_binary( - name = "_wheel_scripts_311", - srcs = ["server.py"], - dep_group = "images", - python_version = "3.11", - deps = ["@pypi_oci_py_image_layer//build"], - ) - py_binary( - name = "_wheel_scripts_312", - srcs = ["server.py"], - dep_group = "images", - python_version = "3.12", - deps = ["@pypi_oci_py_image_layer//build"], - ) - py_layer_tier( - name = "_wheel_scripts_tier", - groups = {"@pip//build": "wheel_scripts"}, - ) - py_image_layer( - name = "_configured_wheel_collision_layers", - binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], - launcher_dir = "/app/bin", - layer_tier = ":_wheel_scripts_tier", - ) - - # Manual action-failure fixture: one custom toolchain resolves under both - # Python configurations and emits distinct artifacts at the same runtime - # short_path. The interpreter mapper must participate in global validation. - _configured_runtime( - name = "_configured_runtime", - minor = select({ - ":_python_3_11": 11, - "//conditions:default": 12, - }), - ) - native.toolchain( - name = "_configured_python_toolchain", - toolchain = ":_configured_runtime", - toolchain_type = "@bazel_tools//tools/python:toolchain_type", - ) - py_binary( - name = "_configured_interpreter_311", - srcs = ["server.py"], - python_version = "3.11", - ) - py_binary( - name = "_configured_interpreter_312", - srcs = ["server.py"], - python_version = "3.12", - ) - py_binary( - name = "_configured_interpreter_peer_311", - srcs = ["server.py"], - python_version = "3.11", - ) - py_layer_tier( - name = "_configured_interpreter_tier", - interpreter_group = "interpreter", - ) - py_image_layer( - name = "_configured_interpreter_collision_layers", - binaries = [":_configured_interpreter_311", ":_configured_interpreter_312"], - launcher_dir = "/app/bin", - layer_tier = ":_configured_interpreter_tier", - ) - py_image_layer( - name = "_configured_interpreter_shared_layers", - binaries = [":_configured_interpreter_311", ":_configured_interpreter_peer_311"], - launcher_dir = "/app/bin", - layer_tier = ":_configured_interpreter_tier", - ) diff --git a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml index 052378d34..19ab498b9 100644 --- a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml +++ b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml @@ -6,7 +6,7 @@ commandTests: command: /app/bin/my_app_bin expectedOutput: ["Hello rules_py - 3.14"] - - name: additional launcher uses its own interpreter and pip closure + - name: secondary launcher uses its own interpreter and pip closure exitCode: 0 command: /app/bin/my_app_worker_bin expectedOutput: ["worker ok 12 0.4.6 1.2.0 grouped"] diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh deleted file mode 100755 index 76caa30b3..000000000 --- a/e2e/cases/oci/test.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# -# The py_image_layer collision fixtures fail while their validation actions run, -# so they cannot be sh_tests under //...; CI runs this script directly. -set -uo pipefail - -cd "$(dirname "$0")/.." || exit 1 - -BAZEL="${BAZEL:-bazel}" -PKG="//oci/py_image_layer" -output_log="$(mktemp)" -trap 'rm -f "$output_log"' EXIT - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -expect_diagnostic() { - local expected="$1" - - if ! grep -Fq "$expected" "$output_log"; then - cat "$output_log" >&2 - fail "expected validation diagnostic: $expected" - fi -} - -echo "== cross-layer collisions must fail validation ==" -if "$BAZEL" build --keep_going --output_groups=_validation -- \ - "${PKG}:_expanded_tree_collision_layers" \ - "${PKG}:_rule_group_collision_layers" \ - "${PKG}:_configured_wheel_collision_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected cross-layer collision fixtures to fail validation" -fi -expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_tree/support.py:" -expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/generated_support.py:" -if ! grep -F "py_image_layer runfile collision at ./app.runfiles/" "$output_log" | grep -Fq "/bin/pyproject-build:"; then - cat "$output_log" >&2 - fail "expected wheel-script collision validation diagnostic" -fi - -echo "== distinct interpreters at the same runfile path must fail validation ==" -if "$BAZEL" build \ - --extra_toolchains="${PKG}:_configured_python_toolchain" \ - --output_groups=_validation -- \ - "${PKG}:_configured_interpreter_collision_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected interpreter collision fixture to fail validation" -fi -expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/shared_runtime/bin/python:" - -echo "== identical runfile artifacts must pass validation ==" -if ! "$BAZEL" build \ - --extra_toolchains="${PKG}:_configured_python_toolchain" \ - --output_groups=_validation -- \ - "${PKG}:_configured_interpreter_shared_layers" \ - "${PKG}:_same_rule_group_source_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected shared-artifact fixtures to pass validation" -fi - -echo "PASS: py_image_layer validates cross-layer and interpreter collisions" diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 7bc96203f..1fd5eb05e 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -1,5 +1,4 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") -load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//py:defs.bzl", _py_binary_rule = "py_binary", _py_test_rule = "py_test") load(":py_image_layer.bzl", "py_layer_tier") load(":py_library.bzl", "py_library") @@ -41,19 +40,6 @@ _py_test_rule( main = "py_image_layer_validator_test.py", ) -sh_test( - name = "modify_mtree_test", - srcs = ["modify_mtree_test.sh"], - args = [ - "$(location :modify_mtree.awk)", - "$(location @gawk)", - ], - data = [ - ":modify_mtree.awk", - "@gawk", - ], -) - py_layer_tier( name = "default_layer_tier", visibility = ["//visibility:public"], diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index 7c61e174f..67366fbf7 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -58,98 +58,7 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back return back_steps target } -function normalize_destination(path, count, i, n, part, parts, normalized) { - count = split(path, parts, "/") - n = 0 - for (i = 1; i <= count; i++) { - part = parts[i] - if (part == "" || part == ".") { - continue - } - if (part == "..") { - if (n == 0) { - print "py_image_layer image destination escapes its root: " path > "/dev/stderr" - failed = 1 - exit 1 - } - delete normalized[n--] - continue - } - normalized[++n] = part - } - path = "." - for (i = 1; i <= n; i++) { - path = path "/" normalized[i] - } - return path -} - -function replace_metadata_field(row, pattern, replacement) { - if (!match(row, pattern)) { - return row - } - return substr(row, 1, RSTART) replacement substr(row, RSTART + RLENGTH) -} - { - if ($0 !~ /^#/) { - original_path = $1 - destination = normalize_destination($1) - $0 = destination substr($0, length(original_path) + 1) - current_source = "" - for (field = 2; field <= NF; field++) { - if ($field ~ /^(contents|content|link)=[^ ]+$/) { - current_source = $field - break - } - } - if (current_source == "") { - print "invalid py_image_layer mtree row (missing source): " $0 > "/dev/stderr" - failed = 1 - exit 1 - } - current_source_path = substr(current_source, index(current_source, "=") + 1) - - count = split(destination, destination_parts, "/") - parent = destination_parts[1] - for (j = 2; j < count; j++) { - parent = parent "/" destination_parts[j] - if (parent in destination_rows) { - print "py_image_layer runfile collision at " destination ": " destination_sources[parent] " and " current_source > "/dev/stderr" - failed = 1 - exit 1 - } - } - if (destination in destination_descendants) { - print "py_image_layer runfile collision at " destination_descendants[destination] ": " destination_descendant_sources[destination] " and " current_source > "/dev/stderr" - failed = 1 - exit 1 - } - if (destination in destination_rows) { - if (destination_rows[destination] == $0 || (validate_only && destination_source_paths[destination] == current_source_path)) { - next - } - print "py_image_layer runfile collision at " destination ": " destination_sources[destination] " and " current_source > "/dev/stderr" - failed = 1 - exit 1 - } - destination_rows[destination] = $0 - destination_sources[destination] = current_source - destination_source_paths[destination] = current_source_path - parent = destination_parts[1] - for (j = 2; j < count; j++) { - parent = parent "/" destination_parts[j] - if (!(parent in destination_descendants)) { - destination_descendants[parent] = destination - destination_descendant_sources[parent] = current_source - } - } - } - - if (validate_only) { - next - } - symlink = "" symlink_content = "" # Two markers Starlark emits for paths that could be symlinks: @@ -159,10 +68,17 @@ function replace_metadata_field(row, pattern, replacement) { # might be symlinks Bazel didn't flag (repo-rule-staged ones like # rules_python's `bin/python -> python3.11`). Empty `readlink` means # it's a regular file and the row passes through unchanged. - is_hot_path = ($0 ~ /[[:space:]]type=link([[:space:]]|$)/) && (current_source ~ /^link=/) - is_slow_path = ($0 ~ /[[:space:]]type=file([[:space:]]|$)/) && (current_source ~ /^content=/) + is_hot_path = ($0 ~ /type=link/) && ($0 ~ /link=/) + is_slow_path = ($0 ~ /type=file/) && ($0 ~ /content=/) if (is_hot_path || is_slow_path) { - path = current_source_path + if (is_hot_path) { + match($0, /link=[^ ]+/) + } else { + match($0, /content=[^ ]+/) + } + content_field = substr($0, RSTART, RLENGTH) + split(content_field, parts, "=") + path = parts[2] symlink_map[path] = $1 # Plain `readlink` first: keep its result if relative @@ -235,24 +151,17 @@ function replace_metadata_field(row, pattern, replacement) { } } if (symlink != "") { - line_array[++row_count] = $0 SUBSEP $1 SUBSEP resolved_path + line_array[NR] = $0 SUBSEP $1 SUBSEP resolved_path } else { - line_array[++row_count] = $0 + line_array[NR] = $0 } } END { - if (failed) { - exit 1 - } - if (validate_only) { - print "#mtree" > outfile - exit - } # Buffer rewritten rows, sort byte-wise (asort under LC_ALL=C, set by # the action env), and write to `outfile`. n = 0 - for (i = 1; i <= row_count; i++) { + for (i = 1; i <= NR; i++) { line = line_array[i] if (index(line, SUBSEP) > 0) { split(line, fields, SUBSEP) @@ -271,11 +180,9 @@ END { # Already a relative path linked_to = resolved_path } - original_line = replace_metadata_field(original_line, "[[:space:]]type=[^ ]+", "type=link") - if (original_line ~ /[[:space:]]content=[^ ]+/) { - original_line = replace_metadata_field(original_line, "[[:space:]]content=[^ ]+", "link=" linked_to) - } else { - original_line = replace_metadata_field(original_line, "[[:space:]]link=[^ ]+", "link=" linked_to) + sub(/type=[^ ]+/, "type=link", original_line) + if (!sub(/content=[^ ]+/, "link=" linked_to, original_line)) { + sub(/link=[^ ]+/, "link=" linked_to, original_line) } out_lines[++n] = original_line } else { diff --git a/py/private/modify_mtree_test.sh b/py/private/modify_mtree_test.sh deleted file mode 100755 index f9c5c8d46..000000000 --- a/py/private/modify_mtree_test.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -awk_script="$1" -gawk="$2" - -run_case() { - local name="$1" - local expected="$2" - local input="$3" - local diagnostic="${4:-py_image_layer runfile collision at ./generated_tree/support.py:}" - local output="$TEST_TMPDIR/$name.out" - local error="$TEST_TMPDIR/$name.err" - - if "$gawk" -v "outfile=$output" -f "$awk_script" <<<"$input" 2>"$error"; then - if [[ "$expected" != pass ]]; then - echo "$name: expected collision failure" >&2 - exit 1 - fi - elif [[ "$expected" != fail ]]; then - echo "$name: unexpected failure" >&2 - sed -n '1,80p' "$error" >&2 - exit 1 - fi - - if [[ "$expected" == fail ]] && ! grep -Fq "$diagnostic" "$error"; then - echo "$name: missing collision diagnostic" >&2 - sed -n '1,80p' "$error" >&2 - exit 1 - fi -} - -run_case disjoint pass $'#mtree\n./generated_tree/first.py type=file contents=first\n./generated_tree/second.py type=file contents=second' -run_case ampersand_destination pass $'#mtree\n./assets/nested/../a&b.txt type=file mode=0644 contents=asset' -grep -Fxq './assets/a&b.txt type=file mode=0644 contents=asset' "$TEST_TMPDIR/ampersand_destination.out" -run_case identical pass $'#mtree\n./generated_tree/support.py type=file contents=same\n./generated_tree/support.py type=file contents=same' -[[ $(grep -Fc './generated_tree/support.py ' "$TEST_TMPDIR/identical.out") == 1 ]] -run_case conflicting fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' -run_case interpreter_identical pass $'#mtree\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/cfg/bin/shared_runtime/bin/python\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/cfg/bin/shared_runtime/bin/python' 'py_image_layer runfile collision at ./app.runfiles/_main/shared_runtime/bin/python:' -[[ $(grep -Fc './app.runfiles/_main/shared_runtime/bin/python ' "$TEST_TMPDIR/interpreter_identical.out") == 1 ]] -run_case interpreter_conflicting fail $'#mtree\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/first/bin/shared_runtime/bin/python\n./app.runfiles/_main/shared_runtime/bin/python type=file mode=0755 content=bazel-out/second/bin/shared_runtime/bin/python' 'py_image_layer runfile collision at ./app.runfiles/_main/shared_runtime/bin/python:' -run_case dot_alias fail $'#mtree\n./generated_tree/./support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' -run_case parent_alias fail $'#mtree\n./generated_tree/nested/../support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' -run_case ancestor_first fail $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py/data type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' -run_case descendant_first fail $'#mtree\n./generated_tree/support.py/data type=file contents=first\n./generated_tree/support.py type=file contents=second' 'py_image_layer runfile collision at ./generated_tree/support.py/data:' -run_case above_root fail $'#mtree\n../generated_tree/support.py type=file contents=first' 'py_image_layer image destination escapes its root:' -run_case missing_source fail $'#mtree\n./generated_tree/support.py type=file mode=0644' 'invalid py_image_layer mtree row (missing source):' - -if "$gawk" -v "outfile=$TEST_TMPDIR/validate_only.out" -v validate_only=1 -f "$awk_script" \ - <<< $'#mtree\n./generated_tree/support.py type=file contents=first\n./generated_tree/support.py type=file contents=second' \ - 2>"$TEST_TMPDIR/validate_only.err"; then - echo 'validate_only: expected collision failure' >&2 - exit 1 -fi -grep -Fq 'py_image_layer runfile collision at ./generated_tree/support.py:' "$TEST_TMPDIR/validate_only.err" - -if "$gawk" -v "outfile=$TEST_TMPDIR/validate_destination_decoy.out" -v validate_only=1 -f "$awk_script" \ - <<< $'#mtree\n./generated_tree/content=decoy/contents=also/link=x type=file mode=0755 contents=bazel-out/first/bin/file\n./generated_tree/content=decoy/contents=also/link=x type=file mode=0644 content=bazel-out/second/bin/file' \ - 2>"$TEST_TMPDIR/validate_destination_decoy.err"; then - echo 'validate_destination_decoy: expected collision failure' >&2 - exit 1 -fi -grep -Fq 'py_image_layer runfile collision at ./generated_tree/content=decoy/contents=also/link=x:' "$TEST_TMPDIR/validate_destination_decoy.err" - -ln -s target.py "$TEST_TMPDIR/destination_decoy_link" -"$gawk" -v "outfile=$TEST_TMPDIR/destination_decoy_link.out" -f "$awk_script" \ - <<< "#mtree"$'\n'"./generated_tree/content=decoy/contents=also/link=x type=link mode=0755 link=$TEST_TMPDIR/destination_decoy_link" -grep -Fxq './generated_tree/content=decoy/contents=also/link=x type=link mode=0755 link=target.py' "$TEST_TMPDIR/destination_decoy_link.out" - -"$gawk" -v "outfile=$TEST_TMPDIR/validate_same_source.out" -v validate_only=1 -f "$awk_script" \ - <<< $'#mtree\n./generated_tree/support.py type=file mode=0755 content=same\n./generated_tree/support.py type=file mode=0644 contents=same' diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index c2b43f3a5..b19f624e1 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -510,35 +510,22 @@ def _apply_strip_prefix(sp, strip_prefix, root): return "." + root + sp[len(prefix):] return "./app.runfiles/_main/" + sp -def _normalize_destination(path): - parts = [] - for part in path.split("/"): - if not part or part == ".": - continue - if part == "..": - if not parts: - fail("py_image_layer image destination escapes its root: {}".format(path)) - parts.pop() - continue - parts.append(part) - return "./" + "/".join(parts) - def _source_destination(sp, strip_prefix, root, executable_dsts): executable_dst = executable_dsts.get(sp, None) if executable_dst: - return _normalize_destination(executable_dst) + return executable_dst if sp == "_repo_mapping": # Bazel synthesizes a top-level `_repo_mapping` runfile (no `_main/` # prefix); replicate that placement so runfiles.bash can find it. return "./app.runfiles/_repo_mapping" if sp.startswith("../"): - return _normalize_destination("./app.runfiles/" + sp[3:]) - if strip_prefix: - return _normalize_destination(_apply_strip_prefix(sp, strip_prefix, root)) + return "./app.runfiles/" + sp[3:] for executable_short_path in executable_dsts: - if sp == executable_short_path or sp.startswith(executable_short_path + ".runfiles/"): - return _normalize_destination(_apply_strip_prefix(sp, executable_short_path, root)) - return _normalize_destination("./app.runfiles/_main/" + sp) + if sp.startswith(executable_short_path + ".runfiles/"): + return _apply_strip_prefix(sp, executable_short_path, root) + if strip_prefix: + return _apply_strip_prefix(sp, strip_prefix, root) + return "./app.runfiles/_main/" + sp def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False, executable_dsts = {}): dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) @@ -573,14 +560,6 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex def _make_source_map(strip_prefix, root, maybe_symlink, executable_dsts): return lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) -def _check_runfile_collision(f, dst, runfile_paths): - if f.is_directory: - return - previous = runfile_paths.get(dst, None) - if previous != None and previous != f.path: - fail("py_image_layer runfile collision at {}: {} and {}".format(dst, previous, f.path)) - runfile_paths[dst] = f.path - def _user_file_to_mtree(f, dir_expander): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] @@ -738,35 +717,6 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, use_default_shell_env = False, ) -def _validate_source_mtree(ctx, source_files, source_map, rule_group_files, wheel_files, interpreter_files): - mtree_args = ctx.actions.args() - mtree_args.set_param_file_format("multiline") - mtree_args.use_param_file("%s", use_always = True) - mtree_args.add("#mtree") - mtree_args.add_all(source_files, map_each = source_map, expand_directories = False, allow_closure = True) - for files in rule_group_files: - mtree_args.add_all(files, map_each = _user_file_to_mtree, expand_directories = False) - for files in wheel_files: - mtree_args.add_all(files, map_each = _pkg_file_to_mtree, expand_directories = False) - for files in interpreter_files: - mtree_args.add_all(files, map_each = _interpreter_file_to_mtree, expand_directories = False) - - output = ctx.actions.declare_file(ctx.attr.name + "_source_validation.mtree") - gawk_args = ctx.actions.args() - gawk_args.add("-v", output, format = "outfile=%s") - gawk_args.add("-v", "validate_only=1") - gawk_args.add("-f", ctx.file._awk_script) - ctx.actions.run( - executable = ctx.executable._awk, - inputs = depset(direct = [ctx.file._awk_script], transitive = [source_files] + rule_group_files + wheel_files + interpreter_files), - outputs = [output], - arguments = [gawk_args, mtree_args], - env = {"LC_ALL": "C"}, - mnemonic = "PyImageLayerValidateMtree", - progress_message = "Validating source runfiles for %{output}", - ) - return output - def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, map_each, progress): tar_out = ctx.actions.declare_file(out_name) level = ctx.attr.group_compress_levels.get(group_name, "6") @@ -830,33 +780,23 @@ def _py_image_layer_impl(ctx): else: executable_dsts[executable.short_path] = "" - # Grouped and ungrouped sources share one runfiles tree. Reject different - # configured artifacts that map to the same actual image destination. - runfile_paths = {} - for info in infos: - for entry in info.first_party_layers.to_list(): - for f in entry.files.to_list(): - dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) - _check_runfile_collision(f, dst, runfile_paths) - for f in info.source_files.to_list(): - dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) - _check_runfile_collision(f, dst, runfile_paths) - # 3p pip layers are action-shared across the graph and hard-code their # destination under `./app.runfiles//...`, so the consumer's source - # layer has to land under the same `/app.runfiles/` tree for the launcher - # to find them. Default the binary's `short_path` to `strip_prefix` so the - # natural runfile layout maps onto `./app.runfiles/_main/...` without each - # caller wiring it up. + # layer has to land under the same `/app.runfiles/` tree for each launcher + # to find them. Map every launcher's `.runfiles/` tree into that shared root. all_tars = [] - rule_group_files = [] + source_map = _make_source_map( + strip_prefix, + root, + any([info.interpreter_layer == None for info in infos]), + executable_dsts, + ) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) if dep_label in pip_labels: continue - rule_group_files.append(dep[DefaultInfo].files) tar_out = _declare_group_tar( ctx, bsdtar, @@ -897,7 +837,7 @@ def _py_image_layer_impl(ctx): "{}_{}.tar.gz".format(ctx.attr.name, group_name), group_name, depset(transitive = fp_by_group[group_name]), - _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), + source_map, "Creating first-party layer %s[%s]" % (ctx.label, group_name), ) all_tars.append(tar_out) @@ -959,23 +899,11 @@ def _py_image_layer_impl(ctx): "{}_default.tar.gz".format(ctx.attr.name), "default", depset(transitive = [info.source_files for info in infos]), - _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), + source_map, "Creating source layer for %s" % ctx.label, ) all_tars.append(source_tar) - # Validate one expanded mtree spanning grouped and default sources. Each - # tar sees only its own rows, so this catches TreeArtifact child collisions - # that would otherwise land in different OCI layers. - source_validation = _validate_source_mtree( - ctx, - depset(transitive = [info.source_files for info in infos] + [entry.files for info in infos for entry in info.first_party_layers.to_list()]), - _make_source_map(strip_prefix, root, any([info.interpreter_layer == None for info in infos]), executable_dsts), - rule_group_files, - [pkg.files for pkg in all_pkgs], - [layer.interpreter_files for layer in interpreter_layers.values()], - ) - validation = ctx.actions.declare_file(ctx.attr.name + "_validation.log") validation_args = ctx.actions.args() validation_args.add("--threshold_mb", str(ctx.attr.warn_remote_cache_threshold_mb)) @@ -998,7 +926,7 @@ def _py_image_layer_impl(ctx): OutputGroupInfo( deps = depset(dep_tars), sources = depset([source_tar]), - _validation = depset([validation, source_validation]), + _validation = depset([validation]), ), ] From ae9978664b4bc14b10decf20d3ec1b84830ae073 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 15:30:35 -0400 Subject: [PATCH 14/40] fix(py): preserve scalar image-layer behavior --- .../image_layer_analysis_tests.bzl | 32 +++- py/private/py_image_layer.bzl | 178 +++++++++++++++--- 2 files changed, 188 insertions(+), 22 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index d839687bd..91325dd72 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -1,6 +1,6 @@ """Analysis coverage for invalid multi-launcher image-layer configurations.""" -load("@aspect_rules_py//py:defs.bzl", "py_image_layer") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): @@ -55,3 +55,33 @@ def image_layer_analysis_test_suite(): name = "_python_3_11", flag_values = {"@aspect_rules_py//py/private/interpreter:python_version": "3.11"}, ) + + py_binary( + name = "_wheel_scripts_311", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.11", + deps = ["@pypi_oci_py_image_layer//build"], + ) + py_binary( + name = "_wheel_scripts_312", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.12", + deps = ["@pypi_oci_py_image_layer//build"], + ) + py_layer_tier( + name = "_wheel_scripts_tier", + groups = {"@pip//build": "wheel_scripts"}, + ) + py_image_layer( + name = "_configured_wheel_collision_layers", + binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], + launcher_dir = "/app/bin", + layer_tier = ":_wheel_scripts_tier", + ) + _expected_failure_test( + name = "configured_wheel_collision_test", + expected_error = "actual_install.install:", + target_under_test = ":_configured_wheel_collision_layers", + ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index b19f624e1..4a18715f8 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -19,8 +19,8 @@ Layer tier (groups + compression) is carried by the `py_layer_tier` rule which p Sharing model: - Solo whole-group + subpath-split pip tars: action-shared across every rule using that package (declared at the pip target's namespace). - - Multi-member merged tars: per-rule action, but deterministic content (canonical - mtree, fixed bsdtar options) → remote CAS / OCI registry dedupe by digest. + - Multi-member merged tars: action-shared for scalar binaries; one unioned per-rule + action for binary lists, with deterministic content for CAS / registry dedupe. - First-party grouped tars: per-rule action, one tar per group, collected from matched py_library targets in the binaries' dep closures. - Ungrouped pip packages: squashed by the rule into one per-rule tar. @@ -501,6 +501,76 @@ _layer_aspect = aspect( provides = [_LayerInfo], ) +_MergedLayerInfo = provider( + doc = "Private: closure-filtered merged tars for multi-member groups, produced by _merge_aspect.", + fields = { + "merged_tars": "dict[group_name, File] — one merged tar per multi-member group.", + }, +) + +def _merge_aspect_impl(target, ctx): + info = target[_LayerInfo] + + bucket = {} + seen = {} + for pkg in info.pip_packages.to_list(): + if pkg.label in seen: + continue + seen[pkg.label] = True + if pkg.merge_group != None: + bucket.setdefault(pkg.merge_group, []).append(pkg.files) + + if not bucket: + return [_MergedLayerInfo(merged_tars = {})] + + bsdtar, bsdtar_files = _tar_toolchain(ctx) + plan = ctx.attr._layer_tier[PyLayerTierInfo] + + merged_tars = {} + for group_name in sorted(bucket): + install_dirs = bucket[group_name] + algorithm, level, ext = _compression_for(plan, group_name) + tar_out = ctx.actions.declare_file("_merged_pip_layer_{}{}".format(group_name, ext)) + _run_tar_action( + ctx, + bsdtar, + bsdtar_files, + tar_out, + depset(transitive = install_dirs), + _pkg_file_to_mtree, + algorithm, + level, + {}, + "PyImageMergedLayer", + "Merging %d pip packages into %s[%s]" % (len(install_dirs), target.label, group_name), + ) + merged_tars[group_name] = tar_out + + return [_MergedLayerInfo(merged_tars = merged_tars)] + +_merge_aspect = aspect( + implementation = _merge_aspect_impl, + attr_aspects = [], + attrs = { + "_layer_tier": attr.label( + default = "//py:layer_tier", + providers = [PyLayerTierInfo], + ), + "_awk_script": attr.label( + default = "//py/private:modify_mtree.awk", + allow_single_file = True, + ), + "_awk": attr.label( + default = "@gawk", + cfg = "exec", + executable = True, + ), + }, + toolchains = [_TAR_TOOLCHAIN], + required_aspect_providers = [[_LayerInfo]], + provides = [_MergedLayerInfo], +) + def _apply_strip_prefix(sp, strip_prefix, root): prefix = strip_prefix.replace("\\/", "/") if sp == prefix: @@ -521,7 +591,7 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): if sp.startswith("../"): return "./app.runfiles/" + sp[3:] for executable_short_path in executable_dsts: - if sp.startswith(executable_short_path + ".runfiles/"): + if sp == executable_short_path or sp.startswith(executable_short_path + ".runfiles/"): return _apply_strip_prefix(sp, executable_short_path, root) if strip_prefix: return _apply_strip_prefix(sp, strip_prefix, root) @@ -560,6 +630,46 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex def _make_source_map(strip_prefix, root, maybe_symlink, executable_dsts): return lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) +def _normalize_destination(path): + parts = [] + for part in path.split("/"): + if not part or part == ".": + continue + if part == "..": + if not parts: + fail("py_image_layer image destination escapes its root: {}".format(path)) + parts.pop() + else: + parts.append(part) + return "./" + "/".join(parts) + +def _register_image_destination(f, destination, paths, tree_roots, descendants): + destination = _normalize_destination(destination) + previous = paths.get(destination, None) + if previous != None: + if previous != f.path: + fail("py_image_layer runfile collision at {}: {} and {}".format(destination, previous, f.path)) + return + + parts = destination.split("/") + for end in range(len(parts) - 1, 0, -1): + parent = "/".join(parts[:end]) + tree = tree_roots.get(parent, None) + if tree != None and tree != f.path: + fail("py_image_layer runfile collision at {}: {} and {}".format(destination, tree, f.path)) + + if f.is_directory and destination in descendants: + descendant, source = descendants[destination] + fail("py_image_layer runfile collision at {}: {} and {}".format(descendant, source, f.path)) + + paths[destination] = f.path + if f.is_directory: + tree_roots[destination] = f.path + for end in range(len(parts) - 1, 0, -1): + parent = "/".join(parts[:end]) + if parent not in descendants: + descendants[parent] = (destination, f.path) + def _user_file_to_mtree(f, dir_expander): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] @@ -792,6 +902,29 @@ def _py_image_layer_impl(ctx): executable_dsts, ) + # Multiple configured closures share one runfiles tree. Compare their + # top-level artifacts at analysis time; TreeArtifact children cannot be + # inspected here, so reject any artifact placed below a tree root. + if len(binaries) > 1: + paths = {} + tree_roots = {} + descendants = {} + for info in infos: + for entry in info.first_party_layers.to_list(): + for f in entry.files.to_list(): + _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, tree_roots, descendants) + for f in info.source_files.to_list(): + _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, tree_roots, descendants) + if info.interpreter_files != None: + for f in info.interpreter_files.to_list(): + _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, tree_roots, descendants) + for pkg in all_pkgs: + for f in pkg.files.to_list(): + _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, tree_roots, descendants) + for dep in ctx.attr.groups: + for f in dep[DefaultInfo].files.to_list(): + _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, tree_roots, descendants) + rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) @@ -854,23 +987,26 @@ def _py_image_layer_impl(ctx): all_tars.extend(pip_tars.values()) - for group_name in sorted(merged): - algorithm, level, ext = _compression_for(plan, group_name) - tar_out = ctx.actions.declare_file("{}/merged_pip_layers/{}{}".format(ctx.attr.name, group_name, ext)) - _run_tar_action( - ctx, - bsdtar, - bsdtar_files, - tar_out, - depset(transitive = merged[group_name]), - _pkg_file_to_mtree, - algorithm, - level, - {}, - "PyImageMergedLayer", - "Merging %d pip packages into %s[%s]" % (len(merged[group_name]), ctx.label, group_name), - ) - all_tars.append(tar_out) + if ctx.attr.binary != None: + all_tars.extend([tar for _group_name, tar in sorted(ctx.attr.binary[_MergedLayerInfo].merged_tars.items())]) + else: + for group_name in sorted(merged): + algorithm, level, ext = _compression_for(plan, group_name) + tar_out = ctx.actions.declare_file("{}/merged_pip_layers/{}{}".format(ctx.attr.name, group_name, ext)) + _run_tar_action( + ctx, + bsdtar, + bsdtar_files, + tar_out, + depset(transitive = merged[group_name]), + _pkg_file_to_mtree, + algorithm, + level, + {}, + "PyImageMergedLayer", + "Merging %d pip packages into %s[%s]" % (len(merged[group_name]), ctx.label, group_name), + ) + all_tars.append(tar_out) ungrouped_pkgs = [p for p in all_pkgs if len(p.layers) == 0 and p.merge_group == None] if ungrouped_pkgs: @@ -934,7 +1070,7 @@ _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { "binary": attr.label( - aspects = [_layer_aspect], + aspects = [_layer_aspect, _merge_aspect], ), "binaries": attr.label_list( aspects = [_layer_aspect], From f776568ad7ebaf8aab9b7672b0afb18293ff6f34 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 15:39:12 -0400 Subject: [PATCH 15/40] fix(py): preserve image-layer destination semantics --- e2e/cases/oci/py_image_layer/BUILD.bazel | 6 ++ .../image_layer_analysis_tests.bzl | 61 +++++++++++-- py/private/py_image_layer.bzl | 89 +++++++++++++------ 3 files changed, 121 insertions(+), 35 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index d439da8e3..de76ea44f 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -418,9 +418,15 @@ py_binary( srcs = ["server.py"], ) +py_layer_tier( + name = "server_tier", + strip_prefix = "oci/py_image_layer/server", +) + py_image_layer( name = "server_layers", binary = ":server", + layer_tier = ":server_tier", ) oci_image( diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 91325dd72..ff3eae81f 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -5,15 +5,27 @@ load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): env = analysistest.begin(ctx) - asserts.expect_failure(env, ctx.attr.expected_error) + for expected_error in ctx.attr.expected_errors: + asserts.expect_failure(env, expected_error) return analysistest.end(env) _expected_failure_test = analysistest.make( _expected_failure_impl, - attrs = {"expected_error": attr.string(mandatory = True)}, + attrs = {"expected_errors": attr.string_list(mandatory = True)}, expect_failure = True, ) +def _source_tree_impl(ctx): + out = ctx.actions.declare_directory("piecewise_tree") + ctx.actions.run_shell( + outputs = [out], + command = "mkdir -p \"$1/nested\" && echo tree > \"$1/nested/support.py\"", + arguments = [out.path], + ) + return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] + +_source_tree = rule(implementation = _source_tree_impl) + def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", @@ -21,7 +33,7 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "missing_launcher_dir_test", - expected_error = "py_image_layer with multiple binaries requires launcher_dir", + expected_errors = ["py_image_layer with multiple binaries requires launcher_dir"], target_under_test = ":_missing_launcher_dir_layers", ) @@ -32,7 +44,7 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "relative_launcher_dir_test", - expected_error = "py_image_layer.launcher_dir must be an absolute image path", + expected_errors = ["py_image_layer.launcher_dir must be an absolute image path"], target_under_test = ":_relative_launcher_dir_layers", ) @@ -47,7 +59,7 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "duplicate_launcher_basename_test", - expected_error = "duplicate py_image_layer launcher basename: my_app_bin", + expected_errors = ["duplicate py_image_layer launcher basename: my_app_bin"], target_under_test = ":_duplicate_launcher_layers", ) @@ -82,6 +94,43 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "configured_wheel_collision_test", - expected_error = "actual_install.install:", + expected_errors = [ + "py_image_layer runfile collision at", + "actual_install.install:", + ], target_under_test = ":_configured_wheel_collision_layers", ) + + _source_tree(name = "_piecewise_tree") + py_binary( + name = "_piecewise_tree_bin", + srcs = ["server.py"], + data = [":_piecewise_tree"], + ) + py_layer_tier( + name = "_piecewise_tree_tier", + strip_prefix = "oci/py_image_layer/piecewise_tree/nested", + ) + py_image_layer( + name = "_piecewise_tree_layers", + binaries = [":_piecewise_tree_bin", ":my_app_bin"], + launcher_dir = "/app/bin", + layer_tier = ":_piecewise_tree_tier", + ) + _expected_failure_test( + name = "piecewise_tree_test", + expected_errors = ["py_image_layer cannot map TreeArtifact across strip_prefix"], + target_under_test = ":_piecewise_tree_layers", + ) + + py_image_layer( + name = "_interpreter_group_collision_layers", + binary = ":my_app_bin", + groups = {":worker_support": "interpreter"}, + layer_tier = ":my_app_launchers_tier", + ) + _expected_failure_test( + name = "interpreter_group_collision_test", + expected_errors = ["Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier"], + target_under_test = ":_interpreter_group_collision_layers", + ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 4a18715f8..6d1a50634 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -1,6 +1,7 @@ """py_image_layer — analysis-time grouped OCI layers with globally shared pip tars. -One rule-propagated aspect wires onto the `py_image_layer` binary inputs: +`_layer_aspect` wires onto both `py_image_layer` binary inputs. Scalar binaries +also receive `_merge_aspect` so their merged pip layers remain action-shared. `_layer_aspect` propagates through `deps`/`data`/`actual`. For pip packages it creates aspect-owned per-package tars at the pip target's namespace (globally @@ -641,9 +642,9 @@ def _normalize_destination(path): parts.pop() else: parts.append(part) - return "./" + "/".join(parts) + return "./" + "/".join(parts) if parts else "." -def _register_image_destination(f, destination, paths, tree_roots, descendants): +def _register_image_destination(f, destination, paths, descendants): destination = _normalize_destination(destination) previous = paths.get(destination, None) if previous != None: @@ -654,22 +655,34 @@ def _register_image_destination(f, destination, paths, tree_roots, descendants): parts = destination.split("/") for end in range(len(parts) - 1, 0, -1): parent = "/".join(parts[:end]) - tree = tree_roots.get(parent, None) - if tree != None and tree != f.path: - fail("py_image_layer runfile collision at {}: {} and {}".format(destination, tree, f.path)) + ancestor = paths.get(parent, None) + if ancestor != None: + fail("py_image_layer runfile collision at {}: {} and {}".format(destination, ancestor, f.path)) - if f.is_directory and destination in descendants: + if destination in descendants: descendant, source = descendants[destination] fail("py_image_layer runfile collision at {}: {} and {}".format(descendant, source, f.path)) paths[destination] = f.path - if f.is_directory: - tree_roots[destination] = f.path for end in range(len(parts) - 1, 0, -1): parent = "/".join(parts[:end]) if parent not in descendants: descendants[parent] = (destination, f.path) +def _check_source_tree(f, strip_prefix, executable_dsts): + if not f.is_directory: + return + tree = f.short_path.rstrip("/") + prefix = strip_prefix.replace("\\/", "/") + if prefix and prefix.startswith(tree + "/"): + fail("py_image_layer cannot map TreeArtifact across strip_prefix: {} contains {}".format(tree, prefix)) + for executable_short_path in executable_dsts: + if executable_short_path.startswith(tree + "/"): + fail("py_image_layer cannot map TreeArtifact across launcher: {} contains {}".format(tree, executable_short_path)) + runfiles = executable_short_path + ".runfiles" + if runfiles == tree or runfiles.startswith(tree + "/"): + fail("py_image_layer cannot map TreeArtifact across launcher runfiles: {} contains {}".format(tree, runfiles)) + def _user_file_to_mtree(f, dir_expander): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] @@ -853,15 +866,22 @@ def _py_image_layer_impl(ctx): infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) - # Labels are normalized for tier lookup and can collide across lock universes. - # Concrete install-tree paths identify the configured wheel artifacts. - pkg_by_files = {} - for info in infos: - for pkg in info.pip_packages.to_list(): - key = tuple(sorted([f.path for f in pkg.files.to_list()])) - if key not in pkg_by_files: - pkg_by_files[key] = pkg - all_pkgs = pkg_by_files.values() + if ctx.attr.binary != None: + pkg_by_label = {} + for pkg in infos[0].pip_packages.to_list(): + if pkg.label not in pkg_by_label: + pkg_by_label[pkg.label] = pkg + all_pkgs = pkg_by_label.values() + else: + # Labels are normalized for tier lookup and can collide across lock + # universes. Concrete paths identify each configured wheel artifact. + pkg_by_files = {} + for info in infos: + for pkg in info.pip_packages.to_list(): + key = tuple(sorted([f.path for f in pkg.files.to_list()])) + if key not in pkg_by_files: + pkg_by_files[key] = pkg + all_pkgs = pkg_by_files.values() pip_labels = {pkg.label: True for pkg in all_pkgs} # `_platform_cfg` rewrites the `//py:layer_tier` flag from `attr.layer_tier`, @@ -904,26 +924,29 @@ def _py_image_layer_impl(ctx): # Multiple configured closures share one runfiles tree. Compare their # top-level artifacts at analysis time; TreeArtifact children cannot be - # inspected here, so reject any artifact placed below a tree root. + # inspected here, so reject trees crossing a piecewise-mapped prefix. if len(binaries) > 1: paths = {} - tree_roots = {} descendants = {} for info in infos: for entry in info.first_party_layers.to_list(): for f in entry.files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, tree_roots, descendants) + _check_source_tree(f, strip_prefix, executable_dsts) + _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, descendants) for f in info.source_files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, tree_roots, descendants) + _check_source_tree(f, strip_prefix, executable_dsts) + _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, descendants) if info.interpreter_files != None: for f in info.interpreter_files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, tree_roots, descendants) + _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) for pkg in all_pkgs: for f in pkg.files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, tree_roots, descendants) + _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) for dep in ctx.attr.groups: + if normalize_label(str(dep.label)) in pip_labels: + continue for f in dep[DefaultInfo].files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, tree_roots, descendants) + _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): @@ -943,8 +966,13 @@ def _py_image_layer_impl(ctx): all_tars.append(tar_out) fp_by_group = {} + seen_fp_labels = {} for info in infos: for entry in info.first_party_layers.to_list(): + if ctx.attr.binary != None: + if entry.label in seen_fp_labels: + continue + seen_fp_labels[entry.label] = True fp_by_group.setdefault(entry.group, []).append(entry.files) # Interpreter tars are declared at the configured toolchain, so identical @@ -955,7 +983,9 @@ def _py_image_layer_impl(ctx): layer = info.interpreter_layer interpreter_layers[layer.tar.path] = layer - for group_name in sorted(fp_by_group): + layer_group_names = {group_name: True for group_name in fp_by_group} + layer_group_names.update({layer.group: True for layer in interpreter_layers.values()}) + for group_name in layer_group_names: if group_name in rule_group_names: fail( ("Group %r is declared in both py_image_layer.groups and the active " + @@ -963,6 +993,7 @@ def _py_image_layer_impl(ctx): "deps with a different file layout than first-party sources, so they " + "cannot share a tar.") % group_name, ) + for group_name in sorted(fp_by_group): tar_out = _declare_group_tar( ctx, bsdtar, @@ -982,7 +1013,7 @@ def _py_image_layer_impl(ctx): for pkg in all_pkgs: for layer in pkg.layers: pip_tars[layer.tar.path] = layer.tar - if pkg.merge_group != None: + if ctx.attr.binary == None and pkg.merge_group != None: merged.setdefault(pkg.merge_group, []).append(pkg.files) all_tars.extend(pip_tars.values()) @@ -1137,8 +1168,8 @@ def py_image_layer( binary's dep closure). 3. Solo-group and subpath-split pip tars — built by `_layer_aspect` at each pip target's own namespace; globally shared across every rule using that package. - 4. Multi-member merged tars — one per group from the closure-filtered union - of member install_dirs across all binaries. + 4. Multi-member merged tars — action-shared at a scalar binary or one per + group from the closure-filtered union across a binary list. 5. Ungrouped pip packages → one squashed rule-created tar. 6. Remaining first-party Python source files → the "default" layer. From 60d0cddef96e75ff06f29ff8c2fe2a89ed0180ad Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 15:47:58 -0400 Subject: [PATCH 16/40] fix(py): preserve scalar layer placement and ordering --- e2e/cases/oci/py_image_layer/BUILD.bazel | 31 ++++++++- .../image_layer_analysis_tests.bzl | 50 +++++++++++++- py/private/py_image_layer.bzl | 66 +++++++++++++------ 3 files changed, 124 insertions(+), 23 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index de76ea44f..9969f540f 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -420,7 +420,7 @@ py_binary( py_layer_tier( name = "server_tier", - strip_prefix = "oci/py_image_layer/server", + strip_prefix = "does/not/match", ) py_image_layer( @@ -429,6 +429,35 @@ py_image_layer( layer_tier = ":server_tier", ) +py_layer_tier( + name = "server_directory_tier", + strip_prefix = "oci/py_image_layer", +) + +py_image_layer( + name = "server_directory_layers", + binary = ":server", + layer_tier = ":server_directory_tier", +) + +genrule( + name = "server_directory_sources_listing", + srcs = [":server_directory_layers_only_src"], + outs = ["_server_directory_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + +py_test( + name = "server_directory_sources_test", + srcs = ["assert_source_dedup.py"], + args = [ + "$(rootpath :server_directory_sources_listing)", + "/app/server", + ], + data = [":server_directory_sources_listing"], +) + oci_image( name = "server_image", base = "@ubuntu", diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index ff3eae81f..67a31bf77 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -1,6 +1,6 @@ """Analysis coverage for invalid multi-launcher image-layer configurations.""" -load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): @@ -26,6 +26,16 @@ def _source_tree_impl(ctx): _source_tree = rule(implementation = _source_tree_impl) +def _regular_path_impl(ctx): + out = ctx.actions.declare_file("regular_collision" if ctx.attr.ancestor else "regular_collision/child.py") + ctx.actions.write(out, "regular\n") + return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] + +_regular_path = rule( + implementation = _regular_path_impl, + attrs = {"ancestor": attr.bool()}, +) + def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", @@ -134,3 +144,41 @@ def image_layer_analysis_test_suite(): expected_errors = ["Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier"], target_under_test = ":_interpreter_group_collision_layers", ) + + _regular_path( + name = "_regular_path", + ancestor = select({ + ":_python_3_11": True, + "//conditions:default": False, + }), + ) + py_library( + name = "_regular_path_lib", + srcs = [":_regular_path"], + imports = ["."], + ) + py_binary( + name = "_regular_path_311", + srcs = ["server.py"], + python_version = "3.11", + deps = [":_regular_path_lib"], + ) + py_binary( + name = "_regular_path_312", + srcs = ["server.py"], + python_version = "3.12", + deps = [":_regular_path_lib"], + ) + py_image_layer( + name = "_regular_path_collision_layers", + binaries = [":_regular_path_311", ":_regular_path_312"], + launcher_dir = "/app/bin", + ) + _expected_failure_test( + name = "regular_path_collision_test", + expected_errors = [ + "py_image_layer runfile collision at", + "regular_collision/child.py", + ], + target_under_test = ":_regular_path_collision_layers", + ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 6d1a50634..88ff75684 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -9,8 +9,7 @@ shared across every rule using that package) for solo whole-groups and subpath splits. Members of multi-member whole groups get NO per-package tar — intermediate elided; they just flag `merge_group` on their _LayerInfo struct. First-party PyInfo targets matched by `py_layer_tier.groups` are captured as `first_party_layers` -entries (label, files, group) to be tarred per-group at the binary. Produces -`_LayerInfo`. +entries (label, files, group) for the image-layer rule. Produces `_LayerInfo`. Layer tier (groups + compression) is carried by the `py_layer_tier` rule which produces `PyLayerTierInfo`. Aspects read it via a private `_layer_tier` attr whose default is @@ -219,7 +218,7 @@ def _build_pip_layers(ctx, plan, label, install_dir): Returns (layers, merge_group): layers is a tuple of struct(tar, group) entries for this package; merge_group is set (and layers is empty) iff - the package is deferred to the image-layer rule. + the package is deferred for closure-level merging. """ subpath_for_this = plan.subpath_groups.get(label, {}) whole_group = plan.whole_groups.get(label, None) @@ -572,8 +571,11 @@ _merge_aspect = aspect( provides = [_MergedLayerInfo], ) +def _normalize_strip_prefix(strip_prefix): + return "/".join([part for part in strip_prefix.replace("\\/", "/").split("/") if part]) + def _apply_strip_prefix(sp, strip_prefix, root): - prefix = strip_prefix.replace("\\/", "/") + prefix = _normalize_strip_prefix(strip_prefix) if sp == prefix: return "." + root @@ -592,10 +594,14 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): if sp.startswith("../"): return "./app.runfiles/" + sp[3:] for executable_short_path in executable_dsts: - if sp == executable_short_path or sp.startswith(executable_short_path + ".runfiles/"): + if sp.startswith(executable_short_path + ".runfiles/"): return _apply_strip_prefix(sp, executable_short_path, root) if strip_prefix: - return _apply_strip_prefix(sp, strip_prefix, root) + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination + if sp in executable_dsts: + return _apply_strip_prefix(sp, sp, root) return "./app.runfiles/_main/" + sp def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False, executable_dsts = {}): @@ -673,12 +679,19 @@ def _check_source_tree(f, strip_prefix, executable_dsts): if not f.is_directory: return tree = f.short_path.rstrip("/") - prefix = strip_prefix.replace("\\/", "/") + if tree == "_repo_mapping": + fail("py_image_layer cannot map TreeArtifact at _repo_mapping") + for executable_short_path, executable_dst in executable_dsts.items(): + if tree == executable_short_path: + fail("py_image_layer executable cannot be a TreeArtifact: {}".format(tree)) + if executable_dst and executable_short_path.startswith(tree + "/"): + fail("py_image_layer cannot map TreeArtifact across launcher: {} contains {}".format(tree, executable_short_path)) + if tree.startswith("../"): + return + prefix = _normalize_strip_prefix(strip_prefix) if prefix and prefix.startswith(tree + "/"): fail("py_image_layer cannot map TreeArtifact across strip_prefix: {} contains {}".format(tree, prefix)) for executable_short_path in executable_dsts: - if executable_short_path.startswith(tree + "/"): - fail("py_image_layer cannot map TreeArtifact across launcher: {} contains {}".format(tree, executable_short_path)) runfiles = executable_short_path + ".runfiles" if runfiles == tree or runfiles.startswith(tree + "/"): fail("py_image_layer cannot map TreeArtifact across launcher runfiles: {} contains {}".format(tree, runfiles)) @@ -946,6 +959,7 @@ def _py_image_layer_impl(ctx): if normalize_label(str(dep.label)) in pip_labels: continue for f in dep[DefaultInfo].files.to_list(): + _check_source_tree(f, "", {}) _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} @@ -983,6 +997,12 @@ def _py_image_layer_impl(ctx): layer = info.interpreter_layer interpreter_layers[layer.tar.path] = layer + prebuilt_group_tars = {} + if ctx.attr.binary != None: + for layer in interpreter_layers.values(): + prebuilt_group_tars[layer.group] = layer.tar + fp_by_group.setdefault(layer.group, []) + layer_group_names = {group_name: True for group_name in fp_by_group} layer_group_names.update({layer.group: True for layer in interpreter_layers.values()}) for group_name in layer_group_names: @@ -994,19 +1014,23 @@ def _py_image_layer_impl(ctx): "cannot share a tar.") % group_name, ) for group_name in sorted(fp_by_group): - tar_out = _declare_group_tar( - ctx, - bsdtar, - bsdtar_files, - "{}_{}.tar.gz".format(ctx.attr.name, group_name), - group_name, - depset(transitive = fp_by_group[group_name]), - source_map, - "Creating first-party layer %s[%s]" % (ctx.label, group_name), - ) + if group_name in prebuilt_group_tars: + tar_out = prebuilt_group_tars[group_name] + else: + tar_out = _declare_group_tar( + ctx, + bsdtar, + bsdtar_files, + "{}_{}.tar.gz".format(ctx.attr.name, group_name), + group_name, + depset(transitive = fp_by_group[group_name]), + source_map, + "Creating first-party layer %s[%s]" % (ctx.label, group_name), + ) all_tars.append(tar_out) - all_tars.extend([layer.tar for layer in interpreter_layers.values()]) + if ctx.attr.binary == None: + all_tars.extend([layer.tar for layer in interpreter_layers.values()]) pip_tars = {} merged = {} @@ -1165,7 +1189,7 @@ def py_image_layer( 1. Non-pip deps listed in `groups` → one rule-created tar per group. 2. First-party py_library targets matched by `py_layer_tier.groups` → one rule-created tar per group (aggregated across all matched targets in the - binary's dep closure). + binary inputs' dep closures). 3. Solo-group and subpath-split pip tars — built by `_layer_aspect` at each pip target's own namespace; globally shared across every rule using that package. 4. Multi-member merged tars — action-shared at a scalar binary or one per From a23add5760c097d825c5718fc4aad37f400994a4 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 15:58:08 -0400 Subject: [PATCH 17/40] fix(py): validate expanded image destinations --- .../image_layer_analysis_tests.bzl | 127 ++++++------------ e2e/cases/oci/test.sh | 50 +++++++ py/private/py_image_layer.bzl | 116 ++++------------ py/private/py_image_layer_validator.py | 61 ++++++++- py/private/py_image_layer_validator_test.py | 75 +++++++++++ 5 files changed, 250 insertions(+), 179 deletions(-) create mode 100755 e2e/cases/oci/test.sh diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 67a31bf77..cd34ff42e 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -1,41 +1,19 @@ -"""Analysis coverage for invalid multi-launcher image-layer configurations.""" +"""Analysis and validation fixtures for multi-launcher image layers.""" -load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): env = analysistest.begin(ctx) - for expected_error in ctx.attr.expected_errors: - asserts.expect_failure(env, expected_error) + asserts.expect_failure(env, ctx.attr.expected_error) return analysistest.end(env) _expected_failure_test = analysistest.make( _expected_failure_impl, - attrs = {"expected_errors": attr.string_list(mandatory = True)}, + attrs = {"expected_error": attr.string(mandatory = True)}, expect_failure = True, ) -def _source_tree_impl(ctx): - out = ctx.actions.declare_directory("piecewise_tree") - ctx.actions.run_shell( - outputs = [out], - command = "mkdir -p \"$1/nested\" && echo tree > \"$1/nested/support.py\"", - arguments = [out.path], - ) - return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] - -_source_tree = rule(implementation = _source_tree_impl) - -def _regular_path_impl(ctx): - out = ctx.actions.declare_file("regular_collision" if ctx.attr.ancestor else "regular_collision/child.py") - ctx.actions.write(out, "regular\n") - return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] - -_regular_path = rule( - implementation = _regular_path_impl, - attrs = {"ancestor": attr.bool()}, -) - def image_layer_analysis_test_suite(): py_image_layer( name = "_missing_launcher_dir_layers", @@ -43,7 +21,7 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "missing_launcher_dir_test", - expected_errors = ["py_image_layer with multiple binaries requires launcher_dir"], + expected_error = "py_image_layer with multiple binaries requires launcher_dir", target_under_test = ":_missing_launcher_dir_layers", ) @@ -54,7 +32,7 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "relative_launcher_dir_test", - expected_errors = ["py_image_layer.launcher_dir must be an absolute image path"], + expected_error = "py_image_layer.launcher_dir must be an absolute image path", target_under_test = ":_relative_launcher_dir_layers", ) @@ -69,7 +47,7 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "duplicate_launcher_basename_test", - expected_errors = ["duplicate py_image_layer launcher basename: my_app_bin"], + expected_error = "duplicate py_image_layer launcher basename: my_app_bin", target_under_test = ":_duplicate_launcher_layers", ) @@ -102,35 +80,46 @@ def image_layer_analysis_test_suite(): launcher_dir = "/app/bin", layer_tier = ":_wheel_scripts_tier", ) - _expected_failure_test( - name = "configured_wheel_collision_test", - expected_errors = [ - "py_image_layer runfile collision at", - "actual_install.install:", - ], - target_under_test = ":_configured_wheel_collision_layers", + + py_binary( + name = "_pure_wheel_311", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.11", + deps = ["@pypi_oci_py_image_layer//colorama"], + ) + py_binary( + name = "_pure_wheel_312", + srcs = ["server.py"], + dep_group = "images", + python_version = "3.12", + deps = ["@pypi_oci_py_image_layer//colorama"], + ) + py_image_layer( + name = "_configured_pure_wheel_layers", + binaries = [":_pure_wheel_311", ":_pure_wheel_312"], + launcher_dir = "/app/bin", ) - _source_tree(name = "_piecewise_tree") + native.genrule( + name = "_scalar_launcher_collision_data", + outs = ["bin/_scalar_launcher_collision"], + cmd = "echo data > $@", + ) py_binary( - name = "_piecewise_tree_bin", + name = "_scalar_launcher_collision", srcs = ["server.py"], - data = [":_piecewise_tree"], + data = ["bin/_scalar_launcher_collision"], ) py_layer_tier( - name = "_piecewise_tree_tier", - strip_prefix = "oci/py_image_layer/piecewise_tree/nested", + name = "_scalar_launcher_collision_tier", + strip_prefix = "oci/py_image_layer", ) py_image_layer( - name = "_piecewise_tree_layers", - binaries = [":_piecewise_tree_bin", ":my_app_bin"], + name = "_scalar_launcher_collision_layers", + binary = ":_scalar_launcher_collision", launcher_dir = "/app/bin", - layer_tier = ":_piecewise_tree_tier", - ) - _expected_failure_test( - name = "piecewise_tree_test", - expected_errors = ["py_image_layer cannot map TreeArtifact across strip_prefix"], - target_under_test = ":_piecewise_tree_layers", + layer_tier = ":_scalar_launcher_collision_tier", ) py_image_layer( @@ -141,44 +130,6 @@ def image_layer_analysis_test_suite(): ) _expected_failure_test( name = "interpreter_group_collision_test", - expected_errors = ["Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier"], + expected_error = "Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier", target_under_test = ":_interpreter_group_collision_layers", ) - - _regular_path( - name = "_regular_path", - ancestor = select({ - ":_python_3_11": True, - "//conditions:default": False, - }), - ) - py_library( - name = "_regular_path_lib", - srcs = [":_regular_path"], - imports = ["."], - ) - py_binary( - name = "_regular_path_311", - srcs = ["server.py"], - python_version = "3.11", - deps = [":_regular_path_lib"], - ) - py_binary( - name = "_regular_path_312", - srcs = ["server.py"], - python_version = "3.12", - deps = [":_regular_path_lib"], - ) - py_image_layer( - name = "_regular_path_collision_layers", - binaries = [":_regular_path_311", ":_regular_path_312"], - launcher_dir = "/app/bin", - ) - _expected_failure_test( - name = "regular_path_collision_test", - expected_errors = [ - "py_image_layer runfile collision at", - "regular_collision/child.py", - ], - target_under_test = ":_regular_path_collision_layers", - ) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh new file mode 100755 index 000000000..8650066f7 --- /dev/null +++ b/e2e/cases/oci/test.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Multi-binary destination collisions are discovered by the validation action, +# so the expected-failure case cannot be an sh_test under //.... +set -uo pipefail + +cd "$(dirname "$0")/.." || exit 1 + +BAZEL="${BAZEL:-bazel}" +PKG="//oci/py_image_layer" +output_log="$(mktemp)" +trap 'rm -f "$output_log"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +echo "== versioned pure-wheel children must share an image ==" +if ! "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_configured_pure_wheel_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the two-version pure-wheel image to validate" +fi + +echo "== unversioned wheel scripts must fail shared-image validation ==" +if "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_configured_wheel_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the shared wheel-script destination to fail validation" +fi +if ! grep -F "py_image_layer runfile collision at ./app.runfiles/" "$output_log" | grep -Fq "/bin/pyproject-build:"; then + cat "$output_log" >&2 + fail "expected the pyproject-build destination-collision diagnostic" +fi + +echo "PASS: expanded multi-binary destinations validate correctly" + +echo "== a relocated scalar launcher must not overwrite runfile data ==" +if "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_scalar_launcher_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the relocated scalar launcher destination to fail validation" +fi +if ! grep -Fq "py_image_layer runfile collision at ./app/bin/_scalar_launcher_collision:" "$output_log"; then + cat "$output_log" >&2 + fail "expected the relocated scalar launcher collision diagnostic" +fi + +echo "PASS: relocated scalar launcher destinations validate correctly" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 88ff75684..d1ee1a251 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -637,65 +637,6 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex def _make_source_map(strip_prefix, root, maybe_symlink, executable_dsts): return lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) -def _normalize_destination(path): - parts = [] - for part in path.split("/"): - if not part or part == ".": - continue - if part == "..": - if not parts: - fail("py_image_layer image destination escapes its root: {}".format(path)) - parts.pop() - else: - parts.append(part) - return "./" + "/".join(parts) if parts else "." - -def _register_image_destination(f, destination, paths, descendants): - destination = _normalize_destination(destination) - previous = paths.get(destination, None) - if previous != None: - if previous != f.path: - fail("py_image_layer runfile collision at {}: {} and {}".format(destination, previous, f.path)) - return - - parts = destination.split("/") - for end in range(len(parts) - 1, 0, -1): - parent = "/".join(parts[:end]) - ancestor = paths.get(parent, None) - if ancestor != None: - fail("py_image_layer runfile collision at {}: {} and {}".format(destination, ancestor, f.path)) - - if destination in descendants: - descendant, source = descendants[destination] - fail("py_image_layer runfile collision at {}: {} and {}".format(descendant, source, f.path)) - - paths[destination] = f.path - for end in range(len(parts) - 1, 0, -1): - parent = "/".join(parts[:end]) - if parent not in descendants: - descendants[parent] = (destination, f.path) - -def _check_source_tree(f, strip_prefix, executable_dsts): - if not f.is_directory: - return - tree = f.short_path.rstrip("/") - if tree == "_repo_mapping": - fail("py_image_layer cannot map TreeArtifact at _repo_mapping") - for executable_short_path, executable_dst in executable_dsts.items(): - if tree == executable_short_path: - fail("py_image_layer executable cannot be a TreeArtifact: {}".format(tree)) - if executable_dst and executable_short_path.startswith(tree + "/"): - fail("py_image_layer cannot map TreeArtifact across launcher: {} contains {}".format(tree, executable_short_path)) - if tree.startswith("../"): - return - prefix = _normalize_strip_prefix(strip_prefix) - if prefix and prefix.startswith(tree + "/"): - fail("py_image_layer cannot map TreeArtifact across strip_prefix: {} contains {}".format(tree, prefix)) - for executable_short_path in executable_dsts: - runfiles = executable_short_path + ".runfiles" - if runfiles == tree or runfiles.startswith(tree + "/"): - fail("py_image_layer cannot map TreeArtifact across launcher runfiles: {} contains {}".format(tree, runfiles)) - def _user_file_to_mtree(f, dir_expander): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] @@ -935,33 +876,6 @@ def _py_image_layer_impl(ctx): executable_dsts, ) - # Multiple configured closures share one runfiles tree. Compare their - # top-level artifacts at analysis time; TreeArtifact children cannot be - # inspected here, so reject trees crossing a piecewise-mapped prefix. - if len(binaries) > 1: - paths = {} - descendants = {} - for info in infos: - for entry in info.first_party_layers.to_list(): - for f in entry.files.to_list(): - _check_source_tree(f, strip_prefix, executable_dsts) - _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, descendants) - for f in info.source_files.to_list(): - _check_source_tree(f, strip_prefix, executable_dsts) - _register_image_destination(f, _source_destination(f.short_path, strip_prefix, root, executable_dsts), paths, descendants) - if info.interpreter_files != None: - for f in info.interpreter_files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) - for pkg in all_pkgs: - for f in pkg.files.to_list(): - _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) - for dep in ctx.attr.groups: - if normalize_label(str(dep.label)) in pip_labels: - continue - for f in dep[DefaultInfo].files.to_list(): - _check_source_tree(f, "", {}) - _register_image_destination(f, _source_destination(f.short_path, "", "/", {}), paths, descendants) - rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) @@ -1104,11 +1018,37 @@ def _py_image_layer_impl(ctx): for pkg in ungrouped_pkgs: validation_args.add_all(pkg.files, format_each = pkg.label + "=%s", expand_directories = False) + validation_inputs = [pkg.files for pkg in ungrouped_pkgs] + validation_arguments = [validation_args] + if len(binaries) > 1 or launcher_dir: + # Validate expanded rows whenever launchers are relocated or shared. + # TreeArtifact roots can contain disjoint versioned children, so only + # the production mappers' expanded destinations are authoritative. + source_files = depset(transitive = [info.source_files for info in infos] + [entry.files for info in infos for entry in info.first_party_layers.to_list()]) + rule_group_files = [dep[DefaultInfo].files for dep in ctx.attr.groups if normalize_label(str(dep.label)) not in pip_labels] + wheel_files = [pkg.files for pkg in all_pkgs] + interpreter_files = [info.interpreter_files for info in infos if info.interpreter_files != None] + + mtree_args = ctx.actions.args() + mtree_args.set_param_file_format("multiline") + mtree_args.use_param_file("%s", use_always = True) + mtree_args.add("#mtree") + mtree_args.add_all(source_files, map_each = source_map, expand_directories = False, allow_closure = True) + for files in rule_group_files: + mtree_args.add_all(files, map_each = _user_file_to_mtree, expand_directories = False) + for files in wheel_files: + mtree_args.add_all(files, map_each = _pkg_file_to_mtree, expand_directories = False) + for files in interpreter_files: + mtree_args.add_all(files, map_each = _interpreter_file_to_mtree, expand_directories = False) + validation_args.add("--mtree") + validation_arguments.append(mtree_args) + validation_inputs.extend([source_files] + rule_group_files + wheel_files + interpreter_files) + ctx.actions.run( executable = ctx.executable._validator, - inputs = depset(transitive = [pkg.files for pkg in ungrouped_pkgs]), + inputs = depset(transitive = validation_inputs), outputs = [validation], - arguments = [validation_args], + arguments = validation_arguments, mnemonic = "PyImageLayerValidate", ) diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index 1851dd141..30a7f85ea 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""py_image_layer_validator — validate pip layer sizing for a py_image_layer target. +"""py_image_layer_validator — validate sizing and shared destinations for an image layer. Invoked as a Bazel validation action. Fails (exit 1) with actionable `py_layer_tier` snippets when the squashed pip layer exceeds a size threshold or when the OCI 127-layer hard limit is @@ -9,17 +9,17 @@ py_image_layer_validator --threshold_mb N --output FILE [label=path ...] label=path — one entry per ungrouped pip package; `label` is the canonical pip label (e.g. @pip//numpy), `path` is its install directory / file. + --mtree FILE — expanded mtree rows for relocated or shared launchers. """ from __future__ import annotations import argparse -import contextlib import csv import glob import os import sys -from typing import Dict, List, Optional, Sequence, Tuple +from typing import Dict, Iterable, List, Optional, Sequence, Tuple _OCI_LAYER_HARD_LIMIT = 127 _BINARY_GLOBS = {"*.so", "*.so.*", "*.pyd", "*.dylib", "*.dll"} @@ -229,12 +229,62 @@ def _add_subpath_or_whole( _add_whole_promotion(suggestions, label, size_mb, is_binary, annotation="whole package") +def _mtree_collision(rows: Iterable[str]) -> Optional[str]: + """Return the first conflicting expanded mtree destination, or None.""" + paths: Dict[str, str] = {} + descendants: Dict[str, Tuple[str, str]] = {} + for row in rows: + if not row or row.startswith("#"): + continue + fields = row.split() + destination_parts: List[str] = [] + for part in fields[0].split("/"): + if not part or part == ".": + continue + if part == "..": + if not destination_parts: + return "py_image_layer image destination escapes its root: {}".format(fields[0]) + destination_parts.pop() + else: + destination_parts.append(part) + destination = "./" + "/".join(destination_parts) if destination_parts else "." + source = next( + (field.partition("=")[2] for field in fields[1:] if field.startswith(("contents=", "content=", "link="))), + None, + ) + if source is None: + return "invalid py_image_layer mtree row (missing source): {}".format(row) + + previous = paths.get(destination) + if previous is not None: + if previous != source: + return "py_image_layer runfile collision at {}: {} and {}".format(destination, previous, source) + continue + + parts = destination.split("/") + for end in range(len(parts) - 1, 0, -1): + parent = "/".join(parts[:end]) + if parent in paths: + return "py_image_layer runfile collision at {}: {} and {}".format(destination, paths[parent], source) + if destination in descendants: + descendant, previous = descendants[destination] + return "py_image_layer runfile collision at {}: {} and {}".format(descendant, previous, source) + + paths[destination] = source + for end in range(len(parts) - 1, 0, -1): + parent = "/".join(parts[:end]) + descendants.setdefault(parent, (destination, source)) + + return None + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--threshold_mb", type=int, default=200) parser.add_argument("--layer_count", type=int, default=0) parser.add_argument("--warn_layer_count", type=int, default=90) parser.add_argument("--output", required=True) + parser.add_argument("--mtree") parser.add_argument("pkg_paths", nargs="*", metavar="label=path") args = parser.parse_args() @@ -252,6 +302,11 @@ def main() -> None: pkg_binary = {label: _pkg_is_binary(paths) for label, paths in pkg_path_map.items()} messages: List[str] = [] + if args.mtree: + with open(args.mtree) as mtree: + collision = _mtree_collision(mtree) + if collision: + messages.append("ERROR: " + collision) suggestions = _Suggestions() layer_count_comment_lines: List[str] = [] diff --git a/py/private/py_image_layer_validator_test.py b/py/private/py_image_layer_validator_test.py index 1e9cc759c..82b8cd831 100644 --- a/py/private/py_image_layer_validator_test.py +++ b/py/private/py_image_layer_validator_test.py @@ -13,6 +13,7 @@ _Suggestions, _find_large_files, _glob_for_file, + _mtree_collision, _pkg_is_binary, _pkg_name_from_label, _pkg_size, @@ -21,6 +22,58 @@ ) +def _mtree_row(destination: str, source: str) -> str: + return "{} type=file mode=0755 contents={}".format(destination, source) + + +def test_mtree_identical_source_coalesces() -> None: + row = _mtree_row("./app.runfiles/pkg/module.py", "same/module.py") + assert _mtree_collision(["#mtree", row, row]) is None + + +def test_mtree_conflicting_source_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/module.py", "second/module.py"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_ancestor_then_descendant_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/module.py/data", "second/data"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py/data:" in collision + + +def test_mtree_descendant_then_ancestor_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py/data", "second/data"), + _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py/data:" in collision + + +def test_mtree_normalized_alias_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/nested/../module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/./module.py", "second/module.py"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_disjoint_children_succeed() -> None: + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/python3.11/module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/python3.12/module.py", "second/module.py"), + ]) is None + + # --------------------------------------------------------------------------- # _pkg_name_from_label # --------------------------------------------------------------------------- @@ -264,6 +317,28 @@ def test_main_ok_below_threshold() -> None: os.unlink(out_path) +def test_main_mtree_collision_error() -> None: + with tempfile.NamedTemporaryFile(mode="w", suffix=".params", delete=False) as params: + params.write("#mtree\n") + params.write(_mtree_row("./app.runfiles/pkg/module.py", "first/module.py") + "\n") + params.write(_mtree_row("./app.runfiles/pkg/module.py", "second/module.py") + "\n") + params_path = params.name + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as output: + out_path = output.name + sys.argv = ["validator", "--threshold_mb", "999", "--output", out_path, "--mtree", params_path] + with _capture_stderr(): + try: + main() + exit_code = 0 + except SystemExit as e: + exit_code = e.code + assert exit_code == 1 + with open(out_path) as fh: + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in fh.read() + os.unlink(params_path) + os.unlink(out_path) + + def test_main_layer_count_error() -> None: with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: out_path = f.name From a7c6a343d79c179884e0473d2461e68460c48c26 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 16:04:37 -0400 Subject: [PATCH 18/40] fix(py): align destination validation with emitted layers --- .../_scalar_strip_collision/data.txt | 1 + .../image_layer_analysis_tests.bzl | 16 ++++++++++++++++ e2e/cases/oci/test.sh | 13 +++++++++++++ py/private/py_image_layer.bzl | 18 +++++++++++------- py/private/py_image_layer_validator.py | 2 +- 5 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt diff --git a/e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt b/e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt new file mode 100644 index 000000000..1269488f7 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt @@ -0,0 +1 @@ +data diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index cd34ff42e..3cba3e1fe 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -122,6 +122,22 @@ def image_layer_analysis_test_suite(): layer_tier = ":_scalar_launcher_collision_tier", ) + py_binary( + name = "_scalar_strip_collision", + srcs = ["server.py"], + data = ["_scalar_strip_collision/data.txt"], + ) + py_layer_tier( + name = "_scalar_strip_collision_tier", + root = "/app.runfiles/_main/oci/py_image_layer", + strip_prefix = "oci/py_image_layer", + ) + py_image_layer( + name = "_scalar_strip_collision_layers", + binary = ":_scalar_strip_collision", + layer_tier = ":_scalar_strip_collision_tier", + ) + py_image_layer( name = "_interpreter_group_collision_layers", binary = ":my_app_bin", diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 8650066f7..c27d6d104 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -48,3 +48,16 @@ if ! grep -Fq "py_image_layer runfile collision at ./app/bin/_scalar_launcher_co fi echo "PASS: relocated scalar launcher destinations validate correctly" + +echo "== a scalar strip prefix must not turn its launcher into a file-prefix conflict ==" +if "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_scalar_strip_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the scalar strip-prefix destination to fail validation" +fi +if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/_scalar_strip_collision/data.txt:" "$output_log"; then + cat "$output_log" >&2 + fail "expected the scalar strip-prefix collision diagnostic" +fi + +echo "PASS: scalar strip-prefix destinations validate correctly" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index d1ee1a251..b983c7f72 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -596,10 +596,9 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): for executable_short_path in executable_dsts: if sp.startswith(executable_short_path + ".runfiles/"): return _apply_strip_prefix(sp, executable_short_path, root) - if strip_prefix: - destination = _apply_strip_prefix(sp, strip_prefix, root) - if destination != "./app.runfiles/_main/" + sp: - return destination + prefix = _normalize_strip_prefix(strip_prefix) + if prefix and (sp == prefix or sp.startswith(prefix + ".runfiles/") or sp.startswith(prefix + "/")): + return _apply_strip_prefix(sp, prefix, root) if sp in executable_dsts: return _apply_strip_prefix(sp, sp, root) return "./app.runfiles/_main/" + sp @@ -1020,11 +1019,16 @@ def _py_image_layer_impl(ctx): validation_inputs = [pkg.files for pkg in ungrouped_pkgs] validation_arguments = [validation_args] - if len(binaries) > 1 or launcher_dir: - # Validate expanded rows whenever launchers are relocated or shared. + if len(binaries) > 1 or launcher_dir or _normalize_strip_prefix(strip_prefix): + # Validate expanded rows whenever source destinations can be shared or remapped. # TreeArtifact roots can contain disjoint versioned children, so only # the production mappers' expanded destinations are authoritative. - source_files = depset(transitive = [info.source_files for info in infos] + [entry.files for info in infos for entry in info.first_party_layers.to_list()]) + source_files = depset(transitive = [info.source_files for info in infos] + [ + files + for group_name, file_sets in fp_by_group.items() + if group_name not in prebuilt_group_tars + for files in file_sets + ]) rule_group_files = [dep[DefaultInfo].files for dep in ctx.attr.groups if normalize_label(str(dep.label)) not in pip_labels] wheel_files = [pkg.files for pkg in all_pkgs] interpreter_files = [info.interpreter_files for info in infos if info.interpreter_files != None] diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index 30a7f85ea..e31f4149d 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -9,7 +9,7 @@ py_image_layer_validator --threshold_mb N --output FILE [label=path ...] label=path — one entry per ungrouped pip package; `label` is the canonical pip label (e.g. @pip//numpy), `path` is its install directory / file. - --mtree FILE — expanded mtree rows for relocated or shared launchers. + --mtree FILE — expanded mtree rows for shared or remapped source destinations. """ from __future__ import annotations From 7d921a8a9c32747ff529ee2bc35a88d5421894af Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 16:12:01 -0400 Subject: [PATCH 19/40] fix(py): use the deepest runfiles prefix --- .../image_layer_analysis_tests.bzl | 49 +++++++++++++++++++ e2e/cases/oci/test.sh | 17 +++++++ py/private/py_image_layer.bzl | 7 ++- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 3cba3e1fe..234d94b5f 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -138,6 +138,55 @@ def image_layer_analysis_test_suite(): layer_tier = ":_scalar_strip_collision_tier", ) + py_binary( + name = "_nested_prefix/foo", + srcs = ["server.py"], + python_version = "3.11", + ) + native.genrule( + name = "_nested_prefix_runfile", + outs = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], + cmd = "echo data > $@", + ) + py_binary( + name = "_nested_prefix/foo.runfiles/worker", + srcs = ["server.py"], + data = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], + python_version = "3.12", + ) + py_layer_tier( + name = "_nested_prefix_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_nested_prefix_layers", + binaries = [ + ":_nested_prefix/foo", + ":_nested_prefix/foo.runfiles/worker", + ], + launcher_dir = "/app/bin", + layer_tier = ":_nested_prefix_tier", + ) + py_image_layer( + name = "_nested_prefix_reversed_layers", + binaries = [ + ":_nested_prefix/foo.runfiles/worker", + ":_nested_prefix/foo", + ], + launcher_dir = "/app/bin", + layer_tier = ":_nested_prefix_tier", + ) + native.genrule( + name = "_nested_prefix_sources_listing", + srcs = [ + ":_nested_prefix_layers_only_src", + ":_nested_prefix_reversed_layers_only_src", + ], + outs = ["_nested_prefix_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + py_image_layer( name = "_interpreter_group_collision_layers", binary = ":my_app_bin", diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index c27d6d104..7e0897b61 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -61,3 +61,20 @@ if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_i fi echo "PASS: scalar strip-prefix destinations validate correctly" + +echo "== nested launcher prefixes must share the same runfiles layout in either input order ==" +if ! "$BAZEL" build -- "${PKG}:_nested_prefix_sources_listing" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the nested launcher source layers to build" +fi +listing="bazel-bin/oci/py_image_layer/_nested_prefix_sources.listing" +if test "$(grep -Fxc './app.runfiles/_main/nested/data.txt' "$listing")" -ne 2; then + cat "$listing" >&2 + fail "expected the nested runfile in the shared layout for both launcher orders" +fi +if grep -Fq './app.runfiles/worker.runfiles/' "$listing"; then + cat "$listing" >&2 + fail "nested launcher prefix leaked into the shared runfiles layout" +fi + +echo "PASS: nested launcher prefixes share the same runfiles layout" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index b983c7f72..92c333c5e 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -593,9 +593,12 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): return "./app.runfiles/_repo_mapping" if sp.startswith("../"): return "./app.runfiles/" + sp[3:] + runfiles_prefix = None for executable_short_path in executable_dsts: - if sp.startswith(executable_short_path + ".runfiles/"): - return _apply_strip_prefix(sp, executable_short_path, root) + if sp.startswith(executable_short_path + ".runfiles/") and (runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix)): + runfiles_prefix = executable_short_path + if runfiles_prefix != None: + return _apply_strip_prefix(sp, runfiles_prefix, root) prefix = _normalize_strip_prefix(strip_prefix) if prefix and (sp == prefix or sp.startswith(prefix + ".runfiles/") or sp.startswith(prefix + "/")): return _apply_strip_prefix(sp, prefix, root) From a583e357492ee8c1a4d07786a193ebd69f3d4179 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 16:28:06 -0400 Subject: [PATCH 20/40] chore(py): align image-layer fixtures with typing checks --- e2e/cases/oci/py_image_layer/assert_source_dedup.py | 2 +- e2e/cases/oci/py_image_layer/worker_support.py | 2 +- py/private/py_image_layer_validator.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/assert_source_dedup.py b/e2e/cases/oci/py_image_layer/assert_source_dedup.py index 1cf2b17a9..19ffff026 100644 --- a/e2e/cases/oci/py_image_layer/assert_source_dedup.py +++ b/e2e/cases/oci/py_image_layer/assert_source_dedup.py @@ -3,7 +3,7 @@ import sys -def main(argv): +def main(argv: list[str]) -> None: listing, sentinels = argv[1], argv[2:] with open(listing, encoding="utf-8") as f: entries = f.read().splitlines() diff --git a/e2e/cases/oci/py_image_layer/worker_support.py b/e2e/cases/oci/py_image_layer/worker_support.py index 0a1b5bc05..440629a6a 100644 --- a/e2e/cases/oci/py_image_layer/worker_support.py +++ b/e2e/cases/oci/py_image_layer/worker_support.py @@ -1,2 +1,2 @@ -def support(): +def support() -> str: return "grouped" diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index e31f4149d..ea3752a5b 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import contextlib import csv import glob import os From 41586fd4e21b6a77e43078681fd6208a19236090 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 16:48:45 -0400 Subject: [PATCH 21/40] fix(py): drop unreachable nested runfiles handling --- .../image_layer_analysis_tests.bzl | 49 ------------------- e2e/cases/oci/test.sh | 17 ------- py/private/py_image_layer.bzl | 7 +-- 3 files changed, 2 insertions(+), 71 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 234d94b5f..3cba3e1fe 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -138,55 +138,6 @@ def image_layer_analysis_test_suite(): layer_tier = ":_scalar_strip_collision_tier", ) - py_binary( - name = "_nested_prefix/foo", - srcs = ["server.py"], - python_version = "3.11", - ) - native.genrule( - name = "_nested_prefix_runfile", - outs = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], - cmd = "echo data > $@", - ) - py_binary( - name = "_nested_prefix/foo.runfiles/worker", - srcs = ["server.py"], - data = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], - python_version = "3.12", - ) - py_layer_tier( - name = "_nested_prefix_tier", - interpreter_group = "interpreter", - ) - py_image_layer( - name = "_nested_prefix_layers", - binaries = [ - ":_nested_prefix/foo", - ":_nested_prefix/foo.runfiles/worker", - ], - launcher_dir = "/app/bin", - layer_tier = ":_nested_prefix_tier", - ) - py_image_layer( - name = "_nested_prefix_reversed_layers", - binaries = [ - ":_nested_prefix/foo.runfiles/worker", - ":_nested_prefix/foo", - ], - launcher_dir = "/app/bin", - layer_tier = ":_nested_prefix_tier", - ) - native.genrule( - name = "_nested_prefix_sources_listing", - srcs = [ - ":_nested_prefix_layers_only_src", - ":_nested_prefix_reversed_layers_only_src", - ], - outs = ["_nested_prefix_sources.listing"], - cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", - toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], - ) - py_image_layer( name = "_interpreter_group_collision_layers", binary = ":my_app_bin", diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 7e0897b61..c27d6d104 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -61,20 +61,3 @@ if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_i fi echo "PASS: scalar strip-prefix destinations validate correctly" - -echo "== nested launcher prefixes must share the same runfiles layout in either input order ==" -if ! "$BAZEL" build -- "${PKG}:_nested_prefix_sources_listing" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected the nested launcher source layers to build" -fi -listing="bazel-bin/oci/py_image_layer/_nested_prefix_sources.listing" -if test "$(grep -Fxc './app.runfiles/_main/nested/data.txt' "$listing")" -ne 2; then - cat "$listing" >&2 - fail "expected the nested runfile in the shared layout for both launcher orders" -fi -if grep -Fq './app.runfiles/worker.runfiles/' "$listing"; then - cat "$listing" >&2 - fail "nested launcher prefix leaked into the shared runfiles layout" -fi - -echo "PASS: nested launcher prefixes share the same runfiles layout" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 92c333c5e..b983c7f72 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -593,12 +593,9 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): return "./app.runfiles/_repo_mapping" if sp.startswith("../"): return "./app.runfiles/" + sp[3:] - runfiles_prefix = None for executable_short_path in executable_dsts: - if sp.startswith(executable_short_path + ".runfiles/") and (runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix)): - runfiles_prefix = executable_short_path - if runfiles_prefix != None: - return _apply_strip_prefix(sp, runfiles_prefix, root) + if sp.startswith(executable_short_path + ".runfiles/"): + return _apply_strip_prefix(sp, executable_short_path, root) prefix = _normalize_strip_prefix(strip_prefix) if prefix and (sp == prefix or sp.startswith(prefix + ".runfiles/") or sp.startswith(prefix + "/")): return _apply_strip_prefix(sp, prefix, root) From da24ca0e1132658ad350614836805601aeb406fb Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 17:26:45 -0400 Subject: [PATCH 22/40] fix(py): preserve nested runfiles on Bazel 8 --- e2e/cases/oci/py_image_layer/BUILD.bazel | 56 ------------------- .../image_layer_analysis_tests.bzl | 53 ++++++++++++++++++ e2e/cases/oci/test.sh | 18 ++++++ py/private/py_image_layer.bzl | 43 ++++++-------- 4 files changed, 87 insertions(+), 83 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 9969f540f..329b64863 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -132,21 +132,6 @@ py_image_layer( layer_tier = ":my_app_multi_tier", ) -py_layer_tier( - name = "my_app_merged_name_tier", - groups = { - "@pip//colorama": "third_party", - "@pip//pyproject_hooks": "third_party", - }, -) - -py_image_layer( - name = "my_app_merged_name_layers", - binary = ":my_app_multi_bin", - groups = {":worker_support": "merged_pip_layer_third_party"}, - layer_tier = ":my_app_merged_name_tier", -) - # Exercise the actual multi-launcher flow with separate lock universes and two # interpreter versions. colorama has the same normalized label in both # closures, but distinct 3.14/3.12 install trees; both launchers must resolve @@ -262,22 +247,10 @@ py_image_layer( launcher_dir = "/app/bin", ) -py_binary( - name = "my_app_compat_layers_binary", - srcs = ["server.py"], -) - -py_image_layer( - name = "my_app_compat_layers", - binary = ":my_app_compat_layers_binary", -) - build_test( name = "my_app_binary_api_test", targets = [ ":my_app_binaries_layers", - ":my_app_compat_layers", - ":my_app_merged_name_layers", ":my_app_select_layers", ], ) @@ -429,35 +402,6 @@ py_image_layer( layer_tier = ":server_tier", ) -py_layer_tier( - name = "server_directory_tier", - strip_prefix = "oci/py_image_layer", -) - -py_image_layer( - name = "server_directory_layers", - binary = ":server", - layer_tier = ":server_directory_tier", -) - -genrule( - name = "server_directory_sources_listing", - srcs = [":server_directory_layers_only_src"], - outs = ["_server_directory_sources.listing"], - cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", - toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], -) - -py_test( - name = "server_directory_sources_test", - srcs = ["assert_source_dedup.py"], - args = [ - "$(rootpath :server_directory_sources_listing)", - "/app/server", - ], - data = [":server_directory_sources_listing"], -) - oci_image( name = "server_image", base = "@ubuntu", diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 3cba3e1fe..c1bd4491a 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -1,6 +1,7 @@ """Analysis and validation fixtures for multi-launcher image layers.""" load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier") +load("@bazel_features//:features.bzl", "bazel_features") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") def _expected_failure_impl(ctx): @@ -138,6 +139,58 @@ def image_layer_analysis_test_suite(): layer_tier = ":_scalar_strip_collision_tier", ) + # Bazel 8 permits nested runfiles outputs. Bazel 9 rejects this topology + # before analysis, so keep the real longest-prefix regression Bazel-8-only. + if not bazel_features.rules.merkle_cache_v2: + py_binary( + name = "_nested_prefix/foo", + srcs = ["server.py"], + python_version = "3.11", + ) + native.genrule( + name = "_nested_prefix_runfile", + outs = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], + cmd = "echo data > $@", + ) + py_binary( + name = "_nested_prefix/foo.runfiles/worker", + srcs = ["server.py"], + data = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], + python_version = "3.12", + ) + py_layer_tier( + name = "_nested_prefix_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_nested_prefix_layers", + binaries = [ + ":_nested_prefix/foo", + ":_nested_prefix/foo.runfiles/worker", + ], + launcher_dir = "/app/bin", + layer_tier = ":_nested_prefix_tier", + ) + py_image_layer( + name = "_nested_prefix_reversed_layers", + binaries = [ + ":_nested_prefix/foo.runfiles/worker", + ":_nested_prefix/foo", + ], + launcher_dir = "/app/bin", + layer_tier = ":_nested_prefix_tier", + ) + native.genrule( + name = "_nested_prefix_sources_listing", + srcs = [ + ":_nested_prefix_layers_only_src", + ":_nested_prefix_reversed_layers_only_src", + ], + outs = ["_nested_prefix_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + py_image_layer( name = "_interpreter_group_collision_layers", binary = ":my_app_bin", diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index c27d6d104..8e7c29beb 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -61,3 +61,21 @@ if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_i fi echo "PASS: scalar strip-prefix destinations validate correctly" + +if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then + echo "== nested launcher prefixes must share the same runfiles layout in either input order ==" + if ! "$BAZEL" build -- "${PKG}:_nested_prefix_sources_listing" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the nested launcher source layers to build" + fi + listing="bazel-bin/oci/py_image_layer/_nested_prefix_sources.listing" + if test "$(grep -Fxc './app.runfiles/_main/nested/data.txt' "$listing")" -ne 2; then + cat "$listing" >&2 + fail "expected the nested runfile in the shared layout for both launcher orders" + fi + if grep -Fq './app.runfiles/worker.runfiles/' "$listing"; then + cat "$listing" >&2 + fail "nested launcher prefix leaked into the shared runfiles layout" + fi + echo "PASS: nested launcher prefixes share the same runfiles layout" +fi diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index b983c7f72..ddf0bbc1d 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -593,9 +593,12 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): return "./app.runfiles/_repo_mapping" if sp.startswith("../"): return "./app.runfiles/" + sp[3:] + runfiles_prefix = None for executable_short_path in executable_dsts: - if sp.startswith(executable_short_path + ".runfiles/"): - return _apply_strip_prefix(sp, executable_short_path, root) + if sp.startswith(executable_short_path + ".runfiles/") and (runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix)): + runfiles_prefix = executable_short_path + if runfiles_prefix != None: + return _apply_strip_prefix(sp, runfiles_prefix, root) prefix = _normalize_strip_prefix(strip_prefix) if prefix and (sp == prefix or sp.startswith(prefix + ".runfiles/") or sp.startswith(prefix + "/")): return _apply_strip_prefix(sp, prefix, root) @@ -633,9 +636,6 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex ] return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_dsts) -def _make_source_map(strip_prefix, root, maybe_symlink, executable_dsts): - return lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, maybe_symlink, executable_dsts) - def _user_file_to_mtree(f, dir_expander): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] @@ -819,22 +819,15 @@ def _py_image_layer_impl(ctx): infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) - if ctx.attr.binary != None: - pkg_by_label = {} - for pkg in infos[0].pip_packages.to_list(): - if pkg.label not in pkg_by_label: - pkg_by_label[pkg.label] = pkg - all_pkgs = pkg_by_label.values() - else: - # Labels are normalized for tier lookup and can collide across lock - # universes. Concrete paths identify each configured wheel artifact. - pkg_by_files = {} - for info in infos: - for pkg in info.pip_packages.to_list(): - key = tuple(sorted([f.path for f in pkg.files.to_list()])) - if key not in pkg_by_files: - pkg_by_files[key] = pkg - all_pkgs = pkg_by_files.values() + # For multiple binaries, normalized labels can collide across lock + # universes. Concrete paths identify each configured wheel artifact. + pkg_by_key = {} + for info in infos: + for pkg in info.pip_packages.to_list(): + key = pkg.label if ctx.attr.binary != None else tuple(sorted([f.path for f in pkg.files.to_list()])) + if key not in pkg_by_key: + pkg_by_key[key] = pkg + all_pkgs = pkg_by_key.values() pip_labels = {pkg.label: True for pkg in all_pkgs} # `_platform_cfg` rewrites the `//py:layer_tier` flag from `attr.layer_tier`, @@ -868,12 +861,8 @@ def _py_image_layer_impl(ctx): # layer has to land under the same `/app.runfiles/` tree for each launcher # to find them. Map every launcher's `.runfiles/` tree into that shared root. all_tars = [] - source_map = _make_source_map( - strip_prefix, - root, - any([info.interpreter_layer == None for info in infos]), - executable_dsts, - ) + source_maybe_symlink = any([info.interpreter_layer == None for info in infos]) + source_map = lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, source_maybe_symlink, executable_dsts) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} for dep, group_name in ctx.attr.groups.items(): From d73140271fe331836c67b9c756736c1ffe05a8a4 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Mon, 20 Jul 2026 17:53:59 -0400 Subject: [PATCH 23/40] fix(py): preserve scalar image-layer mapping --- .../image_layer_analysis_tests.bzl | 39 +++++++++++++++++++ e2e/cases/oci/test.sh | 30 ++++++++++++++ py/private/py_image_layer.bzl | 8 ++-- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index c1bd4491a..f260e205d 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -139,6 +139,45 @@ def image_layer_analysis_test_suite(): layer_tier = ":_scalar_strip_collision_tier", ) + py_layer_tier( + name = "_scalar_default_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_scalar_default_layers", + binary = ":my_app_peer_bin", + layer_tier = ":_scalar_default_tier", + ) + py_image_layer( + name = "_scalar_default_binaries_layers", + binaries = [":my_app_peer_bin"], + layer_tier = ":_scalar_default_tier", + ) + native.genrule( + name = "_scalar_default_sources_listing", + srcs = [ + ":_scalar_default_binaries_layers_only_src", + ":_scalar_default_layers_only_src", + ], + outs = ["_scalar_default_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + + py_binary( + name = "_scalar_root_collision", + srcs = ["server.py"], + ) + py_layer_tier( + name = "_scalar_root_collision_tier", + root = "/app.runfiles/_main/oci/py_image_layer/server.py", + ) + py_image_layer( + name = "_scalar_root_collision_layers", + binary = ":_scalar_root_collision", + layer_tier = ":_scalar_root_collision_tier", + ) + # Bazel 8 permits nested runfiles outputs. Bazel 9 rejects this topology # before analysis, so keep the real longest-prefix regression Bazel-8-only. if not bazel_features.rules.merkle_cache_v2: diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 8e7c29beb..fd855f693 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -62,6 +62,36 @@ fi echo "PASS: scalar strip-prefix destinations validate correctly" +echo "== a default scalar strip prefix must retain executable descendants ==" +if ! "$BAZEL" build -- "${PKG}:_scalar_default_sources_listing" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the scalar source layer to build" +fi +listing="bazel-bin/oci/py_image_layer/_scalar_default_sources.listing" +if test "$(grep -Fxc './app/config.json' "$listing")" -ne 2; then + cat "$listing" >&2 + fail "expected both scalar executable descendants at ./app/config.json" +fi +if grep -Fq './app.runfiles/_main/oci/py_image_layer/my_app_peer_bin/config.json' "$listing"; then + cat "$listing" >&2 + fail "scalar executable descendant leaked into the shared runfiles layout" +fi + +echo "PASS: default scalar strip-prefix descendants are preserved" + +echo "== a custom scalar root must not overwrite runfile data ==" +if "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_scalar_root_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the custom scalar-root destination to fail validation" +fi +if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/server.py:" "$output_log"; then + cat "$output_log" >&2 + fail "expected the custom scalar-root collision diagnostic" +fi + +echo "PASS: custom scalar-root destinations validate correctly" + if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then echo "== nested launcher prefixes must share the same runfiles layout in either input order ==" if ! "$BAZEL" build -- "${PKG}:_nested_prefix_sources_listing" >"$output_log" 2>&1; then diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index ddf0bbc1d..a36d6f5a6 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -595,8 +595,10 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): return "./app.runfiles/" + sp[3:] runfiles_prefix = None for executable_short_path in executable_dsts: - if sp.startswith(executable_short_path + ".runfiles/") and (runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix)): - runfiles_prefix = executable_short_path + if (sp.startswith(executable_short_path + ".runfiles/") or + (not strip_prefix and not executable_dsts[executable_short_path] and sp.startswith(executable_short_path + "/"))): + if runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix): + runfiles_prefix = executable_short_path if runfiles_prefix != None: return _apply_strip_prefix(sp, runfiles_prefix, root) prefix = _normalize_strip_prefix(strip_prefix) @@ -1008,7 +1010,7 @@ def _py_image_layer_impl(ctx): validation_inputs = [pkg.files for pkg in ungrouped_pkgs] validation_arguments = [validation_args] - if len(binaries) > 1 or launcher_dir or _normalize_strip_prefix(strip_prefix): + if len(binaries) > 1 or launcher_dir or _normalize_strip_prefix(strip_prefix) or root != "/app": # Validate expanded rows whenever source destinations can be shared or remapped. # TreeArtifact roots can contain disjoint versioned children, so only # the production mappers' expanded destinations are authoritative. From fd1ede590653ca9203e68407b4a916c57a4b907e Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Tue, 21 Jul 2026 11:14:33 -0400 Subject: [PATCH 24/40] refactor(py): simplify multi-binary image layering --- e2e/cases/oci/py_image_layer/BUILD.bazel | 15 --- .../oci/py_image_layer/assert_source_dedup.py | 23 ---- .../image_layer_analysis_tests.bzl | 85 +++++--------- e2e/cases/oci/test.sh | 109 ++++++++---------- py/private/py_image_layer.bzl | 33 ++---- 5 files changed, 88 insertions(+), 177 deletions(-) delete mode 100644 e2e/cases/oci/py_image_layer/assert_source_dedup.py diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 329b64863..2c0950070 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -208,21 +208,6 @@ genrule( toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], ) -py_test( - name = "my_app_shared_sources_test", - srcs = ["assert_source_dedup.py"], - args = [ - "$(rootpath :my_app_shared_sources_listing)", - "/branding/__init__.py", - "/branding/palette.txt", - "/lib/python3.11/os.py", - "/app/bin/my_app_bin", - "/app/bin/my_app_peer_bin", - "/oci/py_image_layer/my_app_peer_bin/config.json", - ], - data = [":my_app_shared_sources_listing"], -) - py_image_layer( name = "my_app_select_layers", binary = select({ diff --git a/e2e/cases/oci/py_image_layer/assert_source_dedup.py b/e2e/cases/oci/py_image_layer/assert_source_dedup.py deleted file mode 100644 index 19ffff026..000000000 --- a/e2e/cases/oci/py_image_layer/assert_source_dedup.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Assert that each shared source-layer sentinel is present exactly once.""" - -import sys - - -def main(argv: list[str]) -> None: - listing, sentinels = argv[1], argv[2:] - with open(listing, encoding="utf-8") as f: - entries = f.read().splitlines() - - failures = [] - for sentinel in sentinels: - count = sum(entry.endswith(sentinel) for entry in entries) - if count != 1: - failures.append("{}: expected once, found {}".format(sentinel, count)) - if failures: - sys.exit("\n".join(failures)) - - print("assert_source_dedup: ok ({})".format(", ".join(sentinels))) - - -if __name__ == "__main__": - main(sys.argv) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index f260e205d..64389ed2d 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -15,62 +15,53 @@ _expected_failure_test = analysistest.make( expect_failure = True, ) -def image_layer_analysis_test_suite(): - py_image_layer( - name = "_missing_launcher_dir_layers", - binaries = [":my_app_bin", ":my_app_worker_bin"], - ) +def _image_layer_failure(name, expected_error, **kwargs): + target = "_{}_layers".format(name) + py_image_layer(name = target, **kwargs) _expected_failure_test( - name = "missing_launcher_dir_test", - expected_error = "py_image_layer with multiple binaries requires launcher_dir", - target_under_test = ":_missing_launcher_dir_layers", + name = name + "_test", + expected_error = expected_error, + target_under_test = ":" + target, ) - py_image_layer( - name = "_relative_launcher_dir_layers", +def image_layer_analysis_test_suite(): + _image_layer_failure( + name = "missing_launcher_dir", + expected_error = "py_image_layer with multiple binaries requires launcher_dir", binaries = [":my_app_bin", ":my_app_worker_bin"], - launcher_dir = "app/bin", ) - _expected_failure_test( - name = "relative_launcher_dir_test", + _image_layer_failure( + name = "relative_launcher_dir", expected_error = "py_image_layer.launcher_dir must be an absolute image path", - target_under_test = ":_relative_launcher_dir_layers", + binaries = [":my_app_bin", ":my_app_worker_bin"], + launcher_dir = "app/bin", ) native.alias( name = "_my_app_bin_alias", actual = ":my_app_bin", ) - py_image_layer( - name = "_duplicate_launcher_layers", + _image_layer_failure( + name = "duplicate_launcher_basename", + expected_error = "duplicate py_image_layer launcher basename: my_app_bin", binaries = [":my_app_bin", ":_my_app_bin_alias"], launcher_dir = "////", ) - _expected_failure_test( - name = "duplicate_launcher_basename_test", - expected_error = "duplicate py_image_layer launcher basename: my_app_bin", - target_under_test = ":_duplicate_launcher_layers", - ) native.config_setting( name = "_python_3_11", flag_values = {"@aspect_rules_py//py/private/interpreter:python_version": "3.11"}, ) - py_binary( - name = "_wheel_scripts_311", - srcs = ["server.py"], - dep_group = "images", - python_version = "3.11", - deps = ["@pypi_oci_py_image_layer//build"], - ) - py_binary( - name = "_wheel_scripts_312", - srcs = ["server.py"], - dep_group = "images", - python_version = "3.12", - deps = ["@pypi_oci_py_image_layer//build"], - ) + for prefix, package in [("wheel_scripts", "build"), ("pure_wheel", "colorama")]: + for version in ["3.11", "3.12"]: + py_binary( + name = "_{}_{}".format(prefix, version.replace(".", "")), + srcs = ["server.py"], + dep_group = "images", + python_version = version, + deps = ["@pypi_oci_py_image_layer//" + package], + ) py_layer_tier( name = "_wheel_scripts_tier", groups = {"@pip//build": "wheel_scripts"}, @@ -82,20 +73,6 @@ def image_layer_analysis_test_suite(): layer_tier = ":_wheel_scripts_tier", ) - py_binary( - name = "_pure_wheel_311", - srcs = ["server.py"], - dep_group = "images", - python_version = "3.11", - deps = ["@pypi_oci_py_image_layer//colorama"], - ) - py_binary( - name = "_pure_wheel_312", - srcs = ["server.py"], - dep_group = "images", - python_version = "3.12", - deps = ["@pypi_oci_py_image_layer//colorama"], - ) py_image_layer( name = "_configured_pure_wheel_layers", binaries = [":_pure_wheel_311", ":_pure_wheel_312"], @@ -230,14 +207,10 @@ def image_layer_analysis_test_suite(): toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], ) - py_image_layer( - name = "_interpreter_group_collision_layers", + _image_layer_failure( + name = "interpreter_group_collision", + expected_error = "Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier", binary = ":my_app_bin", groups = {":worker_support": "interpreter"}, layer_tier = ":my_app_launchers_tier", ) - _expected_failure_test( - name = "interpreter_group_collision_test", - expected_error = "Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier", - target_under_test = ":_interpreter_group_collision_layers", - ) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index fd855f693..c76648cb0 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -16,6 +16,25 @@ fail() { exit 1 } +expect_diagnostic() { + if ! grep -Fq "$1" "$output_log"; then + cat "$output_log" >&2 + fail "expected validation diagnostic: $1" + fi +} + +expect_listing_count() { + local listing="$1" + local suffix="$2" + local expected="$3" + local actual + actual="$(awk -v suffix="$suffix" 'substr($0, length($0) - length(suffix) + 1) == suffix { count++ } END { print count + 0 }' "$listing")" + if test "$actual" -ne "$expected"; then + cat "$listing" >&2 + fail "$suffix: expected $expected, found $actual" + fi +} + echo "== versioned pure-wheel children must share an image ==" if ! "$BAZEL" build --output_groups=_validation -- \ "${PKG}:_configured_pure_wheel_layers" >"$output_log" 2>&1; then @@ -23,74 +42,47 @@ if ! "$BAZEL" build --output_groups=_validation -- \ fail "expected the two-version pure-wheel image to validate" fi -echo "== unversioned wheel scripts must fail shared-image validation ==" -if "$BAZEL" build --output_groups=_validation -- \ - "${PKG}:_configured_wheel_collision_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected the shared wheel-script destination to fail validation" -fi -if ! grep -F "py_image_layer runfile collision at ./app.runfiles/" "$output_log" | grep -Fq "/bin/pyproject-build:"; then - cat "$output_log" >&2 - fail "expected the pyproject-build destination-collision diagnostic" -fi - -echo "PASS: expanded multi-binary destinations validate correctly" - -echo "== a relocated scalar launcher must not overwrite runfile data ==" -if "$BAZEL" build --output_groups=_validation -- \ - "${PKG}:_scalar_launcher_collision_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected the relocated scalar launcher destination to fail validation" -fi -if ! grep -Fq "py_image_layer runfile collision at ./app/bin/_scalar_launcher_collision:" "$output_log"; then - cat "$output_log" >&2 - fail "expected the relocated scalar launcher collision diagnostic" -fi - -echo "PASS: relocated scalar launcher destinations validate correctly" - -echo "== a scalar strip prefix must not turn its launcher into a file-prefix conflict ==" -if "$BAZEL" build --output_groups=_validation -- \ - "${PKG}:_scalar_strip_collision_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected the scalar strip-prefix destination to fail validation" -fi -if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/_scalar_strip_collision/data.txt:" "$output_log"; then +echo "== remapped destinations must fail validation ==" +if "$BAZEL" build --keep_going --output_groups=_validation -- \ + "${PKG}:_configured_wheel_collision_layers" \ + "${PKG}:_scalar_launcher_collision_layers" \ + "${PKG}:_scalar_strip_collision_layers" \ + "${PKG}:_scalar_root_collision_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 - fail "expected the scalar strip-prefix collision diagnostic" + fail "expected remapped destinations to fail validation" fi +expect_diagnostic "/bin/pyproject-build:" +expect_diagnostic "py_image_layer runfile collision at ./app/bin/_scalar_launcher_collision:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/_scalar_strip_collision/data.txt:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/server.py:" -echo "PASS: scalar strip-prefix destinations validate correctly" +echo "PASS: expanded and remapped destinations validate correctly" -echo "== a default scalar strip prefix must retain executable descendants ==" -if ! "$BAZEL" build -- "${PKG}:_scalar_default_sources_listing" >"$output_log" 2>&1; then +echo "== source closures must preserve scalar and shared layouts ==" +if ! "$BAZEL" build -- \ + "${PKG}:_scalar_default_sources_listing" \ + "${PKG}:my_app_shared_sources_listing" >"$output_log" 2>&1; then cat "$output_log" >&2 - fail "expected the scalar source layer to build" + fail "expected source-layer listings to build" fi listing="bazel-bin/oci/py_image_layer/_scalar_default_sources.listing" -if test "$(grep -Fxc './app/config.json' "$listing")" -ne 2; then - cat "$listing" >&2 - fail "expected both scalar executable descendants at ./app/config.json" -fi +expect_listing_count "$listing" "/app/config.json" 2 if grep -Fq './app.runfiles/_main/oci/py_image_layer/my_app_peer_bin/config.json' "$listing"; then cat "$listing" >&2 fail "scalar executable descendant leaked into the shared runfiles layout" fi +listing="bazel-bin/oci/py_image_layer/_my_app_shared_sources.listing" +for suffix in \ + /branding/__init__.py \ + /branding/palette.txt \ + /lib/python3.11/os.py \ + /app/bin/my_app_bin \ + /app/bin/my_app_peer_bin \ + /oci/py_image_layer/my_app_peer_bin/config.json; do + expect_listing_count "$listing" "$suffix" 1 +done -echo "PASS: default scalar strip-prefix descendants are preserved" - -echo "== a custom scalar root must not overwrite runfile data ==" -if "$BAZEL" build --output_groups=_validation -- \ - "${PKG}:_scalar_root_collision_layers" >"$output_log" 2>&1; then - cat "$output_log" >&2 - fail "expected the custom scalar-root destination to fail validation" -fi -if ! grep -Fq "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/server.py:" "$output_log"; then - cat "$output_log" >&2 - fail "expected the custom scalar-root collision diagnostic" -fi - -echo "PASS: custom scalar-root destinations validate correctly" +echo "PASS: scalar and shared source layouts are preserved" if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then echo "== nested launcher prefixes must share the same runfiles layout in either input order ==" @@ -99,10 +91,7 @@ if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then fail "expected the nested launcher source layers to build" fi listing="bazel-bin/oci/py_image_layer/_nested_prefix_sources.listing" - if test "$(grep -Fxc './app.runfiles/_main/nested/data.txt' "$listing")" -ne 2; then - cat "$listing" >&2 - fail "expected the nested runfile in the shared layout for both launcher orders" - fi + expect_listing_count "$listing" "/app.runfiles/_main/nested/data.txt" 2 if grep -Fq './app.runfiles/worker.runfiles/' "$listing"; then cat "$listing" >&2 fail "nested launcher prefix leaked into the shared runfiles layout" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index a36d6f5a6..938d2ed27 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -171,7 +171,6 @@ _LayerInfo = provider( "source_files": "depset[File] — ungrouped first-party Python source files.", "pip_packages": "depset[struct] — fully transitive pip packages with per-package layers.", "first_party_layers": "depset[struct(label, files, group)] — first-party PyInfo targets matched by py_layer_tier.groups.", - "interpreter_files": "depset[File] — interpreter runfiles, populated only on the py toolchain pass for the binary-branch skip filter.", "interpreter_layer": "struct(tar, group, interpreter_files) | None — prebuilt interpreter layer tar + its group name + the files used to build it, declared at the toolchain target's namespace so the tar action-shares across every py_image_layer using that toolchain config.", }, ) @@ -331,7 +330,6 @@ def _layer_aspect_impl(target, ctx): source_files = depset(), pip_packages = depset(), first_party_layers = depset(), - interpreter_files = interp_depset, interpreter_layer = interp_layer, )] @@ -359,14 +357,12 @@ def _layer_aspect_impl(target, ctx): transitive = transitive_pkgs, ), first_party_layers = depset(transitive = transitive_fp), - interpreter_files = depset(), interpreter_layer = None, )] own_source = [] own_fp = [] interpreter_layer = None - interpreter_files = None kind = ctx.rule.kind # The binary being layered must be a rules_py py_binary (it carries rules_py's @@ -388,7 +384,6 @@ def _layer_aspect_impl(target, ctx): source_files = depset(transitive = transitive_source), pip_packages = depset(transitive = transitive_pkgs), first_party_layers = depset(transitive = transitive_fp), - interpreter_files = interp.interpreter_files if interp != None else depset(), interpreter_layer = interp, )] @@ -423,10 +418,7 @@ def _layer_aspect_impl(target, ctx): if PY_TOOLCHAIN in ctx.rule.toolchains: py_tc = ctx.rule.toolchains[PY_TOOLCHAIN] if _LayerInfo in py_tc: - tc_info = py_tc[_LayerInfo] - if tc_info.interpreter_layer != None: - interpreter_layer = tc_info.interpreter_layer - interpreter_files = tc_info.interpreter_files + interpreter_layer = py_tc[_LayerInfo].interpreter_layer # Binaries walk their runfiles for the source layer, filtering out bytes already # shipping in their own pip / fp-group / interpreter layers. @@ -447,12 +439,10 @@ def _layer_aspect_impl(target, ctx): # that built the venv rather than relying on its own toolchain resolution. interp_paths = {} for interp_layer in transitive_interp: - if interp_layer.interpreter_files != None: - for f in interp_layer.interpreter_files.to_list(): - interp_paths[f.path] = True + for f in interp_layer.interpreter_files.to_list(): + interp_paths[f.path] = True if transitive_interp: interpreter_layer = transitive_interp[0] - interpreter_files = transitive_interp[0].interpreter_files runfiles_files = target[DefaultInfo].default_runfiles.files.to_list() filtered = [f for f in runfiles_files if f.path not in skip_paths and f.path not in interp_paths] @@ -474,7 +464,6 @@ def _layer_aspect_impl(target, ctx): source_files = depset(transitive = transitive_source + own_source), pip_packages = depset(transitive = transitive_pkgs), first_party_layers = depset(direct = own_fp, transitive = transitive_fp), - interpreter_files = interpreter_files, interpreter_layer = interpreter_layer, )] @@ -571,11 +560,8 @@ _merge_aspect = aspect( provides = [_MergedLayerInfo], ) -def _normalize_strip_prefix(strip_prefix): - return "/".join([part for part in strip_prefix.replace("\\/", "/").split("/") if part]) - def _apply_strip_prefix(sp, strip_prefix, root): - prefix = _normalize_strip_prefix(strip_prefix) + prefix = strip_prefix.replace("\\/", "/") if sp == prefix: return "." + root @@ -601,9 +587,10 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): runfiles_prefix = executable_short_path if runfiles_prefix != None: return _apply_strip_prefix(sp, runfiles_prefix, root) - prefix = _normalize_strip_prefix(strip_prefix) - if prefix and (sp == prefix or sp.startswith(prefix + ".runfiles/") or sp.startswith(prefix + "/")): - return _apply_strip_prefix(sp, prefix, root) + if strip_prefix: + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination if sp in executable_dsts: return _apply_strip_prefix(sp, sp, root) return "./app.runfiles/_main/" + sp @@ -1010,7 +997,7 @@ def _py_image_layer_impl(ctx): validation_inputs = [pkg.files for pkg in ungrouped_pkgs] validation_arguments = [validation_args] - if len(binaries) > 1 or launcher_dir or _normalize_strip_prefix(strip_prefix) or root != "/app": + if len(binaries) > 1 or launcher_dir or strip_prefix or root != "/app": # Validate expanded rows whenever source destinations can be shared or remapped. # TreeArtifact roots can contain disjoint versioned children, so only # the production mappers' expanded destinations are authoritative. @@ -1022,7 +1009,7 @@ def _py_image_layer_impl(ctx): ]) rule_group_files = [dep[DefaultInfo].files for dep in ctx.attr.groups if normalize_label(str(dep.label)) not in pip_labels] wheel_files = [pkg.files for pkg in all_pkgs] - interpreter_files = [info.interpreter_files for info in infos if info.interpreter_files != None] + interpreter_files = [layer.interpreter_files for layer in interpreter_layers.values()] mtree_args = ctx.actions.args() mtree_args.set_param_file_format("multiline") From 15089450d5920451270ca7e377a124051a66ff20 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Tue, 21 Jul 2026 21:52:07 -0400 Subject: [PATCH 25/40] fix(py): default multi-binary launcher directory --- e2e/cases/oci/py_image_layer/BUILD.bazel | 7 ++++--- .../oci/py_image_layer/image_layer_analysis_tests.bzl | 5 ----- .../oci/py_image_layer/server_image_command_test.yaml | 2 +- e2e/cases/oci/test.sh | 5 +++-- py/private/py_image_layer.bzl | 11 ++++++----- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 2c0950070..018e24633 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -176,7 +176,6 @@ py_image_layer( ":my_app_bin", ":my_app_worker_alias", ], - launcher_dir = "/app/bin", layer_tier = ":my_app_launchers_tier", ) @@ -197,7 +196,7 @@ py_image_layer( ":my_app_bin", ":my_app_peer_bin", ], - launcher_dir = "/app/bin", + launcher_dir = "/custom/bin", ) genrule( @@ -384,13 +383,15 @@ py_layer_tier( py_image_layer( name = "server_layers", binary = ":server", + launcher_dir = "/custom/bin", layer_tier = ":server_tier", ) oci_image( name = "server_image", base = "@ubuntu", - entrypoint = ["/app"], + entrypoint = ["/custom/bin/server"], + env = {"RUNFILES_DIR": "/app.runfiles"}, tars = [":server_layers"], ) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 64389ed2d..e49e53719 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -25,11 +25,6 @@ def _image_layer_failure(name, expected_error, **kwargs): ) def image_layer_analysis_test_suite(): - _image_layer_failure( - name = "missing_launcher_dir", - expected_error = "py_image_layer with multiple binaries requires launcher_dir", - binaries = [":my_app_bin", ":my_app_worker_bin"], - ) _image_layer_failure( name = "relative_launcher_dir", expected_error = "py_image_layer.launcher_dir must be an absolute image path", diff --git a/e2e/cases/oci/py_image_layer/server_image_command_test.yaml b/e2e/cases/oci/py_image_layer/server_image_command_test.yaml index 357c7ebe5..1aec2609f 100644 --- a/e2e/cases/oci/py_image_layer/server_image_command_test.yaml +++ b/e2e/cases/oci/py_image_layer/server_image_command_test.yaml @@ -3,5 +3,5 @@ schemaVersion: 2.0.0 commandTests: - name: server binary runs correctly (issue #1120 regression) exitCode: 0 - command: /app + command: /custom/bin/server expectedOutput: ["server ok"] diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index c76648cb0..d473fcd6c 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -66,6 +66,7 @@ if ! "$BAZEL" build -- \ fail "expected source-layer listings to build" fi listing="bazel-bin/oci/py_image_layer/_scalar_default_sources.listing" +expect_listing_count "$listing" "/app" 2 expect_listing_count "$listing" "/app/config.json" 2 if grep -Fq './app.runfiles/_main/oci/py_image_layer/my_app_peer_bin/config.json' "$listing"; then cat "$listing" >&2 @@ -76,8 +77,8 @@ for suffix in \ /branding/__init__.py \ /branding/palette.txt \ /lib/python3.11/os.py \ - /app/bin/my_app_bin \ - /app/bin/my_app_peer_bin \ + /custom/bin/my_app_bin \ + /custom/bin/my_app_peer_bin \ /oci/py_image_layer/my_app_peer_bin/config.json; do expect_listing_count "$listing" "$suffix" 1 done diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 938d2ed27..952b6a6d5 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -825,10 +825,10 @@ def _py_image_layer_impl(ctx): root = plan.root strip_prefix = plan.strip_prefix launcher_dir = ctx.attr.launcher_dir + if len(binaries) > 1 and not launcher_dir: + launcher_dir = "/app/bin" if launcher_dir: launcher_dir = launcher_dir.rstrip("/") or "/" - if len(binaries) > 1 and not launcher_dir: - fail("py_image_layer with multiple binaries requires launcher_dir") if launcher_dir and not launcher_dir.startswith("/"): fail("py_image_layer.launcher_dir must be an absolute image path") @@ -1054,7 +1054,7 @@ _py_image_layer = rule( ), "launcher_dir": attr.string( default = "", - doc = "Absolute image directory for binary launchers. Required with multiple binaries.", + doc = "Absolute image directory for binary launchers. Defaults to /app/bin with multiple binaries.", ), "groups": attr.label_keyed_string_dict(default = {}), "group_execution_requirements": attr.string_list_dict(default = {}), @@ -1136,8 +1136,9 @@ def py_image_layer( layer_tier: Optional py_layer_tier target pinned for this rule. Sets the `@aspect_rules_py//py:layer_tier` label_flag via the rule transition, overriding any command-line value for this rule's subgraph. - launcher_dir: Absolute image directory for the binary launchers. Required - with multiple binaries. Set RUNFILES_DIR=/app.runfiles in the image. + launcher_dir: Absolute image directory for the binary launchers. Defaults + to /app/bin with multiple binaries. Set RUNFILES_DIR=/app.runfiles in + the image. binaries: Alternative to binary. A nonempty list of py_binary targets to include in the image. **kwargs: Forwarded to inner rule. From 14b4201588cde9febf87c2957735f28be63d7c3a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 22 Jul 2026 22:35:39 -0400 Subject: [PATCH 26/40] fix(py): validate image file contents Configured wheels for different interpreters can emit byte-identical scripts at the same image destination. Treat matching regular files as one entry after verifying their contents and mtree metadata, while rejecting type and metadata conflicts. Keep custom launcher coverage separate from the original default /app regression so both layouts execute. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 66 ++++++++++++++++--- e2e/cases/oci/py_image_layer/launcher.py | 13 ++++ .../launchers_image_command_test.yaml | 6 +- .../server_custom_image_command_test.yaml | 7 ++ .../server_image_command_test.yaml | 2 +- e2e/cases/oci/py_image_layer/worker.py | 4 +- py/private/py_image_layer_validator.py | 39 ++++++++--- py/private/py_image_layer_validator_test.py | 53 +++++++++++++++ 8 files changed, 166 insertions(+), 24 deletions(-) create mode 100644 e2e/cases/oci/py_image_layer/launcher.py create mode 100644 e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 018e24633..b9aca2032 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -142,6 +142,17 @@ py_library( imports = ["."], ) +py_binary( + name = "my_app_launcher_bin", + srcs = ["launcher.py"], + dep_group = "images", + python_version = "3.11", + deps = [ + "@pypi_oci_py_image_layer//build", + "@pypi_oci_py_image_layer//colorama", + ], +) + py_binary( name = "my_app_worker_bin", srcs = ["worker.py"], @@ -149,6 +160,7 @@ py_binary( python_version = "3.12", deps = [ ":worker_support", + "@pypi_oci_py_venv_image_layer//build", "@pypi_oci_py_venv_image_layer//colorama", "@pypi_oci_py_venv_image_layer//pyproject_hooks", ], @@ -163,6 +175,7 @@ py_layer_tier( name = "my_app_launchers_tier", groups = { "//oci/py_image_layer:worker_support": "worker_support", + "@pip//build": "third_party", "@pip//colorama": "third_party", "@pip//pyproject_hooks": "third_party", }, @@ -173,7 +186,7 @@ py_layer_tier( py_image_layer( name = "my_app_launchers_layers", binaries = [ - ":my_app_bin", + ":my_app_launcher_bin", ":my_app_worker_alias", ], layer_tier = ":my_app_launchers_tier", @@ -234,7 +247,9 @@ py_image_layer( build_test( name = "my_app_binary_api_test", targets = [ + ":_configured_wheel_collision_layers", ":my_app_binaries_layers", + ":my_app_launchers_layers", ":my_app_select_layers", ], ) @@ -375,23 +390,15 @@ py_binary( srcs = ["server.py"], ) -py_layer_tier( - name = "server_tier", - strip_prefix = "does/not/match", -) - py_image_layer( name = "server_layers", binary = ":server", - launcher_dir = "/custom/bin", - layer_tier = ":server_tier", ) oci_image( name = "server_image", base = "@ubuntu", - entrypoint = ["/custom/bin/server"], - env = {"RUNFILES_DIR": "/app.runfiles"}, + entrypoint = ["/app"], tars = [":server_layers"], ) @@ -411,3 +418,42 @@ container_structure_test( "requires-docker", ], ) + +# Exercise scalar launcher relocation independently from the default `/app` +# regression above. +py_layer_tier( + name = "server_custom_tier", + strip_prefix = "does/not/match", +) + +py_image_layer( + name = "server_custom_layers", + binary = ":server", + launcher_dir = "/custom/bin", + layer_tier = ":server_custom_tier", +) + +oci_image( + name = "server_custom_image", + base = "@ubuntu", + entrypoint = ["/custom/bin/server"], + env = {"RUNFILES_DIR": "/app.runfiles"}, + tars = [":server_custom_layers"], +) + +platform_transition_filegroup( + name = "amd64_server_custom_image", + srcs = [":server_custom_image"], + target_platform = ":x86_64_linux", +) + +container_structure_test( + name = "server_custom_image_command_test", + args = ["--verbosity=debug"], + configs = ["server_custom_image_command_test.yaml"], + image = ":amd64_server_custom_image", + platform = "linux/amd64", + tags = [ + "requires-docker", + ], +) diff --git a/e2e/cases/oci/py_image_layer/launcher.py b/e2e/cases/oci/py_image_layer/launcher.py new file mode 100644 index 000000000..a69bb03d8 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/launcher.py @@ -0,0 +1,13 @@ +import sys + +import build +import colorama + +if __name__ == "__main__": + print( + "launcher ok {} {} {}".format( + sys.version_info.minor, + build.__version__, + colorama.__version__, + ) + ) diff --git a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml index 19ab498b9..39ed26645 100644 --- a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml +++ b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml @@ -3,10 +3,10 @@ schemaVersion: 2.0.0 commandTests: - name: primary launcher runs from launcher_dir exitCode: 0 - command: /app/bin/my_app_bin - expectedOutput: ["Hello rules_py - 3.14"] + command: /app/bin/my_app_launcher_bin + expectedOutput: ["launcher ok 11 1.4.0 0.4.6"] - name: secondary launcher uses its own interpreter and pip closure exitCode: 0 command: /app/bin/my_app_worker_bin - expectedOutput: ["worker ok 12 0.4.6 1.2.0 grouped"] + expectedOutput: ["worker ok 12 1.4.0 0.4.6 1.2.0 grouped"] diff --git a/e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml b/e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml new file mode 100644 index 000000000..a051e5520 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml @@ -0,0 +1,7 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: server binary runs from a custom launcher directory + exitCode: 0 + command: /custom/bin/server + expectedOutput: ["server ok"] diff --git a/e2e/cases/oci/py_image_layer/server_image_command_test.yaml b/e2e/cases/oci/py_image_layer/server_image_command_test.yaml index 1aec2609f..357c7ebe5 100644 --- a/e2e/cases/oci/py_image_layer/server_image_command_test.yaml +++ b/e2e/cases/oci/py_image_layer/server_image_command_test.yaml @@ -3,5 +3,5 @@ schemaVersion: 2.0.0 commandTests: - name: server binary runs correctly (issue #1120 regression) exitCode: 0 - command: /custom/bin/server + command: /app expectedOutput: ["server ok"] diff --git a/e2e/cases/oci/py_image_layer/worker.py b/e2e/cases/oci/py_image_layer/worker.py index 19065ab37..79e73265e 100644 --- a/e2e/cases/oci/py_image_layer/worker.py +++ b/e2e/cases/oci/py_image_layer/worker.py @@ -1,13 +1,15 @@ import sys +import build import colorama import pyproject_hooks from worker_support import support if __name__ == "__main__": print( - "worker ok {} {} {} {}".format( + "worker ok {} {} {} {} {}".format( sys.version_info.minor, + build.__version__, colorama.__version__, pyproject_hooks.__version__, support(), diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index ea3752a5b..d0afe5cd8 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -17,6 +17,7 @@ import argparse import contextlib import csv +import filecmp import glob import os import sys @@ -232,7 +233,7 @@ def _add_subpath_or_whole( def _mtree_collision(rows: Iterable[str]) -> Optional[str]: """Return the first conflicting expanded mtree destination, or None.""" - paths: Dict[str, str] = {} + paths: Dict[str, Tuple[str, str, str, Tuple[str, ...]]] = {} descendants: Dict[str, Tuple[str, str]] = {} for row in rows: if not row or row.startswith("#"): @@ -249,29 +250,49 @@ def _mtree_collision(rows: Iterable[str]) -> Optional[str]: else: destination_parts.append(part) destination = "./" + "/".join(destination_parts) if destination_parts else "." - source = next( - (field.partition("=")[2] for field in fields[1:] if field.startswith(("contents=", "content=", "link="))), + entry_type = next( + (field.partition("=")[2] for field in fields[1:] if field.startswith("type=")), None, ) - if source is None: + source_field = next( + (field for field in fields[1:] if field.startswith(("contents=", "content=", "link="))), + None, + ) + if entry_type is None or source_field is None: return "invalid py_image_layer mtree row (missing source): {}".format(row) + source_kind, _, source = source_field.partition("=") + metadata = tuple(sorted(field for field in fields[1:] if field != source_field)) + entry = (entry_type, source_kind, source, metadata) previous = paths.get(destination) if previous is not None: - if previous != source: - return "py_image_layer runfile collision at {}: {} and {}".format(destination, previous, source) - continue + if previous == entry: + continue + previous_type, previous_kind, previous_source, previous_metadata = previous + if ( + entry_type == previous_type == "file" + and source_kind == previous_kind == "contents" + and metadata == previous_metadata + ): + with contextlib.suppress(OSError): + if filecmp.cmp(previous_source, source, shallow=False): + continue + return "py_image_layer runfile collision at {}: {} and {}".format( + destination, previous_source, source + ) parts = destination.split("/") for end in range(len(parts) - 1, 0, -1): parent = "/".join(parts[:end]) if parent in paths: - return "py_image_layer runfile collision at {}: {} and {}".format(destination, paths[parent], source) + return "py_image_layer runfile collision at {}: {} and {}".format( + destination, paths[parent][2], source + ) if destination in descendants: descendant, previous = descendants[destination] return "py_image_layer runfile collision at {}: {} and {}".format(descendant, previous, source) - paths[destination] = source + paths[destination] = entry for end in range(len(parts) - 1, 0, -1): parent = "/".join(parts[:end]) descendants.setdefault(parent, (destination, source)) diff --git a/py/private/py_image_layer_validator_test.py b/py/private/py_image_layer_validator_test.py index 82b8cd831..fb51a32d7 100644 --- a/py/private/py_image_layer_validator_test.py +++ b/py/private/py_image_layer_validator_test.py @@ -40,6 +40,59 @@ def test_mtree_conflicting_source_fails() -> None: assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision +def test_mtree_identical_file_contents_coalesce() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + for path in (first, second): + with open(path, "w") as file: + file.write("identical") + + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", first), + _mtree_row("./app.runfiles/pkg/module.py", second), + ]) is None + + +def test_mtree_different_file_contents_fail() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + with open(first, "w") as file: + file.write("first") + with open(second, "w") as file: + file.write("second") + + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", first), + _mtree_row("./app.runfiles/pkg/module.py", second), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_file_and_symlink_types_fail() -> None: + destination = "./app.runfiles/pkg/module.py" + source = "same/module.py" + collision = _mtree_collision([ + _mtree_row(destination, source), + "{} type=link mode=0755 link={}".format(destination, source), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_different_modes_fail() -> None: + destination = "./app.runfiles/pkg/module.py" + source = "same/module.py" + collision = _mtree_collision([ + _mtree_row(destination, source), + _mtree_row(destination, source).replace("mode=0755", "mode=0644"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + def test_mtree_ancestor_then_descendant_fails() -> None: collision = _mtree_collision([ _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), From 96571573aedc165d3fbaf02742eaa533f4d3e17b Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 22 Jul 2026 22:44:28 -0400 Subject: [PATCH 27/40] test(py): accept identical wheel destinations The configured Python 3.11 and 3.12 wheels install byte-identical console scripts, so their combined image is valid. Require that fixture to pass validation while retaining the three genuine destination collision failures. --- e2e/cases/oci/test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index d473fcd6c..115d3f67c 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -35,23 +35,22 @@ expect_listing_count() { fi } -echo "== versioned pure-wheel children must share an image ==" +echo "== versioned wheel children must share an image when destinations agree ==" if ! "$BAZEL" build --output_groups=_validation -- \ - "${PKG}:_configured_pure_wheel_layers" >"$output_log" 2>&1; then + "${PKG}:_configured_pure_wheel_layers" \ + "${PKG}:_configured_wheel_collision_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 - fail "expected the two-version pure-wheel image to validate" + fail "expected the two-version wheel images to validate" fi echo "== remapped destinations must fail validation ==" if "$BAZEL" build --keep_going --output_groups=_validation -- \ - "${PKG}:_configured_wheel_collision_layers" \ "${PKG}:_scalar_launcher_collision_layers" \ "${PKG}:_scalar_strip_collision_layers" \ "${PKG}:_scalar_root_collision_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 fail "expected remapped destinations to fail validation" fi -expect_diagnostic "/bin/pyproject-build:" expect_diagnostic "py_image_layer runfile collision at ./app/bin/_scalar_launcher_collision:" expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/_scalar_strip_collision/data.txt:" expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/server.py:" From fb22e293be0e42e41800947163c7e335dc771e10 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 22 Jul 2026 23:40:37 -0400 Subject: [PATCH 28/40] fix(py): exclude rule groups from sources Rule-level group files can also enter a binary runfiles closure. Remove those configured artifacts from the default source layer so each destination is emitted by exactly one tar. --- .../image_layer_analysis_tests.bzl | 26 +++++++++++++++++++ e2e/cases/oci/test.sh | 15 +++++++++++ py/private/py_image_layer.bzl | 24 ++++++++++++++--- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index e49e53719..b094878e4 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -74,6 +74,32 @@ def image_layer_analysis_test_suite(): launcher_dir = "/app/bin", ) + native.filegroup( + name = "_rule_group_assets", + srcs = ["my_app_peer_bin/config.json"], + ) + py_binary( + name = "_rule_group_source_bin", + srcs = ["server.py"], + data = [":_rule_group_assets"], + ) + py_image_layer( + name = "_rule_group_source_layers", + binaries = [ + ":_rule_group_source_bin", + ":my_app_bin", + ], + groups = {":_rule_group_assets": "assets"}, + launcher_dir = "/app/bin", + ) + native.genrule( + name = "_rule_group_sources_listing", + srcs = [":_rule_group_source_layers"], + outs = ["_rule_group_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + native.genrule( name = "_scalar_launcher_collision_data", outs = ["bin/_scalar_launcher_collision"], diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 115d3f67c..ee7b7b145 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -57,6 +57,21 @@ expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/ echo "PASS: expanded and remapped destinations validate correctly" +echo "== rule-group files must be excluded from the default source layer ==" +if ! "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_rule_group_source_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the explicitly grouped source to validate" +fi +if ! "$BAZEL" build -- "${PKG}:_rule_group_sources_listing" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the explicitly grouped source listing to build" +fi +listing="bazel-bin/oci/py_image_layer/_rule_group_sources.listing" +expect_listing_count "$listing" "/my_app_peer_bin/config.json" 1 + +echo "PASS: explicitly grouped sources are emitted once" + echo "== source closures must preserve scalar and shared layouts ==" if ! "$BAZEL" build -- \ "${PKG}:_scalar_default_sources_listing" \ diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 952b6a6d5..29b4ed6e4 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -854,17 +854,23 @@ def _py_image_layer_impl(ctx): source_map = lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, source_maybe_symlink, executable_dsts) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} + rule_group_files = [] + rule_group_paths = {} for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) if dep_label in pip_labels: continue + files = dep[DefaultInfo].files + rule_group_files.append(files) + for f in files.to_list(): + rule_group_paths[f.path] = True tar_out = _declare_group_tar( ctx, bsdtar, bsdtar_files, "{}_{}.tar.gz".format(ctx.attr.name, group_name), group_name, - dep[DefaultInfo].files, + files, _user_file_to_mtree, "Creating image layer %s[%s]" % (ctx.label, group_name), ) @@ -972,6 +978,17 @@ def _py_image_layer_impl(ctx): # snapshot here to avoid double-bookkeeping during construction. dep_tars = list(all_tars) + source_files = depset(transitive = [info.source_files for info in infos]) + if rule_group_paths: + # Rule groups are unavailable to the binary aspect, so its source + # closure still contains these files. Subtract them here, where the + # group tars are declared, and keep action inputs aligned with contents. + source_files = depset(direct = [ + f + for f in source_files.to_list() + if f.path not in rule_group_paths + ]) + # Keep the complete source closure in one tar. `modify_mtree.awk` can only # rewrite a Bazel-tree symlink when its target row is in the same mtree. source_tar = _declare_group_tar( @@ -980,7 +997,7 @@ def _py_image_layer_impl(ctx): bsdtar_files, "{}_default.tar.gz".format(ctx.attr.name), "default", - depset(transitive = [info.source_files for info in infos]), + source_files, source_map, "Creating source layer for %s" % ctx.label, ) @@ -1001,13 +1018,12 @@ def _py_image_layer_impl(ctx): # Validate expanded rows whenever source destinations can be shared or remapped. # TreeArtifact roots can contain disjoint versioned children, so only # the production mappers' expanded destinations are authoritative. - source_files = depset(transitive = [info.source_files for info in infos] + [ + source_files = depset(transitive = [source_files] + [ files for group_name, file_sets in fp_by_group.items() if group_name not in prebuilt_group_tars for files in file_sets ]) - rule_group_files = [dep[DefaultInfo].files for dep in ctx.attr.groups if normalize_label(str(dep.label)) not in pip_labels] wheel_files = [pkg.files for pkg in all_pkgs] interpreter_files = [layer.interpreter_files for layer in interpreter_layers.values()] From 0612c6265285c3293e9645e7547efaa338f89ee8 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 22 Jul 2026 23:48:10 -0400 Subject: [PATCH 29/40] fix(py): preserve binary runfile launchers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A listed binary may also be another launcher’s data dependency. Emit that executable at both its relocated entrypoint and its logical runfiles key, and select each binary’s own interpreter layer instead of the first interpreter found through data. --- e2e/cases/oci/py_image_layer/BUILD.bazel | 10 +++++ e2e/cases/oci/py_image_layer/launcher.py | 7 +++ .../launchers_image_command_test.yaml | 4 +- e2e/cases/oci/test.sh | 6 +++ py/private/py_image_layer.bzl | 45 ++++++++++++++++--- 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index b9aca2032..f3b53ab08 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -145,9 +145,11 @@ py_library( py_binary( name = "my_app_launcher_bin", srcs = ["launcher.py"], + data = [":my_app_worker_bin"], dep_group = "images", python_version = "3.11", deps = [ + "@bazel_tools//tools/python/runfiles", "@pypi_oci_py_image_layer//build", "@pypi_oci_py_image_layer//colorama", ], @@ -192,6 +194,14 @@ py_image_layer( layer_tier = ":my_app_launchers_tier", ) +genrule( + name = "my_app_launchers_sources_listing", + srcs = [":my_app_launchers_layers_only_src"], + outs = ["_my_app_launchers_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + # Default-tier source closures share the interpreter, branding source, and # data. The listing test proves the unioned source tar ships each byte once. py_binary( diff --git a/e2e/cases/oci/py_image_layer/launcher.py b/e2e/cases/oci/py_image_layer/launcher.py index a69bb03d8..b882cafc1 100644 --- a/e2e/cases/oci/py_image_layer/launcher.py +++ b/e2e/cases/oci/py_image_layer/launcher.py @@ -1,7 +1,9 @@ +import subprocess import sys import build import colorama +from bazel_tools.tools.python.runfiles import runfiles if __name__ == "__main__": print( @@ -11,3 +13,8 @@ colorama.__version__, ) ) + worker = runfiles.Create().Rlocation( + "_main/oci/py_image_layer/my_app_worker_bin" + ) + assert worker is not None + subprocess.run([worker], check=True) diff --git a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml index 39ed26645..b46a6801d 100644 --- a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml +++ b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml @@ -4,7 +4,9 @@ commandTests: - name: primary launcher runs from launcher_dir exitCode: 0 command: /app/bin/my_app_launcher_bin - expectedOutput: ["launcher ok 11 1.4.0 0.4.6"] + expectedOutput: + - "launcher ok 11 1.4.0 0.4.6" + - "worker ok 12 1.4.0 0.4.6 1.2.0 grouped" - name: secondary launcher uses its own interpreter and pip closure exitCode: 0 diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index ee7b7b145..9cf17aa0d 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -75,6 +75,7 @@ echo "PASS: explicitly grouped sources are emitted once" echo "== source closures must preserve scalar and shared layouts ==" if ! "$BAZEL" build -- \ "${PKG}:_scalar_default_sources_listing" \ + "${PKG}:my_app_launchers_sources_listing" \ "${PKG}:my_app_shared_sources_listing" >"$output_log" 2>&1; then cat "$output_log" >&2 fail "expected source-layer listings to build" @@ -86,6 +87,11 @@ if grep -Fq './app.runfiles/_main/oci/py_image_layer/my_app_peer_bin/config.json cat "$listing" >&2 fail "scalar executable descendant leaked into the shared runfiles layout" fi +listing="bazel-bin/oci/py_image_layer/_my_app_launchers_sources.listing" +expect_listing_count "$listing" "/app/bin/my_app_launcher_bin" 1 +expect_listing_count "$listing" "/app/bin/my_app_worker_bin" 1 +expect_listing_count "$listing" "/app.runfiles/_main/oci/py_image_layer/my_app_worker_bin" 1 +expect_listing_count "$listing" "/app.runfiles/_main/oci/py_image_layer/my_app_launcher_bin" 0 listing="bazel-bin/oci/py_image_layer/_my_app_shared_sources.listing" for suffix in \ /branding/__init__.py \ diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 29b4ed6e4..a858cbc83 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -441,8 +441,9 @@ def _layer_aspect_impl(target, ctx): for interp_layer in transitive_interp: for f in interp_layer.interpreter_files.to_list(): interp_paths[f.path] = True - if transitive_interp: - interpreter_layer = transitive_interp[0] + venv = getattr(ctx.rule.attr, "venv", None) + if venv != None and _LayerInfo in venv: + interpreter_layer = venv[_LayerInfo].interpreter_layer runfiles_files = target[DefaultInfo].default_runfiles.files.to_list() filtered = [f for f in runfiles_files if f.path not in skip_paths and f.path not in interp_paths] @@ -615,7 +616,7 @@ def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_ f.path.replace(" ", "\\040"), ) -def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, executable_dsts): +def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, executable_dsts, runfile_executable_paths): # 0755 throughout: keeps launcher/interpreter/venv shims executable; Bazel # doesn't expose per-input source mode for us to propagate. if f.is_directory: @@ -623,7 +624,17 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink, executable_dsts) for child in dir_expander.expand(f) ] - return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_dsts) + entry = _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_dsts) + if f.path not in runfile_executable_paths: + return entry + return [ + entry, + _file_to_mtree_entry( + f, + "0755", + maybe_symlink = maybe_symlink, + ), + ] def _user_file_to_mtree(f, dir_expander): if f.is_directory: @@ -834,8 +845,10 @@ def _py_image_layer_impl(ctx): launcher_names = {} executable_dsts = {} - for binary in binaries: + executable_owner_by_path = {} + for index, binary in enumerate(binaries): executable = binary[DefaultInfo].files_to_run.executable + executable_owner_by_path[executable.path] = index launcher_name = executable.basename if launcher_dir: if launcher_name in launcher_names: @@ -845,13 +858,33 @@ def _py_image_layer_impl(ctx): else: executable_dsts[executable.short_path] = "" + runfile_executable_paths = {} + if launcher_dir and len(binaries) > 1: + for index, binary in enumerate(binaries): + for f in binary[DefaultInfo].default_runfiles.files.to_list(): + owner = executable_owner_by_path.get(f.path, None) + + # A launcher consumed through another binary's runfiles needs + # both its relocated entrypoint and the logical key resolved + # by that consumer. + if owner != None and owner != index: + runfile_executable_paths[f.path] = True + # 3p pip layers are action-shared across the graph and hard-code their # destination under `./app.runfiles//...`, so the consumer's source # layer has to land under the same `/app.runfiles/` tree for each launcher # to find them. Map every launcher's `.runfiles/` tree into that shared root. all_tars = [] source_maybe_symlink = any([info.interpreter_layer == None for info in infos]) - source_map = lambda f, d: _source_file_to_mtree(f, d, strip_prefix, root, source_maybe_symlink, executable_dsts) + source_map = lambda f, d: _source_file_to_mtree( + f, + d, + strip_prefix, + root, + source_maybe_symlink, + executable_dsts, + runfile_executable_paths, + ) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} rule_group_files = [] From cb320612efc9db0ec007742c370743c6c0ed076c Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 00:34:47 -0400 Subject: [PATCH 30/40] fix(py): preserve multi-binary image metadata Normalize mtree source paths and compare only regular files across content marker forms. When rule groups take source-owned files, preserve their modes and make group-owned destinations available to source symlink rewriting without duplicating payload bytes. --- .../image_layer_analysis_tests.bzl | 74 ++++++++++++--- e2e/cases/oci/test.sh | 11 +-- py/private/modify_mtree.awk | 32 +++++-- py/private/py_image_layer.bzl | 77 ++++++++++----- py/private/py_image_layer_validator.py | 44 ++++++++- py/private/py_image_layer_validator_test.py | 94 ++++++++++++++++++- 6 files changed, 275 insertions(+), 57 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index b094878e4..c5564295c 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -15,6 +15,16 @@ _expected_failure_test = analysistest.make( expect_failure = True, ) +def _target_file_symlink_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = output, target_file = ctx.file.target) + return [DefaultInfo(files = depset([output]))] + +_target_file_symlink = rule( + implementation = _target_file_symlink_impl, + attrs = {"target": attr.label(allow_single_file = True, mandatory = True)}, +) + def _image_layer_failure(name, expected_error, **kwargs): target = "_{}_layers".format(name) py_image_layer(name = target, **kwargs) @@ -74,29 +84,71 @@ def image_layer_analysis_test_suite(): launcher_dir = "/app/bin", ) + native.genrule( + name = "_grouped_tool", + outs = ["grouped/tool.sh"], + cmd = "printf '#!/bin/sh\\nprintf grouped-ok\\n' > $@", + executable = True, + ) + native.genrule( + name = "_grouped_payload", + outs = ["grouped/payload.txt"], + cmd = "printf grouped-payload > $@", + ) + native.genrule( + name = "_group_only_file", + outs = ["grouped/ordinary.txt"], + cmd = "printf ordinary > $@", + ) + _target_file_symlink( + name = "_grouped_payload_link", + target = ":_grouped_payload", + ) native.filegroup( - name = "_rule_group_assets", - srcs = ["my_app_peer_bin/config.json"], + name = "_grouped_assets", + srcs = [ + ":_group_only_file", + ":_grouped_payload", + ":_grouped_tool", + ], ) py_binary( - name = "_rule_group_source_bin", + name = "_grouped_source_bin", srcs = ["server.py"], - data = [":_rule_group_assets"], + data = [ + ":_grouped_payload", + ":_grouped_payload_link", + ":_grouped_tool", + ], ) py_image_layer( - name = "_rule_group_source_layers", + name = "_grouped_source_layers", binaries = [ - ":_rule_group_source_bin", + ":_grouped_source_bin", ":my_app_bin", ], - groups = {":_rule_group_assets": "assets"}, + groups = {":_grouped_assets": "assets"}, launcher_dir = "/app/bin", ) native.genrule( - name = "_rule_group_sources_listing", - srcs = [":_rule_group_source_layers"], - outs = ["_rule_group_sources.listing"], - cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + name = "_grouped_source_runtime_test", + srcs = [":_grouped_source_layers"], + outs = ["_grouped_source_runtime_test.ok"], + cmd = """ +set -eu +root="$(@D)/_grouped_source_runtime_test.root" +mkdir -p "$$root" +for archive in $(SRCS); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$root" +done +prefix="$$root/app.runfiles/_main/oci/py_image_layer" +test "$$("$$prefix/grouped/tool.sh")" = grouped-ok +test ! -x "$$prefix/grouped/ordinary.txt" +test "$$(cat "$$prefix/_grouped_payload_link")" = grouped-payload +count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/payload.txt$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +touch $@ +""", toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], ) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 9cf17aa0d..bcc97bd2c 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -59,18 +59,17 @@ echo "PASS: expanded and remapped destinations validate correctly" echo "== rule-group files must be excluded from the default source layer ==" if ! "$BAZEL" build --output_groups=_validation -- \ - "${PKG}:_rule_group_source_layers" >"$output_log" 2>&1; then + "${PKG}:_grouped_source_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 fail "expected the explicitly grouped source to validate" fi -if ! "$BAZEL" build -- "${PKG}:_rule_group_sources_listing" >"$output_log" 2>&1; then + +if ! "$BAZEL" build -- "${PKG}:_grouped_source_runtime_test" >"$output_log" 2>&1; then cat "$output_log" >&2 - fail "expected the explicitly grouped source listing to build" + fail "expected grouped modes and cross-layer symlink to survive extraction" fi -listing="bazel-bin/oci/py_image_layer/_rule_group_sources.listing" -expect_listing_count "$listing" "/my_app_peer_bin/config.json" 1 -echo "PASS: explicitly grouped sources are emitted once" +echo "PASS: explicitly grouped sources preserve modes and symlink targets without duplicate bytes" echo "== source closures must preserve scalar and shared layouts ==" if ! "$BAZEL" build -- \ diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index 67366fbf7..a2dd359d1 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -2,9 +2,9 @@ # `type=link` rows, so bsdtar preserves them as symlinks instead of # following and inlining the target bytes. # -# Forked from @tar.bzl//tar/private:preserve_symlinks.awk and tracking -# https://github.com/bazel-contrib/tar.bzl/pull/115. One behavioural -# difference remains: +# Forked from @tar.bzl//tar/private:preserve_symlinks.awk. The upstream +# implementation in https://github.com/bazel-contrib/tar.bzl/pull/115 lacks +# both cross-layer target mappings and one path-normalization fix retained here. # # The `bazel-out/` vs `external/` strip is exclusive (`if` / `else if`) # rather than two sequential `sub`s. Without this, paths matching @@ -13,9 +13,6 @@ # over-stripped down to `external//...`, miss the # `symlink_map` lookup, and dangle inside the OCI layer. # -# Send a follow-up PR to bazel-contrib/tar.bzl once #115 -# lands so this fork can retire. -# # Invoked from `_run_tar_action` in [py_image_layer.bzl](py_image_layer.bzl) # via `ctx.executable._awk` pinned to `@gawk` — the END block uses # `asort()` and `print > outfile`, both gawk extensions. @@ -58,7 +55,22 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back return back_steps target } +function decode_mtree_path(path) { + gsub(/\\040/, " ", path) + return path +} + { + if (ARGIND != source_argind) { + match($0, /(contents|content|link)=[^ ]+/) + if (RSTART != 0) { + content_field = substr($0, RSTART, RLENGTH) + split(content_field, parts, "=") + symlink_map[decode_mtree_path(parts[2])] = $1 + } + next + } + symlink = "" symlink_content = "" # Two markers Starlark emits for paths that could be symlinks: @@ -78,7 +90,7 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back } content_field = substr($0, RSTART, RLENGTH) split(content_field, parts, "=") - path = parts[2] + path = decode_mtree_path(parts[2]) symlink_map[path] = $1 # Plain `readlink` first: keep its result if relative @@ -151,9 +163,9 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back } } if (symlink != "") { - line_array[NR] = $0 SUBSEP $1 SUBSEP resolved_path + line_array[++source_line_count] = $0 SUBSEP $1 SUBSEP resolved_path } else { - line_array[NR] = $0 + line_array[++source_line_count] = $0 } } @@ -161,7 +173,7 @@ END { # Buffer rewritten rows, sort byte-wise (asort under LC_ALL=C, set by # the action env), and write to `outfile`. n = 0 - for (i = 1; i <= NR; i++) { + for (i = 1; i <= source_line_count; i++) { line = line_array[i] if (index(line, SUBSEP) > 0) { split(line, fields, SUBSEP) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index a858cbc83..ad7886d2e 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -636,10 +636,11 @@ def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, ex ), ] -def _user_file_to_mtree(f, dir_expander): +def _user_file_to_mtree(f, dir_expander, source_owned_paths): if f.is_directory: return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] - return _file_to_mtree_entry(f, "0644") + mode = "0755" if f.path in source_owned_paths else "0644" + return _file_to_mtree_entry(f, mode) def _should_skip_pkg_path(p): return ( @@ -742,7 +743,7 @@ _platform_cfg = transition( outputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier"], ) -def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, compress, level, reqs, mnemonic, progress_msg): +def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, compress, level, reqs, mnemonic, progress_msg, symlink_mappings = None): # mtree (param file) → gawk (readlinks `type=link`/`type=file content=` # rows; `contents=` rows pass through; END buffers, asort-sorts, and # writes the sorted mtree to a file) → bsdtar consumes the file. @@ -759,12 +760,29 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, gawk_args = ctx.actions.args() gawk_args.add("-v", sorted_mtree, format = "outfile=%s") + gawk_args.add("-v", "1", format = "source_argind=%s") gawk_args.add("-f", awk_script) + gawk_arguments = [gawk_args, mtree_args] + gawk_inputs = [files_depset] + if symlink_mappings != None: + mapping_args = ctx.actions.args() + mapping_args.set_param_file_format("multiline") + mapping_args.use_param_file("%s", use_always = True) + mapping_args.add("#mtree") + mapping_args.add_all( + symlink_mappings.files, + map_each = symlink_mappings.map_each, + expand_directories = False, + allow_closure = True, + ) + gawk_arguments.append(mapping_args) + if symlink_mappings.tree_files: + gawk_inputs.append(depset(direct = symlink_mappings.tree_files)) ctx.actions.run( executable = awk, - inputs = depset(direct = [awk_script], transitive = [files_depset]), + inputs = depset(direct = [awk_script], transitive = gawk_inputs), outputs = [sorted_mtree], - arguments = [gawk_args, mtree_args], + arguments = gawk_arguments, # LC_ALL=C makes gawk's asort byte-stable. env = {"LC_ALL": "C"}, mnemonic = mnemonic + "Mtree", @@ -793,7 +811,7 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, use_default_shell_env = False, ) -def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, map_each, progress): +def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, map_each, progress, symlink_mappings = None): tar_out = ctx.actions.declare_file(out_name) level = ctx.attr.group_compress_levels.get(group_name, "6") reqs = _parse_exec_requirements(ctx.attr.group_execution_requirements.get(group_name, [])) @@ -809,6 +827,7 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m reqs, "PyImageLayer", progress, + symlink_mappings, ) return tar_out @@ -889,6 +908,8 @@ def _py_image_layer_impl(ctx): rule_group_names = {gname: True for gname in ctx.attr.groups.values()} rule_group_files = [] rule_group_paths = {} + rule_group_tree_files = [] + rule_groups = [] for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) if dep_label in pip_labels: @@ -897,6 +918,25 @@ def _py_image_layer_impl(ctx): rule_group_files.append(files) for f in files.to_list(): rule_group_paths[f.path] = True + if f.is_directory: + rule_group_tree_files.append(f) + rule_groups.append((group_name, files)) + + source_files = depset(transitive = [info.source_files for info in infos]) + source_owned_group_paths = {} + if rule_group_paths: + # The aspect cannot see rule-level groups, so decide ownership from the + # source closure once, then remove grouped bytes from the source tar. + ungrouped_source_files = [] + for f in source_files.to_list(): + if f.path in rule_group_paths: + source_owned_group_paths[f.path] = True + else: + ungrouped_source_files.append(f) + source_files = depset(direct = ungrouped_source_files) + + rule_group_map = lambda f, d: _user_file_to_mtree(f, d, source_owned_group_paths) + for group_name, files in rule_groups: tar_out = _declare_group_tar( ctx, bsdtar, @@ -904,7 +944,7 @@ def _py_image_layer_impl(ctx): "{}_{}.tar.gz".format(ctx.attr.name, group_name), group_name, files, - _user_file_to_mtree, + rule_group_map, "Creating image layer %s[%s]" % (ctx.label, group_name), ) all_tars.append(tar_out) @@ -1011,19 +1051,13 @@ def _py_image_layer_impl(ctx): # snapshot here to avoid double-bookkeeping during construction. dep_tars = list(all_tars) - source_files = depset(transitive = [info.source_files for info in infos]) - if rule_group_paths: - # Rule groups are unavailable to the binary aspect, so its source - # closure still contains these files. Subtract them here, where the - # group tars are declared, and keep action inputs aligned with contents. - source_files = depset(direct = [ - f - for f in source_files.to_list() - if f.path not in rule_group_paths - ]) - - # Keep the complete source closure in one tar. `modify_mtree.awk` can only - # rewrite a Bazel-tree symlink when its target row is in the same mtree. + symlink_mappings = None + if rule_group_files: + symlink_mappings = struct( + files = depset(transitive = rule_group_files), + map_each = rule_group_map, + tree_files = rule_group_tree_files, + ) source_tar = _declare_group_tar( ctx, bsdtar, @@ -1033,6 +1067,7 @@ def _py_image_layer_impl(ctx): source_files, source_map, "Creating source layer for %s" % ctx.label, + symlink_mappings, ) all_tars.append(source_tar) @@ -1066,7 +1101,7 @@ def _py_image_layer_impl(ctx): mtree_args.add("#mtree") mtree_args.add_all(source_files, map_each = source_map, expand_directories = False, allow_closure = True) for files in rule_group_files: - mtree_args.add_all(files, map_each = _user_file_to_mtree, expand_directories = False) + mtree_args.add_all(files, map_each = rule_group_map, expand_directories = False, allow_closure = True) for files in wheel_files: mtree_args.add_all(files, map_each = _pkg_file_to_mtree, expand_directories = False) for files in interpreter_files: diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index d0afe5cd8..277028aa1 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -20,6 +20,7 @@ import filecmp import glob import os +import stat import sys from typing import Dict, Iterable, List, Optional, Sequence, Tuple @@ -231,6 +232,39 @@ def _add_subpath_or_whole( _add_whole_promotion(suggestions, label, size_mb, is_binary, annotation="whole package") +def _decode_mtree_path(path: str) -> str: + return path.replace("\\040", " ") + + +def _comparable_file(source_kind: str, encoded_source: str) -> Optional[str]: + source = _decode_mtree_path(encoded_source) + if source_kind == "contents": + return source if os.path.isfile(source) else None + if source_kind != "content": + return None + + try: + mode = os.lstat(source).st_mode + if stat.S_ISREG(mode): + return source + if not stat.S_ISLNK(mode): + return None + + # Bazel sandboxes expose action inputs through an absolute symlink whose + # suffix repeats the exec path. Peel only that synthetic hop: another + # symlink at the target is the logical artifact that modify_mtree.awk + # preserves and therefore is not comparable as a regular file. + sandbox_target = os.readlink(source) + suffix = os.sep + source + if not os.path.isabs(sandbox_target) or not sandbox_target.endswith(suffix): + return None + if stat.S_ISLNK(os.lstat(sandbox_target).st_mode): + return None + return sandbox_target if stat.S_ISREG(os.lstat(sandbox_target).st_mode) else None + except OSError: + return None + + def _mtree_collision(rows: Iterable[str]) -> Optional[str]: """Return the first conflicting expanded mtree destination, or None.""" paths: Dict[str, Tuple[str, str, str, Tuple[str, ...]]] = {} @@ -271,12 +305,14 @@ def _mtree_collision(rows: Iterable[str]) -> Optional[str]: previous_type, previous_kind, previous_source, previous_metadata = previous if ( entry_type == previous_type == "file" - and source_kind == previous_kind == "contents" and metadata == previous_metadata ): - with contextlib.suppress(OSError): - if filecmp.cmp(previous_source, source, shallow=False): - continue + previous_file = _comparable_file(previous_kind, previous_source) + source_file = _comparable_file(source_kind, source) + if previous_file is not None and source_file is not None: + with contextlib.suppress(OSError): + if filecmp.cmp(previous_file, source_file, shallow=False): + continue return "py_image_layer runfile collision at {}: {} and {}".format( destination, previous_source, source ) diff --git a/py/private/py_image_layer_validator_test.py b/py/private/py_image_layer_validator_test.py index fb51a32d7..b8a7d6a44 100644 --- a/py/private/py_image_layer_validator_test.py +++ b/py/private/py_image_layer_validator_test.py @@ -22,12 +22,12 @@ ) -def _mtree_row(destination: str, source: str) -> str: - return "{} type=file mode=0755 contents={}".format(destination, source) +def _mtree_row(destination: str, source: str, marker: str = "contents") -> str: + return "{} type=file mode=0755 {}={}".format(destination, marker, source.replace(" ", "\\040")) -def test_mtree_identical_source_coalesces() -> None: - row = _mtree_row("./app.runfiles/pkg/module.py", "same/module.py") +def test_mtree_identical_link_source_coalesces() -> None: + row = "./app.runfiles/pkg/module.py type=link mode=0755 link=same/module.py" assert _mtree_collision(["#mtree", row, row]) is None @@ -40,7 +40,7 @@ def test_mtree_conflicting_source_fails() -> None: assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision -def test_mtree_identical_file_contents_coalesce() -> None: +def test_mtree_identical_regular_files_coalesce_across_markers() -> None: with tempfile.TemporaryDirectory() as directory: first = os.path.join(directory, "first") second = os.path.join(directory, "second") @@ -48,12 +48,96 @@ def test_mtree_identical_file_contents_coalesce() -> None: with open(path, "w") as file: file.write("identical") + for first_marker, second_marker in [ + ("contents", "contents"), + ("content", "content"), + ("content", "contents"), + ]: + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", first, first_marker), + _mtree_row("./app.runfiles/pkg/module.py", second, second_marker), + ]) is None, (first_marker, second_marker) + + +def test_mtree_identical_file_contents_with_spaces_coalesce() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first file") + second = os.path.join(directory, "second file") + for path in (first, second): + with open(path, "w") as file: + file.write("identical") + assert _mtree_collision([ _mtree_row("./app.runfiles/pkg/module.py", first), _mtree_row("./app.runfiles/pkg/module.py", second), ]) is None +def test_mtree_sandbox_wrapped_regular_content_coalesces() -> None: + with ( + tempfile.TemporaryDirectory(dir=".") as wrapper_directory, + tempfile.TemporaryDirectory() as target_root, + ): + wrapper_directory = os.path.relpath(wrapper_directory) + source = os.path.join(wrapper_directory, "wrapped") + sandbox_target = os.path.join(target_root, source) + os.makedirs(os.path.dirname(sandbox_target), exist_ok=True) + with open(sandbox_target, "w") as file: + file.write("identical") + + os.symlink(sandbox_target, source) + direct = os.path.join(wrapper_directory, "direct") + with open(direct, "w") as file: + file.write("identical") + + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", source, "content"), + _mtree_row("./app.runfiles/pkg/module.py", direct), + ]) is None + + +def test_mtree_sandbox_wrapped_logical_symlink_fails() -> None: + with ( + tempfile.TemporaryDirectory(dir=".") as wrapper_directory, + tempfile.TemporaryDirectory() as target_root, + ): + wrapper_directory = os.path.relpath(wrapper_directory) + source = os.path.join(wrapper_directory, "wrapped") + sandbox_target = os.path.join(target_root, source) + os.makedirs(os.path.dirname(sandbox_target), exist_ok=True) + actual = os.path.join(os.path.dirname(sandbox_target), "actual") + with open(actual, "w") as file: + file.write("identical") + os.symlink("actual", sandbox_target) + os.symlink(sandbox_target, source) + direct = os.path.join(wrapper_directory, "direct") + with open(direct, "w") as file: + file.write("identical") + + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", source, "content"), + _mtree_row("./app.runfiles/pkg/module.py", direct), + ]) + assert collision is not None + + +def test_mtree_logical_content_symlink_fails() -> None: + with tempfile.TemporaryDirectory() as directory: + target = os.path.join(directory, "target") + link = os.path.join(directory, "link") + direct = os.path.join(directory, "direct") + for path in (target, direct): + with open(path, "w") as file: + file.write("identical") + os.symlink("target", link) + + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", link, "content"), + _mtree_row("./app.runfiles/pkg/module.py", direct), + ]) + assert collision is not None + + def test_mtree_different_file_contents_fail() -> None: with tempfile.TemporaryDirectory() as directory: first = os.path.join(directory, "first") From 887d959ae5b0e6e2a12adc3db8c02f7d16935a9a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 01:10:02 -0400 Subject: [PATCH 31/40] fix(py): preserve multi-binary runfiles A multi-binary layer shares one runfiles tree across every relocated launcher. Keep grouped launchers at their relocated destinations, merge each launcher repository mapping, and normalize external and custom-root runfiles into that tree. Compare configured regular files and authored relative symlinks by their final content so duplicate configured artifacts do not become false collisions. --- .../image_layer_analysis_tests.bzl | 101 +++++++++++++++++- e2e/cases/oci/test.sh | 8 ++ py/private/BUILD.bazel | 1 + py/private/merge_repo_mappings.awk | 34 ++++++ py/private/py_image_layer.bzl | 79 +++++++++++--- py/private/py_image_layer_validator.py | 60 +++++++---- py/private/py_image_layer_validator_test.py | 65 +++++++++++ py/tests/internal-deps/adder/BUILD.bazel | 18 +++- .../internal-deps/adder/external_launcher.py | 4 + 9 files changed, 324 insertions(+), 46 deletions(-) create mode 100644 py/private/merge_repo_mappings.awk create mode 100644 py/tests/internal-deps/adder/external_launcher.py diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index c5564295c..90bb92e23 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -25,6 +25,16 @@ _target_file_symlink = rule( attrs = {"target": attr.label(allow_single_file = True, mandatory = True)}, ) +def _relative_symlink_impl(ctx): + output = ctx.actions.declare_symlink(ctx.label.name) + ctx.actions.symlink(output = output, target_path = ctx.attr.target_path) + return [DefaultInfo(files = depset([output]))] + +_relative_symlink = rule( + implementation = _relative_symlink_impl, + attrs = {"target_path": attr.string(mandatory = True)}, +) + def _image_layer_failure(name, expected_error, **kwargs): target = "_{}_layers".format(name) py_image_layer(name = target, **kwargs) @@ -84,6 +94,80 @@ def image_layer_analysis_test_suite(): launcher_dir = "/app/bin", ) + native.genrule( + name = "_repo_mapping_shared_target", + outs = ["repo_mapping/shared.txt"], + cmd = "printf shared-link-ok > $@", + ) + _relative_symlink( + name = "_repo_mapping_shared_link", + target_path = "repo_mapping/shared.txt", + ) + py_binary( + name = "_repo_mapping_images_bin", + srcs = ["server.py"], + data = [ + ":_repo_mapping_shared_link", + ":_repo_mapping_shared_target", + ], + dep_group = "images", + python_version = "3.11", + deps = ["@pypi_oci_py_image_layer//colorama"], + ) + py_binary( + name = "_repo_mapping_venv_bin", + srcs = ["server.py"], + data = [ + ":_repo_mapping_shared_link", + ":_repo_mapping_shared_target", + ], + dep_group = "venv_images", + python_version = "3.12", + deps = ["@pypi_oci_py_venv_image_layer//colorama"], + ) + py_layer_tier( + name = "_repo_mapping_tier", + root = "/srv", + ) + py_image_layer( + name = "_repo_mapping_layers", + binaries = [ + ":_repo_mapping_images_bin", + ":_repo_mapping_venv_bin", + "@aspect_rules_py//py/tests/internal-deps/adder:external_launcher", + ], + launcher_dir = "/app/bin", + layer_tier = ":_repo_mapping_tier", + ) + native.genrule( + name = "_repo_mapping_runtime_test", + srcs = [":_repo_mapping_layers"], + outs = ["_repo_mapping_runtime_test.ok"], + cmd = """ +set -eu +root="$(@D)/_repo_mapping_runtime_test.root" +mkdir -p "$$root" +for archive in $(SRCS); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$root" +done +mapping="$$root/app.runfiles/_repo_mapping" +count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app.runfiles\\/_repo_mapping$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +grep -Fq ',whl_install__images__colorama__0_4_6,' "$$mapping" +grep -Fq ',whl_install__venv_images__colorama__0_4_6,' "$$mapping" +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_repo_mapping_images_bin" > "$$root/images.out" +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_repo_mapping_venv_bin" > "$$root/venv.out" +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/external_launcher" > "$$root/external.out" +test "$$(cat "$$root/images.out")" = "server ok" +test "$$(cat "$$root/venv.out")" = "server ok" +test "$$(cat "$$root/external.out")" = "external 5" +test -L "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_shared_link" +test "$$(cat "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_shared_link")" = "shared-link-ok" +touch $@ +""", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + native.genrule( name = "_grouped_tool", outs = ["grouped/tool.sh"], @@ -127,12 +211,18 @@ def image_layer_analysis_test_suite(): ":_grouped_source_bin", ":my_app_bin", ], - groups = {":_grouped_assets": "assets"}, + groups = { + ":_grouped_assets": "assets", + ":_grouped_source_bin": "launcher", + }, launcher_dir = "/app/bin", ) native.genrule( name = "_grouped_source_runtime_test", - srcs = [":_grouped_source_layers"], + srcs = [ + ":_grouped_source_layers_no_src", + ":_grouped_source_layers_only_src", + ], outs = ["_grouped_source_runtime_test.ok"], cmd = """ set -eu @@ -145,8 +235,15 @@ prefix="$$root/app.runfiles/_main/oci/py_image_layer" test "$$("$$prefix/grouped/tool.sh")" = grouped-ok test ! -x "$$prefix/grouped/ordinary.txt" test "$$(cat "$$prefix/_grouped_payload_link")" = grouped-payload +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" +test "$$(cat "$$root/launcher.out")" = "server ok" count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/payload.txt$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_source_layers_no_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app\\/bin\\/_grouped_source_bin$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +if for archive in $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | grep -q '/app/bin/_grouped_source_bin$$'; then + exit 1 +fi touch $@ """, toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index bcc97bd2c..ea33ebe03 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -71,6 +71,14 @@ fi echo "PASS: explicitly grouped sources preserve modes and symlink targets without duplicate bytes" +echo "== multi-binary repository mappings must be merged ==" +if ! "$BAZEL" build -- "${PKG}:_repo_mapping_runtime_test" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected repository mappings from both binaries to survive extraction" +fi + +echo "PASS: multi-binary repository mappings are merged" + echo "== source closures must preserve scalar and shared layouts ==" if ! "$BAZEL" build -- \ "${PKG}:_scalar_default_sources_listing" \ diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 1fd5eb05e..c66a29305 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -10,6 +10,7 @@ exports_files( "pytest_main.py", # Template read by the py_unittest_test rule. "unittest_main.py", + "merge_repo_mappings.awk", "modify_mtree.awk", ], visibility = ["//visibility:public"], diff --git a/py/private/merge_repo_mappings.awk b/py/private/merge_repo_mappings.awk new file mode 100644 index 000000000..0ea9a502f --- /dev/null +++ b/py/private/merge_repo_mappings.awk @@ -0,0 +1,34 @@ +# Merge Bazel's three-column repository mappings without silently choosing one +# target for an apparent repository name. +BEGIN { + FS = "," +} + +NF != 3 { + printf "invalid repository mapping at %s:%d: expected 3 columns, got %d\n", FILENAME, FNR, NF > "/dev/stderr" + failed = 1 + exit 1 +} + +{ + key = $1 SUBSEP $2 + if (key in mappings && mappings[key] != $3) { + printf "conflicting repository mapping for %s,%s: %s and %s\n", $1, $2, mappings[key], $3 > "/dev/stderr" + failed = 1 + exit 1 + } + mappings[key] = $3 + rows[$0] = 1 +} + +END { + if (failed) { + exit 1 + } + printf "" > output + count = asorti(rows, sorted) + for (i = 1; i <= count; i++) { + print sorted[i] >> output + } + close(output) +} diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index ad7886d2e..d09dddd73 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -450,17 +450,6 @@ def _layer_aspect_impl(target, ctx): if filtered: own_source.append(depset(direct = filtered)) - # bzlmod-canonical → apparent repo-name translation manifest. - # `runfiles.bash`/`rlocation` need this at `.runfiles/_repo_mapping` - # to resolve apparent labels (e.g. `aspect_rules_py/...`) to the canonical - # repo names (`aspect_rules_py+/...`) under which files are actually staged. - # Without it the launcher fails with "runfiles.bash initializer cannot find - # bazel_tools/tools/bash/runfiles/runfiles.bash" the moment any rlocation - # call is evaluated. - repo_mapping = getattr(target[DefaultInfo].default_runfiles, "repo_mapping_manifest", None) - if repo_mapping != None: - own_source.append(depset(direct = [repo_mapping])) - return [_LayerInfo( source_files = depset(transitive = transitive_source + own_source), pip_packages = depset(transitive = transitive_pkgs), @@ -578,8 +567,6 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): # Bazel synthesizes a top-level `_repo_mapping` runfile (no `_main/` # prefix); replicate that placement so runfiles.bash can find it. return "./app.runfiles/_repo_mapping" - if sp.startswith("../"): - return "./app.runfiles/" + sp[3:] runfiles_prefix = None for executable_short_path in executable_dsts: if (sp.startswith(executable_short_path + ".runfiles/") or @@ -587,7 +574,10 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): if runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix): runfiles_prefix = executable_short_path if runfiles_prefix != None: - return _apply_strip_prefix(sp, runfiles_prefix, root) + runfiles_root = "/app" if executable_dsts[runfiles_prefix] else root + return _apply_strip_prefix(sp, runfiles_prefix, runfiles_root) + if sp.startswith("../"): + return "./app.runfiles/" + sp[3:] if strip_prefix: destination = _apply_strip_prefix(sp, strip_prefix, root) if destination != "./app.runfiles/_main/" + sp: @@ -616,7 +606,23 @@ def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_ f.path.replace(" ", "\\040"), ) -def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink, executable_dsts, runfile_executable_paths): +def _source_file_to_mtree( + f, + dir_expander, + strip_prefix, + root, + maybe_symlink, + executable_dsts, + runfile_executable_paths, + repo_mapping_path): + if f.path == repo_mapping_path: + return _file_to_mtree_entry( + f, + "0755", + f.short_path, + "/app.runfiles/_repo_mapping", + ) + # 0755 throughout: keeps launcher/interpreter/venv shims executable; Bazel # doesn't expose per-input source mode for us to propagate. if f.is_directory: @@ -865,9 +871,14 @@ def _py_image_layer_impl(ctx): launcher_names = {} executable_dsts = {} executable_owner_by_path = {} + repo_mappings_by_path = {} for index, binary in enumerate(binaries): - executable = binary[DefaultInfo].files_to_run.executable + files_to_run = binary[DefaultInfo].files_to_run + executable = files_to_run.executable executable_owner_by_path[executable.path] = index + repo_mapping = files_to_run.repo_mapping_manifest + if repo_mapping != None: + repo_mappings_by_path[repo_mapping.path] = repo_mapping launcher_name = executable.basename if launcher_dir: if launcher_name in launcher_names: @@ -889,6 +900,31 @@ def _py_image_layer_impl(ctx): if owner != None and owner != index: runfile_executable_paths[f.path] = True + # Each manifest describes one launcher's runfiles closure. The image shares + # one runfiles root, so its manifest must resolve apparent names from all of + # the explicitly listed launchers. + repo_mappings = repo_mappings_by_path.values() + repo_mapping = None + if len(repo_mappings) == 1: + repo_mapping = repo_mappings[0] + elif len(repo_mappings) > 1: + repo_mapping = ctx.actions.declare_file(ctx.attr.name + "/_repo_mapping") + args = ctx.actions.args() + args.add("-v") + args.add("output=" + repo_mapping.path) + args.add("-f") + args.add(ctx.file._repo_mapping_merger) + args.add_all(repo_mappings) + ctx.actions.run( + executable = ctx.executable._awk, + inputs = depset(direct = [ctx.file._repo_mapping_merger] + repo_mappings), + outputs = [repo_mapping], + arguments = [args], + env = {"LC_ALL": "C"}, + mnemonic = "PyImageRepoMapping", + progress_message = "Merging repository mappings for %s" % ctx.label, + ) + # 3p pip layers are action-shared across the graph and hard-code their # destination under `./app.runfiles//...`, so the consumer's source # layer has to land under the same `/app.runfiles/` tree for each launcher @@ -903,6 +939,7 @@ def _py_image_layer_impl(ctx): source_maybe_symlink, executable_dsts, runfile_executable_paths, + repo_mapping.path if repo_mapping != None else "", ) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} @@ -923,6 +960,8 @@ def _py_image_layer_impl(ctx): rule_groups.append((group_name, files)) source_files = depset(transitive = [info.source_files for info in infos]) + if repo_mapping != None: + source_files = depset(direct = [repo_mapping], transitive = [source_files]) source_owned_group_paths = {} if rule_group_paths: # The aspect cannot see rule-level groups, so decide ownership from the @@ -935,7 +974,9 @@ def _py_image_layer_impl(ctx): ungrouped_source_files.append(f) source_files = depset(direct = ungrouped_source_files) - rule_group_map = lambda f, d: _user_file_to_mtree(f, d, source_owned_group_paths) + rule_group_map = lambda f, d: ( + source_map(f, d) if f.short_path in executable_dsts else _user_file_to_mtree(f, d, source_owned_group_paths) + ) for group_name, files in rule_groups: tar_out = _declare_group_tar( ctx, @@ -1157,6 +1198,10 @@ _py_image_layer = rule( cfg = "exec", allow_files = True, ), + "_repo_mapping_merger": attr.label( + default = "//py/private:merge_repo_mappings.awk", + allow_single_file = True, + ), "_awk_script": attr.label( default = "//py/private:modify_mtree.awk", allow_single_file = True, diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index 277028aa1..609e5d8ec 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -236,6 +236,18 @@ def _decode_mtree_path(path: str) -> str: return path.replace("\\040", " ") +def _peel_sandbox_symlink(source: str) -> str: + # Bazel's absolute action-input symlink repeats the exec path as a suffix. + # Peel only that transport hop; a relative target is the logical artifact. + if not stat.S_ISLNK(os.lstat(source).st_mode): + return source + target = os.readlink(source) + suffix = os.sep + source + if os.path.isabs(target) and target.endswith(suffix): + return target + return source + + def _comparable_file(source_kind: str, encoded_source: str) -> Optional[str]: source = _decode_mtree_path(encoded_source) if source_kind == "contents": @@ -244,23 +256,25 @@ def _comparable_file(source_kind: str, encoded_source: str) -> Optional[str]: return None try: + source = _peel_sandbox_symlink(source) mode = os.lstat(source).st_mode if stat.S_ISREG(mode): return source - if not stat.S_ISLNK(mode): - return None + return None + except OSError: + return None - # Bazel sandboxes expose action inputs through an absolute symlink whose - # suffix repeats the exec path. Peel only that synthetic hop: another - # symlink at the target is the logical artifact that modify_mtree.awk - # preserves and therefore is not comparable as a regular file. - sandbox_target = os.readlink(source) - suffix = os.sep + source - if not os.path.isabs(sandbox_target) or not sandbox_target.endswith(suffix): - return None - if stat.S_ISLNK(os.lstat(sandbox_target).st_mode): + +def _comparable_symlink_target(source_kind: str, encoded_source: str) -> Optional[str]: + if source_kind not in ("content", "link"): + return None + source = _decode_mtree_path(encoded_source) + try: + source = _peel_sandbox_symlink(source) + if not stat.S_ISLNK(os.lstat(source).st_mode): return None - return sandbox_target if stat.S_ISREG(os.lstat(sandbox_target).st_mode) else None + target = os.readlink(source) + return target if target and not os.path.isabs(target) else None except OSError: return None @@ -303,16 +317,18 @@ def _mtree_collision(rows: Iterable[str]) -> Optional[str]: if previous == entry: continue previous_type, previous_kind, previous_source, previous_metadata = previous - if ( - entry_type == previous_type == "file" - and metadata == previous_metadata - ): - previous_file = _comparable_file(previous_kind, previous_source) - source_file = _comparable_file(source_kind, source) - if previous_file is not None and source_file is not None: - with contextlib.suppress(OSError): - if filecmp.cmp(previous_file, source_file, shallow=False): - continue + if metadata == previous_metadata: + if entry_type == previous_type == "file": + previous_file = _comparable_file(previous_kind, previous_source) + source_file = _comparable_file(source_kind, source) + if previous_file is not None and source_file is not None: + with contextlib.suppress(OSError): + if filecmp.cmp(previous_file, source_file, shallow=False): + continue + previous_target = _comparable_symlink_target(previous_kind, previous_source) + source_target = _comparable_symlink_target(source_kind, source) + if previous_target is not None and previous_target == source_target: + continue return "py_image_layer runfile collision at {}: {} and {}".format( destination, previous_source, source ) diff --git a/py/private/py_image_layer_validator_test.py b/py/private/py_image_layer_validator_test.py index b8a7d6a44..f45cedb86 100644 --- a/py/private/py_image_layer_validator_test.py +++ b/py/private/py_image_layer_validator_test.py @@ -26,11 +26,76 @@ def _mtree_row(destination: str, source: str, marker: str = "contents") -> str: return "{} type=file mode=0755 {}={}".format(destination, marker, source.replace(" ", "\\040")) +def _mtree_link_row(destination: str, source: str, mode: str = "0755") -> str: + return "{} type=link mode={} link={}".format(destination, mode, source.replace(" ", "\\040")) + + def test_mtree_identical_link_source_coalesces() -> None: row = "./app.runfiles/pkg/module.py type=link mode=0755 link=same/module.py" assert _mtree_collision(["#mtree", row, row]) is None +def test_mtree_identical_relative_link_targets_coalesce() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + os.symlink("../target", first) + os.symlink("../target", second) + + assert _mtree_collision([ + _mtree_link_row("./app.runfiles/pkg/module.py", first), + _mtree_link_row("./app.runfiles/pkg/module.py", second), + ]) is None + + +def test_mtree_different_relative_link_targets_fail() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + os.symlink("../first", first) + os.symlink("../second", second) + + assert _mtree_collision([ + _mtree_link_row("./app.runfiles/pkg/module.py", first), + _mtree_link_row("./app.runfiles/pkg/module.py", second), + ]) is not None + + +def test_mtree_identical_relative_link_targets_with_different_modes_fail() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + os.symlink("../target", first) + os.symlink("../target", second) + + assert _mtree_collision([ + _mtree_link_row("./app.runfiles/pkg/module.py", first), + _mtree_link_row("./app.runfiles/pkg/module.py", second, "0644"), + ]) is not None + + +def test_mtree_sandbox_wrapped_logical_symlinks_coalesce() -> None: + with ( + tempfile.TemporaryDirectory(dir=".") as wrapper_directory, + tempfile.TemporaryDirectory() as target_root, + ): + wrapper_directory = os.path.relpath(wrapper_directory) + sources = [ + os.path.join(wrapper_directory, "first"), + os.path.join(wrapper_directory, "second"), + ] + for source in sources: + sandbox_target = os.path.join(target_root, source) + os.makedirs(os.path.dirname(sandbox_target), exist_ok=True) + os.symlink("../target", sandbox_target) + os.symlink(sandbox_target, source) + + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", source, "content") + for source in sources + ]) is None + + def test_mtree_conflicting_source_fails() -> None: collision = _mtree_collision([ _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), diff --git a/py/tests/internal-deps/adder/BUILD.bazel b/py/tests/internal-deps/adder/BUILD.bazel index cf12eff91..1593f6ba2 100644 --- a/py/tests/internal-deps/adder/BUILD.bazel +++ b/py/tests/internal-deps/adder/BUILD.bazel @@ -1,9 +1,10 @@ -"""Deliberate e2e fixture: a py_library from the @aspect_rules_py module consumed by the -OCI e2e cases (//cases/oci/py_image_layer) to verify that py_image_layer correctly -includes cross-module first-party deps. Do not inline into the e2e folder — the point -is to exercise the external-module runfiles path (aspect_rules_py+/...).""" +"""Cross-module first-party fixtures for OCI image-layer tests. -load("//py:defs.bzl", "py_library") +The library covers external source dependencies; the launcher covers external +runfiles paths. Keep both in the @aspect_rules_py module. +""" + +load("//py:defs.bzl", "py_binary", "py_library") py_library( name = "adder", @@ -14,3 +15,10 @@ py_library( imports = [".."], visibility = ["//visibility:public"], ) + +py_binary( + name = "external_launcher", + srcs = ["external_launcher.py"], + visibility = ["//visibility:public"], + deps = [":adder"], +) diff --git a/py/tests/internal-deps/adder/external_launcher.py b/py/tests/internal-deps/adder/external_launcher.py new file mode 100644 index 000000000..d3cfbb85e --- /dev/null +++ b/py/tests/internal-deps/adder/external_launcher.py @@ -0,0 +1,4 @@ +from adder.add import add + +if __name__ == "__main__": + print("external {}".format(add(2, 3))) From cd83dc4c1e7e084abf0bebac07a237450920bb79 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 01:43:54 -0400 Subject: [PATCH 32/40] fix(py): stage scalar repository mappings --- .../snapshots/amd64_layers_listing.yaml | 1 + .../image_layer_analysis_tests.bzl | 21 +++++++++++++++++++ .../snapshots/my_app_layers_fp_listing.yaml | 1 + .../snapshots/my_app_layers_listing.yaml | 1 + .../my_app_layers_multi_listing.yaml | 1 + .../my_app_amd64_layers_listing.yaml | 1 + .../my_app_arm64_layers_listing.yaml | 1 + e2e/cases/oci/test.sh | 8 +++++++ .../snapshots/app_amd64_layers_listing.yaml | 1 + .../snapshots/app_arm64_layers_listing.yaml | 1 + py/private/py_image_layer.bzl | 2 ++ py/tests/internal-deps/adder/BUILD.bazel | 5 ++++- .../internal-deps/adder/external_launcher.py | 6 ++++++ 13 files changed, 49 insertions(+), 1 deletion(-) diff --git a/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml b/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml index a0cf8019f..058b61135 100644 --- a/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml +++ b/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml @@ -12,6 +12,7 @@ files: - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/interpreter-features-tkinter/._check_no_tkinter.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 495 Jan 1 2023 ./app.runfiles/_main/interpreter-features-tkinter/__main__.py - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 90bb92e23..848cbf384 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -129,6 +129,27 @@ def image_layer_analysis_test_suite(): name = "_repo_mapping_tier", root = "/srv", ) + py_image_layer( + name = "_external_scalar_layers", + binary = "@aspect_rules_py//py/tests/internal-deps/adder:external_launcher", + ) + native.genrule( + name = "_external_scalar_runtime_test", + srcs = [":_external_scalar_layers"], + outs = ["_external_scalar_runtime_test.ok"], + cmd = """ +set -eu +root="$(@D)/_external_scalar_runtime_test.root" +mkdir -p "$$root" +for archive in $(SRCS); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$root" +done +RUNFILES_DIR="$$root/app.runfiles" "$$root/app" > "$$root/external.out" +test "$$(cat "$$root/external.out")" = "external 5" +touch $@ +""", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) py_image_layer( name = "_repo_mapping_layers", binaries = [ diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml index 567df3d33..9e05ec5c0 100644 --- a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml @@ -2360,5 +2360,6 @@ files: - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__main__.py - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - -rwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__init__.py - -rwxr-xr-x 0 0 0 32 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/add.py diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml index 80c3953a6..860507f33 100644 --- a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml @@ -34,6 +34,7 @@ files: - -rwxr-xr-x 0 0 0 42 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__init__.py - -rwxr-xr-x 0 0 0 31 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/palette.txt - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml index 893e071fa..f99b82fb1 100644 --- a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml @@ -40,6 +40,7 @@ files: - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_multi_bin.venv/lib/python3.11/site-packages/pyproject_hooks-1.2.0.dist-info -> ../../../../../../../aspect_rules_py++uv+whl_install__images__pyproject_hooks__1_2_0/actual_install.install/lib/python3.11/site-packages/pyproject_hooks-1.2.0.dist-info - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_multi_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__main__.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml index d52d08f52..30b4b927f 100644 --- a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml +++ b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml @@ -35,6 +35,7 @@ files: - -rwxr-xr-x 0 0 0 463 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__main__.py - -rwxr-xr-x 0 0 0 1272 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/my_app_bin.venv - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml index e195be493..96cc9162a 100644 --- a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml +++ b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml @@ -35,6 +35,7 @@ files: - -rwxr-xr-x 0 0 0 463 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__main__.py - -rwxr-xr-x 0 0 0 1272 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/my_app_bin.venv - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_aarch64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_aarch64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_aarch64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index ea33ebe03..8169b992c 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -79,6 +79,14 @@ fi echo "PASS: multi-binary repository mappings are merged" +echo "== scalar external launchers must resolve apparent repository names ==" +if ! "$BAZEL" build -- "${PKG}:_external_scalar_runtime_test" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the scalar external launcher to resolve its runfiles" +fi + +echo "PASS: scalar external launcher resolves its runfiles" + echo "== source closures must preserve scalar and shared layouts ==" if ! "$BAZEL" build -- \ "${PKG}:_scalar_default_sources_listing" \ diff --git a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml index 90fad5bb8..0c2160555 100644 --- a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml +++ b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml @@ -46,6 +46,7 @@ files: - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/lib/python3.12/site-packages/psycopg2_binary.libs -> ../../../../../../../aspect_rules_py++uv+whl_install__crossbuild__psycopg2_binary__2_9_11/actual_install.install/lib/python3.12/site-packages/psycopg2_binary.libs - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 66 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/__main__.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.12 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_x86_64_unknown_linux_gnu/bin/2to3-3.12 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.12 diff --git a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml index 4916b915b..7db44a67d 100644 --- a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml +++ b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml @@ -46,6 +46,7 @@ files: - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/lib/python3.12/site-packages/psycopg2_binary.libs -> ../../../../../../../aspect_rules_py++uv+whl_install__crossbuild__psycopg2_binary__2_9_11/actual_install.install/lib/python3.12/site-packages/psycopg2_binary.libs - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 66 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/__main__.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_aarch64_unknown_linux_gnu/bin/2to3 -> 2to3-3.12 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_aarch64_unknown_linux_gnu/bin/2to3-3.12 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_aarch64_unknown_linux_gnu/bin/idle3 -> idle3.12 diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index d09dddd73..02025b4ec 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -577,6 +577,8 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): runfiles_root = "/app" if executable_dsts[runfiles_prefix] else root return _apply_strip_prefix(sp, runfiles_prefix, runfiles_root) if sp.startswith("../"): + if sp == strip_prefix or sp in executable_dsts: + return _apply_strip_prefix(sp, sp, root) return "./app.runfiles/" + sp[3:] if strip_prefix: destination = _apply_strip_prefix(sp, strip_prefix, root) diff --git a/py/tests/internal-deps/adder/BUILD.bazel b/py/tests/internal-deps/adder/BUILD.bazel index 1593f6ba2..8f9a866b9 100644 --- a/py/tests/internal-deps/adder/BUILD.bazel +++ b/py/tests/internal-deps/adder/BUILD.bazel @@ -20,5 +20,8 @@ py_binary( name = "external_launcher", srcs = ["external_launcher.py"], visibility = ["//visibility:public"], - deps = [":adder"], + deps = [ + ":adder", + "@rules_python//python/runfiles", + ], ) diff --git a/py/tests/internal-deps/adder/external_launcher.py b/py/tests/internal-deps/adder/external_launcher.py index d3cfbb85e..b166b5c11 100644 --- a/py/tests/internal-deps/adder/external_launcher.py +++ b/py/tests/internal-deps/adder/external_launcher.py @@ -1,4 +1,10 @@ +import os + from adder.add import add +from python.runfiles import runfiles if __name__ == "__main__": + adder_path = runfiles.Create().Rlocation("aspect_rules_py/py/tests/internal-deps/adder/add.py") + if adder_path is None or not os.path.exists(adder_path): + raise RuntimeError("could not resolve adder through runfiles") print("external {}".format(add(2, 3))) From c0e878e411b1b196f65f747d514a3c2b34179041 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 01:53:12 -0400 Subject: [PATCH 33/40] fix(py): parse mtree mapping metadata fields --- .../oci/py_image_layer/image_layer_analysis_tests.bzl | 5 +++-- py/private/modify_mtree.awk | 11 ++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 848cbf384..c655985ad 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -197,7 +197,7 @@ touch $@ ) native.genrule( name = "_grouped_payload", - outs = ["grouped/payload.txt"], + outs = ["grouped/content=payload.txt"], cmd = "printf grouped-payload > $@", ) native.genrule( @@ -255,10 +255,11 @@ done prefix="$$root/app.runfiles/_main/oci/py_image_layer" test "$$("$$prefix/grouped/tool.sh")" = grouped-ok test ! -x "$$prefix/grouped/ordinary.txt" +test -L "$$prefix/_grouped_payload_link" test "$$(cat "$$prefix/_grouped_payload_link")" = grouped-payload RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" test "$$(cat "$$root/launcher.out")" = "server ok" -count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/payload.txt$$/ { n++ } END { print n + 0 }')" +count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/content=payload.txt$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 count="$$(for archive in $(locations :_grouped_source_layers_no_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app\\/bin\\/_grouped_source_bin$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index a2dd359d1..f1b676ad7 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -62,11 +62,12 @@ function decode_mtree_path(path) { { if (ARGIND != source_argind) { - match($0, /(contents|content|link)=[^ ]+/) - if (RSTART != 0) { - content_field = substr($0, RSTART, RLENGTH) - split(content_field, parts, "=") - symlink_map[decode_mtree_path(parts[2])] = $1 + for (field = 2; field <= NF; field++) { + if ($field ~ /^(contents|content|link)=[^ ]+$/) { + source_path = substr($field, index($field, "=") + 1) + symlink_map[decode_mtree_path(source_path)] = $1 + break + } } next } From b73ae523f9676710924b7a910784cfe65fbb4e5a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 02:05:15 -0400 Subject: [PATCH 34/40] fix(py): apply external launcher strip prefixes --- .../image_layer_analysis_tests.bzl | 35 ++++++++++++++----- py/private/py_image_layer.bzl | 6 +++- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index c655985ad..61f4630b5 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -129,23 +129,42 @@ def image_layer_analysis_test_suite(): name = "_repo_mapping_tier", root = "/srv", ) + external_launcher = Label("@aspect_rules_py//py/tests/internal-deps/adder:external_launcher") + py_layer_tier( + name = "_external_scalar_tier", + strip_prefix = "../{}/{}".format(external_launcher.workspace_name, external_launcher.package), + ) py_image_layer( name = "_external_scalar_layers", - binary = "@aspect_rules_py//py/tests/internal-deps/adder:external_launcher", + binary = external_launcher, + ) + py_image_layer( + name = "_external_scalar_stripped_layers", + binary = external_launcher, + layer_tier = ":_external_scalar_tier", ) native.genrule( name = "_external_scalar_runtime_test", - srcs = [":_external_scalar_layers"], + srcs = [ + ":_external_scalar_layers", + ":_external_scalar_stripped_layers", + ], outs = ["_external_scalar_runtime_test.ok"], cmd = """ set -eu -root="$(@D)/_external_scalar_runtime_test.root" -mkdir -p "$$root" -for archive in $(SRCS); do - $(BSDTAR_BIN) -xf "$$archive" -C "$$root" +default_root="$(@D)/_external_scalar_runtime_test.default" +stripped_root="$(@D)/_external_scalar_runtime_test.stripped" +mkdir -p "$$default_root" "$$stripped_root" +for archive in $(locations :_external_scalar_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$default_root" done -RUNFILES_DIR="$$root/app.runfiles" "$$root/app" > "$$root/external.out" -test "$$(cat "$$root/external.out")" = "external 5" +for archive in $(locations :_external_scalar_stripped_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$stripped_root" +done +RUNFILES_DIR="$$default_root/app.runfiles" "$$default_root/app" > "$$default_root/external.out" +RUNFILES_DIR="$$stripped_root/app.runfiles" "$$stripped_root/app/external_launcher" > "$$stripped_root/external.out" +test "$$(cat "$$default_root/external.out")" = "external 5" +test "$$(cat "$$stripped_root/external.out")" = "external 5" touch $@ """, toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 02025b4ec..c5d891a05 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -577,7 +577,11 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): runfiles_root = "/app" if executable_dsts[runfiles_prefix] else root return _apply_strip_prefix(sp, runfiles_prefix, runfiles_root) if sp.startswith("../"): - if sp == strip_prefix or sp in executable_dsts: + if strip_prefix and (sp in executable_dsts or sp == strip_prefix): + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination + if sp in executable_dsts: return _apply_strip_prefix(sp, sp, root) return "./app.runfiles/" + sp[3:] if strip_prefix: From 7adce7e039f5fa201be37d718daf5c93e69f914b Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 02:25:26 -0400 Subject: [PATCH 35/40] fix(py): preserve relocated launcher runfiles --- .../py_image_layer/image_layer_analysis_tests.bzl | 15 ++++++++++++--- e2e/cases/oci/test.sh | 9 ++++----- py/private/py_image_layer.bzl | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 61f4630b5..22696dcbe 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -128,6 +128,7 @@ def image_layer_analysis_test_suite(): py_layer_tier( name = "_repo_mapping_tier", root = "/srv", + strip_prefix = "oci/py_image_layer", ) external_launcher = Label("@aspect_rules_py//py/tests/internal-deps/adder:external_launcher") py_layer_tier( @@ -262,15 +263,20 @@ touch $@ srcs = [ ":_grouped_source_layers_no_src", ":_grouped_source_layers_only_src", + ":_scalar_launcher_collision_layers", ], outs = ["_grouped_source_runtime_test.ok"], cmd = """ set -eu root="$(@D)/_grouped_source_runtime_test.root" -mkdir -p "$$root" -for archive in $(SRCS); do +scalar_root="$(@D)/_grouped_source_runtime_test.scalar" +mkdir -p "$$root" "$$scalar_root" +for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -xf "$$archive" -C "$$root" done +for archive in $(locations :_scalar_launcher_collision_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$scalar_root" +done prefix="$$root/app.runfiles/_main/oci/py_image_layer" test "$$("$$prefix/grouped/tool.sh")" = grouped-ok test ! -x "$$prefix/grouped/ordinary.txt" @@ -278,7 +284,10 @@ test -L "$$prefix/_grouped_payload_link" test "$$(cat "$$prefix/_grouped_payload_link")" = grouped-payload RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" test "$$(cat "$$root/launcher.out")" = "server ok" -count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/content=payload.txt$$/ { n++ } END { print n + 0 }')" +RUNFILES_DIR="$$scalar_root/app.runfiles" "$$scalar_root/app/bin/_scalar_launcher_collision" > "$$scalar_root/launcher.out" +test "$$(cat "$$scalar_root/launcher.out")" = "server ok" +test "$$(cat "$$scalar_root/app.runfiles/_main/oci/py_image_layer/bin/_scalar_launcher_collision")" = data +count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/content=payload.txt$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 count="$$(for archive in $(locations :_grouped_source_layers_no_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app\\/bin\\/_grouped_source_bin$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 8169b992c..fc647e0e0 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -35,23 +35,22 @@ expect_listing_count() { fi } -echo "== versioned wheel children must share an image when destinations agree ==" +echo "== configured wheel and relocated scalar images must validate ==" if ! "$BAZEL" build --output_groups=_validation -- \ "${PKG}:_configured_pure_wheel_layers" \ - "${PKG}:_configured_wheel_collision_layers" >"$output_log" 2>&1; then + "${PKG}:_configured_wheel_collision_layers" \ + "${PKG}:_scalar_launcher_collision_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 - fail "expected the two-version wheel images to validate" + fail "expected configured wheel and relocated scalar images to validate" fi echo "== remapped destinations must fail validation ==" if "$BAZEL" build --keep_going --output_groups=_validation -- \ - "${PKG}:_scalar_launcher_collision_layers" \ "${PKG}:_scalar_strip_collision_layers" \ "${PKG}:_scalar_root_collision_layers" >"$output_log" 2>&1; then cat "$output_log" >&2 fail "expected remapped destinations to fail validation" fi -expect_diagnostic "py_image_layer runfile collision at ./app/bin/_scalar_launcher_collision:" expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/_scalar_strip_collision/data.txt:" expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/server.py:" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index c5d891a05..326398a77 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -584,7 +584,7 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): if sp in executable_dsts: return _apply_strip_prefix(sp, sp, root) return "./app.runfiles/" + sp[3:] - if strip_prefix: + if strip_prefix and not any(executable_dsts.values()): destination = _apply_strip_prefix(sp, strip_prefix, root) if destination != "./app.runfiles/_main/" + sp: return destination From 886959383390af2cb43dd7761c1e63dbd91df483 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 02:47:15 -0400 Subject: [PATCH 36/40] fix(py): preserve runfile and mtree paths Keep explicit scalar prefixes effective for sibling runfiles without changing relocated, external, or nonmatching layouts. Parse and replace only mtree metadata fields so marker text and equals signs in destinations cannot corrupt symlink rewriting. --- .../image_layer_analysis_tests.bzl | 36 ++++++++++---- e2e/cases/oci/test.sh | 5 +- py/private/modify_mtree.awk | 48 +++++++++++-------- py/private/py_image_layer.bzl | 4 ++ 4 files changed, 64 insertions(+), 29 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 22696dcbe..0923798f2 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -100,14 +100,14 @@ def image_layer_analysis_test_suite(): cmd = "printf shared-link-ok > $@", ) _relative_symlink( - name = "_repo_mapping_shared_link", + name = "_repo_mapping_link=shared", target_path = "repo_mapping/shared.txt", ) py_binary( name = "_repo_mapping_images_bin", srcs = ["server.py"], data = [ - ":_repo_mapping_shared_link", + ":_repo_mapping_link=shared", ":_repo_mapping_shared_target", ], dep_group = "images", @@ -118,7 +118,7 @@ def image_layer_analysis_test_suite(): name = "_repo_mapping_venv_bin", srcs = ["server.py"], data = [ - ":_repo_mapping_shared_link", + ":_repo_mapping_link=shared", ":_repo_mapping_shared_target", ], dep_group = "venv_images", @@ -202,8 +202,8 @@ RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/external_launcher" > "$$root/ test "$$(cat "$$root/images.out")" = "server ok" test "$$(cat "$$root/venv.out")" = "server ok" test "$$(cat "$$root/external.out")" = "external 5" -test -L "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_shared_link" -test "$$(cat "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_shared_link")" = "shared-link-ok" +test -L "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_link=shared" +test "$$(cat "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_link=shared")" = "shared-link-ok" touch $@ """, toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], @@ -226,7 +226,7 @@ touch $@ cmd = "printf ordinary > $@", ) _target_file_symlink( - name = "_grouped_payload_link", + name = "_grouped_content=payload_link", target = ":_grouped_payload", ) native.filegroup( @@ -242,7 +242,7 @@ touch $@ srcs = ["server.py"], data = [ ":_grouped_payload", - ":_grouped_payload_link", + ":_grouped_content=payload_link", ":_grouped_tool", ], ) @@ -280,8 +280,8 @@ done prefix="$$root/app.runfiles/_main/oci/py_image_layer" test "$$("$$prefix/grouped/tool.sh")" = grouped-ok test ! -x "$$prefix/grouped/ordinary.txt" -test -L "$$prefix/_grouped_payload_link" -test "$$(cat "$$prefix/_grouped_payload_link")" = grouped-payload +test -L "$$prefix/_grouped_content=payload_link" +test "$$(cat "$$prefix/_grouped_content=payload_link")" = grouped-payload RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" test "$$(cat "$$root/launcher.out")" = "server ok" RUNFILES_DIR="$$scalar_root/app.runfiles" "$$scalar_root/app/bin/_scalar_launcher_collision" > "$$scalar_root/launcher.out" @@ -397,6 +397,12 @@ touch $@ py_layer_tier( name = "_nested_prefix_tier", interpreter_group = "interpreter", + strip_prefix = "oci/py_image_layer/_nested_prefix", + ) + py_layer_tier( + name = "_nested_prefix_nonmatching_tier", + interpreter_group = "interpreter", + strip_prefix = "does/not/match", ) py_image_layer( name = "_nested_prefix_layers", @@ -416,11 +422,23 @@ touch $@ launcher_dir = "/app/bin", layer_tier = ":_nested_prefix_tier", ) + py_image_layer( + name = "_nested_prefix_scalar_layers", + binary = ":_nested_prefix/foo.runfiles/worker", + layer_tier = ":_nested_prefix_tier", + ) + py_image_layer( + name = "_nested_prefix_nonmatching_scalar_layers", + binary = ":_nested_prefix/foo.runfiles/worker", + layer_tier = ":_nested_prefix_nonmatching_tier", + ) native.genrule( name = "_nested_prefix_sources_listing", srcs = [ ":_nested_prefix_layers_only_src", + ":_nested_prefix_nonmatching_scalar_layers_only_src", ":_nested_prefix_reversed_layers_only_src", + ":_nested_prefix_scalar_layers_only_src", ], outs = ["_nested_prefix_sources.listing"], cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index fc647e0e0..05c837a18 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -126,7 +126,10 @@ if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then fail "expected the nested launcher source layers to build" fi listing="bazel-bin/oci/py_image_layer/_nested_prefix_sources.listing" - expect_listing_count "$listing" "/app.runfiles/_main/nested/data.txt" 2 + expect_listing_count "$listing" "/app" 1 + expect_listing_count "$listing" "/app.runfiles/_main/nested/data.txt" 3 + expect_listing_count "$listing" "/app/foo.runfiles/worker" 1 + expect_listing_count "$listing" "/app/foo.runfiles/worker.runfiles/_main/nested/data.txt" 1 if grep -Fq './app.runfiles/worker.runfiles/' "$listing"; then cat "$listing" >&2 fail "nested launcher prefix leaked into the shared runfiles layout" diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index f1b676ad7..4d5120d9b 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -60,14 +60,28 @@ function decode_mtree_path(path) { return path } +function replace_metadata_field(row, pattern, replacement) { + if (!match(row, pattern)) { + return row + } + return substr(row, 1, RSTART) replacement substr(row, RSTART + RLENGTH) +} + { + source_field = "" + source_type = "" + for (field = 2; field <= NF; field++) { + if ($field ~ /^(contents|content|link)=[^ ]+$/) { + source_field = $field + } else if ($field ~ /^type=[^ ]+$/) { + source_type = substr($field, index($field, "=") + 1) + } + } + if (ARGIND != source_argind) { - for (field = 2; field <= NF; field++) { - if ($field ~ /^(contents|content|link)=[^ ]+$/) { - source_path = substr($field, index($field, "=") + 1) - symlink_map[decode_mtree_path(source_path)] = $1 - break - } + if (source_field != "") { + source_path = substr(source_field, index(source_field, "=") + 1) + symlink_map[decode_mtree_path(source_path)] = $1 } next } @@ -81,17 +95,11 @@ function decode_mtree_path(path) { # might be symlinks Bazel didn't flag (repo-rule-staged ones like # rules_python's `bin/python -> python3.11`). Empty `readlink` means # it's a regular file and the row passes through unchanged. - is_hot_path = ($0 ~ /type=link/) && ($0 ~ /link=/) - is_slow_path = ($0 ~ /type=file/) && ($0 ~ /content=/) + is_hot_path = source_type == "link" && source_field ~ /^link=/ + is_slow_path = source_type == "file" && source_field ~ /^content=/ if (is_hot_path || is_slow_path) { - if (is_hot_path) { - match($0, /link=[^ ]+/) - } else { - match($0, /content=[^ ]+/) - } - content_field = substr($0, RSTART, RLENGTH) - split(content_field, parts, "=") - path = decode_mtree_path(parts[2]) + source_path = substr(source_field, index(source_field, "=") + 1) + path = decode_mtree_path(source_path) symlink_map[path] = $1 # Plain `readlink` first: keep its result if relative @@ -193,9 +201,11 @@ END { # Already a relative path linked_to = resolved_path } - sub(/type=[^ ]+/, "type=link", original_line) - if (!sub(/content=[^ ]+/, "link=" linked_to, original_line)) { - sub(/link=[^ ]+/, "link=" linked_to, original_line) + original_line = replace_metadata_field(original_line, "[[:space:]]type=[^ ]+", "type=link") + if (original_line ~ /[[:space:]]content=[^ ]+/) { + original_line = replace_metadata_field(original_line, "[[:space:]]content=[^ ]+", "link=" linked_to) + } else { + original_line = replace_metadata_field(original_line, "[[:space:]]link=[^ ]+", "link=" linked_to) } out_lines[++n] = original_line } else { diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 326398a77..2b8ea3dce 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -574,6 +574,10 @@ def _source_destination(sp, strip_prefix, root, executable_dsts): if runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix): runfiles_prefix = executable_short_path if runfiles_prefix != None: + if strip_prefix and not runfiles_prefix.startswith("../") and not executable_dsts[runfiles_prefix]: + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination runfiles_root = "/app" if executable_dsts[runfiles_prefix] else root return _apply_strip_prefix(sp, runfiles_prefix, runfiles_root) if sp.startswith("../"): From bed76baa4c04a5214fc9bd525cac2ac411256676 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 03:08:02 -0400 Subject: [PATCH 37/40] fix(py): resolve grouped symlinks across layers Provide each source and non-prebuilt first-party tar with mapping-only destinations from the other emitted source tiers, without duplicating payload bytes. Detect target-file symlinks in rule groups and exercise both source-to-group and group-to-source links through real extraction. --- .../image_layer_analysis_tests.bzl | 12 +++++- py/private/py_image_layer.bzl | 42 +++++++++++++------ 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 0923798f2..dcf8adb17 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -229,11 +229,15 @@ touch $@ name = "_grouped_content=payload_link", target = ":_grouped_payload", ) + _target_file_symlink( + name = "_grouped_content=asset_link", + target = ":_group_only_file", + ) native.filegroup( name = "_grouped_assets", srcs = [ ":_group_only_file", - ":_grouped_payload", + ":_grouped_content=payload_link", ":_grouped_tool", ], ) @@ -241,8 +245,8 @@ touch $@ name = "_grouped_source_bin", srcs = ["server.py"], data = [ + ":_grouped_content=asset_link", ":_grouped_payload", - ":_grouped_content=payload_link", ":_grouped_tool", ], ) @@ -282,6 +286,8 @@ test "$$("$$prefix/grouped/tool.sh")" = grouped-ok test ! -x "$$prefix/grouped/ordinary.txt" test -L "$$prefix/_grouped_content=payload_link" test "$$(cat "$$prefix/_grouped_content=payload_link")" = grouped-payload +test -L "$$prefix/_grouped_content=asset_link" +test "$$(cat "$$prefix/_grouped_content=asset_link")" = ordinary RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" test "$$(cat "$$root/launcher.out")" = "server ok" RUNFILES_DIR="$$scalar_root/app.runfiles" "$$scalar_root/app/bin/_scalar_launcher_collision" > "$$scalar_root/launcher.out" @@ -289,6 +295,8 @@ test "$$(cat "$$scalar_root/launcher.out")" = "server ok" test "$$(cat "$$scalar_root/app.runfiles/_main/oci/py_image_layer/bin/_scalar_launcher_collision")" = data count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/content=payload.txt$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/ordinary.txt$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 count="$$(for archive in $(locations :_grouped_source_layers_no_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app\\/bin\\/_grouped_source_bin$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 if for archive in $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | grep -q '/app/bin/_grouped_source_bin$$'; then diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 2b8ea3dce..abb545db1 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -653,10 +653,12 @@ def _source_file_to_mtree( ] def _user_file_to_mtree(f, dir_expander, source_owned_paths): + # Rule-level groups may contain declared symlinks that File.is_symlink + # doesn't expose, so every grouped file needs the readlink fallback. if f.is_directory: - return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] + return [_file_to_mtree_entry(child, "0755", maybe_symlink = True) for child in dir_expander.expand(f)] mode = "0755" if f.path in source_owned_paths else "0644" - return _file_to_mtree_entry(f, mode) + return _file_to_mtree_entry(f, mode, maybe_symlink = True) def _should_skip_pkg_path(p): return ( @@ -955,7 +957,6 @@ def _py_image_layer_impl(ctx): rule_group_names = {gname: True for gname in ctx.attr.groups.values()} rule_group_files = [] rule_group_paths = {} - rule_group_tree_files = [] rule_groups = [] for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) @@ -965,8 +966,6 @@ def _py_image_layer_impl(ctx): rule_group_files.append(files) for f in files.to_list(): rule_group_paths[f.path] = True - if f.is_directory: - rule_group_tree_files.append(f) rule_groups.append((group_name, files)) source_files = depset(transitive = [info.source_files for info in infos]) @@ -987,6 +986,30 @@ def _py_image_layer_impl(ctx): rule_group_map = lambda f, d: ( source_map(f, d) if f.short_path in executable_dsts else _user_file_to_mtree(f, d, source_owned_group_paths) ) + + first_party_reference_files = [] + for info in infos: + for entry in info.first_party_layers.to_list(): + first_party_reference_files.append(entry.files) + + # Each source-owned tier may contain a symlink whose target is emitted by + # another tier. Share destination-only rows so every tar can rewrite those + # links without copying the target bytes into that tar. + symlink_mappings = None + if rule_group_files or first_party_reference_files: + reference_files = [source_files] + rule_group_files + first_party_reference_files + reference_tree_files = {} + for files in reference_files: + for f in files.to_list(): + if f.is_directory: + reference_tree_files[f.path] = f + reference_map = lambda f, d: rule_group_map(f, d) if f.path in rule_group_paths else source_map(f, d) + symlink_mappings = struct( + files = depset(transitive = reference_files), + map_each = reference_map, + tree_files = reference_tree_files.values(), + ) + for group_name, files in rule_groups: tar_out = _declare_group_tar( ctx, @@ -997,6 +1020,7 @@ def _py_image_layer_impl(ctx): files, rule_group_map, "Creating image layer %s[%s]" % (ctx.label, group_name), + symlink_mappings, ) all_tars.append(tar_out) @@ -1047,6 +1071,7 @@ def _py_image_layer_impl(ctx): depset(transitive = fp_by_group[group_name]), source_map, "Creating first-party layer %s[%s]" % (ctx.label, group_name), + symlink_mappings, ) all_tars.append(tar_out) @@ -1102,13 +1127,6 @@ def _py_image_layer_impl(ctx): # snapshot here to avoid double-bookkeeping during construction. dep_tars = list(all_tars) - symlink_mappings = None - if rule_group_files: - symlink_mappings = struct( - files = depset(transitive = rule_group_files), - map_each = rule_group_map, - tree_files = rule_group_tree_files, - ) source_tar = _declare_group_tar( ctx, bsdtar, From a82c9dcee5f5ce023cb8cc1001097c5eb25a70b7 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 05:31:20 -0400 Subject: [PATCH 38/40] fix(py): resolve interpreter layer symlinks A rule-group target-file symlink can point into a separately emitted interpreter layer. Include the interpreter destinations in cross-tier mapping rows so extraction preserves a resolvable link while the prebuilt interpreter tar remains the sole payload owner. --- .../image_layer_analysis_tests.bzl | 48 ++++++++++++++++++- py/private/py_image_layer.bzl | 29 +++++++---- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index dcf8adb17..acb081147 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -4,6 +4,8 @@ load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_t load("@bazel_features//:features.bzl", "bazel_features") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +_PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" + def _expected_failure_impl(ctx): env = analysistest.begin(ctx) asserts.expect_failure(env, ctx.attr.expected_error) @@ -25,6 +27,19 @@ _target_file_symlink = rule( attrs = {"target": attr.label(allow_single_file = True, mandatory = True)}, ) +def _interpreter_symlink_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( + output = output, + target_file = ctx.toolchains[_PY_TOOLCHAIN].py3_runtime.interpreter, + ) + return [DefaultInfo(files = depset([output]))] + +_interpreter_symlink = rule( + implementation = _interpreter_symlink_impl, + toolchains = [_PY_TOOLCHAIN], +) + def _relative_symlink_impl(ctx): output = ctx.actions.declare_symlink(ctx.label.name) ctx.actions.symlink(output = output, target_path = ctx.attr.target_path) @@ -262,22 +277,45 @@ touch $@ }, launcher_dir = "/app/bin", ) + _interpreter_symlink( + name = "_grouped_interpreter_link", + ) + py_binary( + name = "_grouped_interpreter_bin", + srcs = ["server.py"], + data = [":_grouped_interpreter_link"], + ) + py_layer_tier( + name = "_grouped_interpreter_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_grouped_interpreter_layers", + binary = ":_grouped_interpreter_bin", + groups = {":_grouped_interpreter_link": "interpreter_alias"}, + layer_tier = ":_grouped_interpreter_tier", + ) native.genrule( name = "_grouped_source_runtime_test", srcs = [ ":_grouped_source_layers_no_src", ":_grouped_source_layers_only_src", + ":_grouped_interpreter_layers", ":_scalar_launcher_collision_layers", ], outs = ["_grouped_source_runtime_test.ok"], cmd = """ set -eu root="$(@D)/_grouped_source_runtime_test.root" +interpreter_root="$(@D)/_grouped_source_runtime_test.interpreter" scalar_root="$(@D)/_grouped_source_runtime_test.scalar" -mkdir -p "$$root" "$$scalar_root" +mkdir -p "$$root" "$$interpreter_root" "$$scalar_root" for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -xf "$$archive" -C "$$root" done +for archive in $(locations :_grouped_interpreter_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$interpreter_root" +done for archive in $(locations :_scalar_launcher_collision_layers); do $(BSDTAR_BIN) -xf "$$archive" -C "$$scalar_root" done @@ -288,6 +326,10 @@ test -L "$$prefix/_grouped_content=payload_link" test "$$(cat "$$prefix/_grouped_content=payload_link")" = grouped-payload test -L "$$prefix/_grouped_content=asset_link" test "$$(cat "$$prefix/_grouped_content=asset_link")" = ordinary +interpreter_link="$$interpreter_root/app.runfiles/_main/oci/py_image_layer/_grouped_interpreter_link" +test -L "$$interpreter_link" +test -s "$$interpreter_link" +test "$$("$$interpreter_link" -c 'print("interpreter-link-ok")')" = interpreter-link-ok RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" test "$$(cat "$$root/launcher.out")" = "server ok" RUNFILES_DIR="$$scalar_root/app.runfiles" "$$scalar_root/app/bin/_scalar_launcher_collision" > "$$scalar_root/launcher.out" @@ -297,6 +339,10 @@ count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations test "$$count" -eq 1 count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/ordinary.txt$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_interpreter_layers); do $(BSDTAR_BIN) -tvf "$$archive"; done | awk '$$1 ~ /^-/ && $$NF ~ /\\/bin\\/python3\\.[0-9]+$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_interpreter_layers); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/_grouped_interpreter_link$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 count="$$(for archive in $(locations :_grouped_source_layers_no_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app\\/bin\\/_grouped_source_bin$$/ { n++ } END { print n + 0 }')" test "$$count" -eq 1 if for archive in $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | grep -q '/app/bin/_grouped_source_bin$$'; then diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index abb545db1..e31e7db61 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -992,18 +992,35 @@ def _py_image_layer_impl(ctx): for entry in info.first_party_layers.to_list(): first_party_reference_files.append(entry.files) + # Interpreter tars are declared at the configured toolchain, so identical + # runtimes action-share while distinct interpreter artifacts are retained. + interpreter_layers = {} + for info in infos: + if info.interpreter_layer != None: + layer = info.interpreter_layer + interpreter_layers[layer.tar.path] = layer + # Each source-owned tier may contain a symlink whose target is emitted by # another tier. Share destination-only rows so every tar can rewrite those # links without copying the target bytes into that tar. symlink_mappings = None if rule_group_files or first_party_reference_files: - reference_files = [source_files] + rule_group_files + first_party_reference_files + interpreter_reference_files = [] + interpreter_reference_paths = {} + for layer in interpreter_layers.values(): + files = layer.interpreter_files + interpreter_reference_files.append(files) + for f in files.to_list(): + interpreter_reference_paths[f.path] = True + reference_files = [source_files] + rule_group_files + first_party_reference_files + interpreter_reference_files reference_tree_files = {} for files in reference_files: for f in files.to_list(): if f.is_directory: reference_tree_files[f.path] = f - reference_map = lambda f, d: rule_group_map(f, d) if f.path in rule_group_paths else source_map(f, d) + reference_map = lambda f, d: ( + rule_group_map(f, d) if f.path in rule_group_paths else _interpreter_file_to_mtree(f, d) if f.path in interpreter_reference_paths else source_map(f, d) + ) symlink_mappings = struct( files = depset(transitive = reference_files), map_each = reference_map, @@ -1034,14 +1051,6 @@ def _py_image_layer_impl(ctx): seen_fp_labels[entry.label] = True fp_by_group.setdefault(entry.group, []).append(entry.files) - # Interpreter tars are declared at the configured toolchain, so identical - # runtimes action-share while distinct interpreter artifacts are retained. - interpreter_layers = {} - for info in infos: - if info.interpreter_layer != None: - layer = info.interpreter_layer - interpreter_layers[layer.tar.path] = layer - prebuilt_group_tars = {} if ctx.attr.binary != None: for layer in interpreter_layers.values(): From a2352e3991fabf1c4cc623e365188ee185fe5b29 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 13:15:16 -0400 Subject: [PATCH 39/40] fix(py): unify image layer binary inputs --- e2e/cases/oci/py_image_layer/BUILD.bazel | 11 +----- .../image_layer_analysis_tests.bzl | 8 +++- py/private/py_image_layer.bzl | 39 +++++++++---------- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index f3b53ab08..9ffc3c616 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -230,15 +230,6 @@ genrule( toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], ) -py_image_layer( - name = "my_app_select_layers", - binary = select({ - ":_python_3_11": ":my_app_bin", - "//conditions:default": ":my_app_peer_bin", - }), - layer_tier = ":my_app_tier", -) - py_image_layer( name = "my_app_binaries_layers", binaries = select({ @@ -260,7 +251,7 @@ build_test( ":_configured_wheel_collision_layers", ":my_app_binaries_layers", ":my_app_launchers_layers", - ":my_app_select_layers", + ":_scalar_default_sources_listing", ], ) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index acb081147..f54ce6592 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -411,7 +411,13 @@ touch $@ ":_scalar_default_layers_only_src", ], outs = ["_scalar_default_sources.listing"], - cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + cmd = """ +set -eu +scalar="$$( $(BSDTAR_BIN) -tf $(location :_scalar_default_layers_only_src))" +singleton="$$( $(BSDTAR_BIN) -tf $(location :_scalar_default_binaries_layers_only_src))" +test "$$scalar" = "$$singleton" +printf '%s\\n' "$$scalar" > "$@" +""", toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], ) diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index e31e7db61..9a7be9141 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -1,7 +1,7 @@ """py_image_layer — analysis-time grouped OCI layers with globally shared pip tars. -`_layer_aspect` wires onto both `py_image_layer` binary inputs. Scalar binaries -also receive `_merge_aspect` so their merged pip layers remain action-shared. +`_layer_aspect` and `_merge_aspect` wire onto each `py_image_layer` binary. +Single-binary images reuse action-shared merged pip layers. `_layer_aspect` propagates through `deps`/`data`/`actual`. For pip packages it creates aspect-owned per-package tars at the pip target's namespace (globally @@ -19,8 +19,8 @@ Layer tier (groups + compression) is carried by the `py_layer_tier` rule which p Sharing model: - Solo whole-group + subpath-split pip tars: action-shared across every rule using that package (declared at the pip target's namespace). - - Multi-member merged tars: action-shared for scalar binaries; one unioned per-rule - action for binary lists, with deterministic content for CAS / registry dedupe. + - Multi-member merged tars: action-shared for a single binary; one unioned + per-rule action for multiple binaries, with deterministic content. - First-party grouped tars: per-rule action, one tar per group, collected from matched py_library targets in the binaries' dep closures. - Ungrouped pip packages: squashed by the rule into one per-rule tar. @@ -850,9 +850,10 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m return tar_out def _py_image_layer_impl(ctx): - binaries = ([ctx.attr.binary] if ctx.attr.binary != None else []) + ctx.attr.binaries + binaries = ctx.attr.binaries if not binaries: fail("py_image_layer requires at least one binary") + single_binary = len(binaries) == 1 infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) @@ -861,7 +862,7 @@ def _py_image_layer_impl(ctx): pkg_by_key = {} for info in infos: for pkg in info.pip_packages.to_list(): - key = pkg.label if ctx.attr.binary != None else tuple(sorted([f.path for f in pkg.files.to_list()])) + key = tuple(sorted([f.path for f in pkg.files.to_list()])) if key not in pkg_by_key: pkg_by_key[key] = pkg all_pkgs = pkg_by_key.values() @@ -1045,14 +1046,14 @@ def _py_image_layer_impl(ctx): seen_fp_labels = {} for info in infos: for entry in info.first_party_layers.to_list(): - if ctx.attr.binary != None: + if single_binary: if entry.label in seen_fp_labels: continue seen_fp_labels[entry.label] = True fp_by_group.setdefault(entry.group, []).append(entry.files) prebuilt_group_tars = {} - if ctx.attr.binary != None: + if single_binary: for layer in interpreter_layers.values(): prebuilt_group_tars[layer.group] = layer.tar fp_by_group.setdefault(layer.group, []) @@ -1084,7 +1085,7 @@ def _py_image_layer_impl(ctx): ) all_tars.append(tar_out) - if ctx.attr.binary == None: + if not single_binary: all_tars.extend([layer.tar for layer in interpreter_layers.values()]) pip_tars = {} @@ -1092,13 +1093,13 @@ def _py_image_layer_impl(ctx): for pkg in all_pkgs: for layer in pkg.layers: pip_tars[layer.tar.path] = layer.tar - if ctx.attr.binary == None and pkg.merge_group != None: + if not single_binary and pkg.merge_group != None: merged.setdefault(pkg.merge_group, []).append(pkg.files) all_tars.extend(pip_tars.values()) - if ctx.attr.binary != None: - all_tars.extend([tar for _group_name, tar in sorted(ctx.attr.binary[_MergedLayerInfo].merged_tars.items())]) + if single_binary: + all_tars.extend([tar for _group_name, tar in sorted(binaries[0][_MergedLayerInfo].merged_tars.items())]) else: for group_name in sorted(merged): algorithm, level, ext = _compression_for(plan, group_name) @@ -1208,11 +1209,10 @@ def _py_image_layer_impl(ctx): _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { - "binary": attr.label( - aspects = [_layer_aspect, _merge_aspect], - ), "binaries": attr.label_list( - aspects = [_layer_aspect], + allow_empty = False, + aspects = [_layer_aspect, _merge_aspect], + mandatory = True, ), "launcher_dir": attr.string( default = "", @@ -1280,8 +1280,8 @@ def py_image_layer( binary inputs' dep closures). 3. Solo-group and subpath-split pip tars — built by `_layer_aspect` at each pip target's own namespace; globally shared across every rule using that package. - 4. Multi-member merged tars — action-shared at a scalar binary or one per - group from the closure-filtered union across a binary list. + 4. Multi-member merged tars — action-shared for a single binary or one per + group from the closure-filtered union across multiple binaries. 5. Ungrouped pip packages → one squashed rule-created tar. 6. Remaining first-party Python source files → the "default" layer. @@ -1317,7 +1317,7 @@ def py_image_layer( else: if binary == None: fail("py_image_layer requires binary or binaries") - binaries = [] + binaries = [binary] for key in groups: if _split_glob_key(key)[1] != None: @@ -1328,7 +1328,6 @@ def py_image_layer( _py_image_layer( name = name, - binary = binary, binaries = binaries, launcher_dir = launcher_dir, groups = groups, From b8db863c0ce6a315f20740d0d6a81dfb1e58ed77 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 23 Jul 2026 13:53:11 -0400 Subject: [PATCH 40/40] test(py): correct singleton source listing counts --- e2e/cases/oci/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 05c837a18..2795a3577 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -95,8 +95,8 @@ if ! "$BAZEL" build -- \ fail "expected source-layer listings to build" fi listing="bazel-bin/oci/py_image_layer/_scalar_default_sources.listing" -expect_listing_count "$listing" "/app" 2 -expect_listing_count "$listing" "/app/config.json" 2 +expect_listing_count "$listing" "/app" 1 +expect_listing_count "$listing" "/app/config.json" 1 if grep -Fq './app.runfiles/_main/oci/py_image_layer/my_app_peer_bin/config.json' "$listing"; then cat "$listing" >&2 fail "scalar executable descendant leaked into the shared runfiles layout"