diff --git a/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml b/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml index a0cf8019f..058b61135 100644 --- a/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml +++ b/e2e/cases/interpreter-features-tkinter/snapshots/amd64_layers_listing.yaml @@ -12,6 +12,7 @@ files: - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/interpreter-features-tkinter/._check_no_tkinter.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 495 Jan 1 2023 ./app.runfiles/_main/interpreter-features-tkinter/__main__.py - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index a17d89634..9ffc3c616 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -1,9 +1,10 @@ -load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_test") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library", "py_test") load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@container_structure_test//:defs.bzl", "container_structure_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") load("//tools:asserts.bzl", "assert_tar_listing") +load(":image_layer_analysis_tests.bzl", "image_layer_analysis_test_suite") platform( name = "aarch64_linux", @@ -98,9 +99,9 @@ py_image_layer( ) # Multi-member group coverage: two pip packages mapped to the same group name -# go through the _merge_aspect path (no per-package tars; single merged tar -# from both install_dirs). Binary also adds pyproject_hooks to make the -# closure non-trivial. Zstd-compressed to exercise _compression_ext. +# produce a single merged tar from both install_dirs. Binary also adds +# pyproject_hooks to make the closure non-trivial. Zstd-compressed to exercise +# _compression_ext. py_binary( name = "my_app_multi_bin", srcs = ["__main__.py"], @@ -131,6 +132,153 @@ py_image_layer( layer_tier = ":my_app_multi_tier", ) +# Exercise the actual multi-launcher flow with separate lock universes and two +# interpreter versions. colorama has the same normalized label in both +# closures, but distinct 3.14/3.12 install trees; both launchers must resolve +# their own venv and the unioned dependency layers from the shared runfiles tree. +py_library( + name = "worker_support", + srcs = ["worker_support.py"], + imports = ["."], +) + +py_binary( + name = "my_app_launcher_bin", + srcs = ["launcher.py"], + data = [":my_app_worker_bin"], + dep_group = "images", + python_version = "3.11", + deps = [ + "@bazel_tools//tools/python/runfiles", + "@pypi_oci_py_image_layer//build", + "@pypi_oci_py_image_layer//colorama", + ], +) + +py_binary( + name = "my_app_worker_bin", + srcs = ["worker.py"], + dep_group = "venv_images", + python_version = "3.12", + deps = [ + ":worker_support", + "@pypi_oci_py_venv_image_layer//build", + "@pypi_oci_py_venv_image_layer//colorama", + "@pypi_oci_py_venv_image_layer//pyproject_hooks", + ], +) + +alias( + name = "my_app_worker_alias", + actual = ":my_app_worker_bin", +) + +py_layer_tier( + name = "my_app_launchers_tier", + groups = { + "//oci/py_image_layer:worker_support": "worker_support", + "@pip//build": "third_party", + "@pip//colorama": "third_party", + "@pip//pyproject_hooks": "third_party", + }, + interpreter_group = "interpreter", + strip_prefix = "oci/py_image_layer/my_app_bin", +) + +py_image_layer( + name = "my_app_launchers_layers", + binaries = [ + ":my_app_launcher_bin", + ":my_app_worker_alias", + ], + layer_tier = ":my_app_launchers_tier", +) + +genrule( + name = "my_app_launchers_sources_listing", + srcs = [":my_app_launchers_layers_only_src"], + outs = ["_my_app_launchers_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + +# Default-tier source closures share the interpreter, branding source, and +# data. The listing test proves the unioned source tar ships each byte once. +py_binary( + name = "my_app_peer_bin", + srcs = ["server.py"], + data = ["my_app_peer_bin/config.json"], + dep_group = "images", + python_version = "3.11", + deps = ["//oci/py_image_layer/branding"], +) + +py_image_layer( + name = "my_app_shared_layers", + binaries = [ + ":my_app_bin", + ":my_app_peer_bin", + ], + launcher_dir = "/custom/bin", +) + +genrule( + name = "my_app_shared_sources_listing", + srcs = [":my_app_shared_layers_only_src"], + outs = ["_my_app_shared_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + +py_image_layer( + name = "my_app_binaries_layers", + binaries = select({ + ":_python_3_11": [ + ":my_app_bin", + ":my_app_peer_bin", + ], + "//conditions:default": [ + ":my_app_peer_bin", + ":my_app_bin", + ], + }), + launcher_dir = "/app/bin", +) + +build_test( + name = "my_app_binary_api_test", + targets = [ + ":_configured_wheel_collision_layers", + ":my_app_binaries_layers", + ":my_app_launchers_layers", + ":_scalar_default_sources_listing", + ], +) + +oci_image( + name = "launchers_image", + base = "@ubuntu", + env = {"RUNFILES_DIR": "/app.runfiles"}, + tars = [":my_app_launchers_layers"], +) + +platform_transition_filegroup( + name = "amd64_launchers_image", + srcs = [":launchers_image"], + target_platform = ":x86_64_linux", +) + +container_structure_test( + name = "launchers_image_command_test", + args = ["--verbosity=debug"], + configs = ["launchers_image_command_test.yaml"], + image = ":amd64_launchers_image", + platform = "linux/amd64", + tags = ["requires-docker"], +) + +image_layer_analysis_test_suite() + platform_transition_filegroup( name = "platform_layers", srcs = [":my_app_layers"], @@ -271,3 +419,42 @@ container_structure_test( "requires-docker", ], ) + +# Exercise scalar launcher relocation independently from the default `/app` +# regression above. +py_layer_tier( + name = "server_custom_tier", + strip_prefix = "does/not/match", +) + +py_image_layer( + name = "server_custom_layers", + binary = ":server", + launcher_dir = "/custom/bin", + layer_tier = ":server_custom_tier", +) + +oci_image( + name = "server_custom_image", + base = "@ubuntu", + entrypoint = ["/custom/bin/server"], + env = {"RUNFILES_DIR": "/app.runfiles"}, + tars = [":server_custom_layers"], +) + +platform_transition_filegroup( + name = "amd64_server_custom_image", + srcs = [":server_custom_image"], + target_platform = ":x86_64_linux", +) + +container_structure_test( + name = "server_custom_image_command_test", + args = ["--verbosity=debug"], + configs = ["server_custom_image_command_test.yaml"], + image = ":amd64_server_custom_image", + platform = "linux/amd64", + tags = [ + "requires-docker", + ], +) diff --git a/e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt b/e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt new file mode 100644 index 000000000..1269488f7 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/_scalar_strip_collision/data.txt @@ -0,0 +1 @@ +data diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl new file mode 100644 index 000000000..f54ce6592 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -0,0 +1,514 @@ +"""Analysis and validation fixtures for multi-launcher image layers.""" + +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier") +load("@bazel_features//:features.bzl", "bazel_features") +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") + +_PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" + +def _expected_failure_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, ctx.attr.expected_error) + return analysistest.end(env) + +_expected_failure_test = analysistest.make( + _expected_failure_impl, + attrs = {"expected_error": attr.string(mandatory = True)}, + expect_failure = True, +) + +def _target_file_symlink_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = output, target_file = ctx.file.target) + return [DefaultInfo(files = depset([output]))] + +_target_file_symlink = rule( + implementation = _target_file_symlink_impl, + attrs = {"target": attr.label(allow_single_file = True, mandatory = True)}, +) + +def _interpreter_symlink_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink( + output = output, + target_file = ctx.toolchains[_PY_TOOLCHAIN].py3_runtime.interpreter, + ) + return [DefaultInfo(files = depset([output]))] + +_interpreter_symlink = rule( + implementation = _interpreter_symlink_impl, + toolchains = [_PY_TOOLCHAIN], +) + +def _relative_symlink_impl(ctx): + output = ctx.actions.declare_symlink(ctx.label.name) + ctx.actions.symlink(output = output, target_path = ctx.attr.target_path) + return [DefaultInfo(files = depset([output]))] + +_relative_symlink = rule( + implementation = _relative_symlink_impl, + attrs = {"target_path": attr.string(mandatory = True)}, +) + +def _image_layer_failure(name, expected_error, **kwargs): + target = "_{}_layers".format(name) + py_image_layer(name = target, **kwargs) + _expected_failure_test( + name = name + "_test", + expected_error = expected_error, + target_under_test = ":" + target, + ) + +def image_layer_analysis_test_suite(): + _image_layer_failure( + name = "relative_launcher_dir", + expected_error = "py_image_layer.launcher_dir must be an absolute image path", + binaries = [":my_app_bin", ":my_app_worker_bin"], + launcher_dir = "app/bin", + ) + + native.alias( + name = "_my_app_bin_alias", + actual = ":my_app_bin", + ) + _image_layer_failure( + name = "duplicate_launcher_basename", + expected_error = "duplicate py_image_layer launcher basename: my_app_bin", + binaries = [":my_app_bin", ":_my_app_bin_alias"], + launcher_dir = "////", + ) + + native.config_setting( + name = "_python_3_11", + flag_values = {"@aspect_rules_py//py/private/interpreter:python_version": "3.11"}, + ) + + for prefix, package in [("wheel_scripts", "build"), ("pure_wheel", "colorama")]: + for version in ["3.11", "3.12"]: + py_binary( + name = "_{}_{}".format(prefix, version.replace(".", "")), + srcs = ["server.py"], + dep_group = "images", + python_version = version, + deps = ["@pypi_oci_py_image_layer//" + package], + ) + py_layer_tier( + name = "_wheel_scripts_tier", + groups = {"@pip//build": "wheel_scripts"}, + ) + py_image_layer( + name = "_configured_wheel_collision_layers", + binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], + launcher_dir = "/app/bin", + layer_tier = ":_wheel_scripts_tier", + ) + + py_image_layer( + name = "_configured_pure_wheel_layers", + binaries = [":_pure_wheel_311", ":_pure_wheel_312"], + launcher_dir = "/app/bin", + ) + + native.genrule( + name = "_repo_mapping_shared_target", + outs = ["repo_mapping/shared.txt"], + cmd = "printf shared-link-ok > $@", + ) + _relative_symlink( + name = "_repo_mapping_link=shared", + target_path = "repo_mapping/shared.txt", + ) + py_binary( + name = "_repo_mapping_images_bin", + srcs = ["server.py"], + data = [ + ":_repo_mapping_link=shared", + ":_repo_mapping_shared_target", + ], + dep_group = "images", + python_version = "3.11", + deps = ["@pypi_oci_py_image_layer//colorama"], + ) + py_binary( + name = "_repo_mapping_venv_bin", + srcs = ["server.py"], + data = [ + ":_repo_mapping_link=shared", + ":_repo_mapping_shared_target", + ], + dep_group = "venv_images", + python_version = "3.12", + deps = ["@pypi_oci_py_venv_image_layer//colorama"], + ) + py_layer_tier( + name = "_repo_mapping_tier", + root = "/srv", + strip_prefix = "oci/py_image_layer", + ) + external_launcher = Label("@aspect_rules_py//py/tests/internal-deps/adder:external_launcher") + py_layer_tier( + name = "_external_scalar_tier", + strip_prefix = "../{}/{}".format(external_launcher.workspace_name, external_launcher.package), + ) + py_image_layer( + name = "_external_scalar_layers", + binary = external_launcher, + ) + py_image_layer( + name = "_external_scalar_stripped_layers", + binary = external_launcher, + layer_tier = ":_external_scalar_tier", + ) + native.genrule( + name = "_external_scalar_runtime_test", + srcs = [ + ":_external_scalar_layers", + ":_external_scalar_stripped_layers", + ], + outs = ["_external_scalar_runtime_test.ok"], + cmd = """ +set -eu +default_root="$(@D)/_external_scalar_runtime_test.default" +stripped_root="$(@D)/_external_scalar_runtime_test.stripped" +mkdir -p "$$default_root" "$$stripped_root" +for archive in $(locations :_external_scalar_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$default_root" +done +for archive in $(locations :_external_scalar_stripped_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$stripped_root" +done +RUNFILES_DIR="$$default_root/app.runfiles" "$$default_root/app" > "$$default_root/external.out" +RUNFILES_DIR="$$stripped_root/app.runfiles" "$$stripped_root/app/external_launcher" > "$$stripped_root/external.out" +test "$$(cat "$$default_root/external.out")" = "external 5" +test "$$(cat "$$stripped_root/external.out")" = "external 5" +touch $@ +""", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + py_image_layer( + name = "_repo_mapping_layers", + binaries = [ + ":_repo_mapping_images_bin", + ":_repo_mapping_venv_bin", + "@aspect_rules_py//py/tests/internal-deps/adder:external_launcher", + ], + launcher_dir = "/app/bin", + layer_tier = ":_repo_mapping_tier", + ) + native.genrule( + name = "_repo_mapping_runtime_test", + srcs = [":_repo_mapping_layers"], + outs = ["_repo_mapping_runtime_test.ok"], + cmd = """ +set -eu +root="$(@D)/_repo_mapping_runtime_test.root" +mkdir -p "$$root" +for archive in $(SRCS); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$root" +done +mapping="$$root/app.runfiles/_repo_mapping" +count="$$(for archive in $(SRCS); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app.runfiles\\/_repo_mapping$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +grep -Fq ',whl_install__images__colorama__0_4_6,' "$$mapping" +grep -Fq ',whl_install__venv_images__colorama__0_4_6,' "$$mapping" +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_repo_mapping_images_bin" > "$$root/images.out" +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_repo_mapping_venv_bin" > "$$root/venv.out" +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/external_launcher" > "$$root/external.out" +test "$$(cat "$$root/images.out")" = "server ok" +test "$$(cat "$$root/venv.out")" = "server ok" +test "$$(cat "$$root/external.out")" = "external 5" +test -L "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_link=shared" +test "$$(cat "$$root/app.runfiles/_main/oci/py_image_layer/_repo_mapping_link=shared")" = "shared-link-ok" +touch $@ +""", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + + native.genrule( + name = "_grouped_tool", + outs = ["grouped/tool.sh"], + cmd = "printf '#!/bin/sh\\nprintf grouped-ok\\n' > $@", + executable = True, + ) + native.genrule( + name = "_grouped_payload", + outs = ["grouped/content=payload.txt"], + cmd = "printf grouped-payload > $@", + ) + native.genrule( + name = "_group_only_file", + outs = ["grouped/ordinary.txt"], + cmd = "printf ordinary > $@", + ) + _target_file_symlink( + name = "_grouped_content=payload_link", + target = ":_grouped_payload", + ) + _target_file_symlink( + name = "_grouped_content=asset_link", + target = ":_group_only_file", + ) + native.filegroup( + name = "_grouped_assets", + srcs = [ + ":_group_only_file", + ":_grouped_content=payload_link", + ":_grouped_tool", + ], + ) + py_binary( + name = "_grouped_source_bin", + srcs = ["server.py"], + data = [ + ":_grouped_content=asset_link", + ":_grouped_payload", + ":_grouped_tool", + ], + ) + py_image_layer( + name = "_grouped_source_layers", + binaries = [ + ":_grouped_source_bin", + ":my_app_bin", + ], + groups = { + ":_grouped_assets": "assets", + ":_grouped_source_bin": "launcher", + }, + launcher_dir = "/app/bin", + ) + _interpreter_symlink( + name = "_grouped_interpreter_link", + ) + py_binary( + name = "_grouped_interpreter_bin", + srcs = ["server.py"], + data = [":_grouped_interpreter_link"], + ) + py_layer_tier( + name = "_grouped_interpreter_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_grouped_interpreter_layers", + binary = ":_grouped_interpreter_bin", + groups = {":_grouped_interpreter_link": "interpreter_alias"}, + layer_tier = ":_grouped_interpreter_tier", + ) + native.genrule( + name = "_grouped_source_runtime_test", + srcs = [ + ":_grouped_source_layers_no_src", + ":_grouped_source_layers_only_src", + ":_grouped_interpreter_layers", + ":_scalar_launcher_collision_layers", + ], + outs = ["_grouped_source_runtime_test.ok"], + cmd = """ +set -eu +root="$(@D)/_grouped_source_runtime_test.root" +interpreter_root="$(@D)/_grouped_source_runtime_test.interpreter" +scalar_root="$(@D)/_grouped_source_runtime_test.scalar" +mkdir -p "$$root" "$$interpreter_root" "$$scalar_root" +for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$root" +done +for archive in $(locations :_grouped_interpreter_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$interpreter_root" +done +for archive in $(locations :_scalar_launcher_collision_layers); do + $(BSDTAR_BIN) -xf "$$archive" -C "$$scalar_root" +done +prefix="$$root/app.runfiles/_main/oci/py_image_layer" +test "$$("$$prefix/grouped/tool.sh")" = grouped-ok +test ! -x "$$prefix/grouped/ordinary.txt" +test -L "$$prefix/_grouped_content=payload_link" +test "$$(cat "$$prefix/_grouped_content=payload_link")" = grouped-payload +test -L "$$prefix/_grouped_content=asset_link" +test "$$(cat "$$prefix/_grouped_content=asset_link")" = ordinary +interpreter_link="$$interpreter_root/app.runfiles/_main/oci/py_image_layer/_grouped_interpreter_link" +test -L "$$interpreter_link" +test -s "$$interpreter_link" +test "$$("$$interpreter_link" -c 'print("interpreter-link-ok")')" = interpreter-link-ok +RUNFILES_DIR="$$root/app.runfiles" "$$root/app/bin/_grouped_source_bin" > "$$root/launcher.out" +test "$$(cat "$$root/launcher.out")" = "server ok" +RUNFILES_DIR="$$scalar_root/app.runfiles" "$$scalar_root/app/bin/_scalar_launcher_collision" > "$$scalar_root/launcher.out" +test "$$(cat "$$scalar_root/launcher.out")" = "server ok" +test "$$(cat "$$scalar_root/app.runfiles/_main/oci/py_image_layer/bin/_scalar_launcher_collision")" = data +count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/content=payload.txt$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_source_layers_no_src) $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/grouped\\/ordinary.txt$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_interpreter_layers); do $(BSDTAR_BIN) -tvf "$$archive"; done | awk '$$1 ~ /^-/ && $$NF ~ /\\/bin\\/python3\\.[0-9]+$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_interpreter_layers); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/_grouped_interpreter_link$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +count="$$(for archive in $(locations :_grouped_source_layers_no_src); do $(BSDTAR_BIN) -tf "$$archive"; done | awk '/\\/app\\/bin\\/_grouped_source_bin$$/ { n++ } END { print n + 0 }')" +test "$$count" -eq 1 +if for archive in $(locations :_grouped_source_layers_only_src); do $(BSDTAR_BIN) -tf "$$archive"; done | grep -q '/app/bin/_grouped_source_bin$$'; then + exit 1 +fi +touch $@ +""", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + + native.genrule( + name = "_scalar_launcher_collision_data", + outs = ["bin/_scalar_launcher_collision"], + cmd = "echo data > $@", + ) + py_binary( + name = "_scalar_launcher_collision", + srcs = ["server.py"], + data = ["bin/_scalar_launcher_collision"], + ) + py_layer_tier( + name = "_scalar_launcher_collision_tier", + strip_prefix = "oci/py_image_layer", + ) + py_image_layer( + name = "_scalar_launcher_collision_layers", + binary = ":_scalar_launcher_collision", + launcher_dir = "/app/bin", + layer_tier = ":_scalar_launcher_collision_tier", + ) + + py_binary( + name = "_scalar_strip_collision", + srcs = ["server.py"], + data = ["_scalar_strip_collision/data.txt"], + ) + py_layer_tier( + name = "_scalar_strip_collision_tier", + root = "/app.runfiles/_main/oci/py_image_layer", + strip_prefix = "oci/py_image_layer", + ) + py_image_layer( + name = "_scalar_strip_collision_layers", + binary = ":_scalar_strip_collision", + layer_tier = ":_scalar_strip_collision_tier", + ) + + py_layer_tier( + name = "_scalar_default_tier", + interpreter_group = "interpreter", + ) + py_image_layer( + name = "_scalar_default_layers", + binary = ":my_app_peer_bin", + layer_tier = ":_scalar_default_tier", + ) + py_image_layer( + name = "_scalar_default_binaries_layers", + binaries = [":my_app_peer_bin"], + layer_tier = ":_scalar_default_tier", + ) + native.genrule( + name = "_scalar_default_sources_listing", + srcs = [ + ":_scalar_default_binaries_layers_only_src", + ":_scalar_default_layers_only_src", + ], + outs = ["_scalar_default_sources.listing"], + cmd = """ +set -eu +scalar="$$( $(BSDTAR_BIN) -tf $(location :_scalar_default_layers_only_src))" +singleton="$$( $(BSDTAR_BIN) -tf $(location :_scalar_default_binaries_layers_only_src))" +test "$$scalar" = "$$singleton" +printf '%s\\n' "$$scalar" > "$@" +""", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + + py_binary( + name = "_scalar_root_collision", + srcs = ["server.py"], + ) + py_layer_tier( + name = "_scalar_root_collision_tier", + root = "/app.runfiles/_main/oci/py_image_layer/server.py", + ) + py_image_layer( + name = "_scalar_root_collision_layers", + binary = ":_scalar_root_collision", + layer_tier = ":_scalar_root_collision_tier", + ) + + # Bazel 8 permits nested runfiles outputs. Bazel 9 rejects this topology + # before analysis, so keep the real longest-prefix regression Bazel-8-only. + if not bazel_features.rules.merkle_cache_v2: + py_binary( + name = "_nested_prefix/foo", + srcs = ["server.py"], + python_version = "3.11", + ) + native.genrule( + name = "_nested_prefix_runfile", + outs = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], + cmd = "echo data > $@", + ) + py_binary( + name = "_nested_prefix/foo.runfiles/worker", + srcs = ["server.py"], + data = ["_nested_prefix/foo.runfiles/worker.runfiles/_main/nested/data.txt"], + python_version = "3.12", + ) + py_layer_tier( + name = "_nested_prefix_tier", + interpreter_group = "interpreter", + strip_prefix = "oci/py_image_layer/_nested_prefix", + ) + py_layer_tier( + name = "_nested_prefix_nonmatching_tier", + interpreter_group = "interpreter", + strip_prefix = "does/not/match", + ) + py_image_layer( + name = "_nested_prefix_layers", + binaries = [ + ":_nested_prefix/foo", + ":_nested_prefix/foo.runfiles/worker", + ], + launcher_dir = "/app/bin", + layer_tier = ":_nested_prefix_tier", + ) + py_image_layer( + name = "_nested_prefix_reversed_layers", + binaries = [ + ":_nested_prefix/foo.runfiles/worker", + ":_nested_prefix/foo", + ], + launcher_dir = "/app/bin", + layer_tier = ":_nested_prefix_tier", + ) + py_image_layer( + name = "_nested_prefix_scalar_layers", + binary = ":_nested_prefix/foo.runfiles/worker", + layer_tier = ":_nested_prefix_tier", + ) + py_image_layer( + name = "_nested_prefix_nonmatching_scalar_layers", + binary = ":_nested_prefix/foo.runfiles/worker", + layer_tier = ":_nested_prefix_nonmatching_tier", + ) + native.genrule( + name = "_nested_prefix_sources_listing", + srcs = [ + ":_nested_prefix_layers_only_src", + ":_nested_prefix_nonmatching_scalar_layers_only_src", + ":_nested_prefix_reversed_layers_only_src", + ":_nested_prefix_scalar_layers_only_src", + ], + outs = ["_nested_prefix_sources.listing"], + cmd = "for f in $(SRCS); do $(BSDTAR_BIN) -tf $$f; done > $@", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], + ) + + _image_layer_failure( + name = "interpreter_group_collision", + expected_error = "Group \"interpreter\" is declared in both py_image_layer.groups and the active py_layer_tier", + binary = ":my_app_bin", + groups = {":worker_support": "interpreter"}, + layer_tier = ":my_app_launchers_tier", + ) diff --git a/e2e/cases/oci/py_image_layer/launcher.py b/e2e/cases/oci/py_image_layer/launcher.py new file mode 100644 index 000000000..b882cafc1 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/launcher.py @@ -0,0 +1,20 @@ +import subprocess +import sys + +import build +import colorama +from bazel_tools.tools.python.runfiles import runfiles + +if __name__ == "__main__": + print( + "launcher ok {} {} {}".format( + sys.version_info.minor, + build.__version__, + colorama.__version__, + ) + ) + worker = runfiles.Create().Rlocation( + "_main/oci/py_image_layer/my_app_worker_bin" + ) + assert worker is not None + subprocess.run([worker], check=True) diff --git a/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml new file mode 100644 index 000000000..b46a6801d --- /dev/null +++ b/e2e/cases/oci/py_image_layer/launchers_image_command_test.yaml @@ -0,0 +1,14 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: primary launcher runs from launcher_dir + exitCode: 0 + command: /app/bin/my_app_launcher_bin + expectedOutput: + - "launcher ok 11 1.4.0 0.4.6" + - "worker ok 12 1.4.0 0.4.6 1.2.0 grouped" + + - name: secondary launcher uses its own interpreter and pip closure + exitCode: 0 + command: /app/bin/my_app_worker_bin + expectedOutput: ["worker ok 12 1.4.0 0.4.6 1.2.0 grouped"] diff --git a/e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json b/e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json new file mode 100644 index 000000000..7c111356f --- /dev/null +++ b/e2e/cases/oci/py_image_layer/my_app_peer_bin/config.json @@ -0,0 +1 @@ +{"sentinel": "prefix-data"} diff --git a/e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml b/e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml new file mode 100644 index 000000000..a051e5520 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/server_custom_image_command_test.yaml @@ -0,0 +1,7 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: server binary runs from a custom launcher directory + exitCode: 0 + command: /custom/bin/server + expectedOutput: ["server ok"] diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml index 567df3d33..9e05ec5c0 100644 --- a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_listing.yaml @@ -2360,5 +2360,6 @@ files: - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__main__.py - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - -rwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__init__.py - -rwxr-xr-x 0 0 0 32 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/add.py diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml index 80c3953a6..860507f33 100644 --- a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_listing.yaml @@ -34,6 +34,7 @@ files: - -rwxr-xr-x 0 0 0 42 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__init__.py - -rwxr-xr-x 0 0 0 31 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/palette.txt - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml index 893e071fa..f99b82fb1 100644 --- a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_multi_listing.yaml @@ -40,6 +40,7 @@ files: - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_multi_bin.venv/lib/python3.11/site-packages/pyproject_hooks-1.2.0.dist-info -> ../../../../../../../aspect_rules_py++uv+whl_install__images__pyproject_hooks__1_2_0/actual_install.install/lib/python3.11/site-packages/pyproject_hooks-1.2.0.dist-info - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_multi_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__main__.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_image_layer/worker.py b/e2e/cases/oci/py_image_layer/worker.py new file mode 100644 index 000000000..79e73265e --- /dev/null +++ b/e2e/cases/oci/py_image_layer/worker.py @@ -0,0 +1,17 @@ +import sys + +import build +import colorama +import pyproject_hooks +from worker_support import support + +if __name__ == "__main__": + print( + "worker ok {} {} {} {} {}".format( + sys.version_info.minor, + build.__version__, + colorama.__version__, + pyproject_hooks.__version__, + support(), + ) + ) diff --git a/e2e/cases/oci/py_image_layer/worker_support.py b/e2e/cases/oci/py_image_layer/worker_support.py new file mode 100644 index 000000000..440629a6a --- /dev/null +++ b/e2e/cases/oci/py_image_layer/worker_support.py @@ -0,0 +1,2 @@ +def support() -> str: + return "grouped" diff --git a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml index d52d08f52..30b4b927f 100644 --- a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml +++ b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_amd64_layers_listing.yaml @@ -35,6 +35,7 @@ files: - -rwxr-xr-x 0 0 0 463 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__main__.py - -rwxr-xr-x 0 0 0 1272 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/my_app_bin.venv - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml index e195be493..96cc9162a 100644 --- a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml +++ b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_arm64_layers_listing.yaml @@ -35,6 +35,7 @@ files: - -rwxr-xr-x 0 0 0 463 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__main__.py - -rwxr-xr-x 0 0 0 1272 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/my_app_bin.venv - -rwxr-xr-x 0 0 0 6713 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_aarch64_unknown_linux_gnu/bin/2to3 -> 2to3-3.11 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_aarch64_unknown_linux_gnu/bin/2to3-3.11 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_11_aarch64_unknown_linux_gnu/bin/idle3 -> idle3.11 diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh new file mode 100755 index 000000000..2795a3577 --- /dev/null +++ b/e2e/cases/oci/test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# +# Multi-binary destination collisions are discovered by the validation action, +# so the expected-failure case cannot be an sh_test under //.... +set -uo pipefail + +cd "$(dirname "$0")/.." || exit 1 + +BAZEL="${BAZEL:-bazel}" +PKG="//oci/py_image_layer" +output_log="$(mktemp)" +trap 'rm -f "$output_log"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +expect_diagnostic() { + if ! grep -Fq "$1" "$output_log"; then + cat "$output_log" >&2 + fail "expected validation diagnostic: $1" + fi +} + +expect_listing_count() { + local listing="$1" + local suffix="$2" + local expected="$3" + local actual + actual="$(awk -v suffix="$suffix" 'substr($0, length($0) - length(suffix) + 1) == suffix { count++ } END { print count + 0 }' "$listing")" + if test "$actual" -ne "$expected"; then + cat "$listing" >&2 + fail "$suffix: expected $expected, found $actual" + fi +} + +echo "== configured wheel and relocated scalar images must validate ==" +if ! "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_configured_pure_wheel_layers" \ + "${PKG}:_configured_wheel_collision_layers" \ + "${PKG}:_scalar_launcher_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected configured wheel and relocated scalar images to validate" +fi + +echo "== remapped destinations must fail validation ==" +if "$BAZEL" build --keep_going --output_groups=_validation -- \ + "${PKG}:_scalar_strip_collision_layers" \ + "${PKG}:_scalar_root_collision_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected remapped destinations to fail validation" +fi +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/_scalar_strip_collision/data.txt:" +expect_diagnostic "py_image_layer runfile collision at ./app.runfiles/_main/oci/py_image_layer/server.py:" + +echo "PASS: expanded and remapped destinations validate correctly" + +echo "== rule-group files must be excluded from the default source layer ==" +if ! "$BAZEL" build --output_groups=_validation -- \ + "${PKG}:_grouped_source_layers" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the explicitly grouped source to validate" +fi + +if ! "$BAZEL" build -- "${PKG}:_grouped_source_runtime_test" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected grouped modes and cross-layer symlink to survive extraction" +fi + +echo "PASS: explicitly grouped sources preserve modes and symlink targets without duplicate bytes" + +echo "== multi-binary repository mappings must be merged ==" +if ! "$BAZEL" build -- "${PKG}:_repo_mapping_runtime_test" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected repository mappings from both binaries to survive extraction" +fi + +echo "PASS: multi-binary repository mappings are merged" + +echo "== scalar external launchers must resolve apparent repository names ==" +if ! "$BAZEL" build -- "${PKG}:_external_scalar_runtime_test" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the scalar external launcher to resolve its runfiles" +fi + +echo "PASS: scalar external launcher resolves its runfiles" + +echo "== source closures must preserve scalar and shared layouts ==" +if ! "$BAZEL" build -- \ + "${PKG}:_scalar_default_sources_listing" \ + "${PKG}:my_app_launchers_sources_listing" \ + "${PKG}:my_app_shared_sources_listing" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected source-layer listings to build" +fi +listing="bazel-bin/oci/py_image_layer/_scalar_default_sources.listing" +expect_listing_count "$listing" "/app" 1 +expect_listing_count "$listing" "/app/config.json" 1 +if grep -Fq './app.runfiles/_main/oci/py_image_layer/my_app_peer_bin/config.json' "$listing"; then + cat "$listing" >&2 + fail "scalar executable descendant leaked into the shared runfiles layout" +fi +listing="bazel-bin/oci/py_image_layer/_my_app_launchers_sources.listing" +expect_listing_count "$listing" "/app/bin/my_app_launcher_bin" 1 +expect_listing_count "$listing" "/app/bin/my_app_worker_bin" 1 +expect_listing_count "$listing" "/app.runfiles/_main/oci/py_image_layer/my_app_worker_bin" 1 +expect_listing_count "$listing" "/app.runfiles/_main/oci/py_image_layer/my_app_launcher_bin" 0 +listing="bazel-bin/oci/py_image_layer/_my_app_shared_sources.listing" +for suffix in \ + /branding/__init__.py \ + /branding/palette.txt \ + /lib/python3.11/os.py \ + /custom/bin/my_app_bin \ + /custom/bin/my_app_peer_bin \ + /oci/py_image_layer/my_app_peer_bin/config.json; do + expect_listing_count "$listing" "$suffix" 1 +done + +echo "PASS: scalar and shared source layouts are preserved" + +if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then + echo "== nested launcher prefixes must share the same runfiles layout in either input order ==" + if ! "$BAZEL" build -- "${PKG}:_nested_prefix_sources_listing" >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the nested launcher source layers to build" + fi + listing="bazel-bin/oci/py_image_layer/_nested_prefix_sources.listing" + expect_listing_count "$listing" "/app" 1 + expect_listing_count "$listing" "/app.runfiles/_main/nested/data.txt" 3 + expect_listing_count "$listing" "/app/foo.runfiles/worker" 1 + expect_listing_count "$listing" "/app/foo.runfiles/worker.runfiles/_main/nested/data.txt" 1 + if grep -Fq './app.runfiles/worker.runfiles/' "$listing"; then + cat "$listing" >&2 + fail "nested launcher prefix leaked into the shared runfiles layout" + fi + echo "PASS: nested launcher prefixes share the same runfiles layout" +fi diff --git a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml index 90fad5bb8..0c2160555 100644 --- a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml +++ b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_amd64_layers_listing.yaml @@ -46,6 +46,7 @@ files: - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/lib/python3.12/site-packages/psycopg2_binary.libs -> ../../../../../../../aspect_rules_py++uv+whl_install__crossbuild__psycopg2_binary__2_9_11/actual_install.install/lib/python3.12/site-packages/psycopg2_binary.libs - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 66 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/__main__.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_x86_64_unknown_linux_gnu/bin/2to3 -> 2to3-3.12 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_x86_64_unknown_linux_gnu/bin/2to3-3.12 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_x86_64_unknown_linux_gnu/bin/idle3 -> idle3.12 diff --git a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml index 4916b915b..7db44a67d 100644 --- a/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml +++ b/e2e/cases/uv-deps-650/crossbuild/snapshots/app_arm64_layers_listing.yaml @@ -46,6 +46,7 @@ files: - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/lib/python3.12/site-packages/psycopg2_binary.libs -> ../../../../../../../aspect_rules_py++uv+whl_install__crossbuild__psycopg2_binary__2_9_11/actual_install.install/lib/python3.12/site-packages/psycopg2_binary.libs - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/._app_bin.venv/pyvenv.cfg - -rwxr-xr-x 0 0 0 66 Jan 1 2023 ./app.runfiles/_main/uv-deps-650/crossbuild/__main__.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_aarch64_unknown_linux_gnu/bin/2to3 -> 2to3-3.12 - -rwxr-xr-x 0 0 0 158 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_aarch64_unknown_linux_gnu/bin/2to3-3.12 - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py++python_interpreters+python_3_12_aarch64_unknown_linux_gnu/bin/idle3 -> idle3.12 diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 1fd5eb05e..c66a29305 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -10,6 +10,7 @@ exports_files( "pytest_main.py", # Template read by the py_unittest_test rule. "unittest_main.py", + "merge_repo_mappings.awk", "modify_mtree.awk", ], visibility = ["//visibility:public"], diff --git a/py/private/merge_repo_mappings.awk b/py/private/merge_repo_mappings.awk new file mode 100644 index 000000000..0ea9a502f --- /dev/null +++ b/py/private/merge_repo_mappings.awk @@ -0,0 +1,34 @@ +# Merge Bazel's three-column repository mappings without silently choosing one +# target for an apparent repository name. +BEGIN { + FS = "," +} + +NF != 3 { + printf "invalid repository mapping at %s:%d: expected 3 columns, got %d\n", FILENAME, FNR, NF > "/dev/stderr" + failed = 1 + exit 1 +} + +{ + key = $1 SUBSEP $2 + if (key in mappings && mappings[key] != $3) { + printf "conflicting repository mapping for %s,%s: %s and %s\n", $1, $2, mappings[key], $3 > "/dev/stderr" + failed = 1 + exit 1 + } + mappings[key] = $3 + rows[$0] = 1 +} + +END { + if (failed) { + exit 1 + } + printf "" > output + count = asorti(rows, sorted) + for (i = 1; i <= count; i++) { + print sorted[i] >> output + } + close(output) +} diff --git a/py/private/modify_mtree.awk b/py/private/modify_mtree.awk index 67366fbf7..4d5120d9b 100644 --- a/py/private/modify_mtree.awk +++ b/py/private/modify_mtree.awk @@ -2,9 +2,9 @@ # `type=link` rows, so bsdtar preserves them as symlinks instead of # following and inlining the target bytes. # -# Forked from @tar.bzl//tar/private:preserve_symlinks.awk and tracking -# https://github.com/bazel-contrib/tar.bzl/pull/115. One behavioural -# difference remains: +# Forked from @tar.bzl//tar/private:preserve_symlinks.awk. The upstream +# implementation in https://github.com/bazel-contrib/tar.bzl/pull/115 lacks +# both cross-layer target mappings and one path-normalization fix retained here. # # The `bazel-out/` vs `external/` strip is exclusive (`if` / `else if`) # rather than two sequential `sub`s. Without this, paths matching @@ -13,9 +13,6 @@ # over-stripped down to `external//...`, miss the # `symlink_map` lookup, and dangle inside the OCI layer. # -# Send a follow-up PR to bazel-contrib/tar.bzl once #115 -# lands so this fork can retire. -# # Invoked from `_run_tar_action` in [py_image_layer.bzl](py_image_layer.bzl) # via `ctx.executable._awk` pinned to `@gawk` — the END block uses # `asort()` and `print > outfile`, both gawk extensions. @@ -58,7 +55,37 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back return back_steps target } +function decode_mtree_path(path) { + gsub(/\\040/, " ", path) + return path +} + +function replace_metadata_field(row, pattern, replacement) { + if (!match(row, pattern)) { + return row + } + return substr(row, 1, RSTART) replacement substr(row, RSTART + RLENGTH) +} + { + source_field = "" + source_type = "" + for (field = 2; field <= NF; field++) { + if ($field ~ /^(contents|content|link)=[^ ]+$/) { + source_field = $field + } else if ($field ~ /^type=[^ ]+$/) { + source_type = substr($field, index($field, "=") + 1) + } + } + + if (ARGIND != source_argind) { + if (source_field != "") { + source_path = substr(source_field, index(source_field, "=") + 1) + symlink_map[decode_mtree_path(source_path)] = $1 + } + next + } + symlink = "" symlink_content = "" # Two markers Starlark emits for paths that could be symlinks: @@ -68,17 +95,11 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back # might be symlinks Bazel didn't flag (repo-rule-staged ones like # rules_python's `bin/python -> python3.11`). Empty `readlink` means # it's a regular file and the row passes through unchanged. - is_hot_path = ($0 ~ /type=link/) && ($0 ~ /link=/) - is_slow_path = ($0 ~ /type=file/) && ($0 ~ /content=/) + is_hot_path = source_type == "link" && source_field ~ /^link=/ + is_slow_path = source_type == "file" && source_field ~ /^content=/ if (is_hot_path || is_slow_path) { - if (is_hot_path) { - match($0, /link=[^ ]+/) - } else { - match($0, /content=[^ ]+/) - } - content_field = substr($0, RSTART, RLENGTH) - split(content_field, parts, "=") - path = parts[2] + source_path = substr(source_field, index(source_field, "=") + 1) + path = decode_mtree_path(source_path) symlink_map[path] = $1 # Plain `readlink` first: keep its result if relative @@ -151,9 +172,9 @@ function make_relative_link(path1, path2, i, common, target, relative_path, back } } if (symlink != "") { - line_array[NR] = $0 SUBSEP $1 SUBSEP resolved_path + line_array[++source_line_count] = $0 SUBSEP $1 SUBSEP resolved_path } else { - line_array[NR] = $0 + line_array[++source_line_count] = $0 } } @@ -161,7 +182,7 @@ END { # Buffer rewritten rows, sort byte-wise (asort under LC_ALL=C, set by # the action env), and write to `outfile`. n = 0 - for (i = 1; i <= NR; i++) { + for (i = 1; i <= source_line_count; i++) { line = line_array[i] if (index(line, SUBSEP) > 0) { split(line, fields, SUBSEP) @@ -180,9 +201,11 @@ END { # Already a relative path linked_to = resolved_path } - sub(/type=[^ ]+/, "type=link", original_line) - if (!sub(/content=[^ ]+/, "link=" linked_to, original_line)) { - sub(/link=[^ ]+/, "link=" linked_to, original_line) + original_line = replace_metadata_field(original_line, "[[:space:]]type=[^ ]+", "type=link") + if (original_line ~ /[[:space:]]content=[^ ]+/) { + original_line = replace_metadata_field(original_line, "[[:space:]]content=[^ ]+", "link=" linked_to) + } else { + original_line = replace_metadata_field(original_line, "[[:space:]]link=[^ ]+", "link=" linked_to) } out_lines[++n] = original_line } else { diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index 0ed063c34..9a7be9141 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -1,21 +1,15 @@ """py_image_layer — analysis-time grouped OCI layers with globally shared pip tars. -Two rule-propagated aspects wire onto `py_image_layer.binary`: - - 1. `_layer_aspect` — propagates through `deps`/`data`/`actual`. For pip packages it - creates aspect-owned per-package tars at the pip target's namespace (globally - shared across every rule using that package) for solo whole-groups and subpath - splits. Members of multi-member whole groups get NO per-package tar — intermediate - elided; they just flag `merge_group` on their _LayerInfo struct. First-party - PyInfo targets matched by `py_layer_tier.groups` are captured as `first_party_layers` - entries (label, files, group) to be tarred per-group at the binary. Produces - `_LayerInfo`. - - 2. `_merge_aspect` — runs only on the binary (`attr_aspects = []`), after `_layer_aspect` - via `required_aspect_providers`. Reads `_LayerInfo.pip_packages` (full closure), - buckets install_dirs by `merge_group`, and emits one bsdtar action per group from - raw install_dirs — single pass, no intermediates, content exactly matches closure - (no dep leak). Produces `_MergedLayerInfo` declared at the binary's output namespace. +`_layer_aspect` and `_merge_aspect` wire onto each `py_image_layer` binary. +Single-binary images reuse action-shared merged pip layers. + +`_layer_aspect` propagates through `deps`/`data`/`actual`. For pip packages it +creates aspect-owned per-package tars at the pip target's namespace (globally +shared across every rule using that package) for solo whole-groups and subpath +splits. Members of multi-member whole groups get NO per-package tar — intermediate +elided; they just flag `merge_group` on their _LayerInfo struct. First-party +PyInfo targets matched by `py_layer_tier.groups` are captured as `first_party_layers` +entries (label, files, group) for the image-layer rule. Produces `_LayerInfo`. Layer tier (groups + compression) is carried by the `py_layer_tier` rule which produces `PyLayerTierInfo`. Aspects read it via a private `_layer_tier` attr whose default is @@ -25,10 +19,10 @@ Layer tier (groups + compression) is carried by the `py_layer_tier` rule which p Sharing model: - Solo whole-group + subpath-split pip tars: action-shared across every rule using that package (declared at the pip target's namespace). - - Multi-member merged tars: per-binary action, but deterministic content (canonical - mtree, fixed bsdtar options) → remote CAS / OCI registry dedupe by digest. - - First-party grouped tars: per-binary action, one tar per group, collected from - matched py_library targets in the binary's dep closure. + - Multi-member merged tars: action-shared for a single binary; one unioned + per-rule action for multiple binaries, with deterministic content. + - First-party grouped tars: per-rule action, one tar per group, collected from + matched py_library targets in the binaries' dep closures. - Ungrouped pip packages: squashed by the rule into one per-rule tar. """ @@ -177,7 +171,6 @@ _LayerInfo = provider( "source_files": "depset[File] — ungrouped first-party Python source files.", "pip_packages": "depset[struct] — fully transitive pip packages with per-package layers.", "first_party_layers": "depset[struct(label, files, group)] — first-party PyInfo targets matched by py_layer_tier.groups.", - "interpreter_files": "depset[File] — interpreter runfiles, populated only on the py toolchain pass for the binary-branch skip filter.", "interpreter_layer": "struct(tar, group, interpreter_files) | None — prebuilt interpreter layer tar + its group name + the files used to build it, declared at the toolchain target's namespace so the tar action-shares across every py_image_layer using that toolchain config.", }, ) @@ -224,7 +217,7 @@ def _build_pip_layers(ctx, plan, label, install_dir): Returns (layers, merge_group): layers is a tuple of struct(tar, group) entries for this package; merge_group is set (and layers is empty) iff - the package is deferred to `_merge_aspect`. + the package is deferred for closure-level merging. """ subpath_for_this = plan.subpath_groups.get(label, {}) whole_group = plan.whole_groups.get(label, None) @@ -337,7 +330,6 @@ def _layer_aspect_impl(target, ctx): source_files = depset(), pip_packages = depset(), first_party_layers = depset(), - interpreter_files = interp_depset, interpreter_layer = interp_layer, )] @@ -365,14 +357,12 @@ def _layer_aspect_impl(target, ctx): transitive = transitive_pkgs, ), first_party_layers = depset(transitive = transitive_fp), - interpreter_files = depset(), interpreter_layer = None, )] own_source = [] own_fp = [] interpreter_layer = None - interpreter_files = None kind = ctx.rule.kind # The binary being layered must be a rules_py py_binary (it carries rules_py's @@ -389,12 +379,12 @@ def _layer_aspect_impl(target, ctx): # it is already captured upstream as a pip package via the wheel-leaf # branch. Just propagate transitively for these targets. if kind == "alias": + interp = transitive_interp[0] if transitive_interp else None return [_LayerInfo( source_files = depset(transitive = transitive_source), pip_packages = depset(transitive = transitive_pkgs), first_party_layers = depset(transitive = transitive_fp), - interpreter_files = depset(), - interpreter_layer = None, + interpreter_layer = interp, )] # Skip PyInfo deps (including wheel-leaf targets, which also emit PyInfo) — @@ -428,10 +418,7 @@ def _layer_aspect_impl(target, ctx): if PY_TOOLCHAIN in ctx.rule.toolchains: py_tc = ctx.rule.toolchains[PY_TOOLCHAIN] if _LayerInfo in py_tc: - tc_info = py_tc[_LayerInfo] - if tc_info.interpreter_layer != None: - interpreter_layer = tc_info.interpreter_layer - interpreter_files = tc_info.interpreter_files + interpreter_layer = py_tc[_LayerInfo].interpreter_layer # Binaries walk their runfiles for the source layer, filtering out bytes already # shipping in their own pip / fp-group / interpreter layers. @@ -452,34 +439,21 @@ def _layer_aspect_impl(target, ctx): # that built the venv rather than relying on its own toolchain resolution. interp_paths = {} for interp_layer in transitive_interp: - if interp_layer.interpreter_files != None: - for f in interp_layer.interpreter_files.to_list(): - interp_paths[f.path] = True - if transitive_interp: - interpreter_layer = transitive_interp[0] - interpreter_files = transitive_interp[0].interpreter_files + for f in interp_layer.interpreter_files.to_list(): + interp_paths[f.path] = True + venv = getattr(ctx.rule.attr, "venv", None) + if venv != None and _LayerInfo in venv: + interpreter_layer = venv[_LayerInfo].interpreter_layer runfiles_files = target[DefaultInfo].default_runfiles.files.to_list() filtered = [f for f in runfiles_files if f.path not in skip_paths and f.path not in interp_paths] if filtered: own_source.append(depset(direct = filtered)) - # bzlmod-canonical → apparent repo-name translation manifest. - # `runfiles.bash`/`rlocation` need this at `.runfiles/_repo_mapping` - # to resolve apparent labels (e.g. `aspect_rules_py/...`) to the canonical - # repo names (`aspect_rules_py+/...`) under which files are actually staged. - # Without it the launcher fails with "runfiles.bash initializer cannot find - # bazel_tools/tools/bash/runfiles/runfiles.bash" the moment any rlocation - # call is evaluated. - repo_mapping = getattr(target[DefaultInfo].default_runfiles, "repo_mapping_manifest", None) - if repo_mapping != None: - own_source.append(depset(direct = [repo_mapping])) - return [_LayerInfo( source_files = depset(transitive = transitive_source + own_source), pip_packages = depset(transitive = transitive_pkgs), first_party_layers = depset(direct = own_fp, transitive = transitive_fp), - interpreter_files = interpreter_files, interpreter_layer = interpreter_layer, )] @@ -585,18 +559,45 @@ def _apply_strip_prefix(sp, strip_prefix, root): return "." + root + sp[len(prefix):] return "./app.runfiles/_main/" + sp -def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False): - sp = f.short_path +def _source_destination(sp, strip_prefix, root, executable_dsts): + executable_dst = executable_dsts.get(sp, None) + if executable_dst: + return executable_dst if sp == "_repo_mapping": # Bazel synthesizes a top-level `_repo_mapping` runfile (no `_main/` # prefix); replicate that placement so runfiles.bash can find it. - dst = "./app.runfiles/_repo_mapping" - elif sp.startswith("../"): - dst = "./app.runfiles/" + sp[3:] - elif strip_prefix: - dst = _apply_strip_prefix(sp, strip_prefix, root) - else: - dst = "./app.runfiles/_main/" + sp + return "./app.runfiles/_repo_mapping" + runfiles_prefix = None + for executable_short_path in executable_dsts: + if (sp.startswith(executable_short_path + ".runfiles/") or + (not strip_prefix and not executable_dsts[executable_short_path] and sp.startswith(executable_short_path + "/"))): + if runfiles_prefix == None or len(executable_short_path) > len(runfiles_prefix): + runfiles_prefix = executable_short_path + if runfiles_prefix != None: + if strip_prefix and not runfiles_prefix.startswith("../") and not executable_dsts[runfiles_prefix]: + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination + runfiles_root = "/app" if executable_dsts[runfiles_prefix] else root + return _apply_strip_prefix(sp, runfiles_prefix, runfiles_root) + if sp.startswith("../"): + if strip_prefix and (sp in executable_dsts or sp == strip_prefix): + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination + if sp in executable_dsts: + return _apply_strip_prefix(sp, sp, root) + return "./app.runfiles/" + sp[3:] + if strip_prefix and not any(executable_dsts.values()): + destination = _apply_strip_prefix(sp, strip_prefix, root) + if destination != "./app.runfiles/_main/" + sp: + return destination + if sp in executable_dsts: + return _apply_strip_prefix(sp, sp, root) + return "./app.runfiles/_main/" + sp + +def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_symlink = False, executable_dsts = {}): + dst = _source_destination(f.short_path, strip_prefix, root, executable_dsts) # `f.is_symlink` emits `type=link` (awk readlinks once); `maybe_symlink=True` # emits `type=file content=` (awk readlinks to detect repo-rule-staged @@ -615,20 +616,49 @@ def _file_to_mtree_entry(f, mode = "0644", strip_prefix = "", root = "/", maybe_ f.path.replace(" ", "\\040"), ) -def _source_file_to_mtree(f, dir_expander, strip_prefix, root, maybe_symlink): +def _source_file_to_mtree( + f, + dir_expander, + strip_prefix, + root, + maybe_symlink, + executable_dsts, + runfile_executable_paths, + repo_mapping_path): + if f.path == repo_mapping_path: + return _file_to_mtree_entry( + f, + "0755", + f.short_path, + "/app.runfiles/_repo_mapping", + ) + # 0755 throughout: keeps launcher/interpreter/venv shims executable; Bazel # doesn't expose per-input source mode for us to propagate. if f.is_directory: return [ - _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink) + _file_to_mtree_entry(child, "0755", strip_prefix, root, maybe_symlink, executable_dsts) for child in dir_expander.expand(f) ] - return _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink) + entry = _file_to_mtree_entry(f, "0755", strip_prefix, root, maybe_symlink, executable_dsts) + if f.path not in runfile_executable_paths: + return entry + return [ + entry, + _file_to_mtree_entry( + f, + "0755", + maybe_symlink = maybe_symlink, + ), + ] -def _user_file_to_mtree(f, dir_expander): +def _user_file_to_mtree(f, dir_expander, source_owned_paths): + # Rule-level groups may contain declared symlinks that File.is_symlink + # doesn't expose, so every grouped file needs the readlink fallback. if f.is_directory: - return [_file_to_mtree_entry(child, "0755") for child in dir_expander.expand(f)] - return _file_to_mtree_entry(f, "0644") + return [_file_to_mtree_entry(child, "0755", maybe_symlink = True) for child in dir_expander.expand(f)] + mode = "0755" if f.path in source_owned_paths else "0644" + return _file_to_mtree_entry(f, mode, maybe_symlink = True) def _should_skip_pkg_path(p): return ( @@ -731,7 +761,7 @@ _platform_cfg = transition( outputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier"], ) -def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, compress, level, reqs, mnemonic, progress_msg): +def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, compress, level, reqs, mnemonic, progress_msg, symlink_mappings = None): # mtree (param file) → gawk (readlinks `type=link`/`type=file content=` # rows; `contents=` rows pass through; END buffers, asort-sorts, and # writes the sorted mtree to a file) → bsdtar consumes the file. @@ -748,12 +778,29 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, gawk_args = ctx.actions.args() gawk_args.add("-v", sorted_mtree, format = "outfile=%s") + gawk_args.add("-v", "1", format = "source_argind=%s") gawk_args.add("-f", awk_script) + gawk_arguments = [gawk_args, mtree_args] + gawk_inputs = [files_depset] + if symlink_mappings != None: + mapping_args = ctx.actions.args() + mapping_args.set_param_file_format("multiline") + mapping_args.use_param_file("%s", use_always = True) + mapping_args.add("#mtree") + mapping_args.add_all( + symlink_mappings.files, + map_each = symlink_mappings.map_each, + expand_directories = False, + allow_closure = True, + ) + gawk_arguments.append(mapping_args) + if symlink_mappings.tree_files: + gawk_inputs.append(depset(direct = symlink_mappings.tree_files)) ctx.actions.run( executable = awk, - inputs = depset(direct = [awk_script], transitive = [files_depset]), + inputs = depset(direct = [awk_script], transitive = gawk_inputs), outputs = [sorted_mtree], - arguments = [gawk_args, mtree_args], + arguments = gawk_arguments, # LC_ALL=C makes gawk's asort byte-stable. env = {"LC_ALL": "C"}, mnemonic = mnemonic + "Mtree", @@ -782,7 +829,7 @@ def _run_tar_action(ctx, bsdtar, bsdtar_files, tar_out, files_depset, map_each, use_default_shell_env = False, ) -def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, map_each, progress): +def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, map_each, progress, symlink_mappings = None): tar_out = ctx.actions.declare_file(out_name) level = ctx.attr.group_compress_levels.get(group_name, "6") reqs = _parse_exec_requirements(ctx.attr.group_execution_requirements.get(group_name, [])) @@ -798,80 +845,222 @@ def _declare_group_tar(ctx, bsdtar, bsdtar_files, out_name, group_name, files, m reqs, "PyImageLayer", progress, + symlink_mappings, ) return tar_out def _py_image_layer_impl(ctx): - info = ctx.attr.binary[_LayerInfo] - merged = ctx.attr.binary[_MergedLayerInfo] + binaries = ctx.attr.binaries + if not binaries: + fail("py_image_layer requires at least one binary") + single_binary = len(binaries) == 1 + infos = [binary[_LayerInfo] for binary in binaries] bsdtar, bsdtar_files = _tar_toolchain(ctx) - pkg_by_label = {} - for pkg in info.pip_packages.to_list(): - if pkg.label not in pkg_by_label: - pkg_by_label[pkg.label] = pkg - all_pkgs = pkg_by_label.values() + # For multiple binaries, normalized labels can collide across lock + # universes. Concrete paths identify each configured wheel artifact. + pkg_by_key = {} + for info in infos: + for pkg in info.pip_packages.to_list(): + key = tuple(sorted([f.path for f in pkg.files.to_list()])) + if key not in pkg_by_key: + pkg_by_key[key] = pkg + all_pkgs = pkg_by_key.values() + pip_labels = {pkg.label: True for pkg in all_pkgs} # `_platform_cfg` rewrites the `//py:layer_tier` flag from `attr.layer_tier`, # so `_layer_tier` always resolves to the effective tier. plan = ctx.attr._layer_tier[PyLayerTierInfo] root = plan.root strip_prefix = plan.strip_prefix + launcher_dir = ctx.attr.launcher_dir + if len(binaries) > 1 and not launcher_dir: + launcher_dir = "/app/bin" + if launcher_dir: + launcher_dir = launcher_dir.rstrip("/") or "/" + if launcher_dir and not launcher_dir.startswith("/"): + fail("py_image_layer.launcher_dir must be an absolute image path") + + launcher_names = {} + executable_dsts = {} + executable_owner_by_path = {} + repo_mappings_by_path = {} + for index, binary in enumerate(binaries): + files_to_run = binary[DefaultInfo].files_to_run + executable = files_to_run.executable + executable_owner_by_path[executable.path] = index + repo_mapping = files_to_run.repo_mapping_manifest + if repo_mapping != None: + repo_mappings_by_path[repo_mapping.path] = repo_mapping + launcher_name = executable.basename + if launcher_dir: + if launcher_name in launcher_names: + fail("duplicate py_image_layer launcher basename: {}".format(launcher_name)) + launcher_names[launcher_name] = True + executable_dsts[executable.short_path] = "." + launcher_dir.rstrip("/") + "/" + launcher_name + else: + executable_dsts[executable.short_path] = "" + + runfile_executable_paths = {} + if launcher_dir and len(binaries) > 1: + for index, binary in enumerate(binaries): + for f in binary[DefaultInfo].default_runfiles.files.to_list(): + owner = executable_owner_by_path.get(f.path, None) + + # A launcher consumed through another binary's runfiles needs + # both its relocated entrypoint and the logical key resolved + # by that consumer. + if owner != None and owner != index: + runfile_executable_paths[f.path] = True + + # Each manifest describes one launcher's runfiles closure. The image shares + # one runfiles root, so its manifest must resolve apparent names from all of + # the explicitly listed launchers. + repo_mappings = repo_mappings_by_path.values() + repo_mapping = None + if len(repo_mappings) == 1: + repo_mapping = repo_mappings[0] + elif len(repo_mappings) > 1: + repo_mapping = ctx.actions.declare_file(ctx.attr.name + "/_repo_mapping") + args = ctx.actions.args() + args.add("-v") + args.add("output=" + repo_mapping.path) + args.add("-f") + args.add(ctx.file._repo_mapping_merger) + args.add_all(repo_mappings) + ctx.actions.run( + executable = ctx.executable._awk, + inputs = depset(direct = [ctx.file._repo_mapping_merger] + repo_mappings), + outputs = [repo_mapping], + arguments = [args], + env = {"LC_ALL": "C"}, + mnemonic = "PyImageRepoMapping", + progress_message = "Merging repository mappings for %s" % ctx.label, + ) # 3p pip layers are action-shared across the graph and hard-code their # destination under `./app.runfiles//...`, so the consumer's source - # layer has to land under the same `/app.runfiles/` tree for the launcher - # to find them. Default the binary's `short_path` to `strip_prefix` so the - # natural runfile layout maps onto `./app.runfiles/_main/...` without each - # caller wiring it up. - binary_short_path = ctx.attr.binary[DefaultInfo].files_to_run.executable.short_path - if not strip_prefix: - strip_prefix = binary_short_path - + # layer has to land under the same `/app.runfiles/` tree for each launcher + # to find them. Map every launcher's `.runfiles/` tree into that shared root. all_tars = [] - - # Without a dedicated interpreter tar, repo-rule-staged symlinks like - # rules_python's `bin/python -> python3.11` end up in the source layer - # and need awk's readlink scan to be preserved. - source_maybe_symlink = info.interpreter_layer == None - - def _source_map(f, d): - return _source_file_to_mtree(f, d, strip_prefix, root, source_maybe_symlink) + source_maybe_symlink = any([info.interpreter_layer == None for info in infos]) + source_map = lambda f, d: _source_file_to_mtree( + f, + d, + strip_prefix, + root, + source_maybe_symlink, + executable_dsts, + runfile_executable_paths, + repo_mapping.path if repo_mapping != None else "", + ) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} + rule_group_files = [] + rule_group_paths = {} + rule_groups = [] for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) - if dep_label in pkg_by_label: + if dep_label in pip_labels: continue + files = dep[DefaultInfo].files + rule_group_files.append(files) + for f in files.to_list(): + rule_group_paths[f.path] = True + rule_groups.append((group_name, files)) + + source_files = depset(transitive = [info.source_files for info in infos]) + if repo_mapping != None: + source_files = depset(direct = [repo_mapping], transitive = [source_files]) + source_owned_group_paths = {} + if rule_group_paths: + # The aspect cannot see rule-level groups, so decide ownership from the + # source closure once, then remove grouped bytes from the source tar. + ungrouped_source_files = [] + for f in source_files.to_list(): + if f.path in rule_group_paths: + source_owned_group_paths[f.path] = True + else: + ungrouped_source_files.append(f) + source_files = depset(direct = ungrouped_source_files) + + rule_group_map = lambda f, d: ( + source_map(f, d) if f.short_path in executable_dsts else _user_file_to_mtree(f, d, source_owned_group_paths) + ) + + first_party_reference_files = [] + for info in infos: + for entry in info.first_party_layers.to_list(): + first_party_reference_files.append(entry.files) + + # Interpreter tars are declared at the configured toolchain, so identical + # runtimes action-share while distinct interpreter artifacts are retained. + interpreter_layers = {} + for info in infos: + if info.interpreter_layer != None: + layer = info.interpreter_layer + interpreter_layers[layer.tar.path] = layer + + # Each source-owned tier may contain a symlink whose target is emitted by + # another tier. Share destination-only rows so every tar can rewrite those + # links without copying the target bytes into that tar. + symlink_mappings = None + if rule_group_files or first_party_reference_files: + interpreter_reference_files = [] + interpreter_reference_paths = {} + for layer in interpreter_layers.values(): + files = layer.interpreter_files + interpreter_reference_files.append(files) + for f in files.to_list(): + interpreter_reference_paths[f.path] = True + reference_files = [source_files] + rule_group_files + first_party_reference_files + interpreter_reference_files + reference_tree_files = {} + for files in reference_files: + for f in files.to_list(): + if f.is_directory: + reference_tree_files[f.path] = f + reference_map = lambda f, d: ( + rule_group_map(f, d) if f.path in rule_group_paths else _interpreter_file_to_mtree(f, d) if f.path in interpreter_reference_paths else source_map(f, d) + ) + symlink_mappings = struct( + files = depset(transitive = reference_files), + map_each = reference_map, + tree_files = reference_tree_files.values(), + ) + + for group_name, files in rule_groups: tar_out = _declare_group_tar( ctx, bsdtar, bsdtar_files, "{}_{}.tar.gz".format(ctx.attr.name, group_name), group_name, - dep[DefaultInfo].files, - _user_file_to_mtree, + files, + rule_group_map, "Creating image layer %s[%s]" % (ctx.label, group_name), + symlink_mappings, ) all_tars.append(tar_out) fp_by_group = {} seen_fp_labels = {} - for entry in info.first_party_layers.to_list(): - if entry.label in seen_fp_labels: - continue - seen_fp_labels[entry.label] = True - fp_by_group.setdefault(entry.group, []).append(entry.files) + for info in infos: + for entry in info.first_party_layers.to_list(): + if single_binary: + if entry.label in seen_fp_labels: + continue + seen_fp_labels[entry.label] = True + fp_by_group.setdefault(entry.group, []).append(entry.files) - # Slot the pre-built, action-shared interpreter tar into the fp ordering - # under its group name so users see a stable per-group sort regardless of - # where the tar was declared. prebuilt_group_tars = {} - if info.interpreter_layer != None: - prebuilt_group_tars[info.interpreter_layer.group] = info.interpreter_layer.tar - fp_by_group.setdefault(info.interpreter_layer.group, []) - - for group_name in sorted(fp_by_group): + if single_binary: + for layer in interpreter_layers.values(): + prebuilt_group_tars[layer.group] = layer.tar + fp_by_group.setdefault(layer.group, []) + + layer_group_names = {group_name: True for group_name in fp_by_group} + layer_group_names.update({layer.group: True for layer in interpreter_layers.values()}) + for group_name in layer_group_names: if group_name in rule_group_names: fail( ("Group %r is declared in both py_image_layer.groups and the active " + @@ -879,6 +1068,7 @@ def _py_image_layer_impl(ctx): "deps with a different file layout than first-party sources, so they " + "cannot share a tar.") % group_name, ) + for group_name in sorted(fp_by_group): if group_name in prebuilt_group_tars: tar_out = prebuilt_group_tars[group_name] else: @@ -889,17 +1079,45 @@ def _py_image_layer_impl(ctx): "{}_{}.tar.gz".format(ctx.attr.name, group_name), group_name, depset(transitive = fp_by_group[group_name]), - _source_map, + source_map, "Creating first-party layer %s[%s]" % (ctx.label, group_name), + symlink_mappings, ) all_tars.append(tar_out) + if not single_binary: + all_tars.extend([layer.tar for layer in interpreter_layers.values()]) + + pip_tars = {} + merged = {} for pkg in all_pkgs: for layer in pkg.layers: - all_tars.append(layer.tar) + pip_tars[layer.tar.path] = layer.tar + if not single_binary and pkg.merge_group != None: + merged.setdefault(pkg.merge_group, []).append(pkg.files) + + all_tars.extend(pip_tars.values()) - for _group_name, merged_tar in sorted(merged.merged_tars.items()): - all_tars.append(merged_tar) + if single_binary: + all_tars.extend([tar for _group_name, tar in sorted(binaries[0][_MergedLayerInfo].merged_tars.items())]) + else: + for group_name in sorted(merged): + algorithm, level, ext = _compression_for(plan, group_name) + tar_out = ctx.actions.declare_file("{}/merged_pip_layers/{}{}".format(ctx.attr.name, group_name, ext)) + _run_tar_action( + ctx, + bsdtar, + bsdtar_files, + tar_out, + depset(transitive = merged[group_name]), + _pkg_file_to_mtree, + algorithm, + level, + {}, + "PyImageMergedLayer", + "Merging %d pip packages into %s[%s]" % (len(merged[group_name]), ctx.label, group_name), + ) + all_tars.append(tar_out) ungrouped_pkgs = [p for p in all_pkgs if len(p.layers) == 0 and p.merge_group == None] if ungrouped_pkgs: @@ -925,9 +1143,10 @@ def _py_image_layer_impl(ctx): bsdtar_files, "{}_default.tar.gz".format(ctx.attr.name), "default", - info.source_files, - _source_map, + source_files, + source_map, "Creating source layer for %s" % ctx.label, + symlink_mappings, ) all_tars.append(source_tar) @@ -940,11 +1159,41 @@ def _py_image_layer_impl(ctx): for pkg in ungrouped_pkgs: validation_args.add_all(pkg.files, format_each = pkg.label + "=%s", expand_directories = False) + validation_inputs = [pkg.files for pkg in ungrouped_pkgs] + validation_arguments = [validation_args] + if len(binaries) > 1 or launcher_dir or strip_prefix or root != "/app": + # Validate expanded rows whenever source destinations can be shared or remapped. + # TreeArtifact roots can contain disjoint versioned children, so only + # the production mappers' expanded destinations are authoritative. + source_files = depset(transitive = [source_files] + [ + files + for group_name, file_sets in fp_by_group.items() + if group_name not in prebuilt_group_tars + for files in file_sets + ]) + wheel_files = [pkg.files for pkg in all_pkgs] + interpreter_files = [layer.interpreter_files for layer in interpreter_layers.values()] + + mtree_args = ctx.actions.args() + mtree_args.set_param_file_format("multiline") + mtree_args.use_param_file("%s", use_always = True) + mtree_args.add("#mtree") + mtree_args.add_all(source_files, map_each = source_map, expand_directories = False, allow_closure = True) + for files in rule_group_files: + mtree_args.add_all(files, map_each = rule_group_map, expand_directories = False, allow_closure = True) + for files in wheel_files: + mtree_args.add_all(files, map_each = _pkg_file_to_mtree, expand_directories = False) + for files in interpreter_files: + mtree_args.add_all(files, map_each = _interpreter_file_to_mtree, expand_directories = False) + validation_args.add("--mtree") + validation_arguments.append(mtree_args) + validation_inputs.extend([source_files] + rule_group_files + wheel_files + interpreter_files) + ctx.actions.run( executable = ctx.executable._validator, - inputs = depset(transitive = [pkg.files for pkg in ungrouped_pkgs]), + inputs = depset(transitive = validation_inputs), outputs = [validation], - arguments = [validation_args], + arguments = validation_arguments, mnemonic = "PyImageLayerValidate", ) @@ -960,9 +1209,14 @@ def _py_image_layer_impl(ctx): _py_image_layer = rule( implementation = _py_image_layer_impl, attrs = { - "binary": attr.label( - mandatory = True, + "binaries": attr.label_list( + allow_empty = False, aspects = [_layer_aspect, _merge_aspect], + mandatory = True, + ), + "launcher_dir": attr.string( + default = "", + doc = "Absolute image directory for binary launchers. Defaults to /app/bin with multiple binaries.", ), "groups": attr.label_keyed_string_dict(default = {}), "group_execution_requirements": attr.string_list_dict(default = {}), @@ -981,6 +1235,10 @@ _py_image_layer = rule( cfg = "exec", allow_files = True, ), + "_repo_mapping_merger": attr.label( + default = "//py/private:merge_repo_mappings.awk", + allow_single_file = True, + ), "_awk_script": attr.label( default = "//py/private:modify_mtree.awk", allow_single_file = True, @@ -997,7 +1255,7 @@ _py_image_layer = rule( def py_image_layer( name, - binary, + binary = None, groups = {}, group_execution_requirements = {}, group_compress_levels = {}, @@ -1005,8 +1263,10 @@ def py_image_layer( warn_layer_count = 90, platform = None, layer_tier = None, + launcher_dir = "", + binaries = None, **kwargs): - """Create OCI-compatible tars from a py_binary or py_venv target. + """Create OCI-compatible tars from one or more py_binary targets. Pip-package grouping + compression is resolved from the `//py:layer_tier` label_flag. Override globally with `--//py:layer_tier=//path:custom_tier`, @@ -1017,17 +1277,17 @@ def py_image_layer( 1. Non-pip deps listed in `groups` → one rule-created tar per group. 2. First-party py_library targets matched by `py_layer_tier.groups` → one rule-created tar per group (aggregated across all matched targets in the - binary's dep closure). + binary inputs' dep closures). 3. Solo-group and subpath-split pip tars — built by `_layer_aspect` at each pip target's own namespace; globally shared across every rule using that package. - 4. Multi-member merged tars — one per group, built by `_merge_aspect` at the - binary's namespace from the closure-filtered union of member install_dirs. + 4. Multi-member merged tars — action-shared for a single binary or one per + group from the closure-filtered union across multiple binaries. 5. Ungrouped pip packages → one squashed rule-created tar. 6. Remaining first-party Python source files → the "default" layer. Args: name: Name of the generated target. - binary: A py_venv or py_binary target. + binary: A py_binary target. groups: Maps a NON-PIP dep label to a group name. Each gets its own rule-created tar. All pip-package grouping (whole-package, subpath, multi-member) belongs in py_layer_tier — subpath glob keys passed here fail loudly. @@ -1042,10 +1302,23 @@ def py_image_layer( layer_tier: Optional py_layer_tier target pinned for this rule. Sets the `@aspect_rules_py//py:layer_tier` label_flag via the rule transition, overriding any command-line value for this rule's subgraph. + launcher_dir: Absolute image directory for the binary launchers. Defaults + to /app/bin with multiple binaries. Set RUNFILES_DIR=/app.runfiles in + the image. + binaries: Alternative to binary. A nonempty list of py_binary targets to + include in the image. **kwargs: Forwarded to inner rule. """ tags = kwargs.pop("tags", []) + ["manual"] + if binaries != None: + if binary != None: + fail("py_image_layer accepts either binary or binaries, not both") + else: + if binary == None: + fail("py_image_layer requires binary or binaries") + binaries = [binary] + for key in groups: if _split_glob_key(key)[1] != None: fail( @@ -1055,7 +1328,8 @@ def py_image_layer( _py_image_layer( name = name, - binary = binary, + binaries = binaries, + launcher_dir = launcher_dir, groups = groups, group_execution_requirements = group_execution_requirements, group_compress_levels = group_compress_levels, diff --git a/py/private/py_image_layer_validator.py b/py/private/py_image_layer_validator.py index 1851dd141..609e5d8ec 100644 --- a/py/private/py_image_layer_validator.py +++ b/py/private/py_image_layer_validator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""py_image_layer_validator — validate pip layer sizing for a py_image_layer target. +"""py_image_layer_validator — validate sizing and shared destinations for an image layer. Invoked as a Bazel validation action. Fails (exit 1) with actionable `py_layer_tier` snippets when the squashed pip layer exceeds a size threshold or when the OCI 127-layer hard limit is @@ -9,6 +9,7 @@ py_image_layer_validator --threshold_mb N --output FILE [label=path ...] label=path — one entry per ungrouped pip package; `label` is the canonical pip label (e.g. @pip//numpy), `path` is its install directory / file. + --mtree FILE — expanded mtree rows for shared or remapped source destinations. """ from __future__ import annotations @@ -16,10 +17,12 @@ import argparse import contextlib import csv +import filecmp import glob import os +import stat import sys -from typing import Dict, List, Optional, Sequence, Tuple +from typing import Dict, Iterable, List, Optional, Sequence, Tuple _OCI_LAYER_HARD_LIMIT = 127 _BINARY_GLOBS = {"*.so", "*.so.*", "*.pyd", "*.dylib", "*.dll"} @@ -229,12 +232,133 @@ def _add_subpath_or_whole( _add_whole_promotion(suggestions, label, size_mb, is_binary, annotation="whole package") +def _decode_mtree_path(path: str) -> str: + return path.replace("\\040", " ") + + +def _peel_sandbox_symlink(source: str) -> str: + # Bazel's absolute action-input symlink repeats the exec path as a suffix. + # Peel only that transport hop; a relative target is the logical artifact. + if not stat.S_ISLNK(os.lstat(source).st_mode): + return source + target = os.readlink(source) + suffix = os.sep + source + if os.path.isabs(target) and target.endswith(suffix): + return target + return source + + +def _comparable_file(source_kind: str, encoded_source: str) -> Optional[str]: + source = _decode_mtree_path(encoded_source) + if source_kind == "contents": + return source if os.path.isfile(source) else None + if source_kind != "content": + return None + + try: + source = _peel_sandbox_symlink(source) + mode = os.lstat(source).st_mode + if stat.S_ISREG(mode): + return source + return None + except OSError: + return None + + +def _comparable_symlink_target(source_kind: str, encoded_source: str) -> Optional[str]: + if source_kind not in ("content", "link"): + return None + source = _decode_mtree_path(encoded_source) + try: + source = _peel_sandbox_symlink(source) + if not stat.S_ISLNK(os.lstat(source).st_mode): + return None + target = os.readlink(source) + return target if target and not os.path.isabs(target) else None + except OSError: + return None + + +def _mtree_collision(rows: Iterable[str]) -> Optional[str]: + """Return the first conflicting expanded mtree destination, or None.""" + paths: Dict[str, Tuple[str, str, str, Tuple[str, ...]]] = {} + descendants: Dict[str, Tuple[str, str]] = {} + for row in rows: + if not row or row.startswith("#"): + continue + fields = row.split() + destination_parts: List[str] = [] + for part in fields[0].split("/"): + if not part or part == ".": + continue + if part == "..": + if not destination_parts: + return "py_image_layer image destination escapes its root: {}".format(fields[0]) + destination_parts.pop() + else: + destination_parts.append(part) + destination = "./" + "/".join(destination_parts) if destination_parts else "." + entry_type = next( + (field.partition("=")[2] for field in fields[1:] if field.startswith("type=")), + None, + ) + source_field = next( + (field for field in fields[1:] if field.startswith(("contents=", "content=", "link="))), + None, + ) + if entry_type is None or source_field is None: + return "invalid py_image_layer mtree row (missing source): {}".format(row) + source_kind, _, source = source_field.partition("=") + metadata = tuple(sorted(field for field in fields[1:] if field != source_field)) + entry = (entry_type, source_kind, source, metadata) + + previous = paths.get(destination) + if previous is not None: + if previous == entry: + continue + previous_type, previous_kind, previous_source, previous_metadata = previous + if metadata == previous_metadata: + if entry_type == previous_type == "file": + previous_file = _comparable_file(previous_kind, previous_source) + source_file = _comparable_file(source_kind, source) + if previous_file is not None and source_file is not None: + with contextlib.suppress(OSError): + if filecmp.cmp(previous_file, source_file, shallow=False): + continue + previous_target = _comparable_symlink_target(previous_kind, previous_source) + source_target = _comparable_symlink_target(source_kind, source) + if previous_target is not None and previous_target == source_target: + continue + return "py_image_layer runfile collision at {}: {} and {}".format( + destination, previous_source, source + ) + + parts = destination.split("/") + for end in range(len(parts) - 1, 0, -1): + parent = "/".join(parts[:end]) + if parent in paths: + return "py_image_layer runfile collision at {}: {} and {}".format( + destination, paths[parent][2], source + ) + if destination in descendants: + descendant, previous = descendants[destination] + return "py_image_layer runfile collision at {}: {} and {}".format(descendant, previous, source) + + paths[destination] = entry + for end in range(len(parts) - 1, 0, -1): + parent = "/".join(parts[:end]) + descendants.setdefault(parent, (destination, source)) + + return None + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--threshold_mb", type=int, default=200) parser.add_argument("--layer_count", type=int, default=0) parser.add_argument("--warn_layer_count", type=int, default=90) parser.add_argument("--output", required=True) + parser.add_argument("--mtree") parser.add_argument("pkg_paths", nargs="*", metavar="label=path") args = parser.parse_args() @@ -252,6 +376,11 @@ def main() -> None: pkg_binary = {label: _pkg_is_binary(paths) for label, paths in pkg_path_map.items()} messages: List[str] = [] + if args.mtree: + with open(args.mtree) as mtree: + collision = _mtree_collision(mtree) + if collision: + messages.append("ERROR: " + collision) suggestions = _Suggestions() layer_count_comment_lines: List[str] = [] diff --git a/py/private/py_image_layer_validator_test.py b/py/private/py_image_layer_validator_test.py index 1e9cc759c..f45cedb86 100644 --- a/py/private/py_image_layer_validator_test.py +++ b/py/private/py_image_layer_validator_test.py @@ -13,6 +13,7 @@ _Suggestions, _find_large_files, _glob_for_file, + _mtree_collision, _pkg_is_binary, _pkg_name_from_label, _pkg_size, @@ -21,6 +22,260 @@ ) +def _mtree_row(destination: str, source: str, marker: str = "contents") -> str: + return "{} type=file mode=0755 {}={}".format(destination, marker, source.replace(" ", "\\040")) + + +def _mtree_link_row(destination: str, source: str, mode: str = "0755") -> str: + return "{} type=link mode={} link={}".format(destination, mode, source.replace(" ", "\\040")) + + +def test_mtree_identical_link_source_coalesces() -> None: + row = "./app.runfiles/pkg/module.py type=link mode=0755 link=same/module.py" + assert _mtree_collision(["#mtree", row, row]) is None + + +def test_mtree_identical_relative_link_targets_coalesce() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + os.symlink("../target", first) + os.symlink("../target", second) + + assert _mtree_collision([ + _mtree_link_row("./app.runfiles/pkg/module.py", first), + _mtree_link_row("./app.runfiles/pkg/module.py", second), + ]) is None + + +def test_mtree_different_relative_link_targets_fail() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + os.symlink("../first", first) + os.symlink("../second", second) + + assert _mtree_collision([ + _mtree_link_row("./app.runfiles/pkg/module.py", first), + _mtree_link_row("./app.runfiles/pkg/module.py", second), + ]) is not None + + +def test_mtree_identical_relative_link_targets_with_different_modes_fail() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + os.symlink("../target", first) + os.symlink("../target", second) + + assert _mtree_collision([ + _mtree_link_row("./app.runfiles/pkg/module.py", first), + _mtree_link_row("./app.runfiles/pkg/module.py", second, "0644"), + ]) is not None + + +def test_mtree_sandbox_wrapped_logical_symlinks_coalesce() -> None: + with ( + tempfile.TemporaryDirectory(dir=".") as wrapper_directory, + tempfile.TemporaryDirectory() as target_root, + ): + wrapper_directory = os.path.relpath(wrapper_directory) + sources = [ + os.path.join(wrapper_directory, "first"), + os.path.join(wrapper_directory, "second"), + ] + for source in sources: + sandbox_target = os.path.join(target_root, source) + os.makedirs(os.path.dirname(sandbox_target), exist_ok=True) + os.symlink("../target", sandbox_target) + os.symlink(sandbox_target, source) + + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", source, "content") + for source in sources + ]) is None + + +def test_mtree_conflicting_source_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/module.py", "second/module.py"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_identical_regular_files_coalesce_across_markers() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + for path in (first, second): + with open(path, "w") as file: + file.write("identical") + + for first_marker, second_marker in [ + ("contents", "contents"), + ("content", "content"), + ("content", "contents"), + ]: + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", first, first_marker), + _mtree_row("./app.runfiles/pkg/module.py", second, second_marker), + ]) is None, (first_marker, second_marker) + + +def test_mtree_identical_file_contents_with_spaces_coalesce() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first file") + second = os.path.join(directory, "second file") + for path in (first, second): + with open(path, "w") as file: + file.write("identical") + + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", first), + _mtree_row("./app.runfiles/pkg/module.py", second), + ]) is None + + +def test_mtree_sandbox_wrapped_regular_content_coalesces() -> None: + with ( + tempfile.TemporaryDirectory(dir=".") as wrapper_directory, + tempfile.TemporaryDirectory() as target_root, + ): + wrapper_directory = os.path.relpath(wrapper_directory) + source = os.path.join(wrapper_directory, "wrapped") + sandbox_target = os.path.join(target_root, source) + os.makedirs(os.path.dirname(sandbox_target), exist_ok=True) + with open(sandbox_target, "w") as file: + file.write("identical") + + os.symlink(sandbox_target, source) + direct = os.path.join(wrapper_directory, "direct") + with open(direct, "w") as file: + file.write("identical") + + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", source, "content"), + _mtree_row("./app.runfiles/pkg/module.py", direct), + ]) is None + + +def test_mtree_sandbox_wrapped_logical_symlink_fails() -> None: + with ( + tempfile.TemporaryDirectory(dir=".") as wrapper_directory, + tempfile.TemporaryDirectory() as target_root, + ): + wrapper_directory = os.path.relpath(wrapper_directory) + source = os.path.join(wrapper_directory, "wrapped") + sandbox_target = os.path.join(target_root, source) + os.makedirs(os.path.dirname(sandbox_target), exist_ok=True) + actual = os.path.join(os.path.dirname(sandbox_target), "actual") + with open(actual, "w") as file: + file.write("identical") + os.symlink("actual", sandbox_target) + os.symlink(sandbox_target, source) + direct = os.path.join(wrapper_directory, "direct") + with open(direct, "w") as file: + file.write("identical") + + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", source, "content"), + _mtree_row("./app.runfiles/pkg/module.py", direct), + ]) + assert collision is not None + + +def test_mtree_logical_content_symlink_fails() -> None: + with tempfile.TemporaryDirectory() as directory: + target = os.path.join(directory, "target") + link = os.path.join(directory, "link") + direct = os.path.join(directory, "direct") + for path in (target, direct): + with open(path, "w") as file: + file.write("identical") + os.symlink("target", link) + + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", link, "content"), + _mtree_row("./app.runfiles/pkg/module.py", direct), + ]) + assert collision is not None + + +def test_mtree_different_file_contents_fail() -> None: + with tempfile.TemporaryDirectory() as directory: + first = os.path.join(directory, "first") + second = os.path.join(directory, "second") + with open(first, "w") as file: + file.write("first") + with open(second, "w") as file: + file.write("second") + + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", first), + _mtree_row("./app.runfiles/pkg/module.py", second), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_file_and_symlink_types_fail() -> None: + destination = "./app.runfiles/pkg/module.py" + source = "same/module.py" + collision = _mtree_collision([ + _mtree_row(destination, source), + "{} type=link mode=0755 link={}".format(destination, source), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_different_modes_fail() -> None: + destination = "./app.runfiles/pkg/module.py" + source = "same/module.py" + collision = _mtree_collision([ + _mtree_row(destination, source), + _mtree_row(destination, source).replace("mode=0755", "mode=0644"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_ancestor_then_descendant_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/module.py/data", "second/data"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py/data:" in collision + + +def test_mtree_descendant_then_ancestor_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/module.py/data", "second/data"), + _mtree_row("./app.runfiles/pkg/module.py", "first/module.py"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py/data:" in collision + + +def test_mtree_normalized_alias_fails() -> None: + collision = _mtree_collision([ + _mtree_row("./app.runfiles/pkg/nested/../module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/./module.py", "second/module.py"), + ]) + assert collision is not None + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in collision + + +def test_mtree_disjoint_children_succeed() -> None: + assert _mtree_collision([ + _mtree_row("./app.runfiles/pkg/python3.11/module.py", "first/module.py"), + _mtree_row("./app.runfiles/pkg/python3.12/module.py", "second/module.py"), + ]) is None + + # --------------------------------------------------------------------------- # _pkg_name_from_label # --------------------------------------------------------------------------- @@ -264,6 +519,28 @@ def test_main_ok_below_threshold() -> None: os.unlink(out_path) +def test_main_mtree_collision_error() -> None: + with tempfile.NamedTemporaryFile(mode="w", suffix=".params", delete=False) as params: + params.write("#mtree\n") + params.write(_mtree_row("./app.runfiles/pkg/module.py", "first/module.py") + "\n") + params.write(_mtree_row("./app.runfiles/pkg/module.py", "second/module.py") + "\n") + params_path = params.name + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as output: + out_path = output.name + sys.argv = ["validator", "--threshold_mb", "999", "--output", out_path, "--mtree", params_path] + with _capture_stderr(): + try: + main() + exit_code = 0 + except SystemExit as e: + exit_code = e.code + assert exit_code == 1 + with open(out_path) as fh: + assert "py_image_layer runfile collision at ./app.runfiles/pkg/module.py:" in fh.read() + os.unlink(params_path) + os.unlink(out_path) + + def test_main_layer_count_error() -> None: with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: out_path = f.name diff --git a/py/tests/internal-deps/adder/BUILD.bazel b/py/tests/internal-deps/adder/BUILD.bazel index cf12eff91..8f9a866b9 100644 --- a/py/tests/internal-deps/adder/BUILD.bazel +++ b/py/tests/internal-deps/adder/BUILD.bazel @@ -1,9 +1,10 @@ -"""Deliberate e2e fixture: a py_library from the @aspect_rules_py module consumed by the -OCI e2e cases (//cases/oci/py_image_layer) to verify that py_image_layer correctly -includes cross-module first-party deps. Do not inline into the e2e folder — the point -is to exercise the external-module runfiles path (aspect_rules_py+/...).""" +"""Cross-module first-party fixtures for OCI image-layer tests. -load("//py:defs.bzl", "py_library") +The library covers external source dependencies; the launcher covers external +runfiles paths. Keep both in the @aspect_rules_py module. +""" + +load("//py:defs.bzl", "py_binary", "py_library") py_library( name = "adder", @@ -14,3 +15,13 @@ py_library( imports = [".."], visibility = ["//visibility:public"], ) + +py_binary( + name = "external_launcher", + srcs = ["external_launcher.py"], + visibility = ["//visibility:public"], + deps = [ + ":adder", + "@rules_python//python/runfiles", + ], +) diff --git a/py/tests/internal-deps/adder/external_launcher.py b/py/tests/internal-deps/adder/external_launcher.py new file mode 100644 index 000000000..b166b5c11 --- /dev/null +++ b/py/tests/internal-deps/adder/external_launcher.py @@ -0,0 +1,10 @@ +import os + +from adder.add import add +from python.runfiles import runfiles + +if __name__ == "__main__": + adder_path = runfiles.Create().Rlocation("aspect_rules_py/py/tests/internal-deps/adder/add.py") + if adder_path is None or not os.path.exists(adder_path): + raise RuntimeError("could not resolve adder through runfiles") + print("external {}".format(add(2, 3)))