Skip to content

Commit 25ad57c

Browse files
authored
fix(uv): allow environment setting for the update action (#3800)
With this PR we allow users to set extra environment variables using `.bazelrc` and to ensure that we can pass UV extra parameters. This should enable users use this with private indexes in a `diff_test` usage scenario. Whilst at it, add integration tests to actually verify that passing works via env vars. Whilst at it also fix the Windows support for the `uv` lock rule. Fixes #3405
1 parent 94ccb8c commit 25ad57c

24 files changed

Lines changed: 825 additions & 74 deletions

.agents/skills/buildkite-get-results/scripts/get_buildkite_results.py

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -94,24 +94,32 @@ def fetch_buildkite_data(build_url):
9494

9595

9696
def download_log(job_url, output_path):
97-
# Construct raw log URL: job_url + "/raw" (Buildkite convention)
98-
# job_url e.g. https://buildkite.com/org/pipeline/builds/14394#job-id
99-
# Wait, the job['path'] gives /org/pipeline/builds/14394#job-id
100-
# We want /org/pipeline/builds/14394/jobs/job-id/raw? No
101-
# The clean URL for a job is https://buildkite.com/org/pipeline/builds/14394/jobs/job-id
102-
# And raw log is https://buildkite.com/org/pipeline/builds/14394/jobs/job-id/raw
103-
104-
# We have full_url e.g. https://buildkite.com/bazel/rules-python-python/builds/14394#019c5cf9-e3cf-468f-a7b1-8f9f5ad4b08c
105-
# We need to transform it.
97+
# job_url looks like:
98+
# https://buildkite.com/bazel/rules-python-python/builds/15594#019e879b-...
99+
# We need to transform it to:
100+
# https://buildkite.com/organizations/bazel/pipelines/rules-python-python/builds/15594/jobs/{job_id}/download.txt
106101

107102
if "#" in job_url:
108103
base, job_id = job_url.split("#")
109-
# Ensure base doesn't end with /
110-
if base.endswith("/"):
111-
base = base[:-1]
112-
113-
# Build raw URL
114-
raw_url = f"{base}/jobs/{job_id}/raw"
104+
base = base.rstrip("/")
105+
106+
# Parse the path segments: https://buildkite.com/org/pipeline/builds/N
107+
# Rebuild with the /organizations/org/pipelines/pipeline/ format which
108+
# supports the /jobs/{id}/download.txt log URL without auth.
109+
parts = base.split("/")
110+
# parts = ["https:", "", "buildkite.com", "org", "pipeline", "builds", "N"]
111+
if len(parts) >= 7 and parts[2] == "buildkite.com":
112+
org = parts[3]
113+
pipeline = parts[4]
114+
build_num = parts[6] if len(parts) >= 7 else ""
115+
raw_url = (
116+
f"https://buildkite.com/organizations/{org}"
117+
f"/pipelines/{pipeline}"
118+
f"/builds/{build_num}"
119+
f"/jobs/{job_id}/download.txt"
120+
)
121+
else:
122+
raw_url = f"{base}/jobs/{job_id}/download.txt"
115123
else:
116124
print(f"Could not parse job URL for download: {job_url}", file=sys.stderr)
117125
return False

.bazelignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,4 @@ tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
3535
tests/integration/local_toolchains/bazel-local_toolchains
3636
tests/integration/py_cc_toolchain_registered/bazel-py_cc_toolchain_registered
3737
tests/integration/toolchain_target_settings/bazel-module_under_test
38+
tests/integration/uv_lock/bazel-uv_lock

.bazelrc.deleted_packages

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ common --deleted_packages=tests/integration/pip_parse/empty
3939
common --deleted_packages=tests/integration/pip_parse_isolated
4040
common --deleted_packages=tests/integration/py_cc_toolchain_registered
4141
common --deleted_packages=tests/integration/toolchain_target_settings
42+
common --deleted_packages=tests/integration/uv_lock
4243
common --deleted_packages=tests/modules/another_module
4344
common --deleted_packages=tests/modules/other
4445
common --deleted_packages=tests/modules/other/nspkg_delta

.gitattributes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
python/features.bzl export-subst
22
tools/publish/*.txt linguist-generated=true
3+
tests/uv/lock/testdata/requirements.txt text eol=lf

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,7 @@ user.bazelrc
5454
# MODULE.bazel.lock is ignored for now as per recommendation from upstream.
5555
# See https://github.com/bazelbuild/bazel/issues/20369
5656
MODULE.bazel.lock
57+
58+
# Buildkite logs
59+
*Windows*.log
60+

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,12 @@ END_UNRELEASED_TEMPLATE
103103
PyPI download and supports PyPI mirror implementations that do not support the root
104104
index functionality. Fixes
105105
([#3769](https://github.com/bazel-contrib/rules_python/pull/3769)).
106+
* (uv) allow user overwrite the build environment using `--action_env` to allow
107+
setting authentication for the index URL.
108+
([#3405](https://github.com/bazel-contrib/rules_python/issues/3405))
109+
* (uv) fix the execution of the `uv pip compile` in the sandbox. Work
110+
towards better supporting `uv` out of the box on our platforms.
111+
([#1975](https://github.com/bazel-contrib/rules_python/issues/1975))
106112

107113
{#v0-0-0-added}
108114
### Added

python/uv/private/lock.bat

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
if defined BUILD_WORKSPACE_DIRECTORY (
2-
set "out=%BUILD_WORKSPACE_DIRECTORY%\{{src_out}}"
3-
) else (
4-
exit /b 1
5-
)
6-
7-
"{{args}}" --output-file "%out%" %*
1+
if defined BUILD_WORKSPACE_DIRECTORY (
2+
set "out=%BUILD_WORKSPACE_DIRECTORY%\{{src_out}}"
3+
) else (
4+
exit /b 1
5+
)
6+
7+
"{{args}}" --output-file "%out%" %*

python/uv/private/lock.bzl

Lines changed: 85 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -117,31 +117,96 @@ def _lock_impl(ctx):
117117
args.run_shell.add("--no-progress")
118118
args.run_shell.add("--quiet")
119119

120+
# Generate a wrapper script that copies the existing output (if any) and
121+
# then runs uv. On POSIX, args are forwarded via exec "$@". On Windows,
122+
# the full command line is embedded in the .bat file with backslash paths
123+
# (CMD doesn't recognize forward slashes in executable paths).
124+
if ctx.attr.is_windows:
125+
ext = ".bat"
126+
lines = ["@echo off"]
127+
else:
128+
ext = ".sh"
129+
lines = ["#!/usr/bin/env bash", "set -euo pipefail"]
130+
131+
python_path = getattr(python, "path", python)
132+
120133
if ctx.files.existing_output:
121-
command = '{python} -c {python_cmd} && "$@"'.format(
122-
python = getattr(python, "path", python),
123-
python_cmd = shell.quote(
124-
"from shutil import copy; copy(\"{src}\", \"{dst}\")".format(
134+
python_cmd = "from shutil import copy; copy(\"{src}\", \"{dst}\")".format(
135+
src = ctx.files.existing_output[0].path,
136+
dst = output.path,
137+
)
138+
if ctx.attr.is_windows:
139+
# In batch files, use "" to escape internal double quotes.
140+
lines.append(
141+
"\"{py}\" -c \"from shutil import copy; copy(\"\"{src}\"\", \"\"{dst}\"\")\"".format(
142+
py = python_path,
125143
src = ctx.files.existing_output[0].path,
126144
dst = output.path,
127145
),
146+
)
147+
else:
148+
lines.append("{py} -c '{cmd}'".format(
149+
py = python_path,
150+
cmd = python_cmd,
151+
))
152+
153+
if ctx.attr.is_windows:
154+
# Build the command line with backslash paths for CMD.
155+
# args.run_info has most args; add the output/progress/quiet
156+
# args that were only added directly to args.run_shell.
157+
def _quote(arg):
158+
if hasattr(arg, "path"):
159+
arg = arg.path.replace("/", "\\")
160+
else:
161+
arg = str(arg)
162+
return '"' + arg.replace('"', '""') + '"'
163+
164+
bat_args = args.run_info + [
165+
"--output-file",
166+
output,
167+
"--no-progress",
168+
"--quiet",
169+
]
170+
lines.append(" ".join([_quote(a) for a in bat_args]))
171+
172+
# Normalize CRLF line endings in the output on Windows.
173+
lines.append(
174+
"\"{py}\" -c \"import pathlib;p=pathlib.Path(r\"\"{dst}\"\");p.write_bytes(p.read_bytes().replace(b'\\r\\n', b'\\n'))\"".format(
175+
py = python_path,
176+
dst = output.path,
128177
),
129178
)
130179
else:
131-
command = '"$@"'
180+
lines.append('exec "$@"')
181+
182+
script = ctx.actions.declare_file(ctx.label.name + "_lock" + ext)
183+
if ctx.attr.is_windows:
184+
content = "\r\n".join(lines) + "\r\n"
185+
else:
186+
content = "\n".join(lines) + "\n"
187+
ctx.actions.write(output = script, content = content, is_executable = True)
132188

133189
srcs = srcs + ctx.files.build_constraints + ctx.files.constraints
134190

135-
ctx.actions.run_shell(
136-
command = command,
191+
ctx.actions.run(
192+
executable = script,
137193
inputs = srcs + ctx.files.existing_output,
138194
mnemonic = "PyRequirementsLockUv",
139195
outputs = [output],
140-
arguments = [args.run_shell],
196+
# On Windows, the command line is embedded directly in the .bat
197+
# script (with backslash paths). On POSIX, args are forwarded via
198+
# exec "$@" in the .sh script.
199+
arguments = [args.run_shell] if not ctx.attr.is_windows else [],
141200
tools = [
142201
uv,
143202
python_files,
203+
script,
144204
],
205+
# User reported being unable to add `--action_env` and get it to work.
206+
# Without this flag.
207+
#
208+
# Ref: https://app.slack.com/client/TA4K1KQ87/CA306CEV6
209+
use_default_shell_env = True,
145210
progress_message = "Creating a requirements.txt with uv: %{label}",
146211
env = ctx.attr.env,
147212
)
@@ -205,6 +270,7 @@ modifications and the locking is not done from scratch.
205270
doc = "Public, see the docs in the macro.",
206271
default = True,
207272
),
273+
"is_windows": attr.bool(mandatory = True),
208274
"output": attr.string(
209275
doc = "Public, see the docs in the macro.",
210276
mandatory = True,
@@ -241,7 +307,7 @@ The string to input for the 'uv pip compile'.
241307
def _lock_run_impl(ctx):
242308
if ctx.attr.is_windows:
243309
path_sep = "\\"
244-
ext = ".exe"
310+
ext = ".bat"
245311
else:
246312
path_sep = "/"
247313
ext = ""
@@ -250,7 +316,12 @@ def _lock_run_impl(ctx):
250316
if hasattr(arg, "short_path"):
251317
arg = arg.short_path
252318

253-
return shell.quote(arg.replace("/", path_sep))
319+
arg = arg.replace("/", path_sep)
320+
if ctx.attr.is_windows:
321+
# On Windows, CMD uses double quotes for quoting, and internal
322+
# double quotes are escaped by doubling them.
323+
return '"' + arg.replace('"', '""') + '"'
324+
return shell.quote(arg)
254325

255326
info = ctx.attr.lock[_RunLockInfo]
256327
executable = ctx.actions.declare_file(ctx.label.name + ext)
@@ -438,6 +509,10 @@ def lock(
438509
env = env,
439510
existing_output = maybe_out,
440511
generate_hashes = generate_hashes,
512+
is_windows = select({
513+
"@platforms//os:windows": True,
514+
"//conditions:default": False,
515+
}),
441516
python_version = python_version,
442517
srcs = srcs,
443518
strip_extras = strip_extras,

python/uv/private/lock_copier.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def main():
5555
"This must be either run as `bazel test` via a `native_test` or similar or via `bazel run`"
5656
)
5757

58-
print(f"cp <bazel-sandbox>/{src} <workspace>/{dst}")
58+
print(f"cp <bazel-sandbox>/{src.as_posix()} <workspace>/{dst}")
5959
build_workspace = Path(environ["BUILD_WORKSPACE_DIRECTORY"])
6060

6161
dst_real_path = build_workspace / dst

tests/integration/BUILD.bazel

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
# limitations under the License.
1414

1515
load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner")
16+
load("//python:py_binary.bzl", "py_binary")
1617
load("//python:py_library.bzl", "py_library")
18+
load("//tests/support:support.bzl", "NOT_WINDOWS")
1719
load(":integration_test.bzl", "rules_python_integration_test")
1820

1921
licenses(["notice"])
@@ -48,6 +50,7 @@ test_suite(
4850
tests = [
4951
"bzlmod_lockfile_test_bazel_9.1.0",
5052
"local_toolchains_test_bazel_self",
53+
"uv_lock_test_bazel_self",
5154
],
5255
)
5356

@@ -111,8 +114,41 @@ rules_python_integration_test(
111114
py_main = "toolchain_target_settings_test.py",
112115
)
113116

117+
rules_python_integration_test(
118+
name = "uv_lock_test",
119+
py_deps = [
120+
"@pypiserver//pypiserver",
121+
":uv_lock_pypi_server_lib",
122+
],
123+
py_main = "uv_lock_test.py",
124+
)
125+
114126
py_library(
115127
name = "runner_lib",
116128
srcs = ["runner.py"],
117129
imports = ["../../"],
118130
)
131+
132+
py_library(
133+
name = "uv_lock_pypi_server_lib",
134+
srcs = ["uv_lock_pypi_server.py"],
135+
imports = ["../../"],
136+
# currently windows is not working due to
137+
# https://github.com/pypiserver/pypiserver/blob/main/pypiserver/config.py#L123
138+
#
139+
# class DEFAULTS:
140+
# ....
141+
# PACKAGE_DIRECTORIES = [pathlib.Path("~/packages").expanduser().resolve()]
142+
# ....
143+
#
144+
# which is loaded through `__init__.py` even though it is not used and breaks because
145+
# in a Windows sandbox one cannot resolve the home directory.
146+
target_compatible_with = NOT_WINDOWS,
147+
deps = ["@pypiserver//pypiserver"],
148+
)
149+
150+
py_binary(
151+
name = "uv_lock_pypi_server",
152+
srcs = ["uv_lock_pypi_server.py"],
153+
deps = [":uv_lock_pypi_server_lib"],
154+
)

0 commit comments

Comments
 (0)