fix(pypi): pass the correct versions to get_index_urls and fix cache invalidation (#3758)

Fix the versions of packages that we are recording to a
`MODULE.bazel.lock`
file facts by passing all of the versions and packages to the
`get_index` function.

Summary:
- Parse ALL requirements files (not just platform-matched), pass all
versions to get_index so lockfile facts are platform-independent.
- Return None when versions mismatch (re-fetch), drop removed packages
so you can start immediately.
- Include files with no matching platforms so packages aren't lost
- Add bzlmod lockfile integration test - Update CI to run new test in
bazel-in-bazel subset - Bump Bazel 9.0.0rc1 → 9.1.0 in bazel-in-bazel
tests

Fixes #3756

(cherry picked from commit 3871306)

Conflicts:
* CHANGELOG.md - trivial fixes
* python/private/pypi/parse_requirements.bzl - refactors to add unused
  arguments
* tests/pypi/parse_requirements/parse_requirements_tests.bzl - similar

Author: aignas

.bazelci/presubmit.yml

@@ -468,17 +468,17 @@ tasks:
     <<: *common_bazelinbazel_config
     name: "tests/integration bazel-in-bazel: macOS (subset)"
     platform: macos_arm64
-    build_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
-    test_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+    build_targets: ["//tests/integration:subset"]
+    test_targets: ["//tests/integration:subset"]
   # The bazelinbazel tests were disabled on Windows to save CI jobs slots, and
   # have bitrotted a bit. For now, just run a subset of what we're most
   # interested in.
   integration_test_bazelinbazel_windows:
     <<: *common_bazelinbazel_config
     name: "tests/integration bazel-in-bazel: Windows (subset)"
     platform: windows
-    build_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
-    test_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+    build_targets: ["//tests/integration:subset"]
+    test_targets: ["//tests/integration:subset"]
 
   integration_test_compile_pip_requirements_ubuntu:
     <<: *reusable_build_test_all

.bazelignore

@@ -30,6 +30,7 @@ gazelle/examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation
 gazelle/examples/bzlmod_build_file_generation/bazel-out
 gazelle/examples/bzlmod_build_file_generation/bazel-testlog
 sphinxdocs
+tests/integration/bzlmod_lockfile/bazel-bzlmod_lockfile
 tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
 tests/integration/local_toolchains/bazel-local_toolchains
 tests/integration/py_cc_toolchain_registered/bazel-py_cc_toolchain_registered

.bazelrc.deleted_packages

@@ -29,6 +29,7 @@ common --deleted_packages=gazelle/modules_mapping
 common --deleted_packages=gazelle/python
 common --deleted_packages=gazelle/pythonconfig
 common --deleted_packages=gazelle/python/private
+common --deleted_packages=tests/integration/bzlmod_lockfile
 common --deleted_packages=tests/integration/compile_pip_requirements
 common --deleted_packages=tests/integration/compile_pip_requirements_test_from_external_repo
 common --deleted_packages=tests/integration/custom_commands

CHANGELOG.md

@@ -47,6 +47,18 @@ BEGIN_UNRELEASED_TEMPLATE
 END_UNRELEASED_TEMPLATE
 -->
 
+{#v2-0-1}
+## [2.0.1] - 2026-05-08
+
+[2.0.1]: https://github.com/bazel-contrib/rules_python/releases/tag/2.0.1
+
+{#v2-0-1-fixed}
+### Fixed
+
+* (pypi) Fix the versions of packages that we are recording to a `MODULE.bazel.lock` file
+  facts by passing all of the versions to the `get_index` function.
+  Fixes [#3756](https://github.com/bazel-contrib/rules_python/issues/3756).
+
 {#v2-0-0}
 ## [2.0.0] - 2026-04-09
 

MODULE.bazel

@@ -358,7 +358,7 @@ bazel_binaries.local(
 )
 bazel_binaries.download(version = "7.7.0")
 bazel_binaries.download(version = "8.5.1")
-bazel_binaries.download(version = "9.0.0rc1")
+bazel_binaries.download(version = "9.1.0")
 use_repo(
     bazel_binaries,
     "bazel_binaries",
@@ -367,7 +367,7 @@ use_repo(
     "bazel_binaries_bazelisk",
     "build_bazel_bazel_7_7_0",
     "build_bazel_bazel_8_5_1",
-    "build_bazel_bazel_9_0_0rc1",
+    "build_bazel_bazel_9_1_0",
     # "build_bazel_bazel_rolling",
     "build_bazel_bazel_self",
 )

python/private/pypi/parse_requirements.bzl

@@ -89,6 +89,7 @@ def parse_requirements(
     evaluate_markers = evaluate_markers or (lambda _ctx, _requirements: {})
     options = {}
     requirements = {}
+    all_files_parsed = {}
     reqs_with_env_markers = {}
     index_url = None
     extra_index_urls = []
@@ -100,6 +101,13 @@ def parse_requirements(
         # needed for the whl_library declarations later.
         parse_result = parse_requirements_txt(contents)
 
+        # Save parsed results from ALL files, even those with no matching
+        # platforms. This ensures the distributions dict (used for index URL
+        # queries) includes packages from all platform files, making the
+        # lockfile facts platform-independent.
+        if file not in all_files_parsed:
+            all_files_parsed[file] = parse_result.requirements
+
         tokenized_options = []
         for opt in parse_result.options:
             for p in opt.split(" "):
@@ -185,13 +193,21 @@ def parse_requirements(
 
     index_urls = {}
     if get_index_urls:
+        # Collect all distributions from all requirements files irrespective
+        # of python_version and platform markers. This ensures that the index
+        # is queried for all packages, not just those matching the current
+        # platform's markers.
         distributions = {}
-        for reqs in requirements_by_platform.values():
-            for req in reqs.values():
-                if req.srcs.url:
+        for entries in all_files_parsed.values():
+            for entry in entries:
+                name, req_line = entry
+                srcs = index_sources(req_line)
+                if srcs.url:
                     continue
+                versions = distributions.setdefault(normalize_name(name), {})
+                versions[srcs.version] = None
 
-                distributions.setdefault(req.distribution, []).append(req.srcs.version)
+        distributions = {k: sorted(v.keys()) for k, v in distributions.items()}
 
         index_urls = get_index_urls(
             ctx,

python/private/pypi/pypi_cache.bzl

@@ -212,6 +212,15 @@ def _get_from_facts(facts, known_facts, index_url, requested_versions, facts_ver
             requested_versions = requested_versions,
         )
         if result:
+            if len(result) != len(requested_versions):
+                # If the results are incomplete, return None, so that we can
+                # fetch sources from the internet again.
+                return None
+
+            # Only persist the accessed (requested) packages. Packages that
+            # exist in known_facts but are not in requested_versions (e.g.
+            # removed from all requirements files) are dropped from the
+            # computed facts so they get cleaned up from the lockfile.
             _store_facts(facts, facts_version, index_url, result)
         return result
 

python/private/pypi/requirements_files_by_platform.bzl

@@ -213,7 +213,6 @@ def requirements_files_by_platform(
                 default_platforms,
                 input_platforms,
             ))
-            continue
 
         if logger:
             logger.debug(lambda: "Configured platforms for file {} are {}".format(file, plats))
@@ -238,4 +237,8 @@ def requirements_files_by_platform(
     for plat, file in requirements.items():
         ret.setdefault(file, []).append(_platform(plat, python_version = python_version))
 
+    for file, _plats in files_by_platform:
+        if file not in ret and _plats != None:
+            ret[file] = []
+
     return ret

tests/integration/BUILD.bazel

@@ -37,6 +37,20 @@ default_test_runner(
     visibility = ["//visibility:public"],
 )
 
+rules_python_integration_test(
+    name = "bzlmod_lockfile_test",
+    bazel_versions = ["9.1.0"],
+)
+
+test_suite(
+    name = "subset",
+    tags = ["manual"],
+    tests = [
+        "bzlmod_lockfile_test_bazel_9.1.0",
+        "local_toolchains_test_bazel_self",
+    ],
+)
+
 rules_python_integration_test(
     name = "compile_pip_requirements_test",
 )

tests/integration/bzlmod_lockfile/.bazelrc

@@ -0,0 +1,9 @@
+# Bazel configuration flags
+
+build --enable_runfiles
+common --lockfile_mode=error
+
+common --experimental_isolated_extension_usages
+
+# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
+try-import %workspace%/user.bazelrc