Render crates into per-alias subpackages

Author: UebelAndre

crate_universe/private/crates_repository.bzl

@@ -114,6 +114,7 @@ def _crates_repository_impl(repository_ctx):
 
     paths_to_track_file = repository_ctx.path("paths-to-track")
     warnings_output_file = repository_ctx.path("warnings-output-file")
+    hub_packages_output_file = repository_ctx.path("hub-packages.json")
 
     # Run the generator
     repository_ctx.report_progress("Generating crate BUILD files.")
@@ -128,6 +129,7 @@ def _crates_repository_impl(repository_ctx):
         nonhermetic_root_bazel_workspace_dir = nonhermetic_root_bazel_workspace_dir,
         paths_to_track_file = paths_to_track_file,
         warnings_output_file = warnings_output_file,
+        hub_packages_output_file = hub_packages_output_file,
         skip_cargo_lockfile_overwrite = repository_ctx.attr.skip_cargo_lockfile_overwrite,
         strip_internal_dependencies_from_cargo_lockfile = repository_ctx.attr.strip_internal_dependencies_from_cargo_lockfile,
         # sysroot = tools.sysroot,

crate_universe/private/generate_utils.bzl

@@ -92,7 +92,7 @@ def get_generator(repository_ctx, host_triple):
 def render_config(
         build_file_template = "//:BUILD.{name}-{version}.bazel",
         crate_label_template = "@{repository}__{name}-{version}//:{target}",
-        crate_alias_template = "//:{name}-{version}",
+        crate_alias_template = "//{name}-{version}",
         crate_repository_template = "{repository}__{name}-{version}",
         crates_module_template = "//:{file}",
         default_alias_rule = "alias",
@@ -102,7 +102,8 @@ def render_config(
         platforms_template = "@rules_rust//rust/platform:{triple}",
         regen_command = None,
         vendor_mode = None,
-        generate_rules_license_metadata = False):
+        generate_rules_license_metadata = False,
+        incompatible_no_root_alias_targets = False):
     """Various settings used to configure rendered outputs
 
     The template parameters each support a select number of format keys. A description of each key
@@ -145,6 +146,11 @@ def render_config(
         regen_command (str, optional): An optional command to demonstrate how generated files should be regenerated.
         vendor_mode (str, optional): An optional configuration for rendirng content to be rendered into repositories.
         generate_rules_license_metadata (bool, optional): Whether to generate rules license metadata
+        incompatible_no_root_alias_targets (bool, optional): Incompatibility flag. Suppresses the top-level
+            `alias()` rules in the hub repository's root `BUILD.bazel` (e.g. `@crate_index//:clap`). Per-alias
+            subpackages (e.g. `@crate_index//clap`) are always emitted, so flipping this flag on lets users
+            keep consuming aliases through the subpackage path while the root version disappears. Planned to
+            flip to default-on in a future release.
 
     Returns:
         string: A json encoded struct to match the Rust `config::RenderConfig` struct
@@ -160,6 +166,7 @@ def render_config(
         generate_cargo_toml_env_vars = generate_cargo_toml_env_vars,
         generate_rules_license_metadata = generate_rules_license_metadata,
         generate_target_compatible_with = generate_target_compatible_with,
+        incompatible_no_root_alias_targets = incompatible_no_root_alias_targets,
         platforms_template = platforms_template,
         regen_command = regen_command,
         vendor_mode = vendor_mode,
@@ -440,6 +447,7 @@ def execute_generator(
         nonhermetic_root_bazel_workspace_dir,
         paths_to_track_file,
         warnings_output_file,
+        hub_packages_output_file,
         skip_cargo_lockfile_overwrite,
         strip_internal_dependencies_from_cargo_lockfile,
         metadata = None,
@@ -456,6 +464,9 @@ def execute_generator(
         nonhermetic_root_bazel_workspace_dir (path): The path to the current workspace root
         paths_to_track_file (path): Path to file where generator should write which files should trigger re-generating as a JSON list.
         warnings_output_file (path): Path to file where generator should write warnings to print.
+        hub_packages_output_file (path): Path to file where the generator should write the names of hub alias
+            subpackages it produced. Always populated; the bzlmod extension reads it to learn which
+            `<name>/BUILD.bazel` files to slurp into the hub repository.
         skip_cargo_lockfile_overwrite (bool): Whether to skip writing the cargo lockfile back after resolving.
             You may want to set this if your dependency versions are maintained externally through a non-trivial set-up.
             But you probably don't want to set this.
@@ -488,6 +499,8 @@ def execute_generator(
         paths_to_track_file,
         "--warnings-output-path",
         warnings_output_file,
+        "--hub-packages-output-path",
+        hub_packages_output_file,
     ]
 
     if generator_label:

crate_universe/src/cli/generate.rs

@@ -97,6 +97,14 @@ pub struct GenerateOptions {
     #[clap(long)]
     pub warnings_output_path: PathBuf,
 
+    /// Path to write a JSON list of hub alias subpackage names. The bzlmod
+    /// extension reads this file to learn which `<name>/BUILD.bazel` files
+    /// to slurp into the hub repository (the per-alias subpackages always
+    /// render; `module_ctx` cannot enumerate directories, so this sidecar
+    /// is how the extension learns the set).
+    #[clap(long)]
+    pub hub_packages_output_path: PathBuf,
+
     /// Whether to skip writing the cargo lockfile back after resolving.
     /// You may want to set this if your dependency versions are maintained externally through a non-trivial set-up.
     /// But you probably don't want to set this.
@@ -130,17 +138,18 @@ pub fn generate(opt: GenerateOptions) -> Result<()> {
                 .apply_label_injection_mapping(&config.label_injection_mapping)?;
 
             // Render build files
-            let outputs = Renderer::new(
+            let renderer = Renderer::new(
                 Arc::new(config.rendering),
                 Arc::new(config.supported_platform_triples),
-            )
-            .render(&context, opt.generator)?;
-
-            // make file paths compatible with bazel labels
-            let normalized_outputs = normalize_cargo_file_paths(outputs, &opt.repository_dir);
-
-            // Write the outputs to disk
-            write_outputs(normalized_outputs, opt.dry_run)?;
+            );
+            render_and_write_outputs(
+                &renderer,
+                &context,
+                opt.generator.clone(),
+                &opt.repository_dir,
+                &opt.hub_packages_output_path,
+                opt.dry_run,
+            )?;
 
             let splicing_manifest = SplicingManifest::try_from_path(&opt.splicing_manifest)?;
 
@@ -225,17 +234,18 @@ pub fn generate(opt: GenerateOptions) -> Result<()> {
     let render_context = context
         .clone()
         .apply_label_injection_mapping(&config.label_injection_mapping)?;
-    let outputs = Renderer::new(
+    let renderer = Renderer::new(
         Arc::new(config.rendering.clone()),
         Arc::new(config.supported_platform_triples.clone()),
-    )
-    .render(&render_context, opt.generator)?;
-
-    // make file paths compatible with bazel labels
-    let normalized_outputs = normalize_cargo_file_paths(outputs, &opt.repository_dir);
-
-    // Write the outputs to disk
-    write_outputs(normalized_outputs, opt.dry_run)?;
+    );
+    render_and_write_outputs(
+        &renderer,
+        &render_context,
+        opt.generator.clone(),
+        &opt.repository_dir,
+        &opt.hub_packages_output_path,
+        opt.dry_run,
+    )?;
 
     // Ensure Bazel lockfiles are written to disk so future generations can be short-circuited.
     if let Some(lockfile) = opt.lockfile {
@@ -286,6 +296,30 @@ fn update_cargo_lockfile(path: &Path, cargo_lockfile: Lockfile) -> Result<()> {
     Ok(())
 }
 
+fn render_and_write_outputs(
+    renderer: &Renderer,
+    context: &Context,
+    generator: Option<Label>,
+    repository_dir: &Path,
+    hub_packages_output_path: &Path,
+    dry_run: bool,
+) -> Result<()> {
+    let rendered = renderer.render_hub(context, generator)?;
+    let normalized_outputs = normalize_cargo_file_paths(rendered.files, repository_dir);
+    write_outputs(normalized_outputs, dry_run)?;
+    write_hub_packages(hub_packages_output_path, &rendered.hub_packages)?;
+    Ok(())
+}
+
+fn write_hub_packages(output_file: &Path, hub_packages: &[String]) -> Result<()> {
+    std::fs::write(
+        output_file,
+        serde_json::to_string(hub_packages).context("Failed to serialize hub alias subpackages")?,
+    )
+    .context("Failed to write hub alias subpackages file")?;
+    Ok(())
+}
+
 fn write_paths_to_track<
     'a,
     SourceAnnotations: Iterator<Item = &'a SourceAnnotation>,

crate_universe/src/config.rs

@@ -62,8 +62,10 @@ pub(crate) struct RenderConfig {
     #[serde(default = "default_crate_label_template")]
     pub(crate) crate_label_template: String,
 
-    /// The pattern to use for a crate alias.
-    /// Eg. `@{repository}//:{name}-{version}-{target}`
+    /// The pattern to use for a crate alias. Defaults to the hub subpackage
+    /// form (`//{name}-{version}`) emitted by the always-on per-alias
+    /// subpackage layout. Override to point `aliases()` /
+    /// `all_crate_deps()` at custom labels.
     #[serde(default = "default_crate_alias_template")]
     pub(crate) crate_alias_template: String,
 
@@ -109,6 +111,27 @@ pub(crate) struct RenderConfig {
     /// Whether to generate cargo_toml_env_vars targets.
     /// This is expected to always be true except for bootstrapping.
     pub(crate) generate_cargo_toml_env_vars: bool,
+
+    /// Incompatibility flag. Suppresses the top-level `alias()` rules in the
+    /// hub repository's root `BUILD.bazel` (e.g. `@crate_index//:clap`).
+    /// Per-alias subpackages (e.g. `@crate_index//clap`) are always emitted,
+    /// so flipping this flag on lets users keep consuming aliases through
+    /// the subpackage path while the root version disappears. Default
+    /// `false`; planned to flip to `true` (and ultimately have the
+    /// root-emitting code removed) in a future release.
+    #[serde(default)]
+    pub(crate) incompatible_no_root_alias_targets: bool,
+
+    /// Internal: true when the rendered output is destined for the
+    /// `crates_vendor` workflow (committed into the user's workspace
+    /// alongside `crates.bzl`). The renderer routes per-alias subpackage
+    /// BUILDs into `_HUB_PACKAGE_BUILDS` inside `crates.bzl` (synthesized
+    /// at fetch time by `crates_vendor_remote_repository`) instead of
+    /// writing them to disk — keeps the vendor tree free of generated
+    /// subdirectories. Defaults `false`; bzlmod / `crates_repository`
+    /// continue to write subpackage BUILDs into the hub repo directly.
+    #[serde(default)]
+    pub(crate) crates_vendor_synthesizes_subpackages: bool,
 }
 
 // Default is manually implemented so that the default values match the default
@@ -131,6 +154,8 @@ impl Default for RenderConfig {
             regen_command: String::default(),
             vendor_mode: Option::default(),
             generate_rules_license_metadata: default_generate_rules_license_metadata(),
+            incompatible_no_root_alias_targets: false,
+            crates_vendor_synthesizes_subpackages: false,
         }
     }
 }
@@ -154,7 +179,11 @@ fn default_crate_label_template() -> String {
 }
 
 fn default_crate_alias_template() -> String {
-    "//:{name}-{version}".to_owned()
+    // Points `aliases()` / `all_crate_deps()` at the per-alias subpackage
+    // layout (e.g. `Label("@crate_index//clap-1.0.0")`). Subpackage `BUILD.bazel`s
+    // are always emitted, so this default works whether or not the user
+    // sets `incompatible_no_root_alias_targets`.
+    "//{name}-{version}".to_owned()
 }
 
 fn default_crate_repository_template() -> String {

crate_universe/src/lockfile.rs

@@ -260,7 +260,7 @@ mod test {
         );
 
         assert_eq!(
-            Digest("edd73970897c01af3bb0e6c9d62f572203dd38a03c189dcca555d463990aa086".to_owned()),
+            Digest("caafd4eaa60b599509522968ce17d6ba07b8118b767b7dddb00bef8d5ddba7fa".to_owned()),
             digest,
         );
     }
@@ -305,7 +305,7 @@ mod test {
         );
 
         assert_eq!(
-            Digest("8a4c1b3bb4c2d6c36e27565e71a13d54cff9490696a492c66a3a37bdd3893edf".to_owned()),
+            Digest("76d987edaa3f6a281133997c7d31a657acfaeb639585d9e8c629996c1fcb94af".to_owned()),
             digest,
         );
     }
@@ -336,7 +336,7 @@ mod test {
         );
 
         assert_eq!(
-            Digest("1e01331686ba1f26f707dc098cd9d21c39d6ccd8e46be03329bb2470d3833e15".to_owned()),
+            Digest("3288cda3c2194f5ca7f1929b74bd13f36eb624e859d77b143d762c60631d700d".to_owned()),
             digest,
         );
     }
@@ -385,7 +385,7 @@ mod test {
         );
 
         assert_eq!(
-            Digest("45ccf7109db2d274420fac521f4736a1fb55450ec60e6df698e1be4dc2c89fad".to_owned()),
+            Digest("ccc875599ade8873ba48496875d29f08772710651b9bf6d428a97f45a559aa2e".to_owned()),
             digest,
         );
     }

crate_universe/src/utils.rs

@@ -42,11 +42,17 @@ pub(crate) fn normalize_cargo_file_paths(
                 .to_str()
                 .expect("All file paths should be strings");
 
-            let path = if original_parent_path_str.contains('+') {
-                let new_parent_file_path = sanitize_repository_name(original_parent_path_str);
+            // Vendor mode pre-creates the source directory (cargo's
+            // `+`-bearing layout) and we rewrite it to be label-friendly.
+            // Other render paths — e.g. the per-alias hub subpackage
+            // `BUILD.bazel`s whose parent name happens to contain a semver `+` —
+            // have nothing pre-created on disk; leave their paths alone so
+            // the directory and the alias name stay in sync.
+            let src = out_dir.join(original_parent_path_str);
+            let path = if original_parent_path_str.contains('+') && src.exists() {
                 std::fs::rename(
-                    out_dir.join(original_parent_path_str),
-                    out_dir.join(new_parent_file_path),
+                    src,
+                    out_dir.join(sanitize_repository_name(original_parent_path_str)),
                 )
                 .expect("Could not rename paths");
                 PathBuf::from(&original_path_str.replace('+', "-"))

crate_universe/tests/integration/cargo_aliases/BUILD.bazel

@@ -19,8 +19,10 @@ rust_test(
     deps = all_crate_deps(normal_dev = True),
 )
 
-# Ensures that Bazel aliases from the crates_repository are actually usable.
+# Ensures that Bazel aliases from the crates_repository are actually usable
+# via the subpackage form. The legacy root alias `@cargo_aliases//:names` is
+# suppressed by this fixture's `incompatible_no_root_alias_targets = True`.
 build_test(
     name = "names_build_test",
-    targets = ["@cargo_aliases//:names"],
+    targets = ["@cargo_aliases//names"],
 )

crate_universe/tests/integration/cargo_aliases/MODULE.bazel

@@ -17,6 +17,10 @@ use_repo(rust, "rust_toolchains")
 register_toolchains("@rust_toolchains//:all")
 
 crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate")
+crate.render_config(
+    incompatible_no_root_alias_targets = True,
+    repositories = ["cargo_aliases"],
+)
 crate.annotation(
     crate = "names",
     repositories = ["cargo_aliases"],