Use internal crate feed in Azure Pipelines

.cargo/config.toml

@@ -0,0 +1,6 @@
+[registries]
+azure-sdk-for-rust = { index = "sparse+https://pkgs.dev.azure.com/azure-sdk/_packaging/azure-sdk-for-rust/Cargo/index/" }
+
+# To use the Azure Artifacts feed, run `eng/scripts/use-registry.rs azure`.
+# [source.crates-io]
+# replace-with = "azure-sdk-for-rust"

.vscode/settings.json

@@ -22,4 +22,11 @@
     "editor.defaultFormatter": "ms-vscode.powershell",
   },
   "cmake.ignoreCMakeListsMissing": true,
+  "yaml.schemas": {
+    "file:///c%3A/Users/heaths/.vscode/extensions/docsmsft.docs-yaml-1.0.5/dist/toc.schema.json": "/toc\\.yml/i",
+    "https://raw.githubusercontent.com/microsoft/azure-pipelines-vscode/master/service-schema.json": [
+      "file:///home/heaths/src/azure-sdk-for-rust2/eng/pipelines/templates/jobs/fetch.yml",
+      "file:///home/heaths/src/azure-sdk-for-rust2/eng/pipelines/fetch.yml"
+    ]
+  },
 }

eng/pipelines/fetch.yml

@@ -0,0 +1,24 @@
+# Manually-triggered pipeline to pull crates from upstream into feed.
+
+trigger: none
+pr: none
+
+extends:
+  template: /eng/pipelines/templates/stages/1es-redirect.yml
+  parameters:
+    stages:
+      - stage: Fetch
+        variables:
+          - template: /eng/pipelines/templates/variables/image.yml
+          - template: /eng/pipelines/templates/variables/rust.yml
+        jobs:
+          - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
+            parameters:
+              JobTemplatePath: /eng/pipelines/templates/jobs/fetch.yml
+              AdditionalParameters: {}
+              MatrixConfigs:
+                - Name: rust_fetch
+                  Path: eng/pipelines/templates/stages/platform-matrix.json
+                  Selection: all
+                  NonSparseParameters: RustToolchainName
+                  GenerateVMJobs: true

eng/pipelines/templates/jobs/fetch.yml

@@ -0,0 +1,52 @@
+parameters:
+  # Required parameters from generate-job-matrix.yml
+  - name: UsePlatformContainer
+    type: boolean
+  - name: OSName
+    type: string
+  - name: Matrix
+    type: object
+  - name: DependsOn
+    type: string
+  - name: CloudConfig
+    type: object
+    default: {}
+
+jobs:
+  - job:
+    displayName: Fetch
+    condition: and(succeeded(), ne(${{ parameters.Matrix }}, '{}'))
+    dependsOn: ${{ parameters.DependsOn }}
+    strategy:
+      matrix: $[ ${{ parameters.Matrix }} ]
+    variables:
+      CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS: artifacts-cargo-credprovider
+    pool:
+      name: $(Pool)
+      # 1es pipeline templates converts `image` to demands: ImageOverride under the hood
+      # which is incompatible with image selection in the default non-1es hosted pools
+      ${{ if eq(parameters.OSName, 'macOS') }}:
+        vmImage: $(OSVmImage)
+      ${{ else }}:
+        image: $(OSVmImage)
+      os: ${{ parameters.OSName }}
+    steps:
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          paths:
+            - "/*"
+      - template: /eng/pipelines/templates/steps/use-rust.yml@self
+        parameters:
+          Toolchain: nightly
+          Condition: and(succeeded(), ne('$(RustToolchainName)', 'nightly'))
+      - template: /eng/pipelines/templates/steps/use-rust.yml@self
+        parameters:
+          Toolchain: $(RustToolchainName)
+      - pwsh: cargo install --locked --index sparse+https://pkgs.dev.azure.com/artifacts-public/PublicTools/_packaging/AzureArtifacts/Cargo/index/ artifacts-cargo-credprovider
+        displayName: Install cargo credential provider
+      - pwsh: cargo login --registry azure-sdk-for-rust
+        displayName: Login to azure-sdk-for-rust registry
+      - pwsh: cargo +nightly -Zscript eng/scripts/use-registry.rs azure
+        displayName: Configure azure-sdk-for-rust registry
+      - pwsh: cargo fetch
+        displayName: Fetch dependencies

eng/pipelines/templates/steps/use-rust.yml

@@ -15,6 +15,9 @@ parameters:
   - name: SetDefault
     type: boolean
     default: true
+  - name: Condition
+    type: string
+    default: succeeded()
 
 steps:
 - pwsh: |
@@ -97,4 +100,5 @@ steps:
     Invoke-LoggedCommand "rustup show"
 
   displayName: "Use Rust ${{ parameters.Toolchain }}"
+  condition: ${{ parameters.Condition }}
   workingDirectory: ${{ parameters.WorkingDirectory }}

eng/scripts/update-cratenames.rs

@@ -4,8 +4,8 @@
 edition = "2021"
 
 [dependencies]
-cargo-util-schemas = "0.1.0"
-toml = "0.8.10"
+cargo-util-schemas = "0.13.0"
+toml = "1.1.2"
 ---
 
 use cargo_util_schemas::manifest::{InheritableDependency, TomlManifest};

eng/scripts/update-pathversions.rs

@@ -5,8 +5,8 @@ edition = "2021"
 description = "In all Cargo.toml files in the repo, for all dependencies that have both path and version properties, update the version property to the version in the package."
 
 [dependencies]
-regex = "1.5"
-toml_edit = "0.22"
+regex = "1.12.3"
+toml_edit = "0.25.10"
 ---
 
 use regex::Regex;
@@ -15,22 +15,20 @@ use std::{env, error::Error, fs, path::PathBuf};
 use toml_edit::{value, DocumentMut, Item, Table};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let add_mode = env::args().nth(1)
+    let add_mode = env::args()
+        .nth(1)
         .map(|arg| match arg.as_str() {
             "add" => true,
             "update" => false,
-            _ => panic!("Invalid mode. Use 'add' or 'update'.")
+            _ => panic!("Invalid mode. Use 'add' or 'update'."),
         })
         .expect("requires 'add' or 'update' mode argument");
 
     let script_root = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
     let repo_root = script_root.join("../..").canonicalize()?;
 
     // find all Cargo.toml files in the repo_root directory
-    let exclude_dirs = vec![
-        repo_root.join("eng"),
-        repo_root.join("target")
-    ];
+    let exclude_dirs = vec![repo_root.join("eng"), repo_root.join("target")];
 
     let toml_files = load_cargo_toml_files(&repo_root, &exclude_dirs)?;
 
@@ -39,7 +37,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for mut toml_file in toml_files {
         let should_add = add_mode && !toml_file.is_publish_disabled;
 
-        update_package_versions(toml_file.document.as_table_mut(), &package_versions, should_add);
+        update_package_versions(
+            toml_file.document.as_table_mut(),
+            &package_versions,
+            should_add,
+        );
 
         // if the toml file has a workspace table, update the workspace table
         if let Some(workspace) = toml_file.document.get_mut("workspace") {
@@ -56,7 +58,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
-fn load_cargo_toml_files(repo_root: &PathBuf, exclude_dirs: &Vec<PathBuf>) -> Result<Vec<TomlInfo>, Box<dyn Error>> {
+fn load_cargo_toml_files(
+    repo_root: &PathBuf,
+    exclude_dirs: &Vec<PathBuf>,
+) -> Result<Vec<TomlInfo>, Box<dyn Error>> {
     let mut toml_paths = Vec::new();
     find_cargo_toml_files(repo_root, exclude_dirs, &mut toml_paths)?;
 
@@ -65,23 +70,33 @@ fn load_cargo_toml_files(repo_root: &PathBuf, exclude_dirs: &Vec<PathBuf>) -> Re
         let content = fs::read_to_string(&path)?;
         let doc = content.parse::<DocumentMut>()?;
         let package_table = doc.get("package").and_then(Item::as_table);
-        let publish_property = package_table.and_then(|table| table.get("publish")).and_then(Item::as_bool);
-        let package_name = package_table.and_then(|table| table.get("name")).and_then(Item::as_str);
-        let package_version = package_table.and_then(|table| table.get("version")).and_then(Item::as_str);
+        let publish_property = package_table
+            .and_then(|table| table.get("publish"))
+            .and_then(Item::as_bool);
+        let package_name = package_table
+            .and_then(|table| table.get("name"))
+            .and_then(Item::as_str);
+        let package_version = package_table
+            .and_then(|table| table.get("version"))
+            .and_then(Item::as_str);
 
         toml_files.push(TomlInfo {
             path,
             package_name: package_name.map(|s| s.to_string()),
             package_version: package_version.map(|s| s.to_string()),
             is_publish_disabled: publish_property == Some(false),
-            document: doc
+            document: doc,
         });
     }
 
     Ok(toml_files)
 }
 
-fn find_cargo_toml_files(dir: &PathBuf, exclude_dirs: &Vec<PathBuf>, toml_paths: &mut Vec<PathBuf>) -> Result<(), Box<dyn Error>> {
+fn find_cargo_toml_files(
+    dir: &PathBuf,
+    exclude_dirs: &Vec<PathBuf>,
+    toml_paths: &mut Vec<PathBuf>,
+) -> Result<(), Box<dyn Error>> {
     for entry in fs::read_dir(dir)? {
         let entry = entry?;
         let path = entry.path();
@@ -103,13 +118,21 @@ fn get_package_versions(toml_files: &Vec<TomlInfo>) -> Vec<(String, String, bool
             continue;
         }
 
-        package_versions.push((toml_file.package_name.clone().unwrap(), toml_file.package_version.clone().unwrap(), toml_file.is_publish_disabled));
+        package_versions.push((
+            toml_file.package_name.clone().unwrap(),
+            toml_file.package_version.clone().unwrap(),
+            toml_file.is_publish_disabled,
+        ));
     }
 
     package_versions
 }
 
-fn update_package_versions(toml: &mut Table, package_versions: &Vec<(String, String, bool)>, add: bool) {
+fn update_package_versions(
+    toml: &mut Table,
+    package_versions: &Vec<(String, String, bool)>,
+    add: bool,
+) {
     // for each dependency table, for each package in package_versions
     // if the package is in the dependency table
     //   if the dependency has both path and version properties, update the version property
@@ -124,7 +147,10 @@ fn update_package_versions(toml: &mut Table, package_versions: &Vec<(String, Str
         for (package, version, is_publish_disabled) in package_versions {
             if let Some(dependency) = table.get_mut(package) {
                 // azure_idenentity will only be a transitive dev-dependency
-                let should_add = add && table_name != "dev-dependencies" && !is_publish_disabled && package != "azure_identity";
+                let should_add = add
+                    && table_name != "dev-dependencies"
+                    && !is_publish_disabled
+                    && package != "azure_identity";
 
                 let has_path_property = dependency.get("path").is_some();
                 let has_version_property = dependency.get("version").is_some();

eng/scripts/use-registry.rs

@@ -0,0 +1,120 @@
+#!/usr/bin/env -S cargo +nightly -Zscript
+---
+[package]
+edition = "2021"
+
+[dependencies]
+toml_edit = "0.25.10"
+---
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+use std::{fs, path::PathBuf, process};
+use toml_edit::{value, DocumentMut, InlineTable, Item, Table, Value};
+
+const REGISTRY: &str = "azure-sdk-for-rust";
+const INDEX: &str =
+    "sparse+https://pkgs.dev.azure.com/azure-sdk/_packaging/azure-sdk-for-rust/Cargo/index/";
+
+fn main() {
+    let mode = match std::env::args().nth(1).as_deref() {
+        Some("azure") => Mode::Azure,
+        Some("crates.io") => Mode::CratesIo,
+        Some(other) => {
+            eprintln!("error: unknown registry \"{other}\", expected \"azure\" or \"crates.io\"");
+            process::exit(1);
+        }
+        None => {
+            eprintln!("usage: use-registry.rs <azure|crates.io>");
+            process::exit(1);
+        }
+    };
+
+    let cargo_home = std::env::var("CARGO_HOME").unwrap_or_else(|_| {
+        eprintln!("error: CARGO_HOME is not set");
+        process::exit(1);
+    });
+    let config_path = PathBuf::from(cargo_home).join("config.toml");
+
+    let content = if config_path.exists() {
+        fs::read_to_string(&config_path).unwrap_or_else(|err| {
+            eprintln!("error: failed to read {}: {err}", config_path.display());
+            process::exit(1);
+        })
+    } else {
+        String::new()
+    };
+
+    let mut doc: DocumentMut = content.parse().unwrap_or_else(|err| {
+        eprintln!("error: failed to parse {}: {err}", config_path.display());
+        process::exit(1);
+    });
+
+    match mode {
+        Mode::Azure => {
+            ensure_registry(&mut doc);
+            set_replace_with(&mut doc);
+        }
+        Mode::CratesIo => {
+            remove_replace_with(&mut doc);
+        }
+    }
+
+    fs::write(&config_path, doc.to_string()).unwrap_or_else(|err| {
+        eprintln!("error: failed to write {}: {err}", config_path.display());
+        process::exit(1);
+    });
+
+    eprintln!("updated {}", config_path.display());
+}
+
+/// Add the azure-sdk-for-rust registry to [registries] if not already present.
+fn ensure_registry(doc: &mut DocumentMut) {
+    let registries = doc
+        .entry("registries")
+        .or_insert_with(|| Item::Table(Table::new()))
+        .as_table_mut()
+        .expect("[registries] should be a table");
+
+    if registries.get(REGISTRY).is_none() {
+        let mut entry = InlineTable::new();
+        entry.insert("index", Value::from(INDEX));
+        registries.insert(REGISTRY, Item::Value(Value::InlineTable(entry)));
+    }
+}
+
+/// Add [source.crates-io] if needed and set replace-with.
+fn set_replace_with(doc: &mut DocumentMut) {
+    let source = doc
+        .entry("source")
+        .or_insert_with(|| {
+            let mut t = Table::new();
+            t.set_implicit(true);
+            Item::Table(t)
+        })
+        .as_table_mut()
+        .expect("[source] should be a table");
+
+    let crates_io = source
+        .entry("crates-io")
+        .or_insert_with(|| Item::Table(Table::new()))
+        .as_table_mut()
+        .expect("[source.crates-io] should be a table");
+
+    crates_io.insert("replace-with", value(REGISTRY));
+}
+
+/// Remove replace-with from [source.crates-io] if present.
+fn remove_replace_with(doc: &mut DocumentMut) {
+    if let Some(source) = doc.get_mut("source").and_then(|s| s.as_table_mut()) {
+        if let Some(crates_io) = source.get_mut("crates-io").and_then(|c| c.as_table_mut()) {
+            crates_io.remove("replace-with");
+        }
+    }
+}
+
+enum Mode {
+    Azure,
+    CratesIo,
+}

eng/scripts/verify-dependencies.rs

@@ -5,9 +5,9 @@ edition = "2021"
 
 [dependencies]
 cargo-util-schemas = "0.11.0"
-serde = { version = "1.0.197", features = ["derive"] }
-serde_json = "1.0.114"
-toml = "1.0.1"
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = "1.0.149"
+toml = "1.1.2"
 ---
 
 use cargo_util_schemas::manifest::{InheritableDependency, TomlDependency, TomlManifest};