diff --git a/CHANGELOG.md b/CHANGELOG.md index ccaf9391..985d6a75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- New `artifacts_download` higher-level module for downloading Universal Packages from Azure DevOps Artifacts. +- Add example for downloading universal packages (`artifacts_download`). + ### Changes - Update `azure_core` and `azure_identity` to 0.34. diff --git a/README.md b/README.md index f31f8152..195333f9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,21 @@ This repo contains: - [autorust](autorust/): A tool to autogenerate the `azure_devops_rust_api` crate from the OpenAPI spec. - [vsts-api-patcher](vsts-api-patcher/): A tool to patch the OpenAPI spec. This modifies the original OpenAPI spec to fix known issues and/or improve the generated code. -- [azure_devops_rust_api](azure_devops_rust_api/): The autogenerated crate. +- [azure_devops_rust_api](azure_devops_rust_api/): The autogenerated crate, plus the hand-written `artifacts_download` module (see below). + +## Artifact downloads + +Most of `azure_devops_rust_api` is auto-generated from the OpenAPI spec and provides thin wrappers +around the Azure DevOps REST API endpoints. The `artifacts_download` module is different: it is a +hand-written, higher-level module that implements the full protocol for downloading +[Universal Packages](https://docs.microsoft.com/en-us/azure/devops/artifacts/universal-packages/universal-packages-overview) +from Azure Artifacts. + +It handles the entire download flow — service URL discovery, package metadata retrieval, blob URL +resolution, chunk download, decompression, and file reassembly — behind a single +`download_universal_package` call. + +See [azure_devops_rust_api/README.md](azure_devops_rust_api/README.md) for usage details. ## Usage of generated `azure_devops_rust_api` crate diff --git a/azure_devops_rust_api/Cargo.toml b/azure_devops_rust_api/Cargo.toml index 6c65335c..14669baf 100644 --- a/azure_devops_rust_api/Cargo.toml +++ b/azure_devops_rust_api/Cargo.toml @@ -52,6 +52,7 @@ no-default-tag = [] accounts = [] approvals_and_checks = [] artifacts = [] +artifacts_download = [] artifacts_package_types = [] audit = [] build = [] @@ -299,3 +300,7 @@ required-features = ["release"] [[example]] name = "member_entitlement_management" required-features = ["member_entitlement_management"] + +[[example]] +name = "download_artifact" +required-features = ["artifacts_download"] diff --git a/azure_devops_rust_api/README.md b/azure_devops_rust_api/README.md index 54835a54..629627cf 100644 --- a/azure_devops_rust_api/README.md +++ b/azure_devops_rust_api/README.md @@ -100,6 +100,53 @@ Example: cargo run --example git_repo_get --features="git" ``` +## Artifact downloads + +In addition to the auto-generated REST API wrappers, the crate includes a higher-level +`artifacts_download` module for downloading [Universal Packages](https://docs.microsoft.com/en-us/azure/devops/artifacts/universal-packages/universal-packages-overview) +from Azure Artifacts. + +Unlike the other modules, `artifacts_download` is not a thin wrapper around a single REST +endpoint. It implements the full dedup-based download protocol used by Azure Artifacts: +discovering service URLs, fetching package metadata, resolving blob IDs, downloading and +decompressing content chunks, and reassembling them into files on disk. + +Enable it with the `artifacts_download` feature: + +```toml +[dependencies] +azure_devops_rust_api = { version = "0.35.0", features = ["artifacts_download"] } +``` + +Example usage (from [examples/download_artifact.rs](examples/download_artifact.rs)): + +```rust + let client = artifacts_download::ClientBuilder::new(credential).build(); + + let metadata = client + .download_universal_package( + &organization, + &project, + &feed, + &package_name, + &version, + &output_path, + ) + .await?; + + println!( + "Downloaded {} v{} ({} bytes) to {:?}", + package_name, metadata.version, metadata.package_size, output_path + ); +``` + +Run the example: + +```sh +cargo run --example download_artifact --features="artifacts_download" -- \ + --feed --name --version --path +``` + ## Issue reporting If you find any issues then please raise them via [Github](https://github.com/microsoft/azure-devops-rust-api/issues). diff --git a/azure_devops_rust_api/examples/download_artifact.rs b/azure_devops_rust_api/examples/download_artifact.rs new file mode 100644 index 00000000..416f34a8 --- /dev/null +++ b/azure_devops_rust_api/examples/download_artifact.rs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Download a Universal Package from Azure Artifacts. +// +// Usage: +// export ADO_ORGANIZATION= +// export ADO_PROJECT= +// cargo run --example download_artifact --features="artifacts_download" -- \ +// --feed --name --version --path + +use anyhow::{Context, Result}; +use azure_devops_rust_api::artifacts_download; +use std::env; +use std::path::PathBuf; + +mod utils; + +// --- CLI argument parsing --- + +struct Args { + organization: String, + project: String, + feed: String, + name: String, + version: String, + path: PathBuf, +} + +fn parse_args() -> Result { + let organization = env::var("ADO_ORGANIZATION").context("Must define ADO_ORGANIZATION")?; + let project = env::var("ADO_PROJECT").context("Must define ADO_PROJECT")?; + + let args: Vec = env::args().collect(); + let mut feed = None; + let mut name = None; + let mut version = None; + let mut path = None; + + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "--feed" => { + feed = Some(args.get(i + 1).context("--feed requires a value")?.clone()); + i += 2; + } + "--name" => { + name = Some(args.get(i + 1).context("--name requires a value")?.clone()); + i += 2; + } + "--version" => { + version = Some( + args.get(i + 1) + .context("--version requires a value")? + .clone(), + ); + i += 2; + } + "--path" => { + path = Some(args.get(i + 1).context("--path requires a value")?.clone()); + i += 2; + } + _ => { + i += 1; + } + } + } + + Ok(Args { + organization, + project, + feed: feed.context("--feed is required")?, + name: name.context("--name is required")?, + version: version.context("--version is required")?, + path: PathBuf::from(path.context("--path is required")?), + }) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = parse_args()?; + let credential = utils::get_credential()?; + + println!( + "Downloading Universal Package: {}@{} from {}/{}", + args.name, args.version, args.organization, args.project + ); + + let client = artifacts_download::ClientBuilder::new(credential).build(); + + let metadata = client + .download_universal_package( + &args.organization, + &args.project, + &args.feed, + &args.name, + &args.version, + &args.path, + ) + .await?; + + println!( + "Downloaded {} v{} ({} bytes) to {:?}", + args.name, metadata.version, metadata.package_size, args.path + ); + + Ok(()) +} diff --git a/azure_devops_rust_api/src/artifacts_download/decompress.rs b/azure_devops_rust_api/src/artifacts_download/decompress.rs new file mode 100644 index 00000000..a7786369 --- /dev/null +++ b/azure_devops_rust_api/src/artifacts_download/decompress.rs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Decompression for Azure DevOps blob store chunks. +//! +//! Azure DevOps blob store uses an LZ77 variant for content chunks. +//! This module implements the decompressor. + +use azure_core::error::{Error, ErrorKind, Result}; + +/// Decompress a chunk compressed with the LZ77 encoding used by +/// Azure DevOps blob store. +/// +/// The format uses: +/// - 32-bit flag indicators with a sentinel bit for tracking consumption +/// - 2-bit literal length encoding (1/2/3/4 byte batches) +/// - 16-bit match references: 13-bit offset + 3-bit length +/// - Nibble-based extended length encoding +pub fn decompress_chunk(compressed: &[u8]) -> Result> { + if compressed.len() < 5 { + return Err(Error::with_message( + ErrorKind::DataConversion, + format!("Compressed data too small: {} bytes", compressed.len()), + )); + } + + let mut output = Vec::with_capacity(compressed.len() * 4); + let mut ci = 0usize; + let mut indicator: i32; + let mut nibble_pos: Option<(usize, bool)> = None; + + // When true, the current indicator MSB is a literal-encoding bit (not a + // decision bit). This happens after reading a fresh indicator whose raw + // MSB was 0 (literal): that bit is consumed by the explicit check, and + // after *2+1 the MSB is the first encoding bit. + let mut fresh_literal; + + let raw = i32::from_le_bytes(compressed[ci..ci + 4].try_into().unwrap()); + ci += 4; + indicator = raw.wrapping_mul(2).wrapping_add(1); + fresh_literal = raw >= 0; + + // If the first decision is match, process it before the main loop. + if raw < 0 { + if ci + 1 >= compressed.len() { + return Ok(output); + } + process_match(compressed, &mut ci, &mut output, &mut nibble_pos)?; + } + + loop { + if ci >= compressed.len() { + break; + } + + if fresh_literal { + fresh_literal = false; + } else if indicator >= 0 { + indicator = indicator.wrapping_mul(2); + } else { + indicator = indicator.wrapping_mul(2); + + if indicator == 0 { + if ci + 3 >= compressed.len() { + break; + } + let raw = i32::from_le_bytes(compressed[ci..ci + 4].try_into().unwrap()); + ci += 4; + indicator = raw.wrapping_mul(2).wrapping_add(1); + if raw >= 0 { + fresh_literal = true; + continue; + } + } + + if ci + 1 >= compressed.len() { + break; + } + process_match(compressed, &mut ci, &mut output, &mut nibble_pos)?; + continue; + } + + // Literal encoding: bits encode length as 1/2/3/4 bytes per batch + loop { + if indicator < 0 { + if ci >= compressed.len() { + return Ok(output); + } + output.push(compressed[ci]); + ci += 1; + break; + } + indicator = indicator.wrapping_mul(2); + if indicator < 0 { + if ci + 1 >= compressed.len() { + return Ok(output); + } + output.extend_from_slice(&compressed[ci..ci + 2]); + ci += 2; + break; + } + indicator = indicator.wrapping_mul(2); + if indicator < 0 { + if ci + 2 >= compressed.len() { + return Ok(output); + } + output.extend_from_slice(&compressed[ci..ci + 3]); + ci += 3; + break; + } + indicator = indicator.wrapping_mul(2); + if ci + 3 >= compressed.len() { + return Ok(output); + } + output.extend_from_slice(&compressed[ci..ci + 4]); + ci += 4; + if indicator < 0 { + break; + } + indicator = indicator.wrapping_mul(2); + } + + // Post-literal shift: consume the "1" bit that ended the literal group + indicator = indicator.wrapping_mul(2); + + if indicator == 0 { + if ci + 3 >= compressed.len() { + break; + } + let raw = i32::from_le_bytes(compressed[ci..ci + 4].try_into().unwrap()); + ci += 4; + indicator = raw.wrapping_mul(2).wrapping_add(1); + if raw >= 0 { + fresh_literal = true; + continue; + } + } + + // Match always follows a literal group + if ci + 1 >= compressed.len() { + break; + } + process_match(compressed, &mut ci, &mut output, &mut nibble_pos)?; + } + + Ok(output) +} + +/// Process a single LZ match: read the 16-bit match descriptor and optional +/// extended length, then copy `match_len` bytes from the output history. +fn process_match( + compressed: &[u8], + ci: &mut usize, + output: &mut Vec, + nibble_pos: &mut Option<(usize, bool)>, +) -> Result<()> { + let v = u16::from_le_bytes(compressed[*ci..*ci + 2].try_into().unwrap()); + *ci += 2; + + let mut match_len = (v & 7) as usize; + let offset = ((v >> 3) as usize) + 1; + + if match_len == 7 { + let nibble_val = if let Some((nib_idx, _)) = nibble_pos.take() { + (compressed[nib_idx] >> 4) as usize + } else { + if *ci >= compressed.len() { + return Err(Error::with_message( + ErrorKind::DataConversion, + "Unexpected end of compressed data in nibble read", + )); + } + let nib_idx = *ci; + *ci += 1; + *nibble_pos = Some((nib_idx, true)); + (compressed[nib_idx] & 0x0F) as usize + }; + + match_len = nibble_val; + if match_len == 15 { + if *ci >= compressed.len() { + return Err(Error::with_message( + ErrorKind::DataConversion, + "Unexpected end of compressed data in length extension", + )); + } + match_len = compressed[*ci] as usize; + *ci += 1; + if match_len == 255 { + if *ci + 1 >= compressed.len() { + return Err(Error::with_message( + ErrorKind::DataConversion, + "Unexpected end of compressed data in 16-bit length", + )); + } + match_len = + u16::from_le_bytes(compressed[*ci..*ci + 2].try_into().unwrap()) as usize; + *ci += 2; + if match_len == 0 { + if *ci + 3 >= compressed.len() { + return Err(Error::with_message( + ErrorKind::DataConversion, + "Unexpected end of compressed data in 32-bit length", + )); + } + match_len = + u32::from_le_bytes(compressed[*ci..*ci + 4].try_into().unwrap()) as usize; + *ci += 4; + } + if match_len < 22 { + return Err(Error::with_message( + ErrorKind::DataConversion, + format!("Invalid extended match length: {}", match_len), + )); + } + match_len -= 22; + } + match_len += 15; + } + match_len += 7; + } + match_len += 3; + + if offset > output.len() { + return Err(Error::with_message( + ErrorKind::DataConversion, + format!( + "Match offset {} exceeds output size {} at compressed pos {}", + offset, + output.len(), + ci + ), + )); + } + let src_start = output.len() - offset; + for i in 0..match_len { + let byte = output[src_start + (i % offset)]; + output.push(byte); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_too_small_input() { + assert!(decompress_chunk(&[]).is_err()); + assert!(decompress_chunk(&[0; 4]).is_err()); + } + + #[test] + fn test_single_literal() { + // Indicator 0x40000000: bit31=0 (literal), bit30=1 (1-byte literal) + let compressed: &[u8] = &[0x00, 0x00, 0x00, 0x40, 0x48]; // literal 'H' + let result = decompress_chunk(compressed).unwrap(); + assert_eq!(result, b"H"); + } + + #[test] + fn test_literal_then_match() { + // Indicator 0x40000000 + literal 'A' + match(offset=1, len=3) -> "AAAA" + let compressed: &[u8] = &[0x00, 0x00, 0x00, 0x40, 0x41, 0x00, 0x00]; + let result = decompress_chunk(compressed).unwrap(); + assert_eq!(result, b"AAAA"); + } + + #[test] + fn test_minimum_valid_input() { + // 5 bytes is the minimum accepted size + let compressed: &[u8] = &[0x00, 0x00, 0x00, 0x40, 0x58]; // literal 'X' + let result = decompress_chunk(compressed).unwrap(); + assert_eq!(result, b"X"); + } + + #[test] + fn test_match_offset_out_of_bounds() { + // First decision is match (bit31=1), match with offset > 0 when output is empty + // raw = 0x80000000, match v=0x0008 -> offset=2, but output is empty -> error + let compressed: &[u8] = &[0x00, 0x00, 0x00, 0x80, 0x08, 0x00]; + assert!(decompress_chunk(compressed).is_err()); + } +} diff --git a/azure_devops_rust_api/src/artifacts_download/mod.rs b/azure_devops_rust_api/src/artifacts_download/mod.rs new file mode 100644 index 00000000..e5d23b43 --- /dev/null +++ b/azure_devops_rust_api/src/artifacts_download/mod.rs @@ -0,0 +1,652 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Download Universal Packages from Azure DevOps Artifacts. +//! +//! This module implements the dedup-based download protocol used by +//! Azure DevOps Artifacts for universal packages. +//! +//! # Protocol overview +//! +//! 1. Discover service URLs via the ResourceAreas API +//! 2. Get package metadata (manifestId, superRootId) from the packaging endpoint +//! 3. Resolve blob IDs to download URLs via the dedup service +//! 4. Download and parse the manifest to get the file/chunk structure +//! 5. Download content chunks, decompress, and reassemble files + +mod decompress; + +use azure_core::error::{Error, ErrorKind, Result, ResultExt}; +use azure_core::http::headers::{self, HeaderValue}; +use azure_core::http::{Method, Request, Url}; +use serde::Deserialize; +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +pub use decompress::decompress_chunk; + +// --- Data structures --- + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ResourceArea { + #[allow(dead_code)] + id: String, + name: String, + location_url: String, +} + +#[derive(Debug, Deserialize)] +struct ResourceAreasResponse { + value: Vec, +} + +/// Package metadata returned by the packaging endpoint. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PackageMetadata { + /// Package version string. + pub version: String, + /// Blob ID of the dedup manifest. + pub manifest_id: String, + /// Blob ID of the super-root node. + pub super_root_id: String, + /// Total package size in bytes. + pub package_size: u64, +} + +/// A file entry in the dedup manifest. +#[derive(Debug, Deserialize)] +pub struct ManifestItem { + /// File path within the package (e.g. "/myfile.bin"). + pub path: String, + /// Reference to the dedup blob for this file. + pub blob: DedupBlobRef, +} + +/// A reference to a dedup blob (hash ID + logical size). +#[derive(Debug, Deserialize)] +pub struct DedupBlobRef { + /// Hex-encoded blob ID with type suffix ("01" = content, "02" = node). + pub id: String, + /// Decompressed size in bytes. + pub size: u64, +} + +/// Parsed manifest listing all files in a package. +#[derive(Debug, Deserialize)] +pub struct Manifest { + /// The files contained in the package. + pub items: Vec, +} + +// --- Client --- + +/// Client for downloading Universal Packages from Azure Artifacts. +#[derive(Clone)] +pub struct Client { + credential: crate::Credential, + scopes: Vec, + pipeline: azure_core::http::Pipeline, +} + +/// Builder for creating an artifacts download [`Client`]. +#[derive(Clone)] +pub struct ClientBuilder { + credential: crate::Credential, + scopes: Option>, + options: azure_core::http::ClientOptions, +} + +impl ClientBuilder { + /// Create a new `ClientBuilder`. + #[must_use] + pub fn new(credential: crate::Credential) -> Self { + Self { + credential, + scopes: None, + options: azure_core::http::ClientOptions::default(), + } + } + + /// Set the authentication scopes. + #[must_use] + pub fn scopes(mut self, scopes: &[&str]) -> Self { + self.scopes = Some(scopes.iter().map(|scope| (*scope).to_owned()).collect()); + self + } + + /// Set the retry options. + #[must_use] + pub fn retry(mut self, retry: impl Into) -> Self { + self.options.retry = retry.into(); + self + } + + /// Set the transport options. + #[must_use] + pub fn transport(mut self, transport: impl Into) -> Self { + self.options.transport = Some(transport.into()); + self + } + + /// Build the [`Client`]. + pub fn build(self) -> Client { + let scopes = self + .scopes + .unwrap_or_else(|| vec![crate::ADO_SCOPE.to_string()]); + let pipeline = azure_core::http::Pipeline::new( + option_env!("CARGO_PKG_NAME"), + option_env!("CARGO_PKG_VERSION"), + self.options, + Vec::new(), + Vec::new(), + None, + ); + Client { + credential: self.credential, + scopes, + pipeline, + } + } +} + +impl Client { + /// Create a new `ClientBuilder`. + #[must_use] + pub fn builder(credential: crate::Credential) -> ClientBuilder { + ClientBuilder::new(credential) + } + + /// Get the authorization header value for the current credential. + async fn auth_header(&self) -> Result { + let scopes: Vec<&str> = self.scopes.iter().map(String::as_str).collect(); + self.credential + .http_authorization_header(&scopes) + .await? + .ok_or_else(|| Error::with_message(ErrorKind::Credential, "No credential configured")) + } + + /// Send a request through the pipeline. + async fn send(&self, request: &mut Request) -> Result { + let context = azure_core::http::Context::default(); + self.pipeline.send(&context, request, None).await + } + + /// Send an authenticated GET request and parse the JSON response. + async fn get_json(&self, url: Url) -> Result { + let mut req = Request::new(url, Method::Get); + let auth = self.auth_header().await?; + req.insert_header(headers::AUTHORIZATION, HeaderValue::from(auth)); + req.insert_header( + headers::ACCEPT, + HeaderValue::from("application/json; api-version=7.1-preview.1"), + ); + req.insert_header("x-tfs-fedauthredirect", HeaderValue::from("Suppress")); + req.set_body(azure_core::Bytes::new()); + + let resp = self.send(&mut req).await?; + let body = resp.into_body(); + serde_json::from_slice(&body).map_err(|e| { + Error::with_error( + ErrorKind::DataConversion, + e, + format!( + "Failed to deserialize response:\n{}", + String::from_utf8_lossy(&body) + ), + ) + }) + } + + /// Send an unauthenticated GET request and return the raw bytes. + async fn get_bytes(&self, url: Url) -> Result> { + let mut req = Request::new(url, Method::Get); + req.set_body(azure_core::Bytes::new()); + let resp = self.send(&mut req).await?; + let body = resp.into_body(); + Ok(body.to_vec()) + } + + // --- Service discovery --- + + /// Discover Azure DevOps service URLs via the ResourceAreas API. + /// Returns a map of service name -> location URL. + pub async fn discover_services(&self, organization: &str) -> Result> { + let url = Url::parse(&format!( + "https://dev.azure.com/{}/_apis/ResourceAreas", + organization + )) + .with_context(ErrorKind::DataConversion, "invalid organization URL")?; + + let areas: ResourceAreasResponse = self.get_json(url).await?; + let map: HashMap = areas + .value + .into_iter() + .map(|a| (a.name.to_lowercase(), a.location_url)) + .collect(); + Ok(map) + } + + /// Find the packages service URL from discovered services. + pub fn find_packages_url(services: &HashMap, organization: &str) -> String { + services + .values() + .find(|url| url.contains("pkgs.")) + .cloned() + .unwrap_or_else(|| format!("https://pkgs.dev.azure.com/{}", organization)) + } + + /// Find the blob/dedup service URL from discovered services. + pub fn find_blob_url(services: &HashMap) -> Result { + services.get("dedup").cloned().ok_or_else(|| { + Error::with_message( + ErrorKind::Other, + "Could not find 'dedup' service in ResourceAreas", + ) + }) + } + + // --- Package metadata --- + + /// Get package download metadata from the packaging endpoint. + pub async fn get_package_metadata( + &self, + packages_url: &str, + project: &str, + feed: &str, + name: &str, + version: &str, + ) -> Result { + let mut url = Url::parse(&format!( + "{}/{}/_packaging/{}/upack/packages/{}/versions/{}", + packages_url.trim_end_matches('/'), + project, + feed, + name, + version, + )) + .with_context(ErrorKind::DataConversion, "invalid package metadata URL")?; + + url.query_pairs_mut().append_pair("intent", "Download"); + self.get_json(url).await + } + + // --- Dedup blob operations --- + + /// Resolve dedup blob IDs to download URLs via the dedup service. + pub async fn resolve_blob_urls( + &self, + blob_service_url: &str, + blob_ids: &[String], + ) -> Result> { + let mut url = Url::parse(&format!( + "{}/_apis/dedup/urls", + blob_service_url.trim_end_matches('/') + )) + .with_context(ErrorKind::DataConversion, "invalid dedup URL")?; + + url.query_pairs_mut().append_pair("allowEdge", "true"); + + let mut req = Request::new(url, Method::Post); + let auth = self.auth_header().await?; + req.insert_header(headers::AUTHORIZATION, HeaderValue::from(auth)); + req.insert_header( + headers::CONTENT_TYPE, + HeaderValue::from("application/json; charset=utf-8; api-version=1.0-preview"), + ); + req.insert_header( + headers::ACCEPT, + HeaderValue::from("application/json; api-version=1.0"), + ); + req.insert_header("x-tfs-fedauthredirect", HeaderValue::from("Suppress")); + let body = azure_core::json::to_json(blob_ids)?; + req.set_body(body); + + let resp = self.send(&mut req).await?; + let body = resp.into_body(); + serde_json::from_slice(&body).map_err(|e| { + Error::with_error( + ErrorKind::DataConversion, + e, + "Failed to parse blob URL response", + ) + }) + } + + /// Download a blob from a SAS URL (no auth required). + pub async fn download_blob(&self, url: &str) -> Result> { + let parsed = + Url::parse(url).with_context(ErrorKind::DataConversion, "invalid blob download URL")?; + self.get_bytes(parsed).await + } + + // --- Manifest parsing --- + + /// Parse the dedup manifest blob (JSON) to extract file entries. + pub fn parse_manifest(data: &[u8]) -> Result { + serde_json::from_slice(data).map_err(|e| { + Error::with_error( + ErrorKind::DataConversion, + e, + "Failed to parse manifest JSON", + ) + }) + } + + /// Parse a dedup node blob (binary format) to extract chunk references. + /// + /// A dedup node (ID ending in "02") contains references to child blobs. + /// The binary format is: + /// - 4-byte header + /// - N entries of: 4-byte metadata + 32-byte hash + /// + /// Content chunk IDs are formed by hex-encoding the 32-byte hash + /// and appending "01" (content type marker). + pub fn parse_dedup_node(data: &[u8]) -> Result> { + const HEADER_SIZE: usize = 4; + const HASH_SIZE: usize = 32; + const METADATA_SIZE: usize = 4; + const ENTRY_SIZE: usize = METADATA_SIZE + HASH_SIZE; + + if data.len() < HEADER_SIZE + ENTRY_SIZE { + return Err(Error::with_message( + ErrorKind::DataConversion, + format!( + "Dedup node blob too small: {} bytes (minimum {})", + data.len(), + HEADER_SIZE + ENTRY_SIZE + ), + )); + } + + let data_portion = data.len() - HEADER_SIZE; + if data_portion % ENTRY_SIZE != 0 { + return Err(Error::with_message( + ErrorKind::DataConversion, + format!( + "Dedup node blob has unexpected size: {} bytes \ + (data portion {} is not a multiple of entry size {})", + data.len(), + data_portion, + ENTRY_SIZE + ), + )); + } + + let num_entries = data_portion / ENTRY_SIZE; + let mut chunk_ids = Vec::with_capacity(num_entries); + + for i in 0..num_entries { + let offset = HEADER_SIZE + i * ENTRY_SIZE; + let hash_bytes = &data[offset + METADATA_SIZE..offset + METADATA_SIZE + HASH_SIZE]; + let hex_hash: String = hash_bytes.iter().map(|b| format!("{:02X}", b)).collect(); + chunk_ids.push(format!("{}01", hex_hash)); + } + + if chunk_ids.is_empty() { + return Err(Error::with_message( + ErrorKind::DataConversion, + format!( + "No chunk references found in dedup node blob ({} bytes)", + data.len() + ), + )); + } + Ok(chunk_ids) + } + + // --- High-level download --- + + /// Download a universal package to the specified output directory. + /// + /// Performs the full download protocol: service discovery, metadata fetch, + /// manifest download, chunk download with decompression, and file assembly. + pub async fn download_universal_package( + &self, + organization: &str, + project: &str, + feed: &str, + name: &str, + version: &str, + output_path: &Path, + ) -> Result { + // Step 1: Discover service URLs + let services = self.discover_services(organization).await?; + let packages_url = Self::find_packages_url(&services, organization); + let blob_service_url = Self::find_blob_url(&services)?; + + // Step 2: Get package metadata + let metadata = self + .get_package_metadata(&packages_url, project, feed, name, version) + .await?; + + // Step 3: Download the manifest blob + let manifest_urls = self + .resolve_blob_urls( + &blob_service_url, + std::slice::from_ref(&metadata.manifest_id), + ) + .await?; + let manifest_url = manifest_urls.get(&metadata.manifest_id).ok_or_else(|| { + Error::with_message(ErrorKind::Other, "Manifest URL not found in response") + })?; + let manifest_data = self.download_blob(manifest_url).await?; + let manifest = Self::parse_manifest(&manifest_data)?; + + // Step 4: Create output directory + std::fs::create_dir_all(output_path).map_err(|e| { + Error::with_error( + ErrorKind::Io, + e, + format!("Failed to create output directory: {:?}", output_path), + ) + })?; + + // Step 5: Download each file + for item in &manifest.items { + let file_root_urls = self + .resolve_blob_urls(&blob_service_url, std::slice::from_ref(&item.blob.id)) + .await?; + let file_root_url = file_root_urls + .get(&item.blob.id) + .ok_or_else(|| Error::with_message(ErrorKind::Other, "File root URL not found"))?; + let file_root_data = self.download_blob(file_root_url).await?; + + let is_node = item.blob.id.ends_with("02"); + + let file_data = if is_node { + let chunk_ids = Self::parse_dedup_node(&file_root_data)?; + let chunk_urls = self + .resolve_blob_urls(&blob_service_url, &chunk_ids) + .await?; + + let mut file_data = Vec::with_capacity(item.blob.size as usize); + for chunk_id in &chunk_ids { + let chunk_url = chunk_urls.get(chunk_id).ok_or_else(|| { + Error::with_message( + ErrorKind::Other, + format!("Chunk URL not found for {}", chunk_id), + ) + })?; + let chunk_data = self.download_blob(chunk_url).await?; + let decompressed = decompress_chunk(&chunk_data)?; + file_data.extend_from_slice(&decompressed); + } + file_data + } else { + file_root_data + }; + + let relative_path = item.path.trim_start_matches('/'); + let file_path = output_path.join(relative_path); + if let Some(parent) = file_path.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + Error::with_error( + ErrorKind::Io, + e, + format!("Failed to create directory: {:?}", parent), + ) + })?; + } + let mut file = std::fs::File::create(&file_path).map_err(|e| { + Error::with_error( + ErrorKind::Io, + e, + format!("Failed to create file: {:?}", file_path), + ) + })?; + file.write_all(&file_data) + .map_err(|e| Error::with_error(ErrorKind::Io, e, "Failed to write file data"))?; + } + + Ok(metadata) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // --- find_packages_url --- + + #[test] + fn test_find_packages_url_with_pkgs_service() { + let mut services = HashMap::new(); + services.insert( + "packaging".to_string(), + "https://pkgs.dev.azure.com/myorg/".to_string(), + ); + services.insert( + "dedup".to_string(), + "https://vsblob.dev.azure.com/myorg/".to_string(), + ); + + let url = Client::find_packages_url(&services, "myorg"); + assert!(url.contains("pkgs.")); + } + + #[test] + fn test_find_packages_url_fallback() { + let services = HashMap::new(); + let url = Client::find_packages_url(&services, "myorg"); + assert_eq!(url, "https://pkgs.dev.azure.com/myorg"); + } + + // --- find_blob_url --- + + #[test] + fn test_find_blob_url_found() { + let mut services = HashMap::new(); + services.insert( + "dedup".to_string(), + "https://vsblob.dev.azure.com/myorg/".to_string(), + ); + + let url = Client::find_blob_url(&services).unwrap(); + assert_eq!(url, "https://vsblob.dev.azure.com/myorg/"); + } + + #[test] + fn test_find_blob_url_missing() { + let services = HashMap::new(); + assert!(Client::find_blob_url(&services).is_err()); + } + + // --- parse_manifest --- + + #[test] + fn test_parse_manifest_valid() { + let json = br#"{"items":[{"path":"/file1.txt","blob":{"id":"ABC01","size":100}},{"path":"/dir/file2.bin","blob":{"id":"DEF02","size":200}}]}"#; + let manifest = Client::parse_manifest(json).unwrap(); + assert_eq!(manifest.items.len(), 2); + assert_eq!(manifest.items[0].path, "/file1.txt"); + assert_eq!(manifest.items[0].blob.id, "ABC01"); + assert_eq!(manifest.items[0].blob.size, 100); + assert_eq!(manifest.items[1].path, "/dir/file2.bin"); + assert_eq!(manifest.items[1].blob.id, "DEF02"); + assert_eq!(manifest.items[1].blob.size, 200); + } + + #[test] + fn test_parse_manifest_empty_items() { + let json = br#"{"items":[]}"#; + let manifest = Client::parse_manifest(json).unwrap(); + assert!(manifest.items.is_empty()); + } + + #[test] + fn test_parse_manifest_invalid_json() { + assert!(Client::parse_manifest(b"not json").is_err()); + } + + #[test] + fn test_parse_manifest_missing_field() { + let json = br#"{"items":[{"path":"/f"}]}"#; + assert!(Client::parse_manifest(json).is_err()); + } + + // --- parse_dedup_node --- + + #[test] + fn test_parse_dedup_node_single_entry() { + // 4-byte header + 1 entry (4-byte meta + 32-byte hash) + let mut data = vec![0x00, 0x01, 0x00, 0x00]; // header + data.extend_from_slice(&[0x00; 4]); // metadata + let hash: Vec = (0..32).collect(); + data.extend_from_slice(&hash); + + let ids = Client::parse_dedup_node(&data).unwrap(); + assert_eq!(ids.len(), 1); + let expected: String = hash + .iter() + .map(|b| format!("{:02X}", b)) + .collect::() + + "01"; + assert_eq!(ids[0], expected); + } + + #[test] + fn test_parse_dedup_node_two_entries() { + let mut data = vec![0x00, 0x01, 0x00, 0x00]; // header + // Entry 1 + data.extend_from_slice(&[0x00; 4]); // metadata + let hash1: Vec = (0..32).collect(); + data.extend_from_slice(&hash1); + // Entry 2 + data.extend_from_slice(&[0x01, 0x00, 0x00, 0x00]); // metadata + let hash2: Vec = (32..64).collect(); + data.extend_from_slice(&hash2); + + let ids = Client::parse_dedup_node(&data).unwrap(); + assert_eq!(ids.len(), 2); + assert!(ids[0].ends_with("01")); + assert!(ids[1].ends_with("01")); + } + + #[test] + fn test_parse_dedup_node_too_small() { + assert!(Client::parse_dedup_node(&[0; 10]).is_err()); + } + + #[test] + fn test_parse_dedup_node_invalid_size() { + // 4 header + 37 bytes (not a multiple of 36) + let data = vec![0u8; 4 + 37]; + assert!(Client::parse_dedup_node(&data).is_err()); + } + + #[test] + fn test_parse_dedup_node_chunk_ids_are_content_type() { + let mut data = vec![0x00; 4]; // header + data.extend_from_slice(&[0x00; 4]); // metadata + data.extend_from_slice(&[0xFF; 32]); // all-FF hash + + let ids = Client::parse_dedup_node(&data).unwrap(); + assert_eq!(ids.len(), 1); + // Should end with "01" (content type), not "02" (node type) + assert!(ids[0].ends_with("01")); + assert_eq!(ids[0].len(), 66); // 64 hex chars + "01" + } +} diff --git a/azure_devops_rust_api/src/lib.rs b/azure_devops_rust_api/src/lib.rs index afc47c04..a28e402d 100644 --- a/azure_devops_rust_api/src/lib.rs +++ b/azure_devops_rust_api/src/lib.rs @@ -22,6 +22,9 @@ pub mod approvals_and_checks; /// Artifacts #[cfg(feature = "artifacts")] pub mod artifacts; +/// Artifacts download (Universal Packages) +#[cfg(feature = "artifacts_download")] +pub mod artifacts_download; /// Artifacts Package Types #[cfg(feature = "artifacts_package_types")] pub mod artifacts_package_types;