Refactor azpysdk apistub command. Extract metadata from API.md.

Author: tjprescott

.github/skills/generate-api-markdown/SKILL.md

@@ -19,5 +19,5 @@ description: Generate an API markdown file and token file using ApiView. Use thi
 1. Navigate to the desired package directory
 2. Run the command:
    ```bash
-   azpysdk apistub --md .
+   azpysdk apistub --md --extract-metadata .
 3. The command outputs the location of the generated markdown file. Provide this file to the user for review.

.github/workflows/src/api-md-consistency/api-md-consistency.js

@@ -44,7 +44,7 @@ function formatIssueSection(title, apiFiles) {
     const packageName = path.basename(packageDir);
     lines.push(`- ${packageDir}`);
     lines.push(`  API.md: ${apiFile}`);
-    lines.push(`  Regenerate: azpysdk apistub --md ${packageName}`);
+    lines.push(`  Regenerate: azpysdk apistub --md --extract-metadata ${packageName}`);
   }
   lines.push("");
   return lines.join("\n");
@@ -88,6 +88,7 @@ module.exports = async function apiMdConsistency({ core }) {
   if (issueCount > 0) {
     const messageParts = [
       "Generated API.md does not match committed API.md, or API.md is missing, for one or more affected packages.",
+      "API.metadata.yml is informational only (for troubleshooting API drift, e.g., parser/runtime differences) and is not part of pass/fail gating.",
       "",
       formatIssueSection("Mismatched packages:", mismatches),
       formatIssueSection("Missing API.md packages:", missing),

eng/scripts/Extract-APIViewMetadata-Python.ps1

@@ -0,0 +1,159 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+
+<#
+.SYNOPSIS
+Extracts Python APIView metadata from API markdown and writes API.metadata.yml.
+
+.DESCRIPTION
+Reads an API markdown file, extracts parser and Python runtime versions from the
+Python APIView metadata header, removes that header from the markdown, trims leading
+blank lines from the markdown body, and writes API.metadata.yml beside the markdown file.
+
+.PARAMETER ApiMarkdownPath
+Optional. Path to API markdown file. If omitted, a markdown file will be resolved
+from OutputPath (prefers API.md, then api.md).
+
+.PARAMETER OutputPath
+Optional. Directory containing API markdown output. Defaults to current directory.
+
+.EXAMPLE
+./Extract-APIViewMetadata-Python.ps1 -OutputPath ./sdk/template/azure-template
+
+.EXAMPLE
+./Extract-APIViewMetadata-Python.ps1 -ApiMarkdownPath ./sdk/template/azure-template/API.md
+#>
+
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $false)]
+    [string]$ApiMarkdownPath,
+
+    [Parameter(Mandatory = $false)]
+    [string]$OutputPath = "."
+)
+
+Set-StrictMode -Version 3
+$ErrorActionPreference = 'Stop'
+
+function Resolve-ApiMarkdownPath {
+    param(
+        [string]$ProvidedPath,
+        [string]$OutputDirectory
+    )
+
+    if ($ProvidedPath) {
+        return $ProvidedPath
+    }
+
+    $resolvedOutput = Resolve-Path -LiteralPath $OutputDirectory -ErrorAction Stop
+    $apiUpper = Join-Path $resolvedOutput.Path "API.md"
+    if (Test-Path -LiteralPath $apiUpper -PathType Leaf) {
+        return $apiUpper
+    }
+
+    $apiLower = Join-Path $resolvedOutput.Path "api.md"
+    if (Test-Path -LiteralPath $apiLower -PathType Leaf) {
+        return $apiLower
+    }
+
+    throw "Could not find API markdown file in '$OutputDirectory'. Expected API.md or api.md."
+}
+
+function Trim-LeadingBlankLines {
+    param([string[]]$Lines)
+
+    $start = 0
+    while ($start -lt $Lines.Count -and [string]::IsNullOrWhiteSpace($Lines[$start])) {
+        $start++
+    }
+
+    if ($start -eq 0) {
+        return $Lines
+    }
+
+    if ($start -ge $Lines.Count) {
+        return @()
+    }
+
+    return $Lines[$start..($Lines.Count - 1)]
+}
+
+function Get-Sha256Hex {
+    param([string]$Text)
+
+    $sha256 = [System.Security.Cryptography.SHA256]::Create()
+    try {
+        $bytes = [System.Text.Encoding]::UTF8.GetBytes($Text)
+        $hashBytes = $sha256.ComputeHash($bytes)
+        return ([System.BitConverter]::ToString($hashBytes)).Replace("-", "").ToLowerInvariant()
+    }
+    finally {
+        $sha256.Dispose()
+    }
+}
+
+$resolvedApiPath = Resolve-ApiMarkdownPath -ProvidedPath $ApiMarkdownPath -OutputDirectory $OutputPath
+if (-not (Test-Path -LiteralPath $resolvedApiPath -PathType Leaf)) {
+    throw "API markdown file not found: $resolvedApiPath"
+}
+
+$metadataPattern = '^# Package is parsed using apiview-stub-generator\(version:([^\)]+)\), Python version:\s*([^\s]+)\s*$'
+
+$fileText = Get-Content -LiteralPath $resolvedApiPath -Raw
+$lineEnding = if ($fileText -match "`r`n") { "`r`n" } else { "`n" }
+$lines = $fileText -split '\r?\n'
+
+$metadata = [ordered]@{}
+$filtered = [System.Collections.Generic.List[string]]::new()
+
+foreach ($line in $lines) {
+    $match = [regex]::Match($line, $metadataPattern)
+    if ($match.Success) {
+        # Alphabetical keys in output YAML.
+        $metadata['parserVersion'] = $match.Groups[1].Value
+        $metadata['pythonVersion'] = $match.Groups[2].Value
+        continue
+    }
+
+    $filtered.Add($line)
+}
+
+# Remove blank lines after opening fence so markdown body starts at namespace.
+if ($filtered.Count -gt 0 -and $filtered[0].StartsWith('```')) {
+    $fence = $filtered[0]
+    $body = Trim-LeadingBlankLines -Lines @($filtered | Select-Object -Skip 1)
+    $rewritten = [System.Collections.Generic.List[string]]::new()
+    $rewritten.Add($fence)
+    foreach ($line in $body) {
+        $rewritten.Add($line)
+    }
+    $filtered = $rewritten
+}
+else {
+    $trimmed = Trim-LeadingBlankLines -Lines @($filtered)
+    $filtered = [System.Collections.Generic.List[string]]::new($trimmed)
+}
+
+$normalizedLinesForHash = @($filtered | ForEach-Object { $_.TrimEnd() })
+$newlineForHash = [string][char]10
+$normalizedTextForHash = $normalizedLinesForHash -join $newlineForHash
+$metadata['apiMdSha256'] = Get-Sha256Hex -Text $normalizedTextForHash
+
+Set-Content -LiteralPath $resolvedApiPath -Value ($filtered -join $lineEnding) -NoNewline -Encoding utf8
+Write-Host "Updated markdown: $resolvedApiPath"
+
+$metadataPath = Join-Path (Split-Path -Parent $resolvedApiPath) "API.metadata.yml"
+if ($metadata.Count -gt 0) {
+    $yamlLines = [System.Collections.Generic.List[string]]::new()
+    foreach ($key in ($metadata.Keys | Sort-Object)) {
+        $yamlLines.Add(("{0}: {1}" -f $key, $metadata[$key]))
+    }
+
+    Set-Content -LiteralPath $metadataPath -Value ($yamlLines -join $lineEnding) -Encoding utf8
+    Write-Host "Generated metadata: $metadataPath"
+}
+elseif (Test-Path -LiteralPath $metadataPath) {
+    Remove-Item -LiteralPath $metadataPath -Force
+    Write-Host "Removed stale metadata: $metadataPath"
+}

eng/tools/azure-sdk-tools/azpysdk/apistub.py

@@ -69,12 +69,30 @@ def register(
             action="store_true",
             help="Generate api.md from the JSON token file using Export-APIViewMarkdown.ps1. Output directory for api.md is the same as the generated token file.",
         )
+        p.add_argument(
+            "--extract-metadata",
+            dest="extract_metadata",
+            default=False,
+            action="store_true",
+            help="Extract language-specific metadata from generated api.md into API.metadata.yml and remove metadata header from api.md.",
+        )
+        p.add_argument(
+            "--install-deps",
+            dest="install_deps",
+            default=False,
+            action="store_true",
+            help="Install dev requirements and apiview dependencies before running. Skipped by default for faster local iteration.",
+        )
         p.set_defaults(func=self.run)
 
     def run(self, args: argparse.Namespace) -> int:
         """Run the apistub check command."""
         logger.info("Running apistub check...")
 
+        if getattr(args, "extract_metadata", False) and not getattr(args, "generate_md", False):
+            logger.error("--extract-metadata requires --md.")
+            return 1
+
         set_envvar_defaults()
         targeted = self.get_targeted_directories(args)
 
@@ -94,22 +112,23 @@ def run(self, args: argparse.Namespace) -> int:
             )
             logger.info(f"Processing {package_name} for apistub check")
 
-            # install dependencies
-            self.install_dev_reqs(executable, args, package_dir)
-
-            try:
-                install_into_venv(
-                    executable,
-                    [
-                        "-r",
-                        os.path.join(REPO_ROOT, "eng", "apiview_reqs.txt"),
-                        "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/",
-                    ],
-                    package_dir,
-                )
-            except CalledProcessError as e:
-                logger.error(f"Failed to install dependencies: {e}")
-                return e.returncode
+            if getattr(args, "install_deps", False):
+                # install dependencies
+                self.install_dev_reqs(executable, args, package_dir)
+
+                try:
+                    install_into_venv(
+                        executable,
+                        [
+                            "-r",
+                            os.path.join(REPO_ROOT, "eng", "apiview_reqs.txt"),
+                            "--index-url=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/",
+                        ],
+                        package_dir,
+                    )
+                except CalledProcessError as e:
+                    logger.error(f"Failed to install dependencies: {e}")
+                    return e.returncode
 
             if not os.getenv("PREBUILT_WHEEL_DIR"):
                 create_package_and_install(
@@ -124,14 +143,15 @@ def run(self, args: argparse.Namespace) -> int:
                     python_executable=executable,
                 )
 
-            self.pip_freeze(executable)
+            if getattr(args, "install_deps", False):
+                self.pip_freeze(executable)
 
             pkg_path = get_package_wheel_path(package_dir)
             pkg_path = os.path.abspath(pkg_path)
 
             dest_dir = getattr(args, "dest_dir", None)
             if dest_dir:
-                out_token_path = os.path.join(os.path.abspath(dest_dir), package_name)
+                out_token_path = os.path.abspath(dest_dir)
                 os.makedirs(out_token_path, exist_ok=True)
             else:
                 out_token_path = os.path.abspath(staging_directory)
@@ -157,6 +177,9 @@ def run(self, args: argparse.Namespace) -> int:
                 if getattr(args, "generate_md", False):
                     token_json_path = os.path.join(out_token_path, f"{package_name}_python.json")
                     md_script = os.path.join(REPO_ROOT, "eng", "common", "scripts", "Export-APIViewMarkdown.ps1")
+                    metadata_script = os.path.join(
+                        REPO_ROOT, "eng", "scripts", "Extract-APIViewMetadata-Python.ps1"
+                    )
                     logger.info(f"Generating api.md for {package_name}")
                     try:
                         result = run(
@@ -168,11 +191,22 @@ def run(self, args: argparse.Namespace) -> int:
                         # pwsh script logs the api.md location
                         if result.stdout:
                             logger.info(result.stdout)
+
+                        if getattr(args, "extract_metadata", False):
+                            logger.info(f"Extracting API metadata for {package_name}")
+                            metadata_result = run(
+                                ["pwsh", metadata_script, "-OutputPath", out_token_path],
+                                check=True,
+                                capture_output=True,
+                                text=True,
+                            )
+                            if metadata_result.stdout:
+                                logger.info(metadata_result.stdout)
                     except FileNotFoundError:
                         logger.error("Failed to generate api.md: pwsh (PowerShell) is not installed or not on PATH.")
                         results.append(1)
                     except CalledProcessError as e:
-                        logger.error(f"Failed to generate api.md (exit code {e.returncode}):")
+                        logger.error(f"Failed to generate api.md or extract metadata (exit code {e.returncode}):")
                         if e.stderr:
                             logger.error(e.stderr)
                         if e.stdout:

eng/tools/azure-sdk-tools/tests/test_apistub.py

@@ -94,7 +94,7 @@ def _make_args(self, dest_dir=None, generate_md=False):
     def test_dest_dir_creates_package_subfolder(
         self, _env, _install, _create, _get_whl, _get_mapping, tmp_path, monkeypatch
     ):
-        """When --dest-dir is given, output should go to <dest_dir>/<package_name>/."""
+        """When --dest-dir is given, output should go directly to <dest_dir>/."""
         monkeypatch.chdir(os.getcwd())
         dest = tmp_path / "output"
         dest.mkdir()
@@ -130,7 +130,7 @@ def fake_pwsh(cmd, **kwargs):
 
             stub.run(self._make_args(dest_dir=str(dest), generate_md=True))
 
-        expected_out = os.path.join(str(dest), "azure-core")
+        expected_out = str(dest)
         assert os.path.isdir(expected_out)
         assert os.path.exists(os.path.join(expected_out, "api.md"))
         assert os.path.exists(os.path.join(expected_out, "azure-core_python.json"))

scripts/api_md_workflow/README.md

@@ -5,6 +5,8 @@ This folder contains the helper scripts used by the GitHub Actions workflows tha
 ## Purpose
 
 The workflow validates that when a pull request changes one or more SDK packages, the committed `API.md` files are still up to date.
+Only `API.md` is diff-gated by this workflow; `API.metadata.yml` is intentionally excluded from mismatch checks.
+Use `API.metadata.yml` as diagnostic context when `API.md` drifts (for example, parser/runtime version differences), but it does not affect pass/fail.
 
 The logic is split between GitHub workflow YAML files and helper scripts in Python and JavaScript.
 
@@ -20,7 +22,7 @@ It runs on pull requests for changes under `sdk/**`.
 - Regenerates `API.md` for those packages.
 - Fails if the generated files differ from the committed files.
 - Fails if an affected package does not have a committed `API.md`.
-- Prints the mismatched or missing packages and the `azpysdk apistub --md` command needed to regenerate each `API.md` file.
+- Prints the mismatched or missing packages and the `azpysdk apistub --md --extract-metadata` command needed to regenerate each `API.md` file.
 
 ## Script Layout
 
@@ -47,7 +49,7 @@ Also writes `count=<n>` to `GITHUB_OUTPUT`.
 
 ### `regenerate.js`
 
-Reads package directories from `API_MD_PACKAGES_FILE` and runs `azpysdk apistub --md <package-name>` for each package.
+Reads package directories from `API_MD_PACKAGES_FILE` and runs `azpysdk apistub --md --extract-metadata <package-name>` for each package.
 
 This script is used by the consistency check.
 
@@ -60,6 +62,8 @@ Reads package directories from `API_MD_PACKAGES_FILE`, checks whether `<package>
 
 Also writes `mismatch_count=<n>`, `missing_count=<n>`, and `issue_count=<n>` to `GITHUB_OUTPUT`.
 
+`API.metadata.yml` is not part of this diff check.
+
 ### `create_api_review_pr.js` and adapters
 
 API review PR creation now uses a shared JavaScript orchestrator with a language adapter boundary:
@@ -96,4 +100,4 @@ Common variables include:
 3. `find_affected.js` determines which packages were touched.
 4. `regenerate.js` rebuilds `API.md` for those packages.
 5. `find_mismatches.js` records any `API.md` drift, including missing or untracked `API.md` files.
-6. If drift is found, the workflow fails and prints the affected packages plus the `azpysdk apistub --md <package-name>` command to regenerate each `API.md` file locally.
+6. If drift is found, the workflow fails and prints the affected packages plus the `azpysdk apistub --md --extract-metadata <package-name>` command to regenerate each `API.md` file locally.

scripts/api_md_workflow/adapters/python.js

@@ -114,7 +114,7 @@ function generateApiForPackage({
     activeLogger.info(`--- Generating API.md on ${refLabel} ---`);
   }
 
-  run("azpysdk", ["apistub", "--md", packageName], {
+  run("azpysdk", ["apistub", "--md", "--extract-metadata", packageName], {
     cwd: repoRoot,
     check: true,
     logger: activeLogger,

scripts/api_md_workflow/find_mismatches.js

@@ -13,6 +13,8 @@ async function main() {
   const mismatches = [];
   const missing = [];
   for (const pkgDir of packages) {
+    // Deliberately scope consistency gating to API.md only.
+    // API.metadata.yml is generated sidecar metadata and is not diff-gated here.
     const apiFile = `${pkgDir}/API.md`;
 
     // Enforce that each affected package has a committed API.md file.