[azpysdk] Refactor apistub so that API consistency files are the default output (#47561)

* Refactor so api consistency files are the default output.

* Remove `--dest-dir` option due to inconsistent/surprising behavior.

Author: tjprescott

.github/skills/generate-api-markdown/SKILL.md

@@ -19,6 +19,7 @@ description: Generate an API markdown file and token file using ApiView. Use thi
 1. Navigate to the desired package directory
 2. Run the command:
    ```bash
-   azpysdk apistub --md --extract-metadata --install-deps --dest-dir . .
+   azpysdk apistub .
    ```
-3. The command outputs the location of the generated markdown file. Provide this file to the user for review.
+3. The command generates `api.md` and `api.metadata.yml` in the package directory, which are the files needed to pass the API consistency check. Provide these files to the user for review.
+4. If the user explicitly asks for the raw APIView token file, run `azpysdk apistub . --token-file` instead.

.github/workflows/src/api-md-consistency/adapters/python.js

@@ -131,7 +131,7 @@ function generateApiForPackage({
     const pythonExecutable = runtimeExecutable || process.env.RUNTIME_EXECUTABLE;
     run(
       pythonExecutable,
-      ["-m", "azpysdk.main", "apistub", "--md", "--extract-metadata", "--dest-dir", packageDir, packageName],
+      ["-m", "azpysdk.main", "apistub", packageName],
       {
         cwd: repoRoot,
         check: true,
@@ -141,7 +141,7 @@ function generateApiForPackage({
     return;
   }
 
-  run("azpysdk", ["apistub", "--md", "--extract-metadata", "--dest-dir", packageDir, packageName], {
+  run("azpysdk", ["apistub", packageName], {
     cwd: repoRoot,
     check: true,
     logger: activeLogger,

.github/workflows/src/api-md-consistency/api-md-consistency.js

@@ -47,8 +47,8 @@ function formatIssueSection(title, apiFiles) {
     lines.push(styleLog(`PACKAGE: ${packageName}`, ANSI.bold, ANSI.cyan));
     lines.push(`PATH:    ${packageDir}`);
     lines.push(`API FILE: ${apiFile}`);
-    lines.push(styleLog("Regenerate from the repository root:", ANSI.bold, ANSI.yellow));
-    lines.push(styleLog(`  azpysdk apistub --md --extract-metadata ${packageName} --dest-dir .`, ANSI.bold, ANSI.yellow));
+    lines.push(styleLog(`Regenerate from the ${packageName} package root:`, ANSI.bold, ANSI.yellow));
+    lines.push(styleLog(`  azpysdk apistub .`, ANSI.bold, ANSI.yellow));
     lines.push("============================================================");
   }
   lines.push("");
@@ -97,7 +97,7 @@ export default async function apiMdConsistency({ core }) {
       "",
       formatIssueSection("Mismatched packages:", mismatches),
       formatIssueSection("Missing required API files:", missing),
-      "To regenerate api.md locally, run the command shown for each package from the repository root.",
+      "To regenerate api.md and api.metadata.yml locally, run the command shown for each package from the repository root.",
     ].filter((part) => part !== "");
 
     core.setFailed(messageParts.join("\n"));

doc/dev/tests.md

@@ -147,7 +147,7 @@ A quick description of the commands above:
 - verifywhl: verifies the wheel contents and manifest
 - verifysdist: verifies the sdist contents and manifest
 - samples: runs all of the samples in the `samples` directory and verifies they are working correctly
-- apistub: runs the [apistubgenerator](https://github.com/Azure/azure-sdk-tools/tree/main/packages/python-packages/apiview-stub-generator) tool on your code
+- apistub: runs the [apistubgenerator](https://github.com/Azure/azure-sdk-tools/tree/main/packages/python-packages/apiview-stub-generator) tool on your code. By default, generates `api.md` and `api.metadata.yml` in the package directory, which are the files needed to pass the API consistency check. Use `--token-file` to generate only the raw APIView token file.
 
 ## The `devtools_testutils` package
 

doc/tool_usage_guide.md

@@ -21,7 +21,7 @@ The following checks are available via the `azpysdk` entrypoint.
 |`pyright`| Runs `pyright` checks or `next-pyright` checks. (based on presence of `--next` argument) | `azpysdk pyright .` |
 |`black`| Runs `black` checks. | `azpysdk black .` |
 |`verifytypes`| Runs `verifytypes` checks. | `azpysdk verifytypes .` |
-|`apistub`| Generates an api stub for the package. | `azpysdk apistub .` |
+|`apistub`| Generates `api.md`, API metadata, or APIView token files for the package. | `azpysdk apistub .` |
 |`bandit`| Runs `bandit` checks, which detect common security issues. | `azpysdk bandit .` |
 |`verifywhl`| Verifies that the root directory in whl is azure, and verifies manifest so that all directories in source are included in sdist. | `azpysdk verifywhl .` |
 |`verifysdist`| Verify directories included in sdist and contents in manifest file. Also ensures that py.typed configuration is correct within the setup.py. | `azpysdk verifysdist .` |

eng/tools/azure-sdk-tools/azpysdk/apistub.py

@@ -57,24 +57,11 @@ def register(
             "apistub", parents=parents, help="Run the apistub check to generate an API stub for a package"
         )
         p.add_argument(
-            "--dest-dir",
-            dest="dest_dir",
-            default=None,
-            help="Destination directory for generated API stub token files.",
-        )
-        p.add_argument(
-            "--md",
-            dest="generate_md",
-            default=False,
-            action="store_true",
-            help="Generate api.md from the JSON token file using Export-APIViewMarkdown.ps1. Output directory for api.md is the same as the generated token file.",
-        )
-        p.add_argument(
-            "--extract-metadata",
-            dest="extract_metadata",
+            "--token-file",
+            dest="token_file",
             default=False,
             action="store_true",
-            help="Extract language-specific metadata from generated api.md into api.metadata.yml and remove metadata header from api.md.",
+            help="Generate only the raw APIView token file.",
         )
         p.add_argument(
             "--install-deps",
@@ -106,9 +93,8 @@ def run(self, args: argparse.Namespace) -> int:
         """Run the apistub check command."""
         logger.info("Running apistub check...")
 
-        if getattr(args, "extract_metadata", False) and not getattr(args, "generate_md", False):
-            logger.error("--extract-metadata requires --md.")
-            return 1
+        token_file = getattr(args, "token_file", False)
+        generate_markdown = not token_file
 
         set_envvar_defaults()
         targeted = self.get_targeted_directories(args)
@@ -160,12 +146,8 @@ def run(self, args: argparse.Namespace) -> int:
             pkg_path = get_package_wheel_path(package_dir)
             pkg_path = os.path.abspath(pkg_path)
 
-            dest_dir = getattr(args, "dest_dir", None)
-            if dest_dir:
-                out_token_path = os.path.abspath(dest_dir)
-                os.makedirs(out_token_path, exist_ok=True)
-            else:
-                out_token_path = os.path.abspath(staging_directory)
+            out_token_path = os.path.abspath(package_dir)
+            os.makedirs(out_token_path, exist_ok=True)
 
             cross_language_mapping_path = get_cross_language_mapping_path(package_dir)
 
@@ -178,15 +160,21 @@ def run(self, args: argparse.Namespace) -> int:
                 cmds.extend(["--out-path", out_token_path])
             if cross_language_mapping_path:
                 cmds.extend(["--mapping-path", cross_language_mapping_path])
-            if getattr(args, "generate_md", False):
+            if generate_markdown:
                 cmds.append("--skip-pylint")
 
             logger.info("Running apistub {}.".format(cmds))
 
             try:
                 self.run_venv_command(executable, cmds, cwd=staging_directory, check=True, immediately_dump=True)
-                if getattr(args, "generate_md", False):
-                    token_json_path = os.path.join(out_token_path, f"{package_name}_python.json")
+                token_json_path = os.path.join(out_token_path, f"{package_name}_python.json")
+                if token_file:
+                    if os.path.exists(token_json_path):
+                        logger.info(f"Generated APIView token file: {token_json_path}")
+                    else:
+                        logger.error(f"Expected APIView token file was not generated: {token_json_path}")
+                        results.append(1)
+                else:
                     md_script = os.path.join(REPO_ROOT, "eng", "common", "scripts", "Export-APIViewMarkdown.ps1")
                     metadata_script = os.path.join(REPO_ROOT, "eng", "scripts", "Extract-APIViewMetadata-Python.ps1")
                     logger.info(f"Generating api.md for {package_name}")
@@ -201,16 +189,15 @@ def run(self, args: argparse.Namespace) -> int:
                         if result.stdout:
                             logger.info(result.stdout)
 
-                        if getattr(args, "extract_metadata", False):
-                            logger.info(f"Extracting API metadata for {package_name}")
-                            metadata_result = run(
-                                ["pwsh", metadata_script, "-OutputPath", out_token_path],
-                                check=True,
-                                capture_output=True,
-                                text=True,
-                            )
-                            if metadata_result.stdout:
-                                logger.info(metadata_result.stdout)
+                        logger.info(f"Extracting API metadata for {package_name}")
+                        metadata_result = run(
+                            ["pwsh", metadata_script, "-OutputPath", out_token_path],
+                            check=True,
+                            capture_output=True,
+                            text=True,
+                        )
+                        if metadata_result.stdout:
+                            logger.info(metadata_result.stdout)
                     except FileNotFoundError:
                         logger.error("Failed to generate api.md: pwsh (PowerShell) is not installed or not on PATH.")
                         results.append(1)

eng/tools/azure-sdk-tools/ci_tools/functions.py

@@ -182,15 +182,19 @@ def glob_packages(glob_string: str, target_root_dir: str) -> List[str]:
     collected_top_level_directories = []
 
     for glob_string in individual_globs:
-        globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py"), recursive=True) + glob.glob(
-            os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py")
+        globbed = (
+            glob.glob(os.path.join(target_root_dir, glob_string, "setup.py"), recursive=True)
+            + glob.glob(os.path.join(target_root_dir, "*/", glob_string, "setup.py"))
+            + glob.glob(os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py"))
         )
         collected_top_level_directories.extend([os.path.dirname(p) for p in globbed])
 
     # handle pyproject.toml separately, as we need to filter them by the presence of a `[project]` section
     for glob_string in individual_globs:
-        globbed = glob.glob(os.path.join(target_root_dir, glob_string, "pyproject.toml"), recursive=True) + glob.glob(
-            os.path.join(target_root_dir, "sdk/*/", glob_string, "pyproject.toml")
+        globbed = (
+            glob.glob(os.path.join(target_root_dir, glob_string, "pyproject.toml"), recursive=True)
+            + glob.glob(os.path.join(target_root_dir, "*/", glob_string, "pyproject.toml"))
+            + glob.glob(os.path.join(target_root_dir, "sdk/*/", glob_string, "pyproject.toml"))
         )
         for p in globbed:
             if get_pyproject(os.path.dirname(p)):

eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -335,11 +335,7 @@ def main(generate_input, generate_output):
                     cmds = [
                         "azpysdk",
                         "apistub",
-                        "--md",
-                        "--extract-metadata",
                         package_name,
-                        "--dest-dir",
-                        package_path.absolute().as_posix(),
                     ]
                     _LOGGER.info(f"generate apiview file for package {package_name}")
                     check_call(

eng/tools/azure-sdk-tools/tests/integration/test_package_discovery.py

@@ -3,7 +3,6 @@
 from ci_tools.parsing import ParsedSetup
 from ci_tools.functions import discover_targeted_packages
 
-
 repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", ".."))
 sdk_root = os.path.join(repo_root, "sdk")
 core_service_root = os.path.join(sdk_root, "core")
@@ -70,6 +69,22 @@ def test_discovery_single_package():
     ]
 
 
+def test_discovery_single_package_from_sdk_root():
+    results = discover_targeted_packages("azure-template", sdk_root, filter_type="Build")
+
+    assert [os.path.basename(result) for result in results] == [
+        "azure-template",
+    ]
+
+
+def test_discovery_single_package_from_repo_root():
+    results = discover_targeted_packages("azure-template", repo_root, filter_type="Build")
+
+    assert [os.path.basename(result) for result in results] == [
+        "azure-template",
+    ]
+
+
 def test_discovery_omit_regression():
     results = discover_targeted_packages("*", core_service_root, filter_type="Regression")