Enforce Conda pyproject.toml metadata in verifywhl (#45012)

* enforce pyproject

* minor

* use logger instead of logging

* dedupe error msg

* check for section field

* only query pypi once

* simplify conda config checking

* format

* minor fix

* ignore yanked versions

* format

* update tests

Author: JennyPng

eng/tools/azure-sdk-tools/azpysdk/verify_whl.py

@@ -1,5 +1,4 @@
 import argparse
-import logging
 import os
 import sys
 import glob
@@ -59,12 +58,16 @@ def extract_whl(dist_dir, version):
 
 
 def verify_whl_root_directory(
-    dist_dir: str, expected_top_level_module: str, parsed_pkg: ParsedSetup, executable: str
+    dist_dir: str,
+    expected_top_level_module: str,
+    parsed_pkg: ParsedSetup,
+    executable: str,
+    pypi_versions: Optional[List[str]] = None,
 ) -> bool:
     # Verify metadata compatibility with prior version
     version: str = parsed_pkg.version
     metadata: Dict[str, Any] = extract_package_metadata(get_path_to_zip(dist_dir, version))
-    prior_version = get_prior_version(parsed_pkg.name, version)
+    prior_version = get_prior_version(parsed_pkg.name, version, pypi_versions=pypi_versions)
     if prior_version:
         if not verify_prior_version_metadata(parsed_pkg.name, prior_version, metadata, executable):
             return False
@@ -77,7 +80,7 @@ def verify_whl_root_directory(
     non_azure_folders = [d for d in root_folders if d != expected_top_level_module and not d.endswith(".dist-info")]
 
     if non_azure_folders:
-        logging.error(
+        logger.error(
             "whl has following incorrect directory at root level [%s]",
             non_azure_folders,
         )
@@ -99,10 +102,52 @@ def should_verify_package(package_name):
     return package_name not in EXCLUDED_PACKAGES and "nspkg" not in package_name and "-mgmt" not in package_name
 
 
-def get_prior_version(package_name: str, current_version: str) -> Optional[str]:
-    """Get prior stable version if it exists, otherwise get prior preview version, else return None."""
+def has_stable_version_on_pypi(all_versions: List[str]) -> bool:
+    """Check if the package has any stable (non-prerelease) version on PyPI."""
     try:
-        all_versions = retrieve_versions_from_pypi(package_name)
+        stable_versions = [pv for v in all_versions if not (pv := Version(v)).is_prerelease and pv > Version("0.0.0")]
+        return len(stable_versions) > 0
+    except Exception:
+        return False
+
+
+def verify_conda_section(
+    package_dir: str, package_name: str, parsed_pkg: ParsedSetup, pypi_versions: List[str]
+) -> bool:
+    """Verify that packages with stable versions on PyPI have [tool.azure-sdk-conda] section in pyproject.toml."""
+    if not has_stable_version_on_pypi(pypi_versions):
+        logger.info(f"Package {package_name} has no stable version on PyPI, skipping conda section check")
+        return True
+
+    pyproject_path = os.path.join(package_dir, "pyproject.toml")
+    if not os.path.exists(pyproject_path):
+        logger.error(f"Package {package_name} has a stable version on PyPI but is missing pyproject.toml")
+        return False
+
+    config = parsed_pkg.get_conda_config()
+    if not config:
+        logger.error(
+            f"Package {package_name} has a stable version on PyPI but is missing "
+            "[tool.azure-sdk-conda] section in pyproject.toml. This section is required to "
+            "specify if the package should be released individually or bundled to Conda."
+        )
+        return False
+    elif "in_bundle" not in config:
+        logger.error(f"[tool.azure-sdk-conda] section in pyproject.toml is missing required field `in_bundle`.")
+        return False
+    logger.info(f"Verified conda section for package {package_name}")
+    return True
+
+
+def get_prior_version(
+    package_name: str, current_version: str, pypi_versions: Optional[List[str]] = None
+) -> Optional[str]:
+    """Get prior stable version if it exists, otherwise get prior preview version, else return None.
+
+    pypi_versions can be optionally passed in to avoid redundant PyPI calls
+    """
+    try:
+        all_versions = pypi_versions if pypi_versions is not None else retrieve_versions_from_pypi(package_name)
         current_ver = Version(current_version)
         prior_versions = [Version(v) for v in all_versions if Version(v) < current_ver]
         if not prior_versions:
@@ -179,7 +224,7 @@ def verify_metadata_compatibility(current_metadata: Dict[str, Any], prior_metada
     repo_urls = ["homepage", "repository"]
     current_keys_lower = {k.lower() for k in current_metadata.keys()}
     if not any(key in current_keys_lower for key in repo_urls):
-        logging.error(f"Current metadata must contain at least one of: {repo_urls}")
+        logger.error(f"Current metadata must contain at least one of: {repo_urls}")
         return False
 
     if not prior_metadata:
@@ -192,7 +237,7 @@ def verify_metadata_compatibility(current_metadata: Dict[str, Any], prior_metada
     is_compatible = prior_keys_filtered.issubset(current_keys)
     if not is_compatible:
         missing_keys = prior_keys_filtered - current_keys
-        logging.error("Metadata compatibility failed. Missing keys: %s", missing_keys)
+        logger.error("Metadata compatibility failed. Missing keys: %s", missing_keys)
     return is_compatible
 
 
@@ -250,11 +295,19 @@ def run(self, args: argparse.Namespace) -> int:
             )
 
             if should_verify_package(package_name):
+                pypi_versions = retrieve_versions_from_pypi(package_name)
+
                 logger.info(f"Verifying whl for package: {package_name}")
-                if verify_whl_root_directory(staging_directory, top_level_module, parsed, executable):
+                if verify_whl_root_directory(
+                    staging_directory, top_level_module, parsed, executable, pypi_versions=pypi_versions
+                ):
                     logger.info(f"Verified whl for package {package_name}")
                 else:
                     logger.error(f"Failed to verify whl for package {package_name}")
                     results.append(1)
 
+                # Verify conda section for packages with stable versions on PyPI
+                if not verify_conda_section(package_dir, package_name, parsed, pypi_versions=pypi_versions):
+                    results.append(1)
+
         return max(results) if results else 0

eng/tools/azure-sdk-tools/pypi_tools/pypi.py

@@ -61,8 +61,11 @@ def get_ordered_versions(self, package_name, filter_by_compatibility=False) -> L
         project = self.project(package_name)
 
         versions: List[Version] = []
-        for package_version in project["releases"].keys():
+        for package_version, files in project["releases"].items():
             try:
+                # Skip yanked versions (no files or all files yanked)
+                if not files or all(f.get("yanked", False) for f in files):
+                    continue
                 versions.append(parse(package_version))
             except InvalidVersion as e:
                 logging.warn(f"Invalid version {package_version} for package {package_name}")

eng/tools/azure-sdk-tools/tests/test_metadata_verification.py

@@ -13,20 +13,14 @@
 import glob
 import sys
 
+from unittest.mock import MagicMock
 from ci_tools.parsing import ParsedSetup
-
-# Import the functions we want to test from the verify modules
-tox_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "tox"))
-if tox_path not in sys.path:
-    sys.path.append(tox_path)
-
-# Also add the azure-sdk-tools path so pypi_tools can be imported
-tools_path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-if tools_path not in sys.path:
-    sys.path.append(tools_path)
-
-from verify_whl import verify_whl_root_directory
-from verify_sdist import verify_sdist
+from azpysdk.verify_whl import (
+    verify_whl_root_directory,
+    has_stable_version_on_pypi,
+    verify_conda_section,
+)
+from azpysdk.verify_sdist import verify_sdist_helper
 
 # Test scenario paths
 scenarios_folder = os.path.join(os.path.dirname(__file__), "integration", "scenarios")
@@ -99,9 +93,13 @@ def test_verify_valid_metadata_passes(package_type, scenario_name, scenario_path
         # Run the appropriate verification function
         if package_type == "wheel":
             expected_module = parsed_pkg.namespace.split(".")[0] if parsed_pkg.namespace else "azure"
-            result = verify_whl_root_directory(os.path.dirname(package_path), expected_module, parsed_pkg)
+            result = verify_whl_root_directory(
+                os.path.dirname(package_path), expected_module, parsed_pkg, sys.executable, pypi_versions=[]
+            )
         else:
-            result = verify_sdist(actual_scenario_path, os.path.dirname(package_path), parsed_pkg)
+            result = verify_sdist_helper(
+                actual_scenario_path, os.path.dirname(package_path), parsed_pkg, sys.executable
+            )
 
         # The valid metadata should pass verification (return True)
         assert result is True, f"verify_{package_type} should return True for valid {scenario_name} metadata scenario"
@@ -142,9 +140,13 @@ def test_verify_invalid_metadata_fails(package_type, scenario_name, scenario_pat
         with caplog.at_level("ERROR"):
             if package_type == "wheel":
                 expected_module = parsed_pkg.namespace.split(".")[0] if parsed_pkg.namespace else "azure"
-                result = verify_whl_root_directory(os.path.dirname(package_path), expected_module, parsed_pkg)
+                result = verify_whl_root_directory(
+                    os.path.dirname(package_path), expected_module, parsed_pkg, sys.executable, pypi_versions=None
+                )
             else:
-                result = verify_sdist(actual_scenario_path, os.path.dirname(package_path), parsed_pkg)
+                result = verify_sdist_helper(
+                    actual_scenario_path, os.path.dirname(package_path), parsed_pkg, sys.executable
+                )
 
         # The invalid metadata should fail verification (return False)
         assert (
@@ -173,3 +175,66 @@ def test_verify_invalid_metadata_fails(package_type, scenario_name, scenario_pat
         # Cleanup dist directory
         if os.path.exists(dist_dir):
             shutil.rmtree(dist_dir)
+
+
+# ======================= has_stable_version_on_pypi tests =======================
+
+
+def test_has_stable_version_on_pypi_with_stable():
+    """Returns True when at least one stable version exists."""
+    assert has_stable_version_on_pypi(["1.0.0", "2.0.0b1", "0.1.0"]) is True
+
+
+def test_has_stable_version_on_pypi_only_previews():
+    """Returns False when all versions are pre-releases."""
+    assert has_stable_version_on_pypi(["1.0.0b1", "2.0.0a3", "0.1.0rc1"]) is False
+
+
+def test_has_stable_version_on_pypi_empty():
+    """Returns False for an empty version list."""
+    assert has_stable_version_on_pypi([]) is False
+
+
+def test_has_stable_version_on_pypi_only_zero():
+    """Returns False when the only stable version is 0.0.0."""
+    assert has_stable_version_on_pypi(["0.0.0"]) is False
+
+
+# ======================= verify_conda_section tests =======================
+
+
+def test_verify_conda_section_skips_when_no_stable_version():
+    """Should return True (pass) when there are no stable versions on PyPI."""
+    parsed_pkg = MagicMock()
+    result = verify_conda_section("/fake/path", "pkg", parsed_pkg, pypi_versions=["1.0.0b1"])
+    assert result is True
+
+
+def test_verify_conda_section_fails_missing_conda_section(tmp_path):
+    """Should fail when pyproject.toml exists but has no [tool.azure-sdk-conda] section."""
+    pyproject = tmp_path / "pyproject.toml"
+    pyproject.write_text("[project]\nname = 'pkg'\n")
+    parsed_pkg = MagicMock()
+    parsed_pkg.get_conda_config.return_value = None
+    result = verify_conda_section(str(tmp_path), "pkg", parsed_pkg, pypi_versions=["1.0.0"])
+    assert result is False
+
+
+def test_verify_conda_section_fails_missing_in_bundle(tmp_path):
+    """Should fail when [tool.azure-sdk-conda] exists but 'in_bundle' key is missing."""
+    pyproject = tmp_path / "pyproject.toml"
+    pyproject.write_text("[tool.azure-sdk-conda]\nsome_other_key = true\n")
+    parsed_pkg = MagicMock()
+    parsed_pkg.get_conda_config.return_value = {"some_other_key": True}
+    result = verify_conda_section(str(tmp_path), "pkg", parsed_pkg, pypi_versions=["1.0.0"])
+    assert result is False
+
+
+def test_verify_conda_section_passes_with_valid_config(tmp_path):
+    """Should pass when [tool.azure-sdk-conda] has 'in_bundle' defined."""
+    pyproject = tmp_path / "pyproject.toml"
+    pyproject.write_text("[tool.azure-sdk-conda]\nin_bundle = true\n")
+    parsed_pkg = MagicMock()
+    parsed_pkg.get_conda_config.return_value = {"in_bundle": True}
+    result = verify_conda_section(str(tmp_path), "pkg", parsed_pkg, pypi_versions=["1.0.0"])
+    assert result is True