Fix stubtest_third_party.py on linux

Previously, when running it on Linux without the `--specified-platforms-only`
flag, it printed spurious warnings and ran stubtest anyway for distributions
that didn't include `linux` as a supported platform.

Rearrange things:

* The `--specified-platforms-only` now works consistently on all platforms.
  Previously, it was ignored on Linux.
* Instead, in CI the flag is added only for non-Linux platforms.
* The note about stubtest not running for a certain platform in CI is now
  printed only after stubtest tested a distribution successfully to avoid
  breaking the output.

Closes: #13641

Author: srittau

.github/workflows/daily.yml

@@ -84,6 +84,7 @@ jobs:
             fi
 
             PYTHON_EXECUTABLE="xvfb-run python"
+            EXTRA_ARGS=""
           else
             if [ "${{ runner.os }}" = "macOS" ] && [ -n "$PACKAGES" ]; then
               brew install -q $PACKAGES
@@ -94,9 +95,10 @@ jobs:
             fi
 
             PYTHON_EXECUTABLE="python"
+            EXTRA_ARGS="--specified-platforms-only"
           fi
 
-          $PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          $PYTHON_EXECUTABLE tests/stubtest_third_party.py $EXTRA_ARGS --num-shards 4 --shard-index ${{ matrix.shard-index }}
 
   stub-uploader:
     name: stub_uploader tests

.github/workflows/stubtest_third_party.yml

@@ -71,6 +71,7 @@ jobs:
               fi
 
               PYTHON_EXECUTABLE="xvfb-run python"
+              EXTRA_ARGS=""
             else
               if [ "${{ runner.os }}" = "macOS" ] && [ -n "$PACKAGES" ]; then
                 echo "Installing Homebrew packages: $PACKAGES"
@@ -83,9 +84,10 @@ jobs:
               fi
 
               PYTHON_EXECUTABLE="python"
+              EXTRA_ARGS="--specified-platforms-only"
             fi
 
-            $PYTHON_EXECUTABLE tests/stubtest_third_party.py --specified-platforms-only $STUBS
+            $PYTHON_EXECUTABLE tests/stubtest_third_party.py $EXTRA_ARGS $STUBS
           else
             echo "Nothing to test"
           fi

lib/ts_utils/metadata.py

@@ -32,6 +32,7 @@
     "read_stubtest_settings",
 ]
 
+DEFAULT_STUBTEST_PLATFORMS = ["linux"]
 
 _STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
 # Some older websites have a bad pattern of using query params for navigation.
@@ -91,7 +92,7 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     choco_dependencies: object = data.get("choco_dependencies", [])
     extras: object = data.get("extras", [])
     ignore_missing_stub: object = data.get("ignore_missing_stub", False)
-    specified_platforms: object = data.get("platforms", ["linux"])
+    specified_platforms: object = data.get("platforms", [])
     stubtest_requirements: object = data.get("stubtest_requirements", [])
 
     assert type(skip) is bool
@@ -109,7 +110,7 @@ def read_stubtest_settings(distribution: str) -> StubtestSettings:
     assert not unrecognised_platforms, f"Unrecognised platforms specified for {distribution!r}: {unrecognised_platforms}"
 
     for platform, dep_key in _STUBTEST_PLATFORM_MAPPING.items():
-        if platform not in specified_platforms:
+        if platform not in (specified_platforms or DEFAULT_STUBTEST_PLATFORMS):
             assert dep_key not in data, (
                 f"Stubtest is not run on {platform} in CI for {distribution!r}, "
                 f"but {dep_key!r} are specified in METADATA.toml"

tests/stubtest_third_party.py

@@ -15,7 +15,13 @@
 from time import time
 from typing import NoReturn
 
-from ts_utils.metadata import NoSuchStubError, get_recursive_requirements, read_metadata
+from ts_utils.metadata import (
+    DEFAULT_STUBTEST_PLATFORMS,
+    NoSuchStubError,
+    StubtestSettings,
+    get_recursive_requirements,
+    read_metadata,
+)
 from ts_utils.paths import STUBS_PATH, allowlists_path, tests_path
 from ts_utils.utils import (
     PYTHON_VERSION,
@@ -46,11 +52,9 @@ def run_stubtest(
         print(colored("skipping", "yellow"))
         return True
 
-    if sys.platform not in stubtest_settings.platforms:
-        if specified_platforms_only:
-            print(colored("skipping (platform not specified in METADATA.toml)", "yellow"))
-            return True
-        print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow"))
+    if should_skip_dist(stubtest_settings, specified_platforms_only=specified_platforms_only):
+        print(colored("skipping (platform not specified in METADATA.toml)", "yellow"))
+        return True
 
     if not metadata.requires_python.contains(PYTHON_VERSION):
         print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow"))
@@ -176,6 +180,14 @@ def run_stubtest(
         else:
             print_time(time() - t)
             print_success_msg()
+
+            if (
+                not stubtest_settings.platforms
+                and sys.platform not in DEFAULT_STUBTEST_PLATFORMS
+                and not specified_platforms_only
+            ):
+                print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI", "yellow"))
+
             if keep_tmp_dir:
                 print_info(f"Virtual environment kept at: {venv_dir}")
     finally:
@@ -188,6 +200,17 @@ def run_stubtest(
     return True
 
 
+def should_skip_dist(stubtest_settings: StubtestSettings, *, specified_platforms_only: bool) -> bool:
+    if stubtest_settings.platforms:
+        # Platforms explicitly specified in METADATA.toml:
+        # skip if the current platform is not in the list.
+        return sys.platform not in stubtest_settings.platforms
+    else:
+        # Platforms not specified in METADATA.toml:
+        # skip if the --specified-platforms-only flag is set.
+        return specified_platforms_only
+
+
 def setup_gdb_stubtest_command(venv_dir: Path, stubtest_cmd: list[str]) -> bool:
     """
     Use wrapper scripts to run stubtest inside gdb.