Merge pull request #23 from ArchiveBox/test-speedups

lots of small test improvements and speedups

Author: pirate

.github/workflows/tests.yml

@@ -113,29 +113,38 @@ jobs:
     - name: Setup pnpm
       uses: pnpm/action-setup@v5.0.0
       with:
-        version: 10
+        version: 10.19.0
 
     - name: Setup Node
       uses: actions/setup-node@v6
       with:
         node-version: '22'
 
-    - name: Setup Yarn (Berry / 4.x via corepack)
+    - name: Setup Yarn (classic + Berry)
       run: |
-        # github-hosted ubuntu runners ship a system-wide yarn 1.22 that
-        # shadows corepack's shim. Wipe it out before enabling corepack so
-        # `yarn` unambiguously resolves to the 4.x shim from node's bin dir.
-        sudo rm -f /usr/local/bin/yarn /usr/local/bin/yarnpkg /usr/bin/yarn /usr/bin/yarnpkg
-        corepack enable
-        corepack prepare yarn@4.13.0 --activate
-        # Pre-download pnpm via corepack so the very first ``pnpm --version``
-        # doesn't pollute logs with "Corepack is about to download ..."
-        # progress output.
-        corepack prepare pnpm@10.17.1 --activate || true
-        which yarn
+        npm install -g yarn@1.22.22
+        if [ "$(uname -s)" = "Darwin" ]; then
+          YARN_BERRY_PREFIX="/opt/homebrew/opt/yarn-berry"
+          YARN_BERRY_ALIAS="/opt/homebrew/bin/yarn-berry"
+        elif [ -d /home/linuxbrew/.linuxbrew/opt ]; then
+          YARN_BERRY_PREFIX="/home/linuxbrew/.linuxbrew/opt/yarn-berry"
+          YARN_BERRY_ALIAS="/home/linuxbrew/.linuxbrew/bin/yarn-berry"
+        else
+          YARN_BERRY_PREFIX="/usr/local/yarn-berry"
+          YARN_BERRY_ALIAS="/usr/local/bin/yarn-berry"
+        fi
+        YARN_BERRY_ALIAS_DIR="$(dirname "$YARN_BERRY_ALIAS")"
+        mkdir -p "$YARN_BERRY_ALIAS_DIR"
+        export PATH="$YARN_BERRY_ALIAS_DIR:$PATH"
+        echo "$YARN_BERRY_ALIAS_DIR" >> "$GITHUB_PATH"
+        npm install --prefix "$YARN_BERRY_PREFIX" @yarnpkg/cli-dist@4.13.0
+        ln -sf "$YARN_BERRY_PREFIX/node_modules/.bin/yarn" "$YARN_BERRY_ALIAS"
+        "$YARN_BERRY_ALIAS" --version | grep -q '^4\.'
+        command -v yarn
         yarn --version
-        yarn --version | grep -q '^4\.' || { echo "ERROR: yarn is not 4.x"; exit 1; }
-        pnpm --version || true
+        command -v yarn-berry
+        yarn-berry --version
+        yarn --version | grep -q '^1\.' || { echo "ERROR: yarn is not 1.x"; exit 1; }
 
     - name: Setup Bun
       uses: oven-sh/setup-bun@v2
@@ -278,29 +287,38 @@ jobs:
     - name: Setup pnpm
       uses: pnpm/action-setup@v5.0.0
       with:
-        version: 10
+        version: 10.19.0
 
     - name: Setup Node
       uses: actions/setup-node@v6
       with:
         node-version: '22'
 
-    - name: Setup Yarn (Berry / 4.x via corepack)
+    - name: Setup Yarn (classic + Berry)
       run: |
-        # github-hosted ubuntu runners ship a system-wide yarn 1.22 that
-        # shadows corepack's shim. Wipe it out before enabling corepack so
-        # `yarn` unambiguously resolves to the 4.x shim from node's bin dir.
-        sudo rm -f /usr/local/bin/yarn /usr/local/bin/yarnpkg /usr/bin/yarn /usr/bin/yarnpkg
-        corepack enable
-        corepack prepare yarn@4.13.0 --activate
-        # Pre-download pnpm via corepack so the very first ``pnpm --version``
-        # doesn't pollute logs with "Corepack is about to download ..."
-        # progress output.
-        corepack prepare pnpm@10.17.1 --activate || true
-        which yarn
+        npm install -g yarn@1.22.22
+        if [ "$(uname -s)" = "Darwin" ]; then
+          YARN_BERRY_PREFIX="/opt/homebrew/opt/yarn-berry"
+          YARN_BERRY_ALIAS="/opt/homebrew/bin/yarn-berry"
+        elif [ -d /home/linuxbrew/.linuxbrew/opt ]; then
+          YARN_BERRY_PREFIX="/home/linuxbrew/.linuxbrew/opt/yarn-berry"
+          YARN_BERRY_ALIAS="/home/linuxbrew/.linuxbrew/bin/yarn-berry"
+        else
+          YARN_BERRY_PREFIX="/usr/local/yarn-berry"
+          YARN_BERRY_ALIAS="/usr/local/bin/yarn-berry"
+        fi
+        YARN_BERRY_ALIAS_DIR="$(dirname "$YARN_BERRY_ALIAS")"
+        mkdir -p "$YARN_BERRY_ALIAS_DIR"
+        export PATH="$YARN_BERRY_ALIAS_DIR:$PATH"
+        echo "$YARN_BERRY_ALIAS_DIR" >> "$GITHUB_PATH"
+        npm install --prefix "$YARN_BERRY_PREFIX" @yarnpkg/cli-dist@4.13.0
+        ln -sf "$YARN_BERRY_PREFIX/node_modules/.bin/yarn" "$YARN_BERRY_ALIAS"
+        "$YARN_BERRY_ALIAS" --version | grep -q '^4\.'
+        command -v yarn
         yarn --version
-        yarn --version | grep -q '^4\.' || { echo "ERROR: yarn is not 4.x"; exit 1; }
-        pnpm --version || true
+        command -v yarn-berry
+        yarn-berry --version
+        yarn --version | grep -q '^1\.' || { echo "ERROR: yarn is not 1.x"; exit 1; }
 
     - name: Setup Bun
       uses: oven-sh/setup-bun@v2

README.md

@@ -855,12 +855,12 @@ Source: [`abx_pkg/binprovider_yarn.py`](./abx_pkg/binprovider_yarn.py) • Tests
 ```python
 INSTALLER_BIN = "yarn"
 PATH = ""                            # prepends <yarn_prefix>/node_modules/.bin
-yarn_prefix = None                   # workspace dir, defaults to ABX_PKG_YARN_ROOT or ~/.cache/abx-pkg/yarn
+yarn_prefix = None                   # workspace dir, defaults to ABX_PKG_YARN_ROOT or ABX_PKG_LIB_DIR/yarn
 cache_dir = user_cache_path("yarn", "abx-pkg") or <system temp>/yarn-cache
 yarn_install_args = []
 ```
 
-- Install root: Yarn 4 / Yarn Berry is workspace-based, so the provider always operates inside a project directory. Set `install_root=Path(...)` or `install_root=Path(...)` for a hermetic workspace; the workspace is auto-initialized with a stub `package.json` and `.yarnrc.yml` (`nodeLinker: node-modules` so binaries land in `<workspace>/node_modules/.bin`). When unset, the provider uses `$ABX_PKG_YARN_ROOT` or `~/.cache/abx-pkg/yarn`.
+- Install root: Yarn 4 / Yarn Berry is workspace-based, so the provider always operates inside a project directory. Set `install_root=Path(...)` or `install_root=Path(...)` for a hermetic workspace; the workspace is auto-initialized with a stub `package.json` and `.yarnrc.yml` (`nodeLinker: node-modules` so binaries land in `<workspace>/node_modules/.bin`). When unset, the provider uses `$ABX_PKG_YARN_ROOT` or `$ABX_PKG_LIB_DIR/yarn`.
 - Auto-switching: none. Honors `YARN_BINARY=/abs/path/to/yarn`. Both Yarn classic (1.x) and Yarn Berry (2+) work for basic install/update/uninstall, but only Yarn 4.10+ supports the security flags.
 - `dry_run`: shared behavior.
 - Security: supports both `min_release_age` and `postinstall_scripts=False`, and hydrates their provider defaults from `ABX_PKG_MIN_RELEASE_AGE` and `ABX_PKG_POSTINSTALL_SCRIPTS`. Both controls require Yarn 4.10+; on older hosts `supports_min_release_age()` / `supports_postinstall_disable()` return `False` and explicit values are logged-and-ignored.
@@ -923,7 +923,7 @@ Source: [`abx_pkg/binprovider_bash.py`](./abx_pkg/binprovider_bash.py) • Tests
 ```python
 INSTALLER_BIN = "sh"
 PATH = ""
-bash_root = $ABX_PKG_BASH_ROOT or ~/.cache/abx-pkg/bash
+bash_root = $ABX_PKG_BASH_ROOT or $ABX_PKG_LIB_DIR/bash
 bash_bin_dir = <bash_root>/bin
 ```
 
@@ -1035,7 +1035,7 @@ Source: [`abx_pkg/binprovider_docker.py`](./abx_pkg/binprovider_docker.py) • T
 ```python
 INSTALLER_BIN = "docker"
 PATH = ""                            # prepends docker_shim_dir
-docker_shim_dir = ($ABX_PKG_DOCKER_ROOT or ~/.cache/abx-pkg/docker) / "bin"
+docker_shim_dir = ($ABX_PKG_DOCKER_ROOT or $ABX_PKG_LIB_DIR/docker) / "bin"
 docker_run_args = ["--rm", "-i"]
 ```
 
@@ -1056,7 +1056,7 @@ Source: [`abx_pkg/binprovider_chromewebstore.py`](./abx_pkg/binprovider_chromewe
 ```python
 INSTALLER_BIN = "node"
 PATH = ""
-extensions_root = $ABX_PKG_CHROMEWEBSTORE_ROOT or ~/.cache/abx-pkg/chromewebstore
+extensions_root = $ABX_PKG_CHROMEWEBSTORE_ROOT or $ABX_PKG_LIB_DIR/chromewebstore
 extensions_dir = <extensions_root>/extensions
 ```
 
@@ -1077,7 +1077,7 @@ Source: [`abx_pkg/binprovider_puppeteer.py`](./abx_pkg/binprovider_puppeteer.py)
 ```python
 INSTALLER_BIN = "puppeteer-browsers"
 PATH = ""
-puppeteer_root = $ABX_PKG_PUPPETEER_ROOT or ~/.cache/abx-pkg/puppeteer
+puppeteer_root = $ABX_PKG_PUPPETEER_ROOT or $ABX_PKG_LIB_DIR/puppeteer
 browser_bin_dir = <puppeteer_root>/bin
 browser_cache_dir = <puppeteer_root>/cache
 ```

abx_pkg/binprovider.py

@@ -13,7 +13,6 @@
 import functools
 import tempfile
 from contextvars import ContextVar
-from types import SimpleNamespace
 
 from typing import (
     Optional,
@@ -453,17 +452,22 @@ def detect_euid(
 
         return candidate_euid if candidate_euid is not None else current_euid
 
-    def get_pw_record(self, uid: int) -> Any:
+    def get_pw_record(self, uid: int) -> pwd.struct_passwd:
         try:
             return pwd.getpwuid(uid)
         except KeyError:
             if uid != os.geteuid():
                 raise
-            return SimpleNamespace(
-                pw_uid=uid,
-                pw_gid=os.getegid(),
-                pw_dir=os.environ.get("HOME", tempfile.gettempdir()),
-                pw_name=os.environ.get("USER") or os.environ.get("LOGNAME") or str(uid),
+            return pwd.struct_passwd(
+                (
+                    os.environ.get("USER") or os.environ.get("LOGNAME") or str(uid),
+                    "x",
+                    uid,
+                    os.getegid(),
+                    "",
+                    os.environ.get("HOME", tempfile.gettempdir()),
+                    os.environ.get("SHELL", "/bin/sh"),
+                ),
             )
 
     @property
@@ -542,7 +546,7 @@ def INSTALLER_BINARY(self, no_cache: bool = False) -> ShallowBinary:
             self.INSTALLER_BIN,
         )
 
-    @computed_field
+    @computed_field(repr=False)
     @property
     def is_valid(self) -> bool:
         try:
@@ -985,6 +989,7 @@ def exec(
         )
         cwd_path = Path(cwd).resolve()
         cmd = [str(bin_abspath), *(str(arg) for arg in cmd)]
+        is_version_probe = len(cmd) == 2 and cmd[1] in {"--version", "-version", "-v"}
         exec_log_prefix = ACTIVE_EXEC_LOG_PREFIX.get()
         if should_log_command:
             if exec_log_prefix:
@@ -1033,7 +1038,7 @@ def drop_privileges():
             except Exception:
                 pass
 
-        if self.dry_run:
+        if self.dry_run and not is_version_probe:
             return subprocess.CompletedProcess(cmd, 0, "", "skipped (dry run)")
 
         kwargs.setdefault("capture_output", True)
@@ -1404,15 +1409,14 @@ def install(
         if self.dry_run:
             # return fake ShallowBinary if we're just doing a dry run
             # no point trying to get real abspath or version if nothing was actually installed
-            return ShallowBinary.model_validate(
-                {
-                    "name": bin_name,
-                    "binprovider": self,
-                    "abspath": Path(shutil.which(bin_name) or UNKNOWN_ABSPATH),
-                    "version": UNKNOWN_VERSION,
-                    "sha256": UNKNOWN_SHA256,
-                    "binproviders": [self],
-                },
+            return ShallowBinary.model_construct(
+                name=bin_name,
+                description=bin_name,
+                loaded_binprovider=self,
+                loaded_abspath=UNKNOWN_ABSPATH,
+                loaded_version=UNKNOWN_VERSION,
+                loaded_sha256=UNKNOWN_SHA256,
+                binproviders=[self],
             )
 
         self.invalidate_cache(bin_name)
@@ -1558,15 +1562,14 @@ def update(
             ACTIVE_EXEC_LOG_PREFIX.reset(exec_log_prefix_token)
 
         if self.dry_run:
-            return ShallowBinary.model_validate(
-                {
-                    "name": bin_name,
-                    "binprovider": self,
-                    "abspath": Path(shutil.which(bin_name) or UNKNOWN_ABSPATH),
-                    "version": UNKNOWN_VERSION,
-                    "sha256": UNKNOWN_SHA256,
-                    "binproviders": [self],
-                },
+            return ShallowBinary.model_construct(
+                name=bin_name,
+                description=bin_name,
+                loaded_binprovider=self,
+                loaded_abspath=UNKNOWN_ABSPATH,
+                loaded_version=UNKNOWN_VERSION,
+                loaded_sha256=UNKNOWN_SHA256,
+                binproviders=[self],
             )
 
         self.invalidate_cache(bin_name)

abx_pkg/binprovider_goget.py

@@ -81,10 +81,7 @@ def detect_euid_to_use(self) -> Self:
     def load_PATH_from_go_env(self) -> Self:
         bin_dir = self.bin_dir
         assert bin_dir is not None
-        if self.install_root != DEFAULT_GOPATH or "bin_dir" in self.model_fields_set:
-            self.PATH = self._merge_PATH(bin_dir)
-        else:
-            self.PATH = self._merge_PATH(bin_dir, PATH=self.PATH)
+        self.PATH = self._merge_PATH(bin_dir, PATH=self.PATH)
         return self
 
     def setup(
@@ -234,6 +231,21 @@ def default_version_handler(
         abspath = abspath or self.get_abspath(bin_name, quiet=True)
         if not abspath:
             return None
+        if str(bin_name) == self.INSTALLER_BIN:
+            version_provider = (
+                self.get_provider_with_overrides(dry_run=False)
+                if self.dry_run
+                else self
+            )
+            proc = version_provider.exec(
+                bin_name=abspath,
+                cmd=["version"],
+                timeout=timeout,
+                quiet=True,
+            )
+            if proc.returncode != 0:
+                return None
+            return SemVer.parse(proc.stdout.strip() or proc.stderr.strip())
         try:
             installer_abspath = self.INSTALLER_BINARY(no_cache=no_cache).loaded_abspath
             assert installer_abspath