diff --git a/src/promptfoo/cli.py b/src/promptfoo/cli.py index 1fdaf64..ca5ae48 100644 --- a/src/promptfoo/cli.py +++ b/src/promptfoo/cli.py @@ -26,16 +26,13 @@ def check_npx_installed() -> bool: def print_installation_help() -> None: - """Print helpful installation instructions for Node.js.""" - print("ERROR: promptfoo requires Node.js to be installed.", file=sys.stderr) - print("", file=sys.stderr) - print("Please install Node.js:", file=sys.stderr) - print(" - macOS: brew install node", file=sys.stderr) - print(" - Ubuntu/Debian: sudo apt install nodejs npm", file=sys.stderr) - print(" - Windows: Download from https://nodejs.org/", file=sys.stderr) - print("", file=sys.stderr) - print("Or use nvm (Node Version Manager):", file=sys.stderr) - print(" https://github.com/nvm-sh/nvm", file=sys.stderr) + """Print contextual installation instructions for Node.js based on the environment.""" + from .environment import detect_environment + from .instructions import get_installation_instructions + + env = detect_environment() + instructions = get_installation_instructions(env) + print(instructions, file=sys.stderr) def _normalize_path(path: str) -> str: diff --git a/src/promptfoo/environment.py b/src/promptfoo/environment.py new file mode 100644 index 0000000..5dd8e91 --- /dev/null +++ b/src/promptfoo/environment.py @@ -0,0 +1,360 @@ +""" +Environment detection for providing contextual Node.js installation instructions. + +This module detects the operating system, Linux distribution, cloud provider, +container environment, CI/CD platform, and Python environment to provide +tailored installation instructions for Node.js. +""" + +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +@dataclass +class Environment: + """Information about the current execution environment.""" + + os_type: str # "linux", "darwin", "windows" + linux_distro: Optional[str] = None # "ubuntu", "debian", "rhel", "fedora", "alpine", "arch", etc. + linux_distro_version: Optional[str] = None # e.g., "22.04", "11", "9" + cloud_provider: Optional[str] = None # "aws", "gcp", "azure" + is_lambda: bool = False # AWS Lambda + is_cloud_function: bool = False # GCP Cloud Functions or Azure Functions + is_docker: bool = False + is_kubernetes: bool = False + is_wsl: bool = False # Windows Subsystem for Linux + is_ci: bool = False + ci_platform: Optional[str] = None # "github", "gitlab", "circleci", "jenkins", etc. + is_venv: bool = False + is_conda: bool = False + has_sudo: bool = False # Best guess if user has sudo access + + +def _detect_linux_distro() -> tuple[Optional[str], Optional[str]]: + """ + Detect Linux distribution and version. + + Returns: + Tuple of (distro_id, version) where distro_id is normalized + (e.g., "ubuntu", "debian", "rhel", "alpine", "arch") + """ + # Define known distros for normalization + known_base_distros = {"ubuntu", "debian", "alpine", "arch", "fedora"} + rhel_family = {"rhel", "centos", "rocky", "almalinux", "ol", "amzn"} + suse_family = {"opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles"} + + # Try /etc/os-release first, then /usr/lib/os-release (per freedesktop spec) + for os_release_path in [Path("/etc/os-release"), Path("/usr/lib/os-release")]: + if os_release_path.exists(): + try: + with open(os_release_path) as f: + os_release = {} + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" in line: + key, _, value = line.partition("=") + # Remove quotes + value = value.strip('"').strip("'") + os_release[key] = value + + distro_id = os_release.get("ID", "").lower() + version = os_release.get("VERSION_ID", "") + id_like = os_release.get("ID_LIKE", "").lower().split() + + # Normalize distro IDs + if distro_id in known_base_distros: + return distro_id, version + elif distro_id in rhel_family: + # Oracle Linux (ol), Amazon Linux (amzn) + return "rhel", version + elif distro_id in suse_family: + return "suse", version + + # Check ID_LIKE for derivative distributions (e.g., Pop!_OS, Raspbian, Mint) + if id_like: + for parent in id_like: + if parent in known_base_distros: + return parent, version + elif parent in rhel_family: + return "rhel", version + elif parent in suse_family: + return "suse", version + + # Return the raw distro_id if we couldn't normalize it + return distro_id, version + except OSError: + pass + + # Fallback: check for specific files + if Path("/etc/debian_version").exists(): + return "debian", None + elif Path("/etc/redhat-release").exists(): + return "rhel", None + elif Path("/etc/alpine-release").exists(): + return "alpine", None + elif Path("/etc/arch-release").exists(): + return "arch", None + + return None, None + + +def _detect_cloud_provider() -> Optional[str]: + """ + Detect if running on a cloud provider. + + Returns: + One of "aws", "gcp", "azure", or None + """ + # AWS detection + # Check for EC2 metadata + if Path("/sys/hypervisor/uuid").exists(): + try: + with open("/sys/hypervisor/uuid") as f: + uuid = f.read().strip() + if uuid.startswith("ec2") or uuid.startswith("EC2"): + return "aws" + except OSError: + pass + + # Check AWS environment variables + if os.getenv("AWS_EXECUTION_ENV") or os.getenv("AWS_REGION"): + return "aws" + + # GCP detection + # Check for GCP metadata + if Path("/sys/class/dmi/id/product_name").exists(): + try: + with open("/sys/class/dmi/id/product_name") as f: + product = f.read().strip() + if "Google" in product or "GCE" in product: + return "gcp" + except OSError: + pass + + # Check GCP environment variables + if os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT"): + return "gcp" + + # Azure detection + if Path("/sys/class/dmi/id/sys_vendor").exists(): + try: + with open("/sys/class/dmi/id/sys_vendor") as f: + vendor = f.read().strip() + # Could be Azure or Hyper-V, check for Azure-specific + if "Microsoft Corporation" in vendor and Path("/var/lib/waagent").exists(): + return "azure" + except OSError: + pass + + # Check Azure environment variables + if os.getenv("AZURE_SUBSCRIPTION_ID") or os.getenv("WEBSITE_INSTANCE_ID"): + return "azure" + + return None + + +def _detect_container() -> tuple[bool, bool]: + """ + Detect if running in a container. + + Returns: + Tuple of (is_docker, is_kubernetes) + """ + is_docker = False + is_kubernetes = False + + # Docker detection + if Path("/.dockerenv").exists(): + is_docker = True + + # Also check cgroup + if Path("/proc/1/cgroup").exists(): + try: + with open("/proc/1/cgroup") as f: + cgroup_content = f.read() + if "docker" in cgroup_content or "containerd" in cgroup_content: + is_docker = True + except OSError: + pass + + # Kubernetes detection + if os.getenv("KUBERNETES_SERVICE_HOST"): + is_kubernetes = True + + return is_docker, is_kubernetes + + +def _detect_wsl() -> bool: + """ + Detect if running in Windows Subsystem for Linux (WSL). + + Returns: + True if running in WSL, False otherwise + """ + # Check for WSL environment variable + if os.getenv("WSL_DISTRO_NAME") or os.getenv("WSL_INTEROP"): + return True + + # Check /proc/version for Microsoft/WSL signatures + if Path("/proc/version").exists(): + try: + with open("/proc/version") as f: + version_info = f.read().lower() + if "microsoft" in version_info or "wsl" in version_info: + return True + except OSError: + pass + + # Check for Windows filesystem mounts (WSL mounts Windows drives at /mnt/) + # This is less reliable but can catch WSL 1 + return Path("/mnt/c").exists() and Path("/proc/version").exists() + + +def _detect_ci() -> tuple[bool, Optional[str]]: + """ + Detect if running in a CI/CD environment. + + Returns: + Tuple of (is_ci, ci_platform) + """ + ci_env_vars = { + "GITHUB_ACTIONS": "github", + "GITLAB_CI": "gitlab", + "CIRCLECI": "circleci", + "JENKINS_HOME": "jenkins", + "TRAVIS": "travis", + "BUILDKITE": "buildkite", + "DRONE": "drone", + "BITBUCKET_BUILD_NUMBER": "bitbucket", + "TEAMCITY_VERSION": "teamcity", + "TF_BUILD": "azure-devops", + } + + for env_var, platform in ci_env_vars.items(): + if os.getenv(env_var): + return True, platform + + # Generic CI detection + if os.getenv("CI"): + return True, None + + return False, None + + +def _detect_python_env() -> tuple[bool, bool]: + """ + Detect Python virtual environment. + + Returns: + Tuple of (is_venv, is_conda) + """ + # venv/virtualenv detection + is_venv = hasattr(sys, "real_prefix") or (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix) + + # Conda detection + is_conda = "CONDA_DEFAULT_ENV" in os.environ or "CONDA_PREFIX" in os.environ + + return is_venv, is_conda + + +def _has_sudo_access() -> bool: + """ + Best-effort check if user likely has sudo access. + + Returns: + True if user is root or likely has sudo, False otherwise + """ + # Unix-like systems + if hasattr(os, "geteuid"): + # Root user + if os.geteuid() == 0: + return True + + # Check if sudo command exists + import shutil + + return shutil.which("sudo") is not None + + # Windows - check if admin (requires elevation detection) + if sys.platform == "win32": + try: + import ctypes + + return ctypes.windll.shell32.IsUserAnAdmin() != 0 + except Exception: + return False + + return False + + +def detect_environment() -> Environment: + """ + Detect the current execution environment. + + Returns: + Environment object with detected platform information + """ + os_type = sys.platform + if os_type.startswith("linux"): + os_type = "linux" + elif os_type == "darwin": + os_type = "darwin" + elif os_type == "win32": + os_type = "windows" + + # Linux-specific detection + linux_distro = None + linux_distro_version = None + if os_type == "linux": + linux_distro, linux_distro_version = _detect_linux_distro() + + # Cloud provider detection + cloud_provider = _detect_cloud_provider() + + # Lambda and Cloud Functions detection + is_lambda = os.getenv("AWS_LAMBDA_FUNCTION_NAME") is not None + is_cloud_function = ( + os.getenv("FUNCTION_NAME") is not None # GCP Cloud Functions + or os.getenv("FUNCTIONS_WORKER_RUNTIME") is not None # Azure Functions + ) + + # Container detection + is_docker, is_kubernetes = False, False + if os_type == "linux": + is_docker, is_kubernetes = _detect_container() + + # WSL detection + is_wsl = False + if os_type == "linux": + is_wsl = _detect_wsl() + + # CI detection + is_ci, ci_platform = _detect_ci() + + # Python environment detection + is_venv, is_conda = _detect_python_env() + + # Sudo detection + has_sudo = _has_sudo_access() + + return Environment( + os_type=os_type, + linux_distro=linux_distro, + linux_distro_version=linux_distro_version, + cloud_provider=cloud_provider, + is_lambda=is_lambda, + is_cloud_function=is_cloud_function, + is_docker=is_docker, + is_kubernetes=is_kubernetes, + is_wsl=is_wsl, + is_ci=is_ci, + ci_platform=ci_platform, + is_venv=is_venv, + is_conda=is_conda, + has_sudo=has_sudo, + ) diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py new file mode 100644 index 0000000..0133f05 --- /dev/null +++ b/src/promptfoo/instructions.py @@ -0,0 +1,429 @@ +""" +Platform-specific Node.js installation instructions. + +Generates tailored installation instructions based on the detected environment. +""" + +from .environment import Environment + + +def get_installation_instructions(env: Environment) -> str: + """ + Generate Node.js installation instructions for the detected environment. + + Args: + env: Detected environment information + + Returns: + Formatted installation instructions as a multi-line string + """ + lines = [] + lines.append("=" * 70) + lines.append("ERROR: promptfoo requires Node.js but it's not installed") + lines.append("=" * 70) + lines.append("") + + # Special cases first (Lambda, Cloud Functions, etc.) + if env.is_lambda: + lines.extend(_get_lambda_instructions()) + return "\n".join(lines) + + if env.is_cloud_function: + lines.extend(_get_cloud_function_instructions(env)) + return "\n".join(lines) + + # CI/CD environment + if env.is_ci: + lines.extend(_get_ci_instructions(env)) + lines.append("") + + # Container environment + if env.is_docker: + lines.extend(_get_docker_instructions(env)) + lines.append("") + + # WSL environment + if env.is_wsl: + lines.extend(_get_wsl_instructions()) + lines.append("") + + # Platform-specific instructions + if env.os_type == "linux": + lines.extend(_get_linux_instructions(env)) + elif env.os_type == "darwin": + lines.extend(_get_macos_instructions()) + elif env.os_type == "windows": + lines.extend(_get_windows_instructions()) + + # Virtual environment alternative + if env.is_venv or env.is_conda: + lines.append("") + lines.extend(_get_venv_instructions()) + + # Direct npx usage + lines.append("") + lines.extend(_get_npx_instructions()) + + return "\n".join(lines) + + +def _get_lambda_instructions() -> list[str]: + """Instructions for AWS Lambda environment.""" + return [ + "You are running in AWS Lambda with a Python runtime.", + "", + "AWS Lambda Python runtimes do not include Node.js. You have options:", + "", + "1. Use a Lambda Layer with Node.js:", + " https://docs.aws.amazon.com/lambda/latest/dg/chapter-layers.html", + "", + "2. Switch to Node.js runtime:", + " https://docs.aws.amazon.com/lambda/latest/dg/lambda-nodejs.html", + "", + "3. Use Lambda container images with both Python and Node.js:", + " https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", + "", + "Note: promptfoo is primarily designed for local development and CI/CD,", + "not for Lambda runtime execution.", + ] + + +def _get_cloud_function_instructions(env: Environment) -> list[str]: + """Instructions for Cloud Functions (GCP/Azure).""" + if env.cloud_provider == "gcp": + return [ + "You are running in Google Cloud Functions with a Python runtime.", + "", + "GCP Cloud Functions Python runtimes do not include Node.js.", + "Consider using Node.js runtime instead:", + " https://cloud.google.com/functions/docs/concepts/nodejs-runtime", + ] + else: # Azure or unknown + return [ + "You are running in Azure Functions with a Python runtime.", + "", + "Azure Functions Python runtimes do not include Node.js.", + "Consider using Node.js runtime instead:", + " https://learn.microsoft.com/en-us/azure/azure-functions/functions-reference-node", + ] + + +def _get_ci_instructions(env: Environment) -> list[str]: + """Instructions for CI/CD environments.""" + lines = ["RUNNING IN CI/CD: " + (env.ci_platform or "detected").upper(), ""] + + if env.ci_platform == "github": + lines.extend( + [ + "Add Node.js to your workflow:", + " - uses: actions/setup-node@v4", + " with:", + " node-version: '20'", + ] + ) + elif env.ci_platform == "gitlab": + lines.extend( + [ + "Use a Docker image with Node.js:", + " image: node:20", + "Or install Node.js in before_script:", + " before_script:", + " - apt-get update && apt-get install -y nodejs npm", + ] + ) + elif env.ci_platform == "circleci": + lines.extend( + [ + "Use a CircleCI image with Node.js:", + " docker:", + " - image: cimg/python:3.11-node", + ] + ) + else: + lines.extend( + [ + "Install Node.js in your CI configuration.", + "Most CI platforms provide Node.js images or setup actions.", + ] + ) + + return lines + + +def _get_docker_instructions(env: Environment) -> list[str]: + """Instructions for Docker environments.""" + lines = ["RUNNING IN DOCKER CONTAINER:", ""] + + if env.linux_distro == "alpine": + lines.extend( + [ + "Add to your Dockerfile (Alpine):", + " RUN apk add --no-cache nodejs npm", + ] + ) + elif env.linux_distro in ("ubuntu", "debian"): + lines.extend( + [ + "Add to your Dockerfile (Debian/Ubuntu):", + " RUN apt-get update && \\", + " apt-get install -y nodejs npm && \\", + " rm -rf /var/lib/apt/lists/*", + "", + "Or use NodeSource for newer version:", + " RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \\", + " apt-get install -y nodejs && \\", + " rm -rf /var/lib/apt/lists/*", + ] + ) + else: + lines.extend( + [ + "Add Node.js to your Dockerfile:", + " FROM python:3.11", + " RUN apt-get update && apt-get install -y nodejs npm", + ] + ) + + return lines + + +def _get_wsl_instructions() -> list[str]: + """Instructions for Windows Subsystem for Linux (WSL).""" + return [ + "WINDOWS SUBSYSTEM FOR LINUX (WSL) DETECTED:", + "", + "IMPORTANT: Install Node.js within WSL, not from Windows.", + "Using Windows Node.js from WSL can cause path and performance issues.", + "", + "Recommended approach:", + " 1. Use your Linux distribution's package manager (see below)", + " 2. Or use nvm for version management:", + " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash", + " source ~/.bashrc", + " nvm install 20", + "", + "Tips for WSL:", + " - Store project files in the WSL filesystem (~/), not /mnt/c/", + " - This improves file I/O performance significantly", + " - Use 'wsl --shutdown' to restart WSL if needed", + ] + + +def _get_linux_instructions(env: Environment) -> list[str]: + """Instructions for Linux systems.""" + lines = [] + distro = env.linux_distro + + if distro in ("ubuntu", "debian"): + lines.extend(_get_debian_instructions(env)) + elif distro == "rhel": + lines.extend(_get_rhel_instructions(env)) + elif distro == "alpine": + lines.extend(_get_alpine_instructions()) + elif distro == "arch": + lines.extend(_get_arch_instructions()) + elif distro == "suse": + lines.extend(_get_suse_instructions()) + else: + # Generic Linux instructions + lines.extend(_get_generic_linux_instructions()) + + return lines + + +def _get_debian_instructions(env: Environment) -> list[str]: + """Instructions for Debian/Ubuntu systems.""" + lines = ["UBUNTU/DEBIAN INSTALLATION:", ""] + + if env.has_sudo: + lines.extend( + [ + "Option 1 - Install from NodeSource (recommended for production):", + " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -", + " sudo apt install -y nodejs", + "", + "Option 2 - Install from default repository (may be outdated):", + " sudo apt update", + " sudo apt install -y nodejs npm", + "", + "Option 3 - Install using snap (not recommended for production):", + " sudo snap install node --classic", + " # Note: Snap auto-updates can cause unexpected behavior", + ] + ) + else: + lines.extend( + [ + "You don't have sudo access. Use nvm (Node Version Manager):", + " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash", + " source ~/.bashrc", + " nvm install 20", + ] + ) + + return lines + + +def _get_rhel_instructions(env: Environment) -> list[str]: + """Instructions for RHEL/CentOS/Fedora/Amazon Linux.""" + lines = [] + + # Detect Amazon Linux vs RHEL/CentOS/Fedora + is_amazon_linux = ( + env.linux_distro == "rhel" and env.linux_distro_version and env.linux_distro_version.startswith("202") + ) + + if is_amazon_linux: + lines.extend(["AMAZON LINUX INSTALLATION:", ""]) + if env.has_sudo: + lines.extend( + [ + "Amazon Linux 2023:", + " sudo dnf install -y nodejs", + "", + "Amazon Linux 2:", + " curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo bash -", + " sudo yum install -y nodejs", + ] + ) + else: + lines.extend( + [ + "Use nvm (Node Version Manager) - no sudo needed:", + " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash", + " source ~/.bashrc", + " nvm install 20", + ] + ) + else: + lines.extend(["RHEL/CENTOS/FEDORA INSTALLATION:", ""]) + if env.has_sudo: + lines.extend( + [ + "Using dnf (RHEL 8+/Fedora):", + " sudo dnf install -y nodejs npm", + "", + "Using yum (RHEL 7):", + " sudo yum install -y nodejs npm", + "", + "Or use NodeSource for newer version:", + " curl -fsSL https://rpm.nodesource.com/setup_20.x | sudo bash -", + " sudo yum install -y nodejs", + ] + ) + else: + lines.extend( + [ + "Use nvm (Node Version Manager) - no sudo needed:", + " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash", + " source ~/.bashrc", + " nvm install 20", + ] + ) + + return lines + + +def _get_alpine_instructions() -> list[str]: + """Instructions for Alpine Linux.""" + return [ + "ALPINE LINUX INSTALLATION:", + "", + " apk add --update nodejs npm", + "", + "In Dockerfile:", + " RUN apk add --no-cache nodejs npm", + ] + + +def _get_arch_instructions() -> list[str]: + """Instructions for Arch Linux.""" + return [ + "ARCH LINUX INSTALLATION:", + "", + " sudo pacman -S nodejs npm", + ] + + +def _get_suse_instructions() -> list[str]: + """Instructions for SUSE/openSUSE.""" + return [ + "SUSE/OPENSUSE INSTALLATION:", + "", + " sudo zypper install nodejs npm", + ] + + +def _get_generic_linux_instructions() -> list[str]: + """Fallback instructions for unknown Linux distributions.""" + return [ + "LINUX INSTALLATION:", + "", + "Use your package manager to install Node.js, or use nvm:", + "", + "Option 1 - nvm (Node Version Manager, works on any Linux):", + " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash", + " source ~/.bashrc", + " nvm install 20", + "", + "Option 2 - Download binary from https://nodejs.org/", + ] + + +def _get_macos_instructions() -> list[str]: + """Instructions for macOS.""" + return [ + "MACOS INSTALLATION:", + "", + "Option 1 - Homebrew (recommended):", + " brew install node", + "", + "Option 2 - nvm (Node Version Manager, for version management):", + " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash", + " source ~/.zshrc # or ~/.bashrc", + " nvm install 20", + "", + "Option 3 - Official installer:", + " Download from https://nodejs.org/", + ] + + +def _get_windows_instructions() -> list[str]: + """Instructions for Windows.""" + return [ + "WINDOWS INSTALLATION:", + "", + "Option 1 - Official installer (recommended):", + " Download from https://nodejs.org/", + "", + "Option 2 - winget (Windows 10/11, built-in):", + " winget install OpenJS.NodeJS.LTS # For LTS version", + " # or: winget install OpenJS.NodeJS # For current version", + "", + "Option 3 - Chocolatey:", + " choco install nodejs-lts # For LTS version", + " # or: choco install nodejs # For current version", + "", + "Option 4 - Scoop:", + " scoop install nodejs-lts # For LTS version", + " # or: scoop install nodejs # For current version", + ] + + +def _get_venv_instructions() -> list[str]: + """Instructions for virtual environment users.""" + return [ + "ALTERNATIVE: Install Node.js in your Python virtualenv (no sudo):", + " pip install nodeenv", + " nodeenv -p # Installs Node.js in current virtualenv", + " # Then run promptfoo again", + ] + + +def _get_npx_instructions() -> list[str]: + """Instructions for direct npx usage.""" + return [ + "DIRECT USAGE (bypasses Python wrapper):", + " npx promptfoo@latest eval", + " # This is often faster and always uses the latest version", + ] diff --git a/tests/test_cli.py b/tests/test_cli.py index 1acf1c7..d5611b8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,7 +12,7 @@ import os import subprocess import sys -from typing import Any, Optional +from typing import Optional from unittest.mock import MagicMock import pytest @@ -34,7 +34,6 @@ print_installation_help, ) - # ============================================================================= # Unit Tests for Helper Functions # ============================================================================= @@ -68,15 +67,16 @@ class TestInstallationHelp: """Test installation help message output.""" def test_print_installation_help_outputs_to_stderr(self, capsys: pytest.CaptureFixture) -> None: - """Installation help is printed to stderr with expected content.""" + """Installation help is printed to stderr with platform-specific content.""" print_installation_help() captured = capsys.readouterr() assert captured.out == "" # Nothing to stdout assert "ERROR: promptfoo requires Node.js" in captured.err - assert "brew install node" in captured.err - assert "apt install nodejs npm" in captured.err - assert "nodejs.org" in captured.err - assert "nvm" in captured.err + # Platform-specific instructions will vary by environment + # Just verify that some installation instructions are present + assert "nodejs.org" in captured.err or "install node" in captured.err.lower() + assert "DIRECT USAGE" in captured.err # npx instructions always included + assert "npx promptfoo@latest" in captured.err class TestPathUtilities: @@ -216,10 +216,7 @@ def test_find_external_promptfoo_when_not_in_path(self, monkeypatch: pytest.Monk def test_find_external_promptfoo_when_found(self, monkeypatch: pytest.MonkeyPatch) -> None: """Returns path when promptfoo found and not this wrapper.""" promptfoo_path = "/usr/local/bin/promptfoo" - monkeypatch.setattr( - "shutil.which", - lambda cmd, path=None: promptfoo_path if cmd == "promptfoo" else None - ) + monkeypatch.setattr("shutil.which", lambda cmd, path=None: promptfoo_path if cmd == "promptfoo" else None) monkeypatch.setattr(sys, "argv", ["different-script"]) result = _find_external_promptfoo() assert result == promptfoo_path @@ -343,10 +340,10 @@ def test_main_exits_when_node_not_installed( def test_main_uses_external_promptfoo_when_available(self, monkeypatch: pytest.MonkeyPatch) -> None: """Uses external promptfoo when found and sets wrapper env var.""" monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"]) - monkeypatch.setattr("shutil.which", lambda cmd, path=None: { - "node": "/usr/bin/node", - "promptfoo": "/usr/local/bin/promptfoo" - }.get(cmd)) + monkeypatch.setattr( + "shutil.which", + lambda cmd, path=None: {"node": "/usr/bin/node", "promptfoo": "/usr/local/bin/promptfoo"}.get(cmd), + ) mock_result = subprocess.CompletedProcess([], 0) mock_run = MagicMock(return_value=mock_result) @@ -374,11 +371,14 @@ def test_main_skips_external_when_wrapper_env_set(self, monkeypatch: pytest.Monk """Skips external promptfoo search when wrapper env var is set.""" monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"]) monkeypatch.setenv(_WRAPPER_ENV, "1") - monkeypatch.setattr("shutil.which", lambda cmd, path=None: { - "node": "/usr/bin/node", - "npx": "/usr/bin/npx", - "promptfoo": "/usr/local/bin/promptfoo" - }.get(cmd)) + monkeypatch.setattr( + "shutil.which", + lambda cmd, path=None: { + "node": "/usr/bin/node", + "npx": "/usr/bin/npx", + "promptfoo": "/usr/local/bin/promptfoo", + }.get(cmd), + ) mock_result = subprocess.CompletedProcess([], 0) mock_run = MagicMock(return_value=mock_result) @@ -399,10 +399,9 @@ def test_main_skips_external_when_wrapper_env_set(self, monkeypatch: pytest.Monk def test_main_falls_back_to_npx(self, monkeypatch: pytest.MonkeyPatch) -> None: """Falls back to npx when no external promptfoo found.""" monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"]) - monkeypatch.setattr("shutil.which", lambda cmd, path=None: { - "node": "/usr/bin/node", - "npx": "/usr/bin/npx" - }.get(cmd)) + monkeypatch.setattr( + "shutil.which", lambda cmd, path=None: {"node": "/usr/bin/node", "npx": "/usr/bin/npx"}.get(cmd) + ) mock_result = subprocess.CompletedProcess([], 0) mock_run = MagicMock(return_value=mock_result) @@ -428,9 +427,7 @@ def test_main_exits_when_neither_external_nor_npx_available( ) -> None: """Exits with error when neither external promptfoo nor npx found.""" monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"]) - monkeypatch.setattr("shutil.which", lambda cmd, path=None: { - "node": "/usr/bin/node" - }.get(cmd)) + monkeypatch.setattr("shutil.which", lambda cmd, path=None: {"node": "/usr/bin/node"}.get(cmd)) with pytest.raises(SystemExit) as exc_info: main() @@ -442,10 +439,9 @@ def test_main_exits_when_neither_external_nor_npx_available( def test_main_passes_arguments_correctly(self, monkeypatch: pytest.MonkeyPatch) -> None: """Passes command-line arguments to the subprocess.""" monkeypatch.setattr(sys, "argv", ["promptfoo", "redteam", "run", "--config", "test.yaml"]) - monkeypatch.setattr("shutil.which", lambda cmd, path=None: { - "node": "/usr/bin/node", - "npx": "/usr/bin/npx" - }.get(cmd)) + monkeypatch.setattr( + "shutil.which", lambda cmd, path=None: {"node": "/usr/bin/node", "npx": "/usr/bin/npx"}.get(cmd) + ) mock_result = subprocess.CompletedProcess([], 0) mock_run = MagicMock(return_value=mock_result) @@ -465,10 +461,9 @@ def test_main_passes_arguments_correctly(self, monkeypatch: pytest.MonkeyPatch) def test_main_returns_subprocess_exit_code(self, monkeypatch: pytest.MonkeyPatch) -> None: """Returns the exit code from the subprocess.""" monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"]) - monkeypatch.setattr("shutil.which", lambda cmd, path=None: { - "node": "/usr/bin/node", - "npx": "/usr/bin/npx" - }.get(cmd)) + monkeypatch.setattr( + "shutil.which", lambda cmd, path=None: {"node": "/usr/bin/node", "npx": "/usr/bin/npx"}.get(cmd) + ) # Test non-zero exit code mock_result = subprocess.CompletedProcess([], 42) diff --git a/tests/test_environment.py b/tests/test_environment.py new file mode 100644 index 0000000..95207a7 --- /dev/null +++ b/tests/test_environment.py @@ -0,0 +1,447 @@ +""" +Tests for environment detection. + +This module tests detection of operating systems, Linux distributions, +cloud providers, containers, CI/CD platforms, and Python environments. +""" + +import os +from pathlib import Path +from unittest import mock + +import pytest + +from promptfoo.environment import ( + _detect_ci, + _detect_cloud_provider, + _detect_container, + _detect_linux_distro, + _detect_python_env, + _has_sudo_access, + detect_environment, +) + + +class TestLinuxDistroDetection: + """Test Linux distribution detection.""" + + def test_detect_linux_distro_returns_tuple(self) -> None: + """Linux distro detection returns a tuple.""" + distro, version = _detect_linux_distro() + # Should return tuple even if both None + assert isinstance(distro, (str, type(None))) + assert isinstance(version, (str, type(None))) + + def test_detect_derivative_distro_pop_os(self, tmp_path: Path) -> None: + """Detect Pop!_OS as Ubuntu derivative via ID_LIKE.""" + os_release = tmp_path / "os-release" + os_release.write_text('ID=pop\nVERSION_ID="22.04"\nID_LIKE="ubuntu debian"') + + os_release_data = 'ID=pop\nVERSION_ID="22.04"\nID_LIKE="ubuntu debian"' + + with ( + mock.patch("promptfoo.environment.Path") as mock_path_class, + mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), + ): + + def path_side_effect(path_str: str) -> mock.Mock: + mock_path_obj = mock.Mock() + if path_str == "/etc/os-release": + mock_path_obj.exists.return_value = True + mock_path_obj.__truediv__ = lambda self, other: os_release + # Make the mock path object work with open() + return os_release + else: + mock_path_obj.exists.return_value = False + return mock_path_obj + + mock_path_class.side_effect = path_side_effect + + distro, version = _detect_linux_distro() + assert distro == "ubuntu" # Should resolve to parent via ID_LIKE + assert version == "22.04" + + def test_detect_derivative_distro_raspbian(self, tmp_path: Path) -> None: + """Detect Raspbian as Debian derivative via ID_LIKE.""" + os_release_data = 'ID=raspbian\nVERSION_ID="11"\nID_LIKE=debian' + + with ( + mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), + mock.patch("promptfoo.environment.Path") as mock_path_class, + ): + mock_path_obj = mock.Mock() + mock_path_obj.exists.return_value = True + mock_path_class.return_value = mock_path_obj + + distro, version = _detect_linux_distro() + assert distro == "debian" # Should resolve to parent via ID_LIKE + assert version == "11" + + def test_detect_derivative_distro_linux_mint(self, tmp_path: Path) -> None: + """Detect Linux Mint as Ubuntu derivative via ID_LIKE.""" + os_release_data = 'ID=linuxmint\nVERSION_ID="21"\nID_LIKE="ubuntu debian"' + + with ( + mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), + mock.patch("promptfoo.environment.Path") as mock_path_class, + ): + mock_path_obj = mock.Mock() + mock_path_obj.exists.return_value = True + mock_path_class.return_value = mock_path_obj + + distro, version = _detect_linux_distro() + assert distro == "ubuntu" # Should resolve to first known parent in ID_LIKE + assert version == "21" + + def test_usr_lib_os_release_fallback(self, tmp_path: Path) -> None: + """Detect distro from /usr/lib/os-release if /etc/os-release missing.""" + with mock.patch("promptfoo.environment.Path") as mock_path_class: + + def path_exists_side_effect(path_obj: mock.Mock) -> bool: + # /etc/os-release doesn't exist, /usr/lib/os-release does + if "/etc/os-release" in str(path_obj): + return False + elif "/usr/lib/os-release" in str(path_obj): + return True + return False + + # Create mock Path objects + etc_path = mock.Mock() + etc_path.exists.return_value = False + etc_path.__str__ = lambda self: "/etc/os-release" + + usr_path = mock.Mock() + usr_path.exists.return_value = True + usr_path.__str__ = lambda self: "/usr/lib/os-release" + + def path_constructor(path_str: str) -> mock.Mock: + if path_str == "/etc/os-release": + return etc_path + elif path_str == "/usr/lib/os-release": + return usr_path + return mock.Mock() + + mock_path_class.side_effect = path_constructor + + with mock.patch("builtins.open", mock.mock_open(read_data='ID=ubuntu\nVERSION_ID="22.04"')): + distro, version = _detect_linux_distro() + assert distro == "ubuntu" + assert version == "22.04" + + +class TestCloudProviderDetection: + """Test cloud provider detection.""" + + def test_detect_aws_from_hypervisor_uuid(self, tmp_path: Path) -> None: + """Detect AWS from hypervisor UUID.""" + uuid_file = tmp_path / "uuid" + uuid_file.write_text("ec2e1916-9099-7caf-fd21-012345abcdef\n") + + with mock.patch("promptfoo.environment.Path") as mock_path: + mock_path_instance = mock_path.return_value + mock_path_instance.exists.return_value = True + mock_path_instance.__truediv__.return_value = uuid_file + + with mock.patch("builtins.open", mock.mock_open(read_data="ec2e1916-9099-7caf-fd21-012345abcdef\n")): + provider = _detect_cloud_provider() + assert provider == "aws" + + def test_detect_aws_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect AWS from environment variables.""" + monkeypatch.setenv("AWS_EXECUTION_ENV", "AWS_Lambda_python3.11") + + with mock.patch("promptfoo.environment.Path") as mock_path: + mock_path.return_value.exists.return_value = False + + provider = _detect_cloud_provider() + assert provider == "aws" + + def test_detect_gcp_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect GCP from environment variables.""" + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "my-project") + + with mock.patch("promptfoo.environment.Path") as mock_path: + mock_path.return_value.exists.return_value = False + + provider = _detect_cloud_provider() + assert provider == "gcp" + + def test_detect_azure_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect Azure from environment variables.""" + monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "12345") + + with mock.patch("promptfoo.environment.Path") as mock_path: + mock_path.return_value.exists.return_value = False + + provider = _detect_cloud_provider() + assert provider == "azure" + + def test_no_cloud_provider_detected(self) -> None: + """Return None when no cloud provider is detected.""" + with mock.patch("promptfoo.environment.Path") as mock_path: + mock_path.return_value.exists.return_value = False + with mock.patch.dict(os.environ, {}, clear=True): + provider = _detect_cloud_provider() + assert provider is None + + +class TestContainerDetection: + """Test container detection.""" + + def test_detect_kubernetes_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect Kubernetes from environment variable.""" + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") + + with mock.patch("promptfoo.environment.Path") as mock_path: + mock_path.return_value.exists.return_value = False + + is_docker, is_k8s = _detect_container() + assert is_k8s is True + + def test_detect_container_returns_tuple(self) -> None: + """Container detection returns a tuple of booleans.""" + is_docker, is_k8s = _detect_container() + assert isinstance(is_docker, bool) + assert isinstance(is_k8s, bool) + + +class TestWSLDetection: + """Test WSL detection.""" + + def test_detect_wsl_from_env_var(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect WSL from WSL_DISTRO_NAME environment variable.""" + monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") + + from promptfoo.environment import _detect_wsl + + assert _detect_wsl() is True + + def test_detect_wsl_from_interop_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect WSL from WSL_INTEROP environment variable.""" + monkeypatch.setenv("WSL_INTEROP", "/run/WSL/123_interop") + + from promptfoo.environment import _detect_wsl + + assert _detect_wsl() is True + + def test_no_wsl_detected(self) -> None: + """Return False when not in WSL.""" + with mock.patch.dict(os.environ, {}, clear=True): + from promptfoo.environment import _detect_wsl + + # This will return False unless we're actually in WSL + # Just verify it returns a boolean + result = _detect_wsl() + assert isinstance(result, bool) + + +class TestCIDetection: + """Test CI/CD platform detection.""" + + @pytest.mark.parametrize( + "env_var,expected_platform", + [ + ("GITHUB_ACTIONS", "github"), + ("GITLAB_CI", "gitlab"), + ("CIRCLECI", "circleci"), + ("JENKINS_HOME", "jenkins"), + ("TRAVIS", "travis"), + ("BUILDKITE", "buildkite"), + ], + ) + def test_detect_specific_ci_platforms( + self, env_var: str, expected_platform: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Detect specific CI/CD platforms from environment variables.""" + with mock.patch.dict(os.environ, {}, clear=True): + monkeypatch.setenv(env_var, "true") + is_ci, platform = _detect_ci() + assert is_ci is True + assert platform == expected_platform + + def test_detect_generic_ci(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect generic CI from CI environment variable.""" + with mock.patch.dict(os.environ, {}, clear=True): + monkeypatch.setenv("CI", "true") + is_ci, platform = _detect_ci() + assert is_ci is True + assert platform is None + + def test_no_ci_detected(self) -> None: + """Return False when no CI is detected.""" + with mock.patch.dict(os.environ, {}, clear=True): + is_ci, platform = _detect_ci() + assert is_ci is False + assert platform is None + + +class TestPythonEnvDetection: + """Test Python environment detection.""" + + def test_detect_venv(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect virtualenv from sys.prefix.""" + import sys + + with mock.patch.object(sys, "prefix", "/home/user/venv"), mock.patch.object(sys, "base_prefix", "/usr"): + is_venv, is_conda = _detect_python_env() + assert is_venv is True + + def test_detect_conda(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect conda from environment variable.""" + monkeypatch.setenv("CONDA_DEFAULT_ENV", "base") + is_venv, is_conda = _detect_python_env() + assert is_conda is True + + def test_no_venv_detected(self) -> None: + """Return False when no venv is detected.""" + import sys + + with ( + mock.patch.object(sys, "prefix", "/usr"), + mock.patch.object(sys, "base_prefix", "/usr"), + mock.patch.dict(os.environ, {}, clear=True), + ): + is_venv, is_conda = _detect_python_env() + assert is_venv is False + assert is_conda is False + + +class TestSudoAccess: + """Test sudo access detection.""" + + def test_has_sudo_when_root(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect sudo access when running as root.""" + if hasattr(os, "geteuid"): + with mock.patch("os.geteuid", return_value=0): + assert _has_sudo_access() is True + + def test_has_sudo_when_sudo_command_exists(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect sudo access when sudo command exists.""" + if hasattr(os, "geteuid"): + with ( + mock.patch("os.geteuid", return_value=1000), + mock.patch("shutil.which", return_value="/usr/bin/sudo"), + ): + assert _has_sudo_access() is True + + def test_no_sudo_when_command_missing(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Return False when sudo command doesn't exist.""" + if hasattr(os, "geteuid"): + with mock.patch("os.geteuid", return_value=1000), mock.patch("shutil.which", return_value=None): + assert _has_sudo_access() is False + + +class TestDetectEnvironment: + """Test complete environment detection.""" + + def test_detect_ubuntu_with_docker(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Detect Ubuntu in Docker container.""" + os_release = tmp_path / "os-release" + os_release.write_text('ID=ubuntu\nVERSION_ID="22.04"') + + with ( + mock.patch("sys.platform", "linux"), + mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), + mock.patch("promptfoo.environment._detect_container", return_value=(True, False)), + mock.patch("promptfoo.environment._detect_wsl", return_value=False), + mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), + mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), + mock.patch("promptfoo.environment._detect_python_env", return_value=(True, False)), + mock.patch("promptfoo.environment._has_sudo_access", return_value=False), + ): + env = detect_environment() + + assert env.os_type == "linux" + assert env.linux_distro == "ubuntu" + assert env.linux_distro_version == "22.04" + assert env.is_docker is True + assert env.is_kubernetes is False + assert env.is_wsl is False + assert env.is_venv is True + + def test_detect_macos_environment(self) -> None: + """Detect macOS environment.""" + with ( + mock.patch("sys.platform", "darwin"), + mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), + mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), + mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), + mock.patch("promptfoo.environment._has_sudo_access", return_value=True), + ): + env = detect_environment() + + assert env.os_type == "darwin" + assert env.linux_distro is None + assert env.has_sudo is True + + def test_detect_windows_environment(self) -> None: + """Detect Windows environment.""" + with ( + mock.patch("sys.platform", "win32"), + mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), + mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), + mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), + mock.patch("promptfoo.environment._has_sudo_access", return_value=False), + ): + env = detect_environment() + + assert env.os_type == "windows" + + def test_detect_aws_lambda(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect AWS Lambda environment.""" + monkeypatch.setenv("AWS_LAMBDA_FUNCTION_NAME", "my-function") + + with ( + mock.patch("sys.platform", "linux"), + mock.patch("promptfoo.environment._detect_linux_distro", return_value=("amzn", "2")), + mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), + mock.patch("promptfoo.environment._detect_wsl", return_value=False), + mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), + mock.patch("promptfoo.environment._detect_cloud_provider", return_value="aws"), + mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), + mock.patch("promptfoo.environment._has_sudo_access", return_value=False), + ): + env = detect_environment() + + assert env.is_lambda is True + assert env.cloud_provider == "aws" + + def test_detect_github_actions(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect GitHub Actions environment.""" + monkeypatch.setenv("GITHUB_ACTIONS", "true") + + with ( + mock.patch("sys.platform", "linux"), + mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), + mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), + mock.patch("promptfoo.environment._detect_wsl", return_value=False), + mock.patch("promptfoo.environment._detect_ci", return_value=(True, "github")), + mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), + mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), + mock.patch("promptfoo.environment._has_sudo_access", return_value=True), + ): + env = detect_environment() + + assert env.is_ci is True + assert env.ci_platform == "github" + + def test_detect_wsl_ubuntu(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Detect WSL with Ubuntu.""" + monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") + + with ( + mock.patch("sys.platform", "linux"), + mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), + mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), + mock.patch("promptfoo.environment._detect_wsl", return_value=True), + mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), + mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), + mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), + mock.patch("promptfoo.environment._has_sudo_access", return_value=True), + ): + env = detect_environment() + + assert env.os_type == "linux" + assert env.linux_distro == "ubuntu" + assert env.is_wsl is True + assert env.is_docker is False diff --git a/tests/test_instructions.py b/tests/test_instructions.py new file mode 100644 index 0000000..999f2ce --- /dev/null +++ b/tests/test_instructions.py @@ -0,0 +1,415 @@ +""" +Tests for platform-specific installation instructions. + +This module tests that appropriate instructions are generated for +different platforms and environments. +""" + +from promptfoo.environment import Environment +from promptfoo.instructions import get_installation_instructions + + +class TestLambdaInstructions: + """Test instructions for AWS Lambda.""" + + def test_lambda_instructions(self) -> None: + """Generate Lambda-specific instructions.""" + env = Environment( + os_type="linux", + linux_distro="rhel", + cloud_provider="aws", + is_lambda=True, + ) + + instructions = get_installation_instructions(env) + + assert "AWS Lambda" in instructions + assert "Lambda Layer" in instructions + assert "Node.js runtime" in instructions + + +class TestCloudFunctionInstructions: + """Test instructions for Cloud Functions.""" + + def test_gcp_cloud_function_instructions(self) -> None: + """Generate GCP Cloud Functions instructions.""" + env = Environment( + os_type="linux", + cloud_provider="gcp", + is_cloud_function=True, + ) + + instructions = get_installation_instructions(env) + + assert "Google Cloud Functions" in instructions or "GCP" in instructions + + def test_azure_function_instructions(self) -> None: + """Generate Azure Functions instructions.""" + env = Environment( + os_type="linux", + cloud_provider="azure", + is_cloud_function=True, + ) + + instructions = get_installation_instructions(env) + + assert "Azure Functions" in instructions + + +class TestCIInstructions: + """Test instructions for CI/CD environments.""" + + def test_github_actions_instructions(self) -> None: + """Generate GitHub Actions-specific instructions.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_ci=True, + ci_platform="github", + ) + + instructions = get_installation_instructions(env) + + assert "actions/setup-node" in instructions + assert "GITHUB" in instructions.upper() + + def test_gitlab_ci_instructions(self) -> None: + """Generate GitLab CI instructions.""" + env = Environment( + os_type="linux", + is_ci=True, + ci_platform="gitlab", + ) + + instructions = get_installation_instructions(env) + + assert "gitlab" in instructions.lower() or "GITLAB" in instructions + assert "image:" in instructions or "before_script" in instructions + + def test_circleci_instructions(self) -> None: + """Generate CircleCI instructions.""" + env = Environment( + os_type="linux", + is_ci=True, + ci_platform="circleci", + ) + + instructions = get_installation_instructions(env) + + assert "circleci" in instructions.lower() or "CIRCLECI" in instructions + + +class TestDockerInstructions: + """Test instructions for Docker containers.""" + + def test_docker_alpine_instructions(self) -> None: + """Generate Docker instructions for Alpine.""" + env = Environment( + os_type="linux", + linux_distro="alpine", + is_docker=True, + ) + + instructions = get_installation_instructions(env) + + assert "apk add" in instructions + assert "Dockerfile" in instructions + + def test_docker_ubuntu_instructions(self) -> None: + """Generate Docker instructions for Ubuntu.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_docker=True, + ) + + instructions = get_installation_instructions(env) + + assert "apt-get" in instructions + assert "Dockerfile" in instructions + + +class TestWSLInstructions: + """Test instructions for WSL (Windows Subsystem for Linux).""" + + def test_wsl_instructions(self) -> None: + """Generate WSL-specific instructions.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_wsl=True, + ) + + instructions = get_installation_instructions(env) + + assert "WSL" in instructions or "Windows Subsystem for Linux" in instructions + assert "nvm" in instructions + assert "/mnt/c" in instructions # Should mention Windows filesystem + assert "performance" in instructions.lower() + + def test_wsl_with_ubuntu_shows_both(self) -> None: + """WSL instructions should show both WSL tips and Ubuntu instructions.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_wsl=True, + has_sudo=True, + ) + + instructions = get_installation_instructions(env) + + # Should have WSL-specific guidance + assert "WSL" in instructions + # Should also have Ubuntu/Debian instructions + assert "UBUNTU" in instructions or "DEBIAN" in instructions + + +class TestLinuxInstructions: + """Test instructions for various Linux distributions.""" + + def test_ubuntu_instructions_with_sudo(self) -> None: + """Generate Ubuntu instructions with sudo access.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + has_sudo=True, + ) + + instructions = get_installation_instructions(env) + + assert "UBUNTU/DEBIAN" in instructions + assert "sudo apt" in instructions + assert "NodeSource" in instructions + + def test_ubuntu_instructions_without_sudo(self) -> None: + """Generate Ubuntu instructions without sudo access.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + has_sudo=False, + ) + + instructions = get_installation_instructions(env) + + assert "nvm" in instructions + # Should NOT suggest sudo apt commands when user doesn't have sudo + assert "sudo apt" not in instructions + assert "sudo snap" not in instructions + + def test_debian_instructions(self) -> None: + """Generate Debian instructions.""" + env = Environment( + os_type="linux", + linux_distro="debian", + has_sudo=True, + ) + + instructions = get_installation_instructions(env) + + assert "UBUNTU/DEBIAN" in instructions + assert "apt" in instructions + + def test_rhel_instructions_with_sudo(self) -> None: + """Generate RHEL instructions with sudo.""" + env = Environment( + os_type="linux", + linux_distro="rhel", + has_sudo=True, + ) + + instructions = get_installation_instructions(env) + + assert "RHEL" in instructions or "CENTOS" in instructions or "FEDORA" in instructions + assert "dnf" in instructions or "yum" in instructions + + def test_amazon_linux_instructions(self) -> None: + """Generate Amazon Linux instructions.""" + env = Environment( + os_type="linux", + linux_distro="rhel", + linux_distro_version="2023", + has_sudo=True, + ) + + instructions = get_installation_instructions(env) + + assert "AMAZON LINUX" in instructions + assert "dnf" in instructions or "yum" in instructions + + def test_alpine_instructions(self) -> None: + """Generate Alpine Linux instructions.""" + env = Environment( + os_type="linux", + linux_distro="alpine", + ) + + instructions = get_installation_instructions(env) + + assert "ALPINE" in instructions + assert "apk add" in instructions + + def test_arch_instructions(self) -> None: + """Generate Arch Linux instructions.""" + env = Environment( + os_type="linux", + linux_distro="arch", + ) + + instructions = get_installation_instructions(env) + + assert "ARCH" in instructions + assert "pacman" in instructions + + def test_suse_instructions(self) -> None: + """Generate SUSE instructions.""" + env = Environment( + os_type="linux", + linux_distro="suse", + ) + + instructions = get_installation_instructions(env) + + assert "SUSE" in instructions or "OPENSUSE" in instructions + assert "zypper" in instructions + + def test_generic_linux_instructions(self) -> None: + """Generate generic Linux instructions for unknown distro.""" + env = Environment( + os_type="linux", + linux_distro="unknown", + ) + + instructions = get_installation_instructions(env) + + assert "nvm" in instructions + + +class TestMacOSInstructions: + """Test instructions for macOS.""" + + def test_macos_instructions(self) -> None: + """Generate macOS instructions.""" + env = Environment(os_type="darwin") + + instructions = get_installation_instructions(env) + + assert "MACOS" in instructions + assert "brew install node" in instructions + assert "Official installer" in instructions + assert "nvm" in instructions + assert "nodejs.org" in instructions + + +class TestWindowsInstructions: + """Test instructions for Windows.""" + + def test_windows_instructions(self) -> None: + """Generate Windows instructions.""" + env = Environment(os_type="windows") + + instructions = get_installation_instructions(env) + + assert "WINDOWS" in instructions + assert "winget" in instructions + assert "Chocolatey" in instructions or "choco" in instructions + assert "Scoop" in instructions + + +class TestVenvInstructions: + """Test virtual environment instructions.""" + + def test_venv_instructions_included(self) -> None: + """Include venv instructions when in virtualenv.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_venv=True, + ) + + instructions = get_installation_instructions(env) + + assert "nodeenv" in instructions + assert "virtualenv" in instructions.lower() + + def test_conda_instructions_included(self) -> None: + """Include venv instructions when in conda.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_conda=True, + ) + + instructions = get_installation_instructions(env) + + assert "nodeenv" in instructions + + +class TestNpxInstructions: + """Test npx direct usage instructions.""" + + def test_npx_instructions_always_included(self) -> None: + """NPX instructions should always be included.""" + env = Environment(os_type="linux", linux_distro="ubuntu") + + instructions = get_installation_instructions(env) + + assert "npx promptfoo@latest" in instructions + assert "DIRECT USAGE" in instructions + + +class TestErrorMessageFormat: + """Test error message formatting.""" + + def test_error_message_has_clear_header(self) -> None: + """Error message should have a clear header.""" + env = Environment(os_type="linux", linux_distro="ubuntu") + + instructions = get_installation_instructions(env) + + assert "ERROR: promptfoo requires Node.js" in instructions + assert "=" * 70 in instructions + + def test_multiline_output(self) -> None: + """Instructions should be multi-line.""" + env = Environment(os_type="linux", linux_distro="ubuntu") + + instructions = get_installation_instructions(env) + + lines = instructions.split("\n") + assert len(lines) > 5 # Should have multiple lines + + +class TestComplexEnvironments: + """Test instructions for complex, combined environments.""" + + def test_docker_github_actions_ubuntu(self) -> None: + """Generate instructions for Docker in GitHub Actions on Ubuntu.""" + env = Environment( + os_type="linux", + linux_distro="ubuntu", + is_docker=True, + is_ci=True, + ci_platform="github", + ) + + instructions = get_installation_instructions(env) + + # Should include both CI and Docker instructions + assert "GITHUB" in instructions.upper() + assert "DOCKER" in instructions.upper() + + def test_aws_ec2_rhel_with_venv(self) -> None: + """Generate instructions for AWS EC2 RHEL with virtualenv.""" + env = Environment( + os_type="linux", + linux_distro="rhel", + cloud_provider="aws", + is_venv=True, + has_sudo=True, + ) + + instructions = get_installation_instructions(env) + + # Should include RHEL and venv instructions + assert "RHEL" in instructions or "CENTOS" in instructions or "FEDORA" in instructions + assert "nodeenv" in instructions diff --git a/uv.lock b/uv.lock index 112d342..5d5136f 100644 --- a/uv.lock +++ b/uv.lock @@ -224,7 +224,7 @@ wheels = [ [[package]] name = "promptfoo" -version = "0.2.0" +version = "0.1.1" source = { editable = "." } [package.optional-dependencies]