diff --git a/AGENTS.md b/AGENTS.md index 469584ce..84085d7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,11 +145,11 @@ Key principles: | File | Purpose | |------------------------------------|------------------------------------| -| `src/vip/cli.py` | CLI entry point: version, verify, cleanup (Connect content + orphaned Workbench sessions via `--workbench-url`), install, uninstall, auth, scaffold commands; `--version` flag | +| `src/vip/cli.py` | CLI entry point: version, verify (including `--basic` to skip `@slow`-tagged scenarios), cleanup (Connect content + orphaned Workbench sessions via `--workbench-url`), install, uninstall, auth, scaffold commands; `--version` flag | | `src/vip/config.py` | TOML config loader and dataclasses | | `src/vip/auth.py` | Interactive and headless browser authentication for OIDC providers; `authenticated_page` opens a headless page from a cached auth session for `vip cleanup --workbench-url` | | `src/vip/idp.py` | IdP login form strategies for headless auth (Keycloak, Okta) | -| `src/vip/plugin.py` | pytest plugin: markers, auto-skip, JSON report output | +| `src/vip/plugin.py` | pytest plugin: markers (including `slow`, used by `verify --basic`), auto-skip, JSON report output | | `src/vip/version.py` | `ProductVersion` parsing/comparison for `min_version` gating; `MINIMUM_SUPPORTED_POSIT_TEAM` support floor (powers `vip version`) | | `src/vip/workbench_ui.py` | Browser-driven Workbench session-cleanup sweep (`quit_vip_sessions_via_ui`), shared by the per-test cleanup fixture and `vip cleanup --workbench-url` | | `src/vip/reporting.py` | Report data model for Quarto templates | diff --git a/README.md b/README.md index a1d8bcf3..bc6acc28 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,12 @@ cp vip.toml.example vip.toml # edit with your deployment details vip verify --config vip.toml ``` +Fast confidence check — skip the detailed/long-running checks: + +```bash +vip verify --config vip.toml --basic +``` + ## Uninstalling To reverse what `vip install` (or `just setup`) did: diff --git a/docs/test-architecture.md b/docs/test-architecture.md index fd5a97aa..43cc7253 100644 --- a/docs/test-architecture.md +++ b/docs/test-architecture.md @@ -245,6 +245,18 @@ Deliberately **not** scaled: Values < 1.0 are valid and useful for speeding up CI smoke checks (`VIP_TIMEOUT_SCALE=0.5`). +## Fast confidence checks with `--basic` + +`vip verify --basic` runs only the core subset of tests, excluding detailed or +long-running checks tagged `@slow`. Today the `@slow` set covers the heavier +Workbench checks — IDE extension installation, job execution, Git operations, +and publishing to Connect — so a `--basic` run still exercises auth, IDE launch, +sessions, runtime versions, packages, data sources, and Chronicle. + +`--basic` composes with `--categories`: `vip verify --categories workbench --basic` +runs the Workbench category minus its `@slow` scenarios. The mechanism is +product-agnostic — tag any feature `@slow` to exclude it from basic runs. + ## Adding a New Test: Layer-by-Layer Checklist 1. **Layer 1 -- Feature file**: Write the Gherkin scenario with a `@product` tag. Focus on business intent, not implementation. diff --git a/pyproject.toml b/pyproject.toml index 977a4266..8eedcad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -172,6 +172,7 @@ markers = [ "performance: performance validation tests (opt-in; excluded by default)", "security: security validation tests", "config_hygiene: checks of VIP's own configuration (opt-in; excluded by default)", + "slow: detailed/long-running checks; excluded by --basic", "min_version(product, version): only run when product meets minimum version", "if_applicable: test is skipped when the feature is not configured", "api_auth: test requires only an API key, not browser credentials", diff --git a/selftests/test_cli_verify.py b/selftests/test_cli_verify.py index 2e2b5e9d..cadc0f87 100644 --- a/selftests/test_cli_verify.py +++ b/selftests/test_cli_verify.py @@ -35,6 +35,7 @@ def _make_args(**overrides) -> argparse.Namespace: "headless_auth": False, "idp": None, "performance_tests": False, + "basic": False, "insecure": False, "ca_bundle": None, } @@ -70,6 +71,18 @@ def _capture_cmd(args: argparse.Namespace) -> list[str]: return cmd +def _marker_expr(cmd: list[str]) -> str: + """Return the value passed to pytest's -m in the assembled command. + + ``cmd[0:2]`` is always ``[sys.executable, "-m"]`` (invoking pytest as a + module), so the marker expression is the value after the *second* + ``-m`` occurrence. + """ + first = cmd.index("-m") + second = cmd.index("-m", first + 1) + return cmd[second + 1] + + def _vip_tests_path() -> str: """Return the resolved vip_tests package path for assertions.""" from importlib.util import find_spec @@ -601,6 +614,41 @@ def test_explicit_category_overrides_performance_flag(self, tmp_path): assert self._marker_expr(cmd) == "connect" +class TestBasicFlag: + """--basic appends 'not slow' to the pytest marker expression.""" + + def test_basic_adds_not_slow(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("VIP_CONFIG", raising=False) + cmd = _capture_cmd( + _make_args(workbench_url="https://wb.example.com", no_auth=True, basic=True) + ) + assert "not slow" in _marker_expr(cmd) + + def test_no_basic_has_no_not_slow(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("VIP_CONFIG", raising=False) + cmd = _capture_cmd( + _make_args(workbench_url="https://wb.example.com", no_auth=True, basic=False) + ) + assert "not slow" not in _marker_expr(cmd) + + def test_basic_composes_with_categories(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("VIP_CONFIG", raising=False) + cmd = _capture_cmd( + _make_args( + workbench_url="https://wb.example.com", + no_auth=True, + categories="workbench", + basic=True, + ) + ) + expr = _marker_expr(cmd) + assert "workbench" in expr + assert "not slow" in expr + + class TestExtraKeepFromArgs: """_extra_keep_from_args mirrors --performance-tests into the marker expression.""" diff --git a/selftests/test_gherkin.py b/selftests/test_gherkin.py index 83c95daa..56f2be3a 100644 --- a/selftests/test_gherkin.py +++ b/selftests/test_gherkin.py @@ -109,3 +109,15 @@ def test_scenario_outline(self, tmp_path): assert len(result["scenarios"]) == 1 assert result["scenarios"][0]["title"] == "Deploy content" assert len(result["scenarios"][0]["steps"]) == 3 + + +def test_slow_membership_matches_decision(): + from pathlib import Path + + wb = Path(__file__).parent.parent / "src" / "vip_tests" / "workbench" + slow = ["test_ide_extensions", "test_jobs", "test_git_ops", "test_publish_to_connect"] + for name in slow: + text = (wb / f"{name}.feature").read_text() + assert "@slow" in text, f"{name}.feature should be tagged @slow" + # Chronicle is deliberately kept in the basic run. + assert "@slow" not in (wb / "test_chronicle.feature").read_text() diff --git a/selftests/test_plugin.py b/selftests/test_plugin.py index 9083e7c2..9d38448a 100644 --- a/selftests/test_plugin.py +++ b/selftests/test_plugin.py @@ -247,6 +247,19 @@ def test_needs_connect(): result.assert_outcomes() result.stdout.fnmatch_lines(["*1 deselected*"]) + def test_slow_deselected_by_not_slow(self, selftest_pytester): + selftest_pytester.makepyfile( + """ + import pytest + + @pytest.mark.slow + def test_a_slow_check(): + assert True + """ + ) + result = selftest_pytester.runpytest("--vip-config=vip.toml", "-m", "not slow", "-v") + result.stdout.fnmatch_lines(["*1 deselected*"]) + def test_progress_indicator_recolored_per_line(self, selftest_pytester): """After a failure, a later passing line's [x%] stays green, not red. @@ -699,6 +712,10 @@ def test_markers_registered(self, selftest_pytester): ] ) + def test_slow_marker_registered(self, selftest_pytester): + result = selftest_pytester.runpytest("--markers") + result.stdout.fnmatch_lines(["*slow: *"]) + def test_interactive_auth_option_registered(self, selftest_pytester): """--interactive-auth appears in help output.""" result = selftest_pytester.runpytest("--help") diff --git a/selftests/test_workbench_ordering.py b/selftests/test_workbench_ordering.py index f49778fe..273c5c73 100644 --- a/selftests/test_workbench_ordering.py +++ b/selftests/test_workbench_ordering.py @@ -29,7 +29,19 @@ _NODEID_RE = re.compile(r"^\S+::\S+$") -def _collect_workbench_nodeids(tmp_path: Path) -> list[str]: +def _slow_workbench_stems() -> set[str]: + """Return the stems of Workbench feature files tagged ``@slow``. + + Derived from the files themselves (not hardcoded) so the deselection + test below self-maintains as the ``@slow`` set changes. The intended + membership is locked separately in ``test_gherkin.py``. + """ + stems = {f.stem for f in _WORKBENCH_DIR.glob("*.feature") if "@slow" in f.read_text()} + assert stems, "expected at least one @slow-tagged Workbench feature file" + return stems + + +def _collect_workbench_nodeids(tmp_path: Path, marker_expr: str | None = None) -> list[str]: """Run ``pytest --collect-only`` over the Workbench suite and return node ids in order. Workbench and Connect must both be "configured" (a URL present) or every @@ -37,6 +49,9 @@ def _collect_workbench_nodeids(tmp_path: Path) -> list[str]: gate (``_should_deselect_for_product``) -- ``test_publish_to_connect`` requires both products, and an unconfigured product means "no test at all", not "collected but skipped". + + Pass *marker_expr* to add a ``-m`` filter (e.g. ``"not slow"``, what + ``vip verify --basic`` applies) and observe which scenarios survive it. """ config_path = tmp_path / "vip.toml" config_path.write_text( @@ -44,16 +59,20 @@ def _collect_workbench_nodeids(tmp_path: Path) -> list[str]: '[connect]\nurl = "https://connect.example.com"\n' ) + cmd = [ + sys.executable, + "-m", + "pytest", + str(_WORKBENCH_DIR), + "--collect-only", + "-q", + f"--vip-config={config_path}", + ] + if marker_expr is not None: + cmd.extend(["-m", marker_expr]) + result = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - str(_WORKBENCH_DIR), - "--collect-only", - "-q", - f"--vip-config={config_path}", - ], + cmd, cwd=_REPO_ROOT, capture_output=True, text=True, @@ -118,3 +137,32 @@ def test_ide_extensions_is_last_workbench_file(tmp_path: Path) -> None: f"test_ide_extensions must be the last Workbench test file collected, " f"got {last_file!r} last: {nodeids}" ) + + +def test_basic_marker_deselects_slow_workbench_features(tmp_path: Path) -> None: + """End-to-end: ``-m "not slow"`` (what ``vip verify --basic`` applies) must + drop exactly the @slow-tagged Workbench features and keep the basic ones. + + This exercises the full chain the other selftests only cover in isolation: + real ``.feature`` files -> pytest-bdd Gherkin-tag->pytest-marker conversion + -> ``-m`` deselection. It would catch a pytest-bdd change (or a stray + ``pytest_bdd_apply_tag`` hook) that silently stopped converting ``@slow``, + which would turn ``--basic`` into a no-op with no other failing test. + """ + slow_files = _slow_workbench_stems() + all_files = {_file_of(n) for n in _collect_workbench_nodeids(tmp_path)} + basic_nodeids = _collect_workbench_nodeids(tmp_path, marker_expr="not slow") + basic_files = {_file_of(n) for n in basic_nodeids} + + # Sanity: unfiltered, the @slow files really do collect (so a broken + # collection can't let the deselection assertion pass vacuously). + assert slow_files <= all_files, ( + f"expected all @slow files unfiltered; missing {slow_files - all_files}" + ) + # The filter drops every @slow file ... + leaked = slow_files & basic_files + assert not leaked, f"@slow files leaked into a `not slow` run: {leaked}" + # ... and keeps a known basic feature (Chronicle stays in the basic run). + assert "test_chronicle" in basic_files, ( + f"test_chronicle should remain in a basic run: {sorted(basic_files)}" + ) diff --git a/src/vip/cli.py b/src/vip/cli.py index 05b4ec1d..f79099b5 100644 --- a/src/vip/cli.py +++ b/src/vip/cli.py @@ -499,9 +499,12 @@ def run_verify(args: argparse.Namespace) -> None: for ext in args.extensions or []: cmd.append(f"--vip-extensions={ext}") if args.categories: - cmd.extend(["-m", _normalize_categories(args.categories)]) + marker_expr = _normalize_categories(args.categories) else: - cmd.extend(["-m", _default_marker_expr(_extra_keep_from_args(args))]) + marker_expr = _default_marker_expr(_extra_keep_from_args(args)) + if getattr(args, "basic", False): + marker_expr = f"({marker_expr}) and not slow" if marker_expr else "not slow" + cmd.extend(["-m", marker_expr]) if args.filter_expr: cmd.extend(["-k", args.filter_expr]) @@ -1379,6 +1382,14 @@ def main() -> None: help="Include performance tests in the default selection (excluded otherwise). " "Has no effect when --categories is also specified.", ) + verify_parser.add_argument( + "--basic", + action="store_true", + default=False, + help="Run only the basic subset; exclude detailed/long-running checks " + "tagged @slow (IDE extensions, jobs, git ops, publish to Connect). " + "Composes with --categories.", + ) verify_parser.add_argument( "-f", "--filter", diff --git a/src/vip/plugin.py b/src/vip/plugin.py index b4cfe90f..4d80a405 100644 --- a/src/vip/plugin.py +++ b/src/vip/plugin.py @@ -162,6 +162,10 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "config_hygiene: checks of VIP's own configuration (opt-in; excluded by default)", ) + config.addinivalue_line( + "markers", + "slow: detailed/long-running checks; excluded by --basic", + ) config.addinivalue_line( "markers", "min_version(product, version): skip when product is below the specified version", diff --git a/src/vip_tests/workbench/test_git_ops.feature b/src/vip_tests/workbench/test_git_ops.feature index 32f11639..9f91775f 100644 --- a/src/vip_tests/workbench/test_git_ops.feature +++ b/src/vip_tests/workbench/test_git_ops.feature @@ -1,4 +1,4 @@ -@workbench +@workbench @slow Feature: Git operations from Workbench sessions Validate that users inside a Workbench session can clone, commit, and push to a Git repository through IDE terminals (RStudio, VS Code, Positron). diff --git a/src/vip_tests/workbench/test_ide_extensions.feature b/src/vip_tests/workbench/test_ide_extensions.feature index ac1fef6d..fdedb17f 100644 --- a/src/vip_tests/workbench/test_ide_extensions.feature +++ b/src/vip_tests/workbench/test_ide_extensions.feature @@ -1,4 +1,4 @@ -@workbench +@workbench @slow Feature: Workbench IDE extensions As a Posit Team administrator I want to verify that required extensions are installed in each IDE diff --git a/src/vip_tests/workbench/test_jobs.feature b/src/vip_tests/workbench/test_jobs.feature index df12e86c..feb6565c 100644 --- a/src/vip_tests/workbench/test_jobs.feature +++ b/src/vip_tests/workbench/test_jobs.feature @@ -1,4 +1,4 @@ -@workbench +@workbench @slow Feature: Workbench job execution Scenario: Background Job runs and completes diff --git a/src/vip_tests/workbench/test_publish_to_connect.feature b/src/vip_tests/workbench/test_publish_to_connect.feature index bec25516..1be14c79 100644 --- a/src/vip_tests/workbench/test_publish_to_connect.feature +++ b/src/vip_tests/workbench/test_publish_to_connect.feature @@ -1,4 +1,4 @@ -@workbench @connect +@workbench @connect @slow Feature: Publish from Workbench to Connect As a Posit Team administrator I want to verify that a user can deploy content from a Workbench session to Connect