feat: add verify --basic flag to run a fast test subset

AGENTS.md

@@ -145,11 +145,11 @@ Key principles:
 
 | File | Purpose |
 |------------------------------------|------------------------------------|
-| `src/vip/cli.py` | CLI entry point: version, verify, cleanup (Connect content + orphaned Workbench sessions via `--workbench-url`), install, uninstall, auth, scaffold commands; `--version` flag |
+| `src/vip/cli.py` | CLI entry point: version, verify (including `--basic` to skip `@slow`-tagged scenarios), cleanup (Connect content + orphaned Workbench sessions via `--workbench-url`), install, uninstall, auth, scaffold commands; `--version` flag |
 | `src/vip/config.py` | TOML config loader and dataclasses |
 | `src/vip/auth.py` | Interactive and headless browser authentication for OIDC providers; `authenticated_page` opens a headless page from a cached auth session for `vip cleanup --workbench-url` |
 | `src/vip/idp.py` | IdP login form strategies for headless auth (Keycloak, Okta) |
-| `src/vip/plugin.py` | pytest plugin: markers, auto-skip, JSON report output |
+| `src/vip/plugin.py` | pytest plugin: markers (including `slow`, used by `verify --basic`), auto-skip, JSON report output |
 | `src/vip/version.py` | `ProductVersion` parsing/comparison for `min_version` gating; `MINIMUM_SUPPORTED_POSIT_TEAM` support floor (powers `vip version`) |
 | `src/vip/workbench_ui.py` | Browser-driven Workbench session-cleanup sweep (`quit_vip_sessions_via_ui`), shared by the per-test cleanup fixture and `vip cleanup --workbench-url` |
 | `src/vip/reporting.py` | Report data model for Quarto templates |

README.md

@@ -61,6 +61,12 @@ cp vip.toml.example vip.toml     # edit with your deployment details
 vip verify --config vip.toml
 ```
 
+Fast confidence check — skip the detailed/long-running checks:
+
+```bash
+vip verify --config vip.toml --basic
+```
+
 ## Uninstalling
 
 To reverse what `vip install` (or `just setup`) did:

docs/test-architecture.md

@@ -245,6 +245,18 @@ Deliberately **not** scaled:
 Values < 1.0 are valid and useful for speeding up CI smoke checks
 (`VIP_TIMEOUT_SCALE=0.5`).
 
+## Fast confidence checks with `--basic`
+
+`vip verify --basic` runs only the core subset of tests, excluding detailed or
+long-running checks tagged `@slow`. Today the `@slow` set covers the heavier
+Workbench checks — IDE extension installation, job execution, Git operations,
+and publishing to Connect — so a `--basic` run still exercises auth, IDE launch,
+sessions, runtime versions, packages, data sources, and Chronicle.
+
+`--basic` composes with `--categories`: `vip verify --categories workbench --basic`
+runs the Workbench category minus its `@slow` scenarios. The mechanism is
+product-agnostic — tag any feature `@slow` to exclude it from basic runs.
+
 ## Adding a New Test: Layer-by-Layer Checklist
 
 1. **Layer 1 -- Feature file**: Write the Gherkin scenario with a `@product` tag. Focus on business intent, not implementation.

pyproject.toml

@@ -172,6 +172,7 @@ markers = [
     "performance: performance validation tests (opt-in; excluded by default)",
     "security: security validation tests",
     "config_hygiene: checks of VIP's own configuration (opt-in; excluded by default)",
+    "slow: detailed/long-running checks; excluded by --basic",
     "min_version(product, version): only run when product meets minimum version",
     "if_applicable: test is skipped when the feature is not configured",
     "api_auth: test requires only an API key, not browser credentials",

selftests/test_cli_verify.py

@@ -35,6 +35,7 @@ def _make_args(**overrides) -> argparse.Namespace:
         "headless_auth": False,
         "idp": None,
         "performance_tests": False,
+        "basic": False,
         "insecure": False,
         "ca_bundle": None,
     }
@@ -70,6 +71,18 @@ def _capture_cmd(args: argparse.Namespace) -> list[str]:
     return cmd
 
 
+def _marker_expr(cmd: list[str]) -> str:
+    """Return the value passed to pytest's -m in the assembled command.
+
+    ``cmd[0:2]`` is always ``[sys.executable, "-m"]`` (invoking pytest as a
+    module), so the marker expression is the value after the *second*
+    ``-m`` occurrence.
+    """
+    first = cmd.index("-m")
+    second = cmd.index("-m", first + 1)
+    return cmd[second + 1]
+
+
 def _vip_tests_path() -> str:
     """Return the resolved vip_tests package path for assertions."""
     from importlib.util import find_spec
@@ -601,6 +614,41 @@ def test_explicit_category_overrides_performance_flag(self, tmp_path):
         assert self._marker_expr(cmd) == "connect"
 
 
+class TestBasicFlag:
+    """--basic appends 'not slow' to the pytest marker expression."""
+
+    def test_basic_adds_not_slow(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        monkeypatch.delenv("VIP_CONFIG", raising=False)
+        cmd = _capture_cmd(
+            _make_args(workbench_url="https://wb.example.com", no_auth=True, basic=True)
+        )
+        assert "not slow" in _marker_expr(cmd)
+
+    def test_no_basic_has_no_not_slow(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        monkeypatch.delenv("VIP_CONFIG", raising=False)
+        cmd = _capture_cmd(
+            _make_args(workbench_url="https://wb.example.com", no_auth=True, basic=False)
+        )
+        assert "not slow" not in _marker_expr(cmd)
+
+    def test_basic_composes_with_categories(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        monkeypatch.delenv("VIP_CONFIG", raising=False)
+        cmd = _capture_cmd(
+            _make_args(
+                workbench_url="https://wb.example.com",
+                no_auth=True,
+                categories="workbench",
+                basic=True,
+            )
+        )
+        expr = _marker_expr(cmd)
+        assert "workbench" in expr
+        assert "not slow" in expr
+
+
 class TestExtraKeepFromArgs:
     """_extra_keep_from_args mirrors --performance-tests into the marker expression."""
 

selftests/test_gherkin.py

@@ -109,3 +109,15 @@ def test_scenario_outline(self, tmp_path):
         assert len(result["scenarios"]) == 1
         assert result["scenarios"][0]["title"] == "Deploy <type> content"
         assert len(result["scenarios"][0]["steps"]) == 3
+
+
+def test_slow_membership_matches_decision():
+    from pathlib import Path
+
+    wb = Path(__file__).parent.parent / "src" / "vip_tests" / "workbench"
+    slow = ["test_ide_extensions", "test_jobs", "test_git_ops", "test_publish_to_connect"]
+    for name in slow:
+        text = (wb / f"{name}.feature").read_text()
+        assert "@slow" in text, f"{name}.feature should be tagged @slow"
+    # Chronicle is deliberately kept in the basic run.
+    assert "@slow" not in (wb / "test_chronicle.feature").read_text()

selftests/test_plugin.py

@@ -247,6 +247,19 @@ def test_needs_connect():
         result.assert_outcomes()
         result.stdout.fnmatch_lines(["*1 deselected*"])
 
+    def test_slow_deselected_by_not_slow(self, selftest_pytester):
+        selftest_pytester.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.slow
+            def test_a_slow_check():
+                assert True
+            """
+        )
+        result = selftest_pytester.runpytest("--vip-config=vip.toml", "-m", "not slow", "-v")
+        result.stdout.fnmatch_lines(["*1 deselected*"])
+
     def test_progress_indicator_recolored_per_line(self, selftest_pytester):
         """After a failure, a later passing line's [x%] stays green, not red.
 
@@ -699,6 +712,10 @@ def test_markers_registered(self, selftest_pytester):
             ]
         )
 
+    def test_slow_marker_registered(self, selftest_pytester):
+        result = selftest_pytester.runpytest("--markers")
+        result.stdout.fnmatch_lines(["*slow: *"])
+
     def test_interactive_auth_option_registered(self, selftest_pytester):
         """--interactive-auth appears in help output."""
         result = selftest_pytester.runpytest("--help")

selftests/test_workbench_ordering.py

@@ -29,31 +29,50 @@
 _NODEID_RE = re.compile(r"^\S+::\S+$")
 
 
-def _collect_workbench_nodeids(tmp_path: Path) -> list[str]:
+def _slow_workbench_stems() -> set[str]:
+    """Return the stems of Workbench feature files tagged ``@slow``.
+
+    Derived from the files themselves (not hardcoded) so the deselection
+    test below self-maintains as the ``@slow`` set changes. The intended
+    membership is locked separately in ``test_gherkin.py``.
+    """
+    stems = {f.stem for f in _WORKBENCH_DIR.glob("*.feature") if "@slow" in f.read_text()}
+    assert stems, "expected at least one @slow-tagged Workbench feature file"
+    return stems
+
+
+def _collect_workbench_nodeids(tmp_path: Path, marker_expr: str | None = None) -> list[str]:
     """Run ``pytest --collect-only`` over the Workbench suite and return node ids in order.
 
     Workbench and Connect must both be "configured" (a URL present) or every
     Workbench test gets deselected outright by the plugin's product-config
     gate (``_should_deselect_for_product``) -- ``test_publish_to_connect``
     requires both products, and an unconfigured product means "no test at
     all", not "collected but skipped".
+
+    Pass *marker_expr* to add a ``-m`` filter (e.g. ``"not slow"``, what
+    ``vip verify --basic`` applies) and observe which scenarios survive it.
     """
     config_path = tmp_path / "vip.toml"
     config_path.write_text(
         '[workbench]\nurl = "https://workbench.example.com"\n\n'
         '[connect]\nurl = "https://connect.example.com"\n'
     )
 
+    cmd = [
+        sys.executable,
+        "-m",
+        "pytest",
+        str(_WORKBENCH_DIR),
+        "--collect-only",
+        "-q",
+        f"--vip-config={config_path}",
+    ]
+    if marker_expr is not None:
+        cmd.extend(["-m", marker_expr])
+
     result = subprocess.run(
-        [
-            sys.executable,
-            "-m",
-            "pytest",
-            str(_WORKBENCH_DIR),
-            "--collect-only",
-            "-q",
-            f"--vip-config={config_path}",
-        ],
+        cmd,
         cwd=_REPO_ROOT,
         capture_output=True,
         text=True,
@@ -118,3 +137,32 @@ def test_ide_extensions_is_last_workbench_file(tmp_path: Path) -> None:
         f"test_ide_extensions must be the last Workbench test file collected, "
         f"got {last_file!r} last: {nodeids}"
     )
+
+
+def test_basic_marker_deselects_slow_workbench_features(tmp_path: Path) -> None:
+    """End-to-end: ``-m "not slow"`` (what ``vip verify --basic`` applies) must
+    drop exactly the @slow-tagged Workbench features and keep the basic ones.
+
+    This exercises the full chain the other selftests only cover in isolation:
+    real ``.feature`` files -> pytest-bdd Gherkin-tag->pytest-marker conversion
+    -> ``-m`` deselection. It would catch a pytest-bdd change (or a stray
+    ``pytest_bdd_apply_tag`` hook) that silently stopped converting ``@slow``,
+    which would turn ``--basic`` into a no-op with no other failing test.
+    """
+    slow_files = _slow_workbench_stems()
+    all_files = {_file_of(n) for n in _collect_workbench_nodeids(tmp_path)}
+    basic_nodeids = _collect_workbench_nodeids(tmp_path, marker_expr="not slow")
+    basic_files = {_file_of(n) for n in basic_nodeids}
+
+    # Sanity: unfiltered, the @slow files really do collect (so a broken
+    # collection can't let the deselection assertion pass vacuously).
+    assert slow_files <= all_files, (
+        f"expected all @slow files unfiltered; missing {slow_files - all_files}"
+    )
+    # The filter drops every @slow file ...
+    leaked = slow_files & basic_files
+    assert not leaked, f"@slow files leaked into a `not slow` run: {leaked}"
+    # ... and keeps a known basic feature (Chronicle stays in the basic run).
+    assert "test_chronicle" in basic_files, (
+        f"test_chronicle should remain in a basic run: {sorted(basic_files)}"
+    )

src/vip/cli.py

@@ -499,9 +499,12 @@ def run_verify(args: argparse.Namespace) -> None:
     for ext in args.extensions or []:
         cmd.append(f"--vip-extensions={ext}")
     if args.categories:
-        cmd.extend(["-m", _normalize_categories(args.categories)])
+        marker_expr = _normalize_categories(args.categories)
     else:
-        cmd.extend(["-m", _default_marker_expr(_extra_keep_from_args(args))])
+        marker_expr = _default_marker_expr(_extra_keep_from_args(args))
+    if getattr(args, "basic", False):
+        marker_expr = f"({marker_expr}) and not slow" if marker_expr else "not slow"
+    cmd.extend(["-m", marker_expr])
     if args.filter_expr:
         cmd.extend(["-k", args.filter_expr])
 
@@ -1379,6 +1382,14 @@ def main() -> None:
         help="Include performance tests in the default selection (excluded otherwise). "
         "Has no effect when --categories is also specified.",
     )
+    verify_parser.add_argument(
+        "--basic",
+        action="store_true",
+        default=False,
+        help="Run only the basic subset; exclude detailed/long-running checks "
+        "tagged @slow (IDE extensions, jobs, git ops, publish to Connect). "
+        "Composes with --categories.",
+    )
     verify_parser.add_argument(
         "-f",
         "--filter",

src/vip/plugin.py

@@ -162,6 +162,10 @@ def pytest_configure(config: pytest.Config) -> None:
         "markers",
         "config_hygiene: checks of VIP's own configuration (opt-in; excluded by default)",
     )
+    config.addinivalue_line(
+        "markers",
+        "slow: detailed/long-running checks; excluded by --basic",
+    )
     config.addinivalue_line(
         "markers",
         "min_version(product, version): skip when product is below the specified version",

src/vip_tests/workbench/test_git_ops.feature

@@ -1,4 +1,4 @@
-@workbench
+@workbench @slow
 Feature: Git operations from Workbench sessions
   Validate that users inside a Workbench session can clone, commit, and push
   to a Git repository through IDE terminals (RStudio, VS Code, Positron).

src/vip_tests/workbench/test_ide_extensions.feature

@@ -1,4 +1,4 @@
-@workbench
+@workbench @slow
 Feature: Workbench IDE extensions
   As a Posit Team administrator
   I want to verify that required extensions are installed in each IDE

src/vip_tests/workbench/test_jobs.feature

@@ -1,4 +1,4 @@
-@workbench
+@workbench @slow
 Feature: Workbench job execution
 
   Scenario: Background Job runs and completes

src/vip_tests/workbench/test_publish_to_connect.feature

@@ -1,4 +1,4 @@
-@workbench @connect
+@workbench @connect @slow
 Feature: Publish from Workbench to Connect
   As a Posit Team administrator
   I want to verify that a user can deploy content from a Workbench session to Connect