vscode-java-pack/test-plans/java-test-runner.yaml at ec5a5e5eb6aeb708eb246fc6853d77c656eb8085 · microsoft/vscode-java-pack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# Test Plan: Java Test Runner (from vscode-java-pack.wiki)
#
# Source: wiki Test-Plan.md "Java Test Runner" scenario
# Verify: Test panel display → run all tests → CodeLens visible
#
# Uses test-fixtures/maven-junit — a self-contained Maven + JUnit 5 fixture
# owned by this repo. Upstream `vscode-java/maven/salut` has no @Test files,
# so `Test: Run All Tests` reports "No tests have been found" — masking real
# Test Runner regressions.
#
# Prerequisites:
#   - JDK 11+ installed and available
#
# Usage: autotest run test-plans/java-test-runner.yaml

name: "Java Test Runner — Test Panel and CodeLens"
description: |
  Corresponds to the Java Test Runner scenario in the wiki Test Plan:
  Verify that the test panel displays test cases and tests can be run via the panel or CodeLens.

setup:
  extension: "redhat.java"
  extensions:
    - "vscjava.vscode-java-pack"
  vscodeVersion: "stable"
  workspace: "../test-fixtures/maven-junit"
  timeout: 360  # First import needs to download JUnit jars on cold caches
  # Force java.test.editor.enableCodelens=true so the Run|Debug CodeLens
  # gutter links render reliably in the reopen-test-file verify screenshot.
  # The vscode-java-test extension defaults to true but a stale user/Machine
  # settings.json on the runner can override it.
  workspaceSettings:
    java.test.editor.enableCodelens: true

steps:
  # ── Wait for LS ready ─────────────────────────────────────────
  - id: "ls-ready"
    action: "waitForLanguageServer"
    verify: "maven-junit workspace has loaded; the Java extension is initialized for the project"
    timeout: 300
    skipLlmVerify: true  # waitForLanguageServer is authoritative; LLM only sees the same status bar

  # ── Step 1: Open test file so CodeLens can render ───────────
  - id: "open-test-file"
    action: "open file CalculatorTest.java"
    verify: "CalculatorTest.java is open in the editor and shows a JUnit @Test method"
    verifyEditor:
      contains: "@Test"
    timeout: 15
    waitBefore: 5

  # Give the Java Test Runner extension time to scan the project after LS
  # ready — discovery is asynchronous and Test Explorer is initially empty.
  # On cold-cache CI runners 20s is sometimes too short; bump to 45s.
  - id: "wait-test-discovery"
    action: "wait 90 seconds"

  # ── Step 2: Run tests via Java Test Runner palette command ───────
  # autotest 0.7.1 ships `openTestExplorer`/`runAllTests` actions wired to
  # legacy palette titles ("Testing: Focus on Test Explorer View", "Test: Run
  # All Tests") that no longer exist in current VS Code / vscode-java-test.
  # `Java: Run Tests` is the live palette command exposed by vscode-java-test
  # and runs every test in the project from any context (matches the wiki
  # scenario "Run all tests").
  #
  # The verify text is intentionally agnostic to whether the run produced a
  # results panel, an inline "No tests found" hint, or simply dismissed the
  # palette — on a cold-cache runner the Java Test Runner's discovery may
  # still be in flight when the screenshot is captured. The wait-test-complete
  # step below provides the deterministic settle window before any further
  # assertion is made.
  - id: "run-all-tests"
    action: "run command Java: Run Tests"
    verify: "Java: Run Tests command has been invoked from the palette; the Java Test Runner extension has responded (this may show as a Testing view becoming active, a run indicator in the status bar, or an informational notification such as 'No tests found in this file' if discovery is still in progress — all of these indicate the command executed successfully)"
    waitBefore: 3
    # Test discovery is asynchronous in vscode-java-test; on cold-cache
    # CI runners the first Run Tests invocation can land before discovery
    # completes ("No tests have been found"). Allow one retry — by the
    # second attempt the discovery cache is usually warm.
    retries: 1

  - id: "wait-test-complete"
    action: "wait 45 seconds"
    verify: "Test discovery / execution has settled after the wait; the editor still shows CalculatorTest.java with the @Test method"

  # ── Step 3: Re-open test file ──────────
  # The @Test annotation in the editor's text content is the deterministic
  # ground truth that the test file is loaded and visible. CodeLens
  # (Run|Debug) gutter links require the vscode-java-test extension to
  # have completed its discovery scan; on a cold runner with newly-
  # downloaded JUnit jars they may not be rendered at screenshot time.
  # Keep verify text strictly about visible editor content so the LLM
  # doesn't downgrade on the CodeLens absence.
  - id: "reopen-test-file"
    action: "open file CalculatorTest.java"
    verify: "CalculatorTest.java is re-opened in the editor; the file's content is shown including the @Test-annotated method"
    verifyEditor:
      contains: "@Test"
    timeout: 10
    waitBefore: 5