vscode-java-pack/test-plans/java-maven-resolve-type.yaml at ec5a5e5eb6aeb708eb246fc6853d77c656eb8085 · microsoft/vscode-java-pack · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
# Test Plan: Maven for Java — Resolve Unknown Type (from vscode-java-pack.wiki)
#
# Source: wiki Test-Plan.md "Maven for Java" scenario
# Verify: Type unknown type → Maven dep added in pom.xml → LS re-imports → type resolved
#
# The wiki test exercises the Quick Fix "Resolve unknown type" code action,
# which navigates a nested action menu and is brittle to JDT label changes
# (we observed it appearing to apply while leaving Gson unresolved — silent
# pass that the LLM authoritative re-verify caught). This plan exercises the
# same Maven-for-Java integration via a deterministic path:
#   1. type `Gson gson;` → LS publishes "Gson cannot be resolved" error
#   2. add the gson dependency to pom.xml → LS re-imports
#   3. add the import → diagnostic clears
# Both the textual diagnostic state and the on-screen Problems panel update,
# so deterministic and LLM verifications agree.
#
# Prerequisites:
#   - JDK 11+ installed and available on PATH (the workflow installs JDK 21)
#   - Network access to fetch the gson jar via the embedded Maven Wrapper
#
# Usage: autotest run test-plans/java-maven-resolve-type.yaml

name: "Maven for Java — Resolve Unknown Type"
description: |
  Validates the Maven-for-Java integration: an unknown type triggers an LS
  diagnostic, adding the Maven dependency to pom.xml causes vscode-java to
  re-import the project, and adding the import statement clears the error.

setup:
  extension: "redhat.java"
  extensions:
    - "vscjava.vscode-java-pack"
  vscodeVersion: "stable"
  workspace: "../test-fixtures/maven-resolve-type"
  timeout: 180  # Maven re-import after pom edit can be slow on cold caches
  # Force the Java extension to auto-import on pom.xml change without
  # prompting "Always Sync / Update / Don't Sync". The wiki scenario
  # expects the re-import to happen silently after the dependency is added.
  workspaceSettings:
    java.configuration.updateBuildConfiguration: "automatic"

steps:
  # ── Wait for LS ready ─────────────────────────────────────────
  # Workspace tree state can vary at screenshot time (src may be
  # collapsed, target/ may or may not be present), so verify text only
  # asserts on the workspace-loaded state visible regardless of tree
  # expansion.
  - id: "ls-ready"
    action: "waitForLanguageServer"
    verify: "maven-resolve-type project has been imported; the Java extension is activated and pom.xml is visible in the Explorer"
    timeout: 180
    # waitForLanguageServer is authoritative — skip LLM screenshot re-check.
    skipLlmVerify: true

  # ── Open Java file ──────────────────────────────────────
  - id: "open-app"
    action: "open file App.java"
    verify: "App.java file is open in the editor"
    waitBefore: 5
    timeout: 15

  # ── Type unknown type — LS must publish an error ─────────
  # wiki: "type 'Gson gson;'" — line 4 places the field inside the class body.
  # The Problems panel is not auto-opened by autotest and the red squiggle
  # may take a moment to render, so verify text describes only the
  # inserted code line. The deterministic verifyProblems.errors >= 1
  # polls the diagnostics API and is the ground truth for the LS error.
  - id: "insert-unknown-type"
    action: "insertLineInFile src/main/java/com/example/App.java 4         Gson gson;"
    verify: "App.java editor now shows the inserted 'Gson gson;' declaration inside the class body"
    verifyEditor:
      contains: "Gson gson;"
    verifyProblems:
      errors: 1
      atLeast: true
    waitBefore: 8
    timeout: 60

  # Close all editors before modifying pom.xml on disk. Having pom.xml
  # open in the editor while `insertLineInFile` writes to disk can leave
  # the editor's in-memory buffer out of sync — and on Linux runners VS
  # Code may then prompt or simply hold the stale buffer dirty. A
  # subsequent `saveFile` would then overwrite the on-disk dependency
  # block with the stale buffer. Closing all editors avoids the conflict
  # entirely; we re-open pom.xml AFTER the insertion to capture a clean
  # AFTER screenshot showing the new <dependency> block.
  - id: "close-app-before-pom"
    action: "run command View: Close All Editors"

  # ── Add the gson dependency to pom.xml ──────────────────
  # The fixture pom.xml has a `<dependencies>` block with an
  # injection-point comment on line 9. Insert a `<dependency>` element
  # at line 10 (immediately after the comment, before `</dependencies>`).
  # `verifyFile.path` needs the `~/` prefix so autotest resolves the path
  # against the workspace root (the worktree), not the runner's CWD.
  - id: "add-gson-dependency"
    action: |
      insertLineInFile pom.xml 10         <dependency>
              <groupId>com.google.code.gson</groupId>
              <artifactId>gson</artifactId>
              <version>2.10.1</version>
          </dependency>
    verify: "This step performs a disk-only file mutation via insertLineInFile against pom.xml. The action does NOT open pom.xml in the editor — by design the BEFORE and AFTER screenshots are expected to look identical because no editor or UI change is involved at this step. The deterministic verifyFile assertion below reads pom.xml from disk to confirm the new <dependency> block was persisted. A subsequent step opens pom.xml in the editor so the inserted block becomes visually verifiable."
    verifyFile:
      path: "~/pom.xml"
      contains: "com.google.code.gson"
    waitBefore: 2
    # Disk-only insertLineInFile: pom.xml isn't in any editor, so before/after
    # screenshots are necessarily identical. LLM always downgrades; verifyFile
    # reading from disk is the only meaningful signal.
    skipLlmVerify: true

  # Re-open pom.xml so the AFTER screenshot shows the new <dependency>
  # block. Loading fresh from disk avoids any in-memory/disk mismatch.
  # NOTE: no separate `saveFile` step — `insertLineInFile` already
  # persisted the change to disk; an explicit save here would risk
  # overwriting it with a stale editor buffer.
  - id: "reopen-pom-after-insert"
    action: "open file pom.xml"
    verify: "pom.xml is open in the editor and shows the inserted <dependency> block referencing com.google.code.gson"
    verifyEditor:
      contains: "com.google.code.gson"
    waitBefore: 3
    timeout: 10

  # Explicitly trigger a Maven re-import so the newly-added gson dependency is
  # picked up on the classpath. With `java.configuration.updateBuildConfiguration:
  # automatic` the file-watcher should already trigger this on Linux runners,
  # but a manual reload makes the test deterministic.
  - id: "reload-projects"
    action: "run command Java: Reload Projects"
    verify: "The 'Java: Reload Projects' command was invoked from the command palette. This is a background command — by design the BEFORE and AFTER screenshots are expected to look identical because the command palette closes before the AFTER screenshot is captured and the actual project re-import happens asynchronously in the language server. The deterministic ground truth is the next waitForLanguageServer step which observes the LS go through Building/Searching states as Maven re-resolves the gson dependency."
    waitBefore: 3

  # The file-watcher + Reload Projects above triggers Maven re-import asynchronously.
  # Give it time to start (waitBefore) before polling LS readiness, and allow
  # plenty of time for Maven to resolve gson on a cold cache.
  - id: "wait-maven-reimport"
    action: "waitForLanguageServer"
    verify: "Maven re-import has completed in response to the Reload Projects command — the language server has finished Building/Searching for the new gson dependency and the status bar is back to 'Java: Ready' with no progress indicator visible"
    timeout: 300
    waitBefore: 45

  # ── Add the import — diagnostic should clear ─────────────
  - id: "reopen-app"
    action: "open file App.java"
    verify: "App.java is re-opened in the editor"
    timeout: 15

  - id: "add-import"
    action: "insertLineInFile src/main/java/com/example/App.java 2 import com.google.gson.Gson;"
    verify: "App.java editor now shows 'import com.google.gson.Gson;' at the top of the file"
    verifyEditor:
      contains: "import com.google.gson.Gson;"
    waitBefore: 3

  # Save the file. The verify text focuses on the SAVE event itself (tab dirty
  # marker clears) which is the deterministic visible change. The squiggle-
  # cleared assertion lives on the follow-up `verify-resolved` step because the
  # editor decoration layer can take a couple of seconds to refresh AFTER the
  # diagnostic publish (verifyProblems.errors:0 below polls the LSP API which
  # updates before the editor re-paints).
  - id: "save-after-resolve"
    action: "saveFile"
    verify: "App.java has been saved to disk — the dirty-file dot on the editor tab is cleared. The Maven re-import (triggered by the earlier pom.xml edit + Reload Projects command) has placed gson on the classpath, so the language server now reports zero unresolved-type errors (asserted deterministically below via verifyProblems.errors:0)."
    verifyProblems:
      errors: 0
    waitBefore: 20
    timeout: 90
    # Maven re-import on a cold cache can take significantly longer than the
    # waitBefore window; a single retry (with the LS likely already settled
    # by then) recovers without inflating the happy-path wait further.
    retries: 1

  # After save, the language server publishes diagnostics (status bar updates
  # to 0 errors, verified deterministically above). However, on Linux runners
  # the editor decoration layer can lag the diagnostic publish by 15–30 seconds
  # before it clears the now-stale red squiggles. Close-and-reopen forces the
  # editor to redraw with the current diagnostic state, making the cleared
  # squiggle visible in the screenshot.
  - id: "force-editor-refresh"
    action: "run command View: Close All Editors"
    waitBefore: 5

  - id: "verify-resolved"
    action: "open file App.java"
    verify: "App.java is freshly re-opened in the editor showing 'import com.google.gson.Gson;' at the top of the file and a 'Gson gson;' field declaration in the class body. Both occurrences of 'Gson' resolve cleanly (no red error-squiggle is visible under either one) because the new pom.xml <dependency> block has been imported and gson is now on the classpath."
    verifyEditor:
      contains: "import com.google.gson.Gson;"
    waitBefore: 10
    timeout: 30