Update webstudio test results view for cli results

Author: ferpasri

agent/resources/skills/testar-cli/SKILL.md

@@ -81,6 +81,12 @@ Use double quotes around target, text, and value when they contain spaces or spe
 6. Stop the session when the goal is finished.
 7. Shutdown the daemon after stopping the session.
 
+After executing one or multiple test goal sessions, always run:
+
+- `shutdownDaemon`
+
+Do this after the final `stopSession ...` command so no daemon or SUT-driver process remains active.
+
 ## Test goal verdicts
 
 Agent CLI executions must finish each test goal with exactly one explicit verdict command:

cli/src/org/testar/cli/CliDaemonServer.java

@@ -533,7 +533,7 @@ private OperatingSystems parseOperatingSystem(String token) {
     List<Verdict> finalVerdictsFromStopRequest(CliRequest request) {
         String verdictName = request.argumentAt(0);
         if (verdictName == null || verdictName.isBlank()) {
-            return List.of(Verdict.OK);
+            return List.of(new Verdict(Verdict.Severity.LLM_COMPLETE, "Manual CLI session completed."));
         }
 
         Verdict.Severity severity = parseFinalVerdictSeverity(verdictName);

cli/test/org/testar/cli/CliDaemonServerTest.java

@@ -143,15 +143,16 @@ public void resolveCliTargetRejectsDesktopAttachConnector() {
     }
 
     @Test
-    public void stopSessionWithoutVerdictDefaultsToOk() {
+    public void stopSessionWithoutVerdictDefaultsToLlmComplete() {
         CliDaemonServer server = new CliDaemonServer();
 
         List<Verdict> verdicts = server.finalVerdictsFromStopRequest(
                 CliRequest.of(CliCommand.STOP_SESSION, List.of())
         );
 
         Assert.assertEquals(1, verdicts.size());
-        Assert.assertEquals(Verdict.OK, verdicts.get(0));
+        Assert.assertEquals(Verdict.Severity.LLM_COMPLETE.getValue(), verdicts.get(0).severity(), 0.0);
+        Assert.assertEquals("Manual CLI session completed.", verdicts.get(0).info());
     }
 
     @Test

docs/architecture_distribution_workspace.md

@@ -154,7 +154,7 @@ Manual CLI execution may use:
 stopSession
 ```
 
-Plain `stopSession` defaults the final session verdict to `OK`.
+Plain `stopSession` defaults the final session verdict to `LLM_COMPLETE` and is intended for manual CLI sessions.
 
 Agent CLI execution must finish each test goal with an explicit LLM verdict:
 
@@ -173,6 +173,17 @@ If reporting is enabled, the final CLI verdict must be written to the generated
 
 The `testar-cli` skill documentation is the operational source that tells agents how to use this contract.
 
+## Output Folder Naming
+
+Generated output folders must include the execution mode after the timestamp.
+
+Examples:
+
+- `2026-06-29_13h23m23s_generate_webdriver_parabank_1`
+- `2026-06-29_13h23m23s_cli_webdriver_parabank_1`
+
+The mode token allows WebStudio to present mode-specific result summaries while keeping one shared output folder.
+
 ## Shared Settings
 
 The same `test.settings` file can contain settings used by different modes.

docs/webstudio/WEBSTUDIO_BUG_BACKLOG.md

@@ -11,7 +11,9 @@ Each bug entry should contain:
 - bug behavior
 - status
 
-## List of Bugs
+## Archived bugs
+
+Bugs fixed during previous designs that no longer map to current UI concepts.
 
 ### WS-001 CLI Buttons Not Recomputed After Workspace Change
 
@@ -36,6 +38,36 @@ Acceptance:
 
 - CLI availability recomputes immediately when workspace changes
 
+### WS-008 View State Model Did Nothing For Unsupported Workspace
+
+- Area: Navigation / State Model Analysis
+- Status: archived after shared runtime refactor
+
+Reproduction:
+
+1. Select a workspace that is not available in either the TESTAR runtime or the CLI runtime.
+2. Click `View State Model`.
+
+Expected:
+
+- WebStudio shows a user-facing dialog explaining why state model analysis cannot be opened for the selected workspace
+
+Bug:
+
+- the button action returned silently when the selected workspace was not available in the TESTAR runtime workspace list
+- the check was too strict because CLI workspaces can also generate state models
+
+Reason archived:
+
+- WebStudio now uses the selected shared workspace/runtime model
+- the separate TESTAR-vs-CLI runtime availability concept no longer maps to the current State Model flow
+- current expected behavior is specified in `WEBSTUDIO_FUNCTIONAL_SPEC.md`
+- current dialog behavior remains covered by `stateModelNavigation.test.js`
+
+## List of Bugs
+
+Bugs fixed which are still relevant to current designs or behaviors.
+
 ### WS-002 Runtime Pages Missing SUT Target Summary
 
 - Area: Generate Mode / Spy Mode
@@ -196,30 +228,59 @@ Acceptance:
 - optional non-dropdown string settings can still render a blank option
 - regression test `settingsSelectOptions.test.js` passes
 
-### WS-008 View State Model Did Nothing For Unsupported Workspace
+### WS-009 Generate And Spy Buttons Disabled By Stale Runtime-Origin Gate
 
-- Area: Navigation / State Model Analysis
+- Area: Generate Mode / Spy Mode
 - Status: fixed
 
 Reproduction:
 
-1. Select a workspace that is not available in either the TESTAR runtime or the CLI runtime.
-2. Click `View State Model`.
+1. Select a workspace after the shared runtime/workspace refactor.
+2. Open Generate Mode or Spy Mode.
+3. Observe the run buttons.
 
 Expected:
 
-- WebStudio shows a user-facing dialog explaining why state model analysis cannot be opened for the selected workspace
+- Generate and Spy start buttons enable when a workspace is selected and no execution is already running
+- workspace origin metadata must not disable scriptless runtime controls
 
 Bug:
 
-- the button action returned silently when the selected workspace was not available in the TESTAR runtime workspace list
-- the check was too strict because CLI workspaces can also generate state models
+- Generate and Spy still required `availableInTestar`
+- workspaces not marked with that old TESTAR-only flag left the buttons disabled
+
+Acceptance:
+
+- Generate and Spy buttons use selected shared workspace availability
+- no selected workspace still disables the buttons
+- saving or active execution still disables the buttons
+- regression test `runtimeModeControls.test.js` passes
+
+### WS-010 LLM Complete Results Shown As Issues
+
+- Area: Test Results / CLI Reports
+- Status: fixed
+
+Reproduction:
+
+1. Run an Agent CLI execution that finishes with `LLM_COMPLETE`.
+2. Open `View Test Results`.
+3. Inspect `Output Results`, `Test Verdict Files`, and success/issue filters.
+
+Expected:
+
+- `LLM_COMPLETE` result files are successful outcomes
+- output folders containing only `LLM_COMPLETE` files are successful
+- `Successful only` includes `LLM_COMPLETE`
+- `Issues only` excludes `LLM_COMPLETE`
+
+Bug:
+
+- shared result listing still treated only `OK` as successful
+- `LLM_COMPLETE` files were marked as failed, causing red rows and incorrect filters
 
 Acceptance:
 
-- incompatible workspaces show an `Unable To Open State Model` dialog
-- missing workspace selection shows an `Unable To Open State Model` dialog
-- TESTAR-compatible workspaces continue to call the state model open API
-- CLI-compatible workspaces continue to call the state model open API
-- state model analysis resolves datastore paths against the selected workspace runtime home
-- regression test `stateModelNavigation.test.js` passes
+- shared result status classification treats `OK` and `LLM_COMPLETE` as successful
+- `LLM_INVALID` and non-success verdicts remain issue outcomes
+- regression test `ResultVerdictStatusTest` passes

docs/webstudio/WEBSTUDIO_FUNCTIONAL_SPEC.md

@@ -307,6 +307,7 @@ Header information:
 
 Runtime behavior:
 
+- run controls are enabled when a shared workspace is selected and no Generate execution is already running
 - start calls the server execution endpoint for scriptless generate
 - stop calls the scriptless stop endpoint
 - polling updates console output and sequence outcomes
@@ -321,6 +322,8 @@ Spy Mode supports:
 - widget hover and selection
 - derived/default/direct actions on widgets
 
+Run controls are enabled when a shared workspace is selected and no Spy execution is already running.
+
 Header information:
 
 - current `SUTConnectorValue`
@@ -364,7 +367,7 @@ Dialog actions:
 
 Manual CLI execution may stop a session with plain `stopSession`.
 
-Plain `stopSession` must finalize the session with an `OK` verdict.
+Plain `stopSession` must finalize the session with an `LLM_COMPLETE` verdict.
 
 Agent CLI execution must stop a session with an explicit LLM verdict:
 
@@ -385,13 +388,24 @@ When CLI reporting is enabled, generated report artifacts must include the final
 
 The Test Results page supports inspection of output folders from the shared TESTAR distribution.
 
+Output result folder names must include the execution mode token after the timestamp:
+
+- `generate` for scriptless Generate mode
+- `cli` for CLI mode
+
+Example names:
+
+- `2026-06-29_13h23m23s_generate_webdriver_parabank_1`
+- `2026-06-29_13h23m23s_cli_webdriver_parabank_1`
+
 ### Output Result Selection
 
 The page must support:
 
 - output result folder selection
-- output result folder sorting
-- output result folder filtering
+- output result folder sorting by date or name
+- output result folder filtering by result type
+- output result folder filtering by execution mode
 - generated file listing for the selected output folder
 - generated file filtering
 - verdict outcome summarization
@@ -415,6 +429,18 @@ The summary should include:
 - sequence status overview
 - enough metadata to understand which run is selected
 
+Generate output summaries use `OK` and `FAILED` labels.
+
+`OK` result files are successful outcomes.
+
+`FAILED` result files are issue outcomes.
+
+CLI output summaries use `COMPLETED` and `INVALID` labels.
+
+`LLM_COMPLETE` result files are successful outcomes.
+
+`LLM_INVALID` result files are issue outcomes.
+
 When a generated file is selected, the right preview area must show the corresponding HTML report.
 
 The user must be able to return from the HTML report preview to the run-level verdict outcome summary.
@@ -442,14 +468,17 @@ Expected empty-state behavior:
 Output result folder filtering should support:
 
 - all folders
-- folders with failures
-- folders with only OK sequences
+- folders with issues
+- folders with only successful sequences
+- all execution modes
+- Generate execution mode
+- CLI execution mode
 
 Generated file filtering should support:
 
 - all files
-- failed files
-- OK files
+- issue files
+- successful files
 
 Filtering must not delete files. It only changes visibility in Web Studio.
 

docs/webstudio/WEBSTUDIO_UX_SPEC.md

@@ -239,9 +239,13 @@ Expected behavior:
 
 - the left panel contains `Output Results`
 - `Output Results` reads from the shared Generate and CLI output folder
-- `Output Results` includes sorting and filtering controls
+- `Output Results` includes sorting by date or name
+- `Output Results` includes filtering by the result type
+- `Output Results` can be filtered by execution mode: All, Generate, or CLI
 - the left panel also contains `Generated Files` for the selected output result
 - selecting an output result folder shows a run-level verdict outcome summary in the right preview area
+- Generate summaries use `OK` and `FAILED` wording
+- CLI summaries use `COMPLETED` and `INVALID` wording
 - selecting an output result folder does not automatically select the first generated file
 - selecting a generated file replaces the summary with the HTML report preview
 - the user can return from HTML report preview to the run-level verdict outcome summary

plugin/src/org/testar/plugin/reporting/SessionReportingManager.java

@@ -208,6 +208,7 @@ private static void configureOutputStructure(Settings settings, String target) {
             OutputStructure.executedSUTname = sanitizeTarget(target);
         }
 
+        OutputStructure.setExecutionModeName("cli");
         OutputStructure.createOutputFolders();
     }
 

reporting/src/org/testar/OutputStructure.java

@@ -25,6 +25,7 @@ private OutputStructure() { }
     public static String startInnerLoopDateString;
 
     public static String executedSUTname;
+    public static String executionModeName = "";
     public static int sequenceInnerLoopCount;
 
     public static String outerLoopOutputDir;
@@ -93,9 +94,18 @@ public static void createOutputSUTname(Settings settings) {
         }
     }
 
+    public static void setExecutionModeName(String modeName) {
+        executionModeName = sanitizeOutputName(modeName);
+    }
+
     public static void createOutputFolders() {
+        String outputName = executedSUTname;
+        if (executionModeName != null && !executionModeName.isEmpty()) {
+            outputName = executionModeName + "_" + outputName;
+        }
+
         outerLoopOutputDir = TestarDirectories.getOutputDir() + File.separator + startOuterLoopDateString
-                + "_" + executedSUTname;
+                + "_" + outputName;
         File runDir = new File(outerLoopOutputDir);
         runDir.mkdirs();
 
@@ -141,6 +151,10 @@ public static void createOutputFolders() {
         }
     }
 
+    private static String sanitizeOutputName(String name) {
+        return name == null ? "" : name.trim().replaceAll("[^A-Za-z0-9_-]", "_");
+    }
+
     private static void reportDirectoryCreationFailure(String purpose, File directory) {
         File parent = directory.getParentFile();
         Path cwd = Paths.get(".").toAbsolutePath().normalize();