Add CLI verdicts when stopping a test session

Author: ferpasri

agent/resources/skills/testar-cli/SKILL.md

@@ -38,7 +38,8 @@ Treat the CLI distribution as an operational environment, not as a source-code w
 - `executeAction click <semanticText>`
 - `executeAction type <semanticText> <inputText>`
 - `executeAction select <semanticText> <value>`
-- `stopSession`
+- `stopSession LLM_COMPLETE <reason>`
+- `stopSession LLM_INVALID <reason>`
 - `shutdownDaemon`
 
 ## Semantic action matching
@@ -68,6 +69,19 @@ Use short meaningful text that is likely to appear in the action description, fo
 6. Stop the session when the goal is finished.
 7. Shutdown the daemon after stopping the session.
 
+## Test goal verdicts
+
+Agent CLI executions must finish each test goal with exactly one explicit verdict command:
+
+- `stopSession LLM_COMPLETE <reason>`
+- `stopSession LLM_INVALID <reason>`
+
+Use `LLM_COMPLETE` only when the requested test goal was completed and the expected result was verified with CLI evidence.
+
+Use `LLM_INVALID` when the expected result is not verified, a bug or invalid state is observed, the goal cannot be completed, or the available evidence is not reliable enough to declare completion.
+
+The `<reason>` must briefly explain the final decision using observed CLI evidence.
+
 ## Resilience and retry behavior rules
 
 Some SUTs may need time to start, load screens, populate state information, or expose derived actions after an action is executed.
@@ -100,7 +114,7 @@ Workspace-driven Windows testing:
 - `testar-cli.bat getDerivedActions`
 - `testar-cli.bat executeAction click Open`
 - `testar-cli.bat executeAction type Editor Writing_text_in_Notepad`
-- `testar-cli.bat stopSession`
+- `testar-cli.bat stopSession LLM_COMPLETE "Notepad opened and text was entered successfully."`
 - `testar-cli.bat shutdownDaemon`
 
 Workspace-driven WebDriver testing:
@@ -113,7 +127,7 @@ Workspace-driven WebDriver testing:
 - `testar-cli.bat executeAction click a_about`
 - `testar-cli.bat executeAction type input_contact This_is_a_message`
 - `testar-cli.bat executeAction select select_amount 99999`
-- `testar-cli.bat stopSession`
+- `testar-cli.bat stopSession LLM_INVALID "There is no confirmation that the transaction was sent successfully."`
 - `testar-cli.bat shutdownDaemon`
 
 Workspace-driven Android testing:
@@ -125,7 +139,7 @@ Workspace-driven Android testing:
 - `testar-cli.bat getDerivedActions`
 - `testar-cli.bat executeAction click Graphics`
 - `testar-cli.bat executeAction type username Writing_text_in_Android`
-- `testar-cli.bat stopSession`
+- `testar-cli.bat stopSession LLM_COMPLETE "The expected Android state was verified."`
 - `testar-cli.bat shutdownDaemon`
 
 Explicit standalone startup wihtout workspace:

cli/src/org/testar/cli/CliDaemonServer.java

@@ -26,6 +26,7 @@
 import org.testar.core.state.State;
 import org.testar.core.state.Widget;
 import org.testar.core.tag.Tags;
+import org.testar.core.verdict.Verdict;
 import org.testar.cli.profile.CliProfileConfiguration;
 import org.testar.cli.profile.CliProfileConfigurationLoader;
 import org.testar.plugin.OperatingSystems;
@@ -105,7 +106,7 @@ private CliResponse handle(CliRequest request) {
             case EXECUTE_ACTION:
                 return executeAction(request);
             case STOP_SESSION:
-                return stopSession();
+                return stopSession(request);
             case SHUTDOWN_DAEMON:
                 return shutdownDaemon();
             case HELP:
@@ -125,9 +126,6 @@ private CliResponse startSession(CliRequest request) {
             lines.add("platform=" + sessionSpec.getOperatingSystem().name().toLowerCase(Locale.ROOT));
             lines.add("pid=" + activeSession.system().get(Tags.PID, -1L));
             lines.add("desc=" + activeSession.system().get(Tags.Desc, ""));
-            for (String warning : preparedSession.profileConfiguration().compatibilityWarnings()) {
-                lines.add("warning=" + warning);
-            }
             return new CliResponse(0, lines);
         } catch (RuntimeException exception) {
             return new CliResponse(1, List.of("startSession failed: " + exception.getMessage()));
@@ -209,15 +207,22 @@ private CliResponse executeAction(CliRequest request) {
         }
     }
 
-    private CliResponse stopSession() {
+    private CliResponse stopSession(CliRequest request) {
         try {
+            List<Verdict> finalVerdicts = finalVerdictsFromStopRequest(request);
             PlatformSession session = requireActiveSession();
             long pid = session.system().get(Tags.PID, -1L);
-            session.stopSystem();
-            closeActiveSession();
+            try {
+                session.stopSystem(finalVerdicts);
+            } finally {
+                closeStoppedSession(session);
+            }
+            Verdict finalVerdict = finalVerdicts.get(0);
             return new CliResponse(0, List.of(
                     "systemStopped",
-                    "pid=" + pid
+                    "pid=" + pid,
+                    "verdict=" + finalVerdict.verdictSeverityTitle(),
+                    "info=" + finalVerdict.info()
             ));
         } catch (RuntimeException exception) {
             return new CliResponse(1, List.of("stopSession failed: " + exception.getMessage()));
@@ -271,6 +276,18 @@ private synchronized void closeActiveSession() {
         }
     }
 
+    private synchronized void closeStoppedSession(PlatformSession session) {
+        try {
+            session.close();
+        } finally {
+            if (activeSession == session) {
+                activeSession = null;
+                activeSessionSpec = null;
+                activePolicyConfiguration = PolicySessionConfiguration.defaults();
+            }
+        }
+    }
+
     private void shutdownIfRequested() {
         if (!shutdownRequested) {
             return;
@@ -400,6 +417,42 @@ private OperatingSystems parseOperatingSystem(String token) {
         throw new IllegalArgumentException("Unsupported platform token: " + token);
     }
 
+    List<Verdict> finalVerdictsFromStopRequest(CliRequest request) {
+        String verdictName = request.argumentAt(0);
+        if (verdictName == null || verdictName.isBlank()) {
+            return List.of(Verdict.OK);
+        }
+
+        Verdict.Severity severity = parseFinalVerdictSeverity(verdictName);
+        String info = buildFinalVerdictInfo(request, severity);
+        return List.of(new Verdict(severity, info));
+    }
+
+    private Verdict.Severity parseFinalVerdictSeverity(String verdictName) {
+        String normalizedVerdictName = verdictName.trim().toUpperCase(Locale.ROOT);
+        if (Verdict.Severity.LLM_COMPLETE.name().equals(normalizedVerdictName)) {
+            return Verdict.Severity.LLM_COMPLETE;
+        }
+        if (Verdict.Severity.LLM_INVALID.name().equals(normalizedVerdictName)) {
+            return Verdict.Severity.LLM_INVALID;
+        }
+
+        throw new IllegalArgumentException(
+                "Unsupported stopSession verdict: " + verdictName + ". Expected LLM_COMPLETE or LLM_INVALID."
+        );
+    }
+
+    private String buildFinalVerdictInfo(CliRequest request, Verdict.Severity severity) {
+        List<String> arguments = request.getArguments();
+        if (arguments.size() <= 1) {
+            return severity == Verdict.Severity.LLM_COMPLETE
+                    ? "Agent completed the test goal."
+                    : "Agent marked the test goal as invalid.";
+        }
+
+        return String.join(" ", arguments.subList(1, arguments.size())).trim();
+    }
+
     private List<String> describeStateWidgets(State state) {
         List<String> descriptions = new ArrayList<>();
         for (Widget widget : state) {

cli/src/org/testar/cli/CliRunner.java

@@ -57,7 +57,7 @@ private void printHelp() {
         output.println("  executeAction click <semanticText>");
         output.println("  executeAction type <semanticText> <inputText>");
         output.println("  executeAction select <semanticText> <value>");
-        output.println("  stopSession");
+        output.println("  stopSession [LLM_COMPLETE|LLM_INVALID <reason>]");
         output.println("  shutdownDaemon");
     }
 

cli/test/org/testar/cli/CliDaemonServerTest.java

@@ -8,6 +8,7 @@
 import org.testar.config.ConfigTags;
 import org.testar.config.settings.Settings;
 import org.testar.config.settings.SettingsDefaults;
+import org.testar.core.verdict.Verdict;
 import org.testar.plugin.OperatingSystems;
 import org.testar.plugin.configuration.PlatformSessionSpecification;
 
@@ -141,6 +142,60 @@ public void resolveCliTargetRejectsDesktopAttachConnector() {
         }
     }
 
+    @Test
+    public void stopSessionWithoutVerdictDefaultsToOk() {
+        CliDaemonServer server = new CliDaemonServer();
+
+        List<Verdict> verdicts = server.finalVerdictsFromStopRequest(
+                CliRequest.of(CliCommand.STOP_SESSION, List.of())
+        );
+
+        Assert.assertEquals(1, verdicts.size());
+        Assert.assertEquals(Verdict.OK, verdicts.get(0));
+    }
+
+    @Test
+    public void stopSessionAcceptsLlmCompleteWithReason() {
+        CliDaemonServer server = new CliDaemonServer();
+
+        List<Verdict> verdicts = server.finalVerdictsFromStopRequest(
+                CliRequest.of(CliCommand.STOP_SESSION, List.of("LLM_COMPLETE", "Login", "verified"))
+        );
+
+        Assert.assertEquals(1, verdicts.size());
+        Assert.assertEquals(Verdict.Severity.LLM_COMPLETE.getValue(), verdicts.get(0).severity(), 0.0);
+        Assert.assertEquals("Login verified", verdicts.get(0).info());
+    }
+
+    @Test
+    public void stopSessionAcceptsLlmInvalidWithReason() {
+        CliDaemonServer server = new CliDaemonServer();
+
+        List<Verdict> verdicts = server.finalVerdictsFromStopRequest(
+                CliRequest.of(CliCommand.STOP_SESSION, List.of("LLM_INVALID", "Welcome message was not shown"))
+        );
+
+        Assert.assertEquals(1, verdicts.size());
+        Assert.assertEquals(Verdict.Severity.LLM_INVALID.getValue(), verdicts.get(0).severity(), 0.0);
+        Assert.assertEquals("Welcome message was not shown", verdicts.get(0).info());
+    }
+
+    @Test
+    public void stopSessionRejectsUnsupportedFinalVerdict() {
+        CliDaemonServer server = new CliDaemonServer();
+
+        try {
+            server.finalVerdictsFromStopRequest(
+                    CliRequest.of(CliCommand.STOP_SESSION, List.of("WARNING", "Not valid for agent finalization"))
+            );
+            Assert.fail("Expected unsupported stopSession verdict failure");
+        } catch (IllegalArgumentException exception) {
+            Assert.assertTrue(exception.getMessage().contains("Unsupported stopSession verdict"));
+            Assert.assertTrue(exception.getMessage().contains("LLM_COMPLETE"));
+            Assert.assertTrue(exception.getMessage().contains("LLM_INVALID"));
+        }
+    }
+
     private static Settings defaultSettings() {
         Properties properties = new Properties();
         properties.setProperty(ConfigTags.SUTConnectorValue.name(), "notepad.exe");

docs/architecture_distribution_workspace.md

@@ -142,7 +142,36 @@ This prevents CLI from failing only because unrelated scriptless-only Java files
 
 Oracle composer entries are scriptless-only.
 
-CLI agent runs should use the agent as the oracle for LLM-complete verdicts, such as `LLM_COMPLETE_VALID` and `LLM_COMPLETE_INVALID`.
+CLI agent runs should use the agent as the oracle for LLM verdicts, such as `LLM_COMPLETE` and `LLM_INVALID`.
+
+## CLI Session Verdict Contract
+
+`stopSession` is the CLI session finalization command.
+
+Manual CLI execution may use:
+
+```text
+stopSession
+```
+
+Plain `stopSession` defaults the final session verdict to `OK`.
+
+Agent CLI execution must finish each test goal with an explicit LLM verdict:
+
+```text
+stopSession LLM_COMPLETE <reason>
+stopSession LLM_INVALID <reason>
+```
+
+`LLM_COMPLETE` means the agent completed the requested test goal and verified the expected result with CLI evidence.
+
+`LLM_INVALID` means the agent reached an invalid result, found a bug, could not verify the expected result, or could not complete the goal with reliable CLI evidence.
+
+The reason is free text and should explain the final decision.
+
+If reporting is enabled, the final CLI verdict must be written to the generated report artifacts.
+
+The `testar-cli` skill documentation is the operational source that tells agents how to use this contract.
 
 ## Shared Settings
 

docs/webstudio/WEBSTUDIO_FUNCTIONAL_SPEC.md

@@ -360,6 +360,27 @@ Dialog actions:
 - `Discard`: restore the last persisted Agent CLI settings in the CLI panel, then continue the pending action
 - `Cancel`: keep the user in CLI mode and abort the pending action
 
+### CLI Verdict Finalization
+
+Manual CLI execution may stop a session with plain `stopSession`.
+
+Plain `stopSession` must finalize the session with an `OK` verdict.
+
+Agent CLI execution must stop a session with an explicit LLM verdict:
+
+- `stopSession LLM_COMPLETE <reason>`
+- `stopSession LLM_INVALID <reason>`
+
+`LLM_COMPLETE` means the agent completed the test goal and verified the expected result.
+
+`LLM_INVALID` means the agent found an invalid result, could not verify the expected result, or could not complete the goal with reliable evidence.
+
+WebStudio must include this finalization rule in the Agent CLI prompt contract.
+
+If an Agent CLI execution ends without an explicit LLM verdict, WebStudio should treat the execution as incomplete or invalid and surface that status to the user.
+
+When CLI reporting is enabled, generated report artifacts must include the final CLI verdict and reason.
+
 ## Test Results
 
 The Test Results page supports inspection of output folders from the shared TESTAR distribution.

plugin/src/org/testar/plugin/DefaultPlatformSession.java

@@ -14,6 +14,7 @@
 import org.testar.core.action.resolver.ResolvedAction;
 import org.testar.core.state.SUT;
 import org.testar.core.state.State;
+import org.testar.core.verdict.Verdict;
 import org.testar.plugin.reporting.SessionReportingManager;
 
 final class DefaultPlatformSession implements PlatformSession {
@@ -88,7 +89,7 @@ public boolean executeAction(Action action) {
     }
 
     @Override
-    public void stopSystem() {
+    public void stopSystem(List<Verdict> finalVerdicts) {
         try {
             State state = services.stateService().getState(system);
             if (reportingEnabled) {
@@ -104,7 +105,7 @@ public void stopSystem() {
             services.systemService().stopSystem(system);
         } finally {
             if (reportingEnabled) {
-                sessionReportingManager.finish();
+                sessionReportingManager.finish(finalVerdicts);
             }
         }
     }

plugin/src/org/testar/plugin/PlatformSession.java

@@ -6,13 +6,14 @@
 
 package org.testar.plugin;
 
-import java.util.Set;
 import java.util.List;
+import java.util.Set;
 
 import org.testar.core.action.Action;
 import org.testar.core.action.resolver.ResolvedAction;
 import org.testar.core.state.SUT;
 import org.testar.core.state.State;
+import org.testar.core.verdict.Verdict;
 
 /**
  * Live platform session holding the started system plus the composed services
@@ -30,7 +31,11 @@ public interface PlatformSession extends AutoCloseable {
 
     boolean executeAction(Action action);
 
-    void stopSystem();
+    default void stopSystem() {
+        stopSystem(List.of(Verdict.OK));
+    }
+
+    void stopSystem(List<Verdict> finalVerdicts);
 
     @Override
     void close();