Merge pull request #468 from TESTARtool/master_verdicts

Refactor Verdict getVerdict to List<Verdict> getVerdicts

Author: ferpasri

.github/workflows/test-windows-webdriver.yml

@@ -49,15 +49,15 @@ jobs:
         name: Java${{ matrix.java }}-runTestWebdriverSuspiciousTagStateModel-artifact
         path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_and_suspicious
 
-    - name: Run webdriver to detect a browser console error in parabank
-      run: ./gradlew runTestWebdriverParabankConsoleError
-    - name: Save runTestWebdriverParabankConsoleError HTML report artifact
+    - name: Run webdriver to detect a browser console error and accessibility warning in parabank
+      run: ./gradlew runTestWebdriverParabankConsoleErrorAndAccessibilityWarning
+    - name: Save runTestWebdriverParabankConsoleErrorAndAccessibilityWarning HTML report artifact
       uses: actions/upload-artifact@v4
       # Only upload GitHub Actions results if this task fails (Can be replaced with 'if: always()')
       if: failure()
       with:
-        name: Java${{ matrix.java }}-runTestWebdriverParabankConsoleError-artifact
-        path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_console_error
+        name: Java${{ matrix.java }}-runTestWebdriverParabankConsoleErrorAndAccessibilityWarning-artifact
+        path: D:/a/TESTAR_dev/TESTAR_dev/testar/target/install/testar/bin/webdriver_console_error_and_accessibility_warning
 
     - name: Run webdriver to login in parabank and detect a welcome suspicious tag
       run: ./gradlew runTestWebdriverParabankLogin

CHANGELOG

@@ -1,3 +1,15 @@
+#TESTAR v2.8.1 (9-Mar-2026)
+- Refactor TESTAR internals from a single Verdict getVerdict to List<Verdict> getVerdicts
+- Report all Verdict issues per state independently
+- Remove Verdict join logic
+- Add a feature and a setting to ignore previously detected Verdicts
+- Remove old disabled Record mode code
+- Move ExtendedOracles logic to DefaultProtocol
+- Remove reflective process-listener handle
+- Fix ExtendedSettingsFile default value
+- Update widget-element description logic
+
+
 #TESTAR v2.7.23 (5-Mar-2026)
 - Update webdriver remote actions descriptions
 - Add webdriver remote actions to LLM conversation

README.md

@@ -110,7 +110,7 @@ Some of the most interesting parameters that can help to integrate TESTAR as an
 
 		ShowVisualSettingsDialogOnStartup -> To run TESTAR without the GUI
 
-		Mode -> TESTAR execution Mode (Spy, Generate, Record, Replay, View)
+		Mode -> TESTAR execution Mode (Spy, Generate, Replay, View)
 
 		SUTConnector & SUTConnectorValue -> The way to link with the desired application to be tested
 

VERSION

@@ -1 +1 @@
-2.7.23
+2.8.1

core/src/org/testar/ProtocolUtil.java

@@ -1,7 +1,7 @@
 /***************************************************************************************************
 *
-* Copyright (c) 2016 - 2025 Universitat Politecnica de Valencia - www.upv.es
-* Copyright (c) 2019 - 2025 Open Universiteit - www.ou.nl
+* Copyright (c) 2016 - 2026 Universitat Politecnica de Valencia - www.upv.es
+* Copyright (c) 2019 - 2026 Open Universiteit - www.ou.nl
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
@@ -49,6 +49,7 @@
 import org.testar.monkey.alayer.Widget;
 
 import java.awt.*;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -251,7 +252,8 @@ public static AWTCanvas getStateshotBinary(State state) {
 		}
 
 		//If the state Shape is not properly obtained, or the State has an error, use full monitor screen
-		if (viewPort == null || (state.get(Tags.OracleVerdict, Verdict.OK).severity() > Verdict.Severity.OK.getValue()))
+		List<Verdict> verdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK));
+		if (viewPort == null || !Verdict.helperAreAllVerdictsOK(verdicts))
 			viewPort = state.get(Tags.Shape, null); // get the SUT process canvas (usually, full monitor screen)
 
 		// Validate viewport dimensions before taking the screenshot

core/src/org/testar/monkey/alayer/Tags.java

@@ -1,7 +1,7 @@
 /***************************************************************************************************
 *
-* Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es
-* Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl
+* Copyright (c) 2013 - 2026 Universitat Politecnica de Valencia - www.upv.es
+* Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
@@ -28,10 +28,6 @@
 * POSSIBILITY OF SUCH DAMAGE.
 *******************************************************************************************************/
 
-/**
- *  @author Sebastian Bauersfeld
- */
-
 package org.testar.monkey.alayer;
 
 import org.testar.CodingManager;
@@ -161,9 +157,9 @@ private Tags() {}
 	/** Usually attached to an object of {@link State}. The value is a screenshot of the state. */
 	public static final Tag<String> ScreenshotPath = from("ScreenshotPath", String.class);
 
-	/** Usually attached to a {@link State} object. The value is an outcome of a test oracle for that state. It is
-	 * used to mark states as 'suspicious' or 'erroneous' */
-	public static final Tag<Verdict> OracleVerdict = from("OracleVerdict", Verdict.class);
+	/** Usually attached to a {@link State} object. The value is a list of outcomes of test oracles for that state. */
+	@SuppressWarnings("unchecked")
+	public static final Tag<List<Verdict>> OracleVerdicts = from("OracleVerdicts", (Class<List<Verdict>>)(Class<?>)List.class);
 
 	/** The standard mouse object. Usually attached to systems */
 	public static final Tag<Mouse> StandardMouse = from("StandardMouse", Mouse.class);

core/src/org/testar/monkey/alayer/Verdict.java

@@ -31,7 +31,9 @@
 package org.testar.monkey.alayer;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import org.testar.monkey.Assert;
 import org.testar.monkey.Util;
@@ -93,12 +95,16 @@ public enum Severity {
 		// WARNING GROUP: ACCESSIBILITY
 		WARNING_ACCESSIBILITY_FAULT(0.4, "WARNING_ACCESSIBILITY_FAULT"),
 
-		/** FAIL (0.5 - 1.0) **/
+		/** FAIL (0.5 - 0.899) **/
 
 		UNREPLAYABLE(0.5, "UNREPLAYABLE"), // Sequence not replayable
 		SUSPICIOUS_TAG(0.8, "SUSPICIOUS_TAG"), // Suspicious tag
 		SUSPICIOUS_PROCESS(0.87, "SUSPICIOUS_PROCESS"), // Suspicious message in the process standard output/error
 		SUSPICIOUS_LOG(0.89, "SUSPICIOUS_LOG"), // Suspicious message in log file or command output (LogOracle)
+
+		/** CRITICAL (0.9 - 1.0) **/
+
+		CRITICAL(0.9, "CRITICAL"),
 		NOT_RESPONDING(0.99999990, "NOT_RESPONDING"), // Unresponsive
 		UNEXPECTEDCLOSE(0.99999999, "UNEXPECTEDCLOSE"), // Crash? Unexpected close?
 		FAIL(1.0, "FAIL");
@@ -193,29 +199,22 @@ public Visualizer visualizer() {
 		return visualizer;
 	}
 
-	@Override
-	public String toString() {
-		return "severity: " + severity + " info: " + info;
-	}
-
 	/**
-	 * Retrieves the verdict result of joining two verdicts.
-	 * @param verdict A verdict to join with current verdict.
-	 * 
-	 * @return A new verdict that is the result of joining the current verdict with the provided verdict.
+	 * Indicates if this verdict is critical (SUT froze or crashed).
+	 *
+	 * @return true if verdict is critical
 	 */
-	public Verdict join(Verdict verdict) {
-		Severity joinedSeverity = Arrays.stream(Severity.values())
-				.filter(s -> s.getValue() == Math.max(this.severity, verdict.severity()))
-				.findFirst()
-				.orElse(Severity.FAIL);
-
-		String joinedInfo = this.info.contains(verdict.info()) ? this.info
-				: (this.severity == Severity.OK.getValue() ? "" : this.info + "\n") + verdict.info();
+	public boolean isCritical() {
+		return this.severity() >= Severity.CRITICAL.getValue();
+	}
 
-		Visualizer joinedVisualizer = Visualizer.join(this.visualizer(), verdict.visualizer());
+	public boolean isCompletion() {
+		return this.severity() == Severity.LLM_COMPLETE.getValue() || this.severity() == Severity.CONDITION_COMPLETE.getValue();
+	}
 
-		return new Verdict(joinedSeverity, joinedInfo, joinedVisualizer);
+	@Override
+	public String toString() {
+		return "severity: " + severity + " info: " + info;
 	}
 
 	/**
@@ -234,4 +233,15 @@ public boolean equals(Object o) {
 				&& this.visualizer.equals(other.visualizer);
 	}
 
+	public static boolean helperAreAllVerdictsOK(List<Verdict> verdicts) {
+		if(verdicts == null || verdicts.isEmpty()) return true;
+
+		for (Verdict verdict : verdicts) {
+			if (verdict.severity() > Severity.OK.getValue()) {
+				return false;
+			}
+		}
+		return true;
+	}
+
 }

core/test/org/testar/monkey/alayer/VerdictTest.java

@@ -35,30 +35,16 @@
 import java.util.Arrays;
 
 import org.junit.Test;
-import org.testar.monkey.alayer.visualizers.RegionsVisualizer;
 import org.testar.monkey.alayer.visualizers.ShapeVisualizer;
 
 public class VerdictTest {
 
-	private final double DELTA = 0;
-
-	private final Visualizer dummyVisualizer = new Visualizer(){
-		private static final long serialVersionUID = -7830649624698071090L;
-		public void run(State s, Canvas c, Pen pen) {}	
-	};
-
 	private final Visualizer failVisualizer = new ShapeVisualizer(
 					Pen.PEN_RED, 
 					Rect.from(0, 0, 10, 10), 
 					"Fail Visualizer", 
 					0.5, 0.5);
 
-	private final Visualizer issueVisualizer = new RegionsVisualizer(
-					Pen.PEN_RED, 
-					Arrays.asList(Rect.from(0, 0, 10, 10)), 
-					"Issue Visualizer", 
-					0.5, 0.5);
-
 	@Test
 	public void testToString() {
 		Verdict v = new Verdict(Verdict.Severity.OK, "This is a test verdict");
@@ -67,54 +53,12 @@ public void testToString() {
 	}
 
 	@Test
-	public void testJoin() {
-		Verdict v1 = new Verdict(Verdict.Severity.OK, "Foo Bar");
-		Verdict v2 = new Verdict(Verdict.Severity.FAIL, "Bar", failVisualizer);
-		Verdict v3 = new Verdict(Verdict.Severity.OK, "Baz", dummyVisualizer);
-		Verdict v4 = new Verdict(Verdict.Severity.FAIL, "Exception", issueVisualizer);
-		Verdict emptyVisualizerVerdict = new Verdict(Verdict.Severity.FAIL, "Empty");
-
-		assertTrue("Joining two Verdicts shall create a new Verdict", 
-				v1 != v1.join(v2));
-
-		assertEquals("Joining two Verdicts shall set the severity to the maximum of both",
-				Verdict.Severity.FAIL.getValue(), v3.join(v2).severity(), DELTA);
-
-		assertEquals("If a Verdict's info contains the info of the Verdict to be joined with, " +
-				"then only the containing info shall be used",
-				"Foo Bar", v1.join(v2).info());
-
-		assertEquals("If a Verdict is OK and its info does not contain the info of the Verdict to be joined with, " +
-				"then the containing info shall be discarded",
-				"Baz", v1.join(v3).info());
-
-		assertEquals("If a Verdict is not OK and its info does not contain the info of the Verdict to be joined with, " +
-				"then both infos shall be included separated by a line break",
-				"Bar\nBaz", v2.join(v3).info());
-
-		assertTrue("Joining an OK and Fail Verdicts shall use the Visualizer of the Verdict with high severity",
-				v2.join(v1).visualizer() == failVisualizer);
-
-		assertTrue("Joining an OK and Fail Verdicts shall use the Visualizer of the Verdict with high severity",
-				v1.join(v2).visualizer() == failVisualizer);
-
-		assertTrue("Joining Fail and Issue Verdicts must contain Fail Shapes",
-				v2.join(v4).visualizer().getShapes().containsAll(failVisualizer.getShapes()));
-
-		assertTrue("Joining Issue and Fail Verdicts must contain Fail Shapes",
-				v4.join(v2).visualizer().getShapes().containsAll(failVisualizer.getShapes()));
-
-		assertTrue("Joining Fail and Issue Verdicts must contain Issue Shapes",
-				v2.join(v4).visualizer().getShapes().containsAll(issueVisualizer.getShapes()));
-
-		assertTrue("Joining Issue and Fail Verdicts must contain Issue Shapes",
-				v4.join(v2).visualizer().getShapes().containsAll(issueVisualizer.getShapes()));
-
-		assertTrue("Joining Fail and emptyVisualizerVerdict Verdicts must equal Fail visualizer",
-				v2.join(emptyVisualizerVerdict).visualizer() == failVisualizer);
+	public void testVerdictHelperAreOK() {
+		Verdict ok = Verdict.OK;
+		Verdict warn = new Verdict(Verdict.Severity.SUSPICIOUS_TAG, "Issue", failVisualizer);
 
-		assertTrue("Joining emptyVisualizerVerdict and Fail Verdicts must equal Fail visualizer",
-				emptyVisualizerVerdict.join(v2).visualizer() == failVisualizer);
+		assertTrue(Verdict.helperAreAllVerdictsOK(Arrays.asList(ok)));
+		assertFalse(Verdict.helperAreAllVerdictsOK(Arrays.asList(warn)));
 	}
 
 	@Test

statemodel/src/org/testar/statemodel/persistence/orientdb/hydrator/ConcreteStateHydrator.java

@@ -1,7 +1,7 @@
 /***************************************************************************************************
  *
- * Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl
- * Copyright (c) 2018 - 2025 Universitat Politecnica de Valencia - www.upv.es
+ * Copyright (c) 2018 - 2026 Open Universiteit - www.ou.nl
+ * Copyright (c) 2018 - 2026 Universitat Politecnica de Valencia - www.upv.es
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -39,6 +39,10 @@
 import org.testar.statemodel.persistence.orientdb.entity.TypeConvertor;
 import org.testar.statemodel.persistence.orientdb.entity.VertexEntity;
 import org.testar.statemodel.persistence.orientdb.util.Validation;
+
+import java.util.Collections;
+import java.util.List;
+
 import org.testar.monkey.alayer.Tag;
 import org.testar.monkey.alayer.TaggableBase;
 import org.testar.monkey.alayer.Tags;
@@ -89,18 +93,13 @@ public void hydrate(VertexEntity target, Object source) throws HydrationExceptio
         }
 
         // get the oracle verdict and transform it into a custom code
-        Verdict verdict = attributes.get(Tags.OracleVerdict, null);
+        List<Verdict> verdicts = attributes.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK));
         int oracleVerdictCode = 4; // default value
-        if (verdict != null) {
-            if (verdict.severity() == Verdict.Severity.OK.getValue()) {
-                oracleVerdictCode = 1;
-            }
-            else if (verdict.severity() == Verdict.Severity.FAIL.getValue()) {
-                oracleVerdictCode = 2;
-            }
-            else if (verdict.severity() > Verdict.Severity.OK.getValue() && verdict.severity() < Verdict.Severity.FAIL.getValue()) {
-                oracleVerdictCode = 3;
-            }
+        if (Verdict.helperAreAllVerdictsOK(verdicts)) {
+            oracleVerdictCode = 1;
+        }
+        else {
+            oracleVerdictCode = 2;
         }
         target.addPropertyValue("oracleVerdictCode", new PropertyValue(OType.INTEGER, oracleVerdictCode));
     }

testar/resources/settings/01_desktop_calculator/Protocol_01_desktop_calculator.java

@@ -1,7 +1,7 @@
 /***************************************************************************************************
  *
- * Copyright (c) 2020 - 2023 Universitat Politecnica de Valencia - www.upv.es
- * Copyright (c) 2020 - 2023 Open Universiteit - www.ou.nl
+ * Copyright (c) 2020 - 2026 Universitat Politecnica de Valencia - www.upv.es
+ * Copyright (c) 2020 - 2026 Open Universiteit - www.ou.nl
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -41,6 +41,7 @@
 import org.testar.protocols.DesktopProtocol;
 import org.testar.settings.Settings;
 
+import java.util.List;
 import java.util.Set;
 
 /**
@@ -118,18 +119,18 @@ protected State getState(SUT system) throws StateBuildException{
 	 * @return oracle verdict, which determines whether the state is erroneous and why.
 	 */
 	@Override
-	protected Verdict getVerdict(State state){
+	protected List<Verdict> getVerdicts(State state){
 		// The super methods implements the implicit online state oracles for:
 		// system crashes
 		// non-responsiveness
 		// suspicious tags
-		Verdict verdict = super.getVerdict(state);
+		List<Verdict> verdicts = super.getVerdicts(state);
 
 		//--------------------------------------------------------
 		// MORE SOPHISTICATED STATE ORACLES CAN BE PROGRAMMED HERE
 		//--------------------------------------------------------
 
-		return verdict;
+		return verdicts;
 	}
 
 	/**