Merge pull request #453 from TESTARtool/llm_development

ferpasri · web-flow · commit 7f68674e65a4 · 2025-10-24T09:23:02.000+02:00
Add reasoning effort to OpenAI API
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,8 @@
+#TESTAR v2.7.13 (24-Oct-2025)
+- Add reasoning effort to OpenAI API
+- Update corresponding llm protocols
+
+
 #TESTAR v2.7.12 (22-Oct-2025)
 - Bump org.seleniumhq.selenium:selenium-java from 4.36.0 to 4.37.0
 - Update devtools dependencies to v141
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.7.12
+2.7.13
diff --git a/testar/resources/settings/03_webdriver_llm_parabank/test.settings b/testar/resources/settings/03_webdriver_llm_parabank/test.settings
@@ -168,7 +168,8 @@ AbstractStateAttributes = WebWidgetId,WebWidgetName
 #################################################################
 
 LlmPlatform = OpenAI
-LlmModel = gpt-4o-mini
+LlmModel = gpt-5-mini
+LlmReasoning = minimal
 LlmHostUrl = https://api.openai.com/v1/chat/completions
 LlmAuthorizationHeader = Bearer %OPENAI_API%
 LlmActionFewshotFile = prompts/fewshot_openai_action.json
diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/test.settings b/testar/resources/settings/webdriver_llm_state_model_evaluator/test.settings
@@ -176,7 +176,8 @@ AbstractStateAttributes = WebWidgetId,WebWidgetName,WebWidgetStyle
 #################################################################
 
 LlmPlatform = OpenAI
-LlmModel = gpt-4o-mini
+LlmModel = gpt-5-mini
+LlmReasoning = minimal
 LlmHostUrl = https://api.openai.com/v1/chat/completions
 LlmAuthorizationHeader = Bearer %OPENAI_API%
 LlmActionFewshotFile = prompts/fewshot_openai_action.json
diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/test.settings b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/test.settings
@@ -176,7 +176,8 @@ AbstractStateAttributes = WebWidgetId,WebWidgetName,WebWidgetStyle
 #################################################################
 
 LlmPlatform = OpenAI
-LlmModel = gpt-4o-mini
+LlmModel = gpt-5-mini
+LlmReasoning = minimal
 LlmHostUrl = https://api.openai.com/v1/chat/completions
 LlmAuthorizationHeader = Bearer %OPENAI_API%
 LlmActionFewshotFile = prompts/fewshot_openai_action.json
diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings
@@ -176,7 +176,8 @@ AbstractStateAttributes = WebWidgetId,WebWidgetName
 #################################################################
 
 LlmPlatform = OpenAI
-LlmModel = gpt-4o-mini
+LlmModel = gpt-5-mini
+LlmReasoning = minimal
 LlmHostUrl = https://api.openai.com/v1/chat/completions
 LlmAuthorizationHeader = Bearer %OPENAI_API%
 LlmActionFewshotFile = prompts/fewshot_openai_action.json
diff --git a/testar/src/org/testar/action/priorization/llm/LlmActionSelector.java b/testar/src/org/testar/action/priorization/llm/LlmActionSelector.java
@@ -66,6 +66,7 @@ public class LlmActionSelector implements IActionSelector {
 
     private final String platform;
     private final String model;
+    private final String reasoning;
     private final String hostUrl;
     private final String authorizationHeader;
     private final String actionFewshotFile;
@@ -79,7 +80,6 @@ public class LlmActionSelector implements IActionSelector {
     private int tokens_used;
     private Integer invalidActions;
 
-    private Gson gson = new Gson();
     private String previousTestGoal = "";
     private LlmTestGoal currentTestGoal;
 
@@ -97,6 +97,7 @@ public LlmActionSelector(Settings settings, IPromptActionGenerator generator) {
 
         this.platform = settings.get(ConfigTags.LlmPlatform);
         this.model = settings.get(ConfigTags.LlmModel);
+        this.reasoning = settings.get(ConfigTags.LlmReasoning);
         this.hostUrl = settings.get(ConfigTags.LlmHostUrl);
         this.authorizationHeader = settings.get(ConfigTags.LlmAuthorizationHeader);
         this.historySize = settings.get(ConfigTags.LlmHistorySize);
@@ -110,7 +111,7 @@ public LlmActionSelector(Settings settings, IPromptActionGenerator generator) {
     }
 
     private void initializeConversation() {
-        conversation = LlmFactory.createLlmConversation(this.platform, this.model, this.temperature);
+        conversation = LlmFactory.createLlmConversation(this.platform, this.model, this.reasoning, this.temperature);
         conversation.initConversation(this.actionFewshotFile);
     }
 
@@ -171,9 +172,9 @@ private Action selectActionWithLlm(State state, Set<Action> actions) {
         logger.log(Level.DEBUG, "Generated prompt: " + prompt);
         conversation.addMessage("user", prompt);
 
-        String conversationJson = gson.toJson(conversation);
+        String conversationJson = conversation.buildRequestBody();
         String llmResponse = getResponseFromLlm(conversationJson);
-        LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(gson);
+        LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(new Gson());
         LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(actions, llmResponse);
 
         switch(llmParseResult.getParseResult()) {
diff --git a/testar/src/org/testar/llm/LlmConversation.java b/testar/src/org/testar/llm/LlmConversation.java
@@ -44,6 +44,8 @@
 public interface LlmConversation {
     static final Logger logger = LogManager.getLogger();
 
+    public String buildRequestBody();
+
     public void initConversation(String fewshotFile);
 
     public void addMessage(String role, String textContent);
diff --git a/testar/src/org/testar/llm/LlmFactory.java b/testar/src/org/testar/llm/LlmFactory.java
@@ -50,10 +50,10 @@ public class LlmFactory {
      * @param temperature Lower values result in more predictable output, usually between 0-1f.
      * @return LlmConversation for the chosen platform.
      */
-    public static LlmConversation createLlmConversation(String platform, String model, float temperature) {
+    public static LlmConversation createLlmConversation(String platform, String model, String reasoning, float temperature) {
         switch (platform) {
         case "OpenAI":
-            return new LlmConversationOpenAI(model, temperature);
+            return new LlmConversationOpenAI(model, reasoning, temperature);
         case "Gemini":
             return new LlmConversationGemini();
         default:
diff --git a/testar/src/org/testar/llm/gemini/LlmConversationGemini.java b/testar/src/org/testar/llm/gemini/LlmConversationGemini.java
@@ -41,6 +41,7 @@
 import org.testar.llm.LlmConversation;
 
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 
 /**
  * Conversation with the Gemini LLM.
@@ -65,6 +66,12 @@ public List<Content> getContents() {
         return contents;
     }
 
+    @Override
+    public String buildRequestBody() {
+        // Serialization for Gemini
+        return new GsonBuilder().create().toJson(this);
+    }
+
     @Override
     public void initConversation(String fewshotFile) {
         try {
diff --git a/testar/src/org/testar/llm/openai/LlmConversationOpenAI.java b/testar/src/org/testar/llm/openai/LlmConversationOpenAI.java
@@ -40,6 +40,8 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
 import com.google.gson.annotations.JsonAdapter;
 import com.google.gson.reflect.TypeToken;
 
@@ -51,15 +53,17 @@ public class LlmConversationOpenAI implements LlmConversation {
     protected static final Logger logger = LogManager.getLogger();
 
     private String model;
-    private String format;
+    private String reasoning_effort;
+    private ResponseFormat response_format;
     private List<Message> messages;
     private float temperature = 0.2f;
     private Integer max_tokens = null;
     private boolean stream = false;
 
-    public LlmConversationOpenAI(String model, float temperature) {
+    public LlmConversationOpenAI(String model, String reasoning_effort, float temperature) {
         this.model = model;
-        this.format = "json";
+        this.reasoning_effort = reasoning_effort;
+        this.response_format = new ResponseFormat("json_object");
         this.messages = new ArrayList<>();
         this.temperature = temperature;
     }
@@ -72,12 +76,20 @@ public void setModel(String model) {
         this.model = model;
     }
 
-    public String getFormat() {
-        return format;
+    public String getReasoningEffort() {
+        return reasoning_effort;
     }
 
-    public void setFormat(String format) {
-        this.format = format;
+    public void setReasoningEffort(String reasoning_effort) {
+        this.reasoning_effort = reasoning_effort;
+    }
+
+    public ResponseFormat getResponseFormat() {
+        return response_format;
+    }
+
+    public void setResponseFormat(ResponseFormat response_format) {
+        this.response_format = response_format;
     }
 
     public List<Message> getMessages() {
@@ -112,6 +124,12 @@ public void setStream(boolean stream) {
         this.stream = stream;
     }
 
+    @Override
+    public String buildRequestBody() {
+        // Serialization for OpenAI
+        return new GsonBuilder().create().toJson(toRequestJson());
+    }
+
     @Override
     public void initConversation(String fewshotFile) {
         try {
@@ -222,4 +240,77 @@ public String getUrl() {
             return url;
         }
     }
+
+    public static class ResponseFormat {
+        private String type;
+        public ResponseFormat(String type) { this.type = type; }
+        public String getType() { return type; }
+    }
+
+    public JsonObject toRequestJson() {
+        JsonObject root = new JsonObject();
+
+        root.addProperty("model", model);
+
+        if (supportsReasoningEffort(model)
+                && reasoning_effort != null
+                && !reasoning_effort.isEmpty()
+                && !"default".equalsIgnoreCase(reasoning_effort)) {
+            root.addProperty("reasoning_effort", reasoning_effort);
+        }
+
+        if (response_format != null && response_format.getType() != null) {
+            JsonObject rf = new JsonObject();
+            rf.addProperty("type", response_format.getType());
+            root.add("response_format", rf);
+        }
+
+        if (!hasFixedTemperature(model)) {
+            root.addProperty("temperature", temperature);
+        }
+
+        if (max_tokens != null) {
+            root.addProperty("max_tokens", max_tokens);
+        }
+
+        root.addProperty("stream", stream);
+
+        JsonArray msgs = new JsonArray();
+        for (Message m : messages) {
+            JsonObject jm = new JsonObject();
+            jm.addProperty("role", m.getRole());
+            JsonArray content = new JsonArray();
+            for (ContentPart p : m.getContent()) {
+                if (p == null || p.getType() == null) continue;
+                JsonObject part = new JsonObject();
+                part.addProperty("type", p.getType());
+                if ("text".equals(p.getType())) {
+                    part.addProperty("text", p.getText() == null ? "" : p.getText());
+                } else if ("image_url".equals(p.getType()) && p.getImage_url() != null) {
+                    JsonObject iu = new JsonObject();
+                    iu.addProperty("url", p.getImage_url().getUrl());
+                    part.add("image_url", iu);
+                }
+                content.add(part);
+            }
+            jm.add("content", content);
+            msgs.add(jm);
+        }
+        root.add("messages", msgs);
+
+        return root;
+    }
+
+    private static boolean supportsReasoningEffort(String model) {
+        if (model == null) return false;
+        String m = model.toLowerCase();
+        return m.startsWith("gpt-5") || m.startsWith("o-") || m.contains("reasoning");
+    }
+
+    private static boolean hasFixedTemperature(String model) {
+        if (model == null) return false;
+        String m = model.toLowerCase();
+        return m.startsWith("gpt-5") || m.startsWith("o-") || m.contains("reasoning");
+    }
+
 }
diff --git a/testar/src/org/testar/monkey/ConfigTags.java b/testar/src/org/testar/monkey/ConfigTags.java
@@ -119,6 +119,9 @@ private ConfigTags() {}
 	public static final Tag<String> LlmModel = Tag.from("LlmModel", String.class,
 			"The model used by the LLM agent");
 
+	public static final Tag<String> LlmReasoning = Tag.from("LlmReasoning", String.class,
+			"The reasoning effort of the LLM model");
+
 	public static final Tag<String> LlmHostUrl = Tag.from("LlmHostUrl", String.class,
 			"The Host URL on which the LLM agent is running");
 
diff --git a/testar/src/org/testar/monkey/Main.java b/testar/src/org/testar/monkey/Main.java
@@ -62,7 +62,7 @@
 
 public class Main {
 
-	public static final String TESTAR_VERSION = "v2.7.12 (22-Oct-2025)";
+	public static final String TESTAR_VERSION = "v2.7.13 (24-Oct-2025)";
 
 	//public static final String TESTAR_DIR_PROPERTY = "DIRNAME"; //Use the OS environment to obtain TESTAR directory
 	public static final String SETTINGS_FILE = "test.settings";
diff --git a/testar/src/org/testar/oracles/llm/LlmOracle.java b/testar/src/org/testar/oracles/llm/LlmOracle.java
@@ -75,6 +75,7 @@ public class LlmOracle implements Oracle {
 
 	private final String platform;
 	private final String model;
+	private final String reasoning;
 	private final String hostUrl;
 	private final String authorizationHeader;
 	private final String fewshotOracleFile;
@@ -85,7 +86,6 @@ public class LlmOracle implements Oracle {
 	private LlmConversation conversation;
 	private int tokens_used;
 
-	private Gson gson = new Gson();
 	private String previousTestGoal = "";
 	private LlmTestGoal currentTestGoal;
 
@@ -94,6 +94,7 @@ public LlmOracle(Settings settings, IPromptOracleGenerator oracleGenerator) {
 
 		this.platform = settings.get(ConfigTags.LlmPlatform);
 		this.model = settings.get(ConfigTags.LlmModel);
+		this.reasoning = settings.get(ConfigTags.LlmReasoning);
 		this.hostUrl = settings.get(ConfigTags.LlmHostUrl);
 		this.authorizationHeader = settings.get(ConfigTags.LlmAuthorizationHeader);
 		this.fewshotOracleFile = settings.get(ConfigTags.LlmOracleFewshotFile);
@@ -122,7 +123,7 @@ public void reset(LlmTestGoal newTestGoal, boolean appendPreviousTestGoal) {
 
 	@Override
 	public void initialize() {
-		conversation = LlmFactory.createLlmConversation(this.platform, this.model, this.temperature);
+		conversation = LlmFactory.createLlmConversation(this.platform, this.model, this.reasoning, this.temperature);
 		conversation.initConversation(this.fewshotOracleFile);
 	}
 
@@ -165,13 +166,13 @@ private Verdict getVerdictWithLlm(State state) {
 			conversation.addMessage("user", prompt);
 		}
 
-		String conversationJson = gson.toJson(conversation);
+		String conversationJson = conversation.buildRequestBody();
 
 		try {
 
 			String llmResponse = getResponseFromLlm(conversationJson);
 
-			LlmVerdict llmVerdict = gson.fromJson(llmResponse, LlmVerdict.class);
+			LlmVerdict llmVerdict = new Gson().fromJson(llmResponse, LlmVerdict.class);
 
 			if(llmVerdict.match()) return new Verdict(Verdict.Severity.LLM_COMPLETE, llmVerdict.getInfo());
 
diff --git a/testar/src/org/testar/settings/SettingsDefaults.java b/testar/src/org/testar/settings/SettingsDefaults.java
@@ -218,6 +218,7 @@ private SettingsDefaults() {}
 		// Settings for LLM agents
 		defaults.add(Pair.from(LlmPlatform, "OpenAI"));
 		defaults.add(Pair.from(LlmModel, ""));
+		defaults.add(Pair.from(LlmReasoning, "default"));
 		defaults.add(Pair.from(LlmHostUrl, "http://192.168.108.242:1234/v1/chat/completions"));
 		defaults.add(Pair.from(LlmAuthorizationHeader, ""));
 		defaults.add(Pair.from(LlmTestGoals, Arrays.asList("Log in with the username john and the password demo\\nThen the message Welcome John Smith is shown")));
diff --git a/testar/src/org/testar/settings/SettingsFileStructure.java b/testar/src/org/testar/settings/SettingsFileStructure.java
@@ -248,6 +248,7 @@ public static String getTestSettingsStructure() {
 				, ""
 				, ConfigTags.LlmPlatform.name() + " = "
 				, ConfigTags.LlmModel.name() + " = "
+				, ConfigTags.LlmReasoning.name() + " = "
 				, ConfigTags.LlmHostUrl.name() + " = "
 				, ConfigTags.LlmAuthorizationHeader.name() + " = "
 				, ConfigTags.LlmActionFewshotFile.name() + " = "
diff --git a/testar/src/org/testar/settings/dialog/LlmPanel.java b/testar/src/org/testar/settings/dialog/LlmPanel.java
