Skip to content

Commit f868876

Browse files
ferpasriferpasri
committed
Enable TextVisualizer for llm input and add TestLlmParseActionResponse
1 parent 3b001ef commit f868876

4 files changed

Lines changed: 456 additions & 160 deletions

File tree

‎core/src/org/testar/monkey/alayer/visualizers/TextVisualizer.java‎

Lines changed: 40 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,34 @@
11
/***************************************************************************************************
2-
*
3-
* Copyright (c) 2013, 2014, 2015, 2016, 2017 Universitat Politecnica de Valencia - www.upv.es
4-
*
5-
* Redistribution and use in source and binary forms, with or without
6-
* modification, are permitted provided that the following conditions are met:
7-
*
8-
* 1. Redistributions of source code must retain the above copyright notice,
9-
* this list of conditions and the following disclaimer.
10-
* 2. Redistributions in binary form must reproduce the above copyright
11-
* notice, this list of conditions and the following disclaimer in the
12-
* documentation and/or other materials provided with the distribution.
13-
* 3. Neither the name of the copyright holder nor the names of its
14-
* contributors may be used to endorse or promote products derived from
15-
* this software without specific prior written permission.
16-
*
17-
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18-
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19-
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20-
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21-
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22-
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23-
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24-
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25-
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26-
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27-
* POSSIBILITY OF SUCH DAMAGE.
28-
*******************************************************************************************************/
2+
*
3+
* Copyright (c) 2013 - 2025 Universitat Politecnica de Valencia - www.upv.es
4+
* Copyright (c) 2018 - 2025 Open Universiteit - www.ou.nl
5+
*
6+
* Redistribution and use in source and binary forms, with or without
7+
* modification, are permitted provided that the following conditions are met:
8+
*
9+
* 1. Redistributions of source code must retain the above copyright notice,
10+
* this list of conditions and the following disclaimer.
11+
* 2. Redistributions in binary form must reproduce the above copyright
12+
* notice, this list of conditions and the following disclaimer in the
13+
* documentation and/or other materials provided with the distribution.
14+
* 3. Neither the name of the copyright holder nor the names of its
15+
* contributors may be used to endorse or promote products derived from
16+
* this software without specific prior written permission.
17+
*
18+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28+
* POSSIBILITY OF SUCH DAMAGE.
29+
*******************************************************************************************************/
2930

30-
31-
/**
32-
* @author Sebastian Bauersfeld
33-
*/
34-
package org.testar.monkey.alayer.visualizers; // refactored by urueda
31+
package org.testar.monkey.alayer.visualizers;
3532

3633
import org.testar.monkey.Assert;
3734
import org.testar.monkey.Pair;
@@ -44,23 +41,31 @@
4441
import org.testar.monkey.alayer.exceptions.PositionException;
4542

4643
public final class TextVisualizer implements Visualizer {
47-
44+
4845
private static final long serialVersionUID = 9156304220974950751L;
4946
final Position pos;
5047
final String text;
5148
final Pen pen;
52-
49+
5350
public TextVisualizer(Position pos, String text, Pen pen){
5451
Assert.notNull(pos, text, pen);
5552
this.pos = pos;
5653
this.text = text;
5754
this.pen = pen;
5855
}
5956

57+
public String getText() {
58+
return text;
59+
}
60+
61+
public TextVisualizer withText(String newText, Pen newPen) {
62+
return new TextVisualizer(this.pos, newText, newPen);
63+
}
64+
6065
public void run(State state, Canvas cv, Pen pen) {
6166
Assert.notNull(state, cv, pen);
6267
pen = Pen.merge(pen, this.pen);
63-
try { // by urueda
68+
try {
6469
Point p = pos.apply(state);
6570
Pair<Double, Double> m = cv.textMetrics(pen, text);
6671
cv.text(pen, p.x() - m.left() / 2, p.y() - m.right() / 2, 0, text);

‎testar/src/org/testar/action/priorization/llm/LlmActionSelector.java‎

Lines changed: 2 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
package org.testar.action.priorization.llm;
3232

3333
import com.google.gson.Gson;
34-
import com.google.gson.JsonParseException;
3534
import org.apache.logging.log4j.Level;
3635
import org.apache.logging.log4j.LogManager;
3736
import org.apache.logging.log4j.Logger;
@@ -46,7 +45,6 @@
4645
import org.testar.monkey.Main;
4746
import org.testar.monkey.alayer.*;
4847
import org.testar.monkey.alayer.actions.*;
49-
import org.testar.monkey.alayer.webdriver.enums.WdTags;
5048
import org.testar.settings.Settings;
5149

5250
import java.io.*;
@@ -175,7 +173,8 @@ private Action selectActionWithLlm(State state, Set<Action> actions) {
175173

176174
String conversationJson = gson.toJson(conversation);
177175
String llmResponse = getResponseFromLlm(conversationJson);
178-
LlmParseActionResult llmParseResult = parseLlmResponse(actions, llmResponse);
176+
LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(gson);
177+
LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(actions, llmResponse);
179178

180179
switch(llmParseResult.getParseResult()) {
181180
case SUCCESS: {
@@ -327,128 +326,6 @@ private String getResponseFromLlm(String requestBody) {
327326
}
328327
}
329328

330-
/**
331-
* Parses the response sent by the LLM and selects an action if the response was valid.
332-
* @param actions Set of actions in the current state.
333-
* @param responseContent The response of the LLM in plaintext.
334-
* @return LlmParseActionResult containing the result of the parse and the action to execute if parsing was successful.
335-
*/
336-
private LlmParseActionResult parseLlmResponse(Set<Action> actions, String responseContent) {
337-
try {
338-
LlmSelectedAction llmSelectedAction = gson.fromJson(sanitizeJsonResponse(responseContent), LlmSelectedAction.class);
339-
340-
Action selectedAction = getActionByIdentifier(actions, llmSelectedAction.getActionId());
341-
342-
// If the selectedAction is a NOP action at this stage, parsing has likely failed.
343-
// Observed to happen when the LLM selects an actionId that does not exist.
344-
if(selectedAction instanceof NOP) {
345-
logger.log(Level.ERROR, "Action AbstractID not found, parsing LLM response has likely failed!: " + responseContent);
346-
return new LlmParseActionResult(null, LlmParseActionResult.ParseResult.INVALID_ACTION);
347-
}
348-
349-
String input = llmSelectedAction.getInput();
350-
Widget widget = selectedAction.get(Tags.OriginWidget);
351-
352-
// For interacting with select combobox web widgets
353-
// A WdSelectListAction is created to change the active value of the combobox
354-
if(Objects.equals(widget.get(WdTags.WebTagName, ""), "select")) {
355-
if(Objects.equals(input, "")) {
356-
return new LlmParseActionResult(null, LlmParseActionResult.ParseResult.SL_MISSING_INPUT);
357-
}
358-
359-
String target = widget.get(WdTags.WebId, "");
360-
WdSelectListAction.JsTargetMethod method;
361-
if (target.isEmpty()) {
362-
logger.warn("elementId is empty for select widget! Using name target method.");
363-
target = widget.get(WdTags.WebName, "");
364-
method = WdSelectListAction.JsTargetMethod.NAME;
365-
} else {
366-
method = WdSelectListAction.JsTargetMethod.ID;
367-
}
368-
369-
return new LlmParseActionResult(new WdSelectListAction(target, input, widget, method),
370-
LlmParseActionResult.ParseResult.SUCCESS);
371-
}
372-
373-
setCompoundActionInputText(selectedAction, input);
374-
return new LlmParseActionResult(selectedAction, LlmParseActionResult.ParseResult.SUCCESS);
375-
376-
} catch(JsonParseException e) {
377-
logger.log(Level.ERROR, "Unable to parse response from LLM to JSON: " + responseContent);
378-
return new LlmParseActionResult(null, LlmParseActionResult.ParseResult.PARSE_FAILED);
379-
} catch(NullPointerException e) {
380-
logger.log(Level.ERROR, "Null response due to LLM parse response error");
381-
return new LlmParseActionResult(null, LlmParseActionResult.ParseResult.COMMUNICATION_FAILURE);
382-
} catch(Exception e) {
383-
logger.log(Level.ERROR, "Exception parsing LLM response");
384-
return new LlmParseActionResult(null, LlmParseActionResult.ParseResult.COMMUNICATION_FAILURE);
385-
}
386-
}
387-
388-
/**
389-
* Try to sanitize markdown-style code fences sometimes generated by LLMs
390-
* @param responseContent
391-
* @return The sanitized LLM response
392-
*/
393-
private String sanitizeJsonResponse(String responseContent) {
394-
if (responseContent == null) return null;
395-
396-
if (responseContent.startsWith("```")) {
397-
logger.log(Level.INFO, String.format("Sanitizing Response: [%s]", responseContent));
398-
// Remove enclosing ```...``` or ```json...```
399-
responseContent = responseContent.replaceAll("(?s)^```(?:json)?\\s*", "");
400-
responseContent = responseContent.replaceAll("(?s)\\s*```$", "");
401-
logger.log(Level.INFO, String.format("Sanitized Response: [%s]", responseContent));
402-
}
403-
404-
return responseContent;
405-
}
406-
407-
/**
408-
* Retrieves an action with given actionId.
409-
* @param actions Set of actions to search.
410-
* @param actionId ActionId to search for.
411-
* @return Requested action if found, NOP Action if not found.
412-
*/
413-
private Action getActionByIdentifier(Set<Action> actions, String actionId) {
414-
for(Action action : actions) {
415-
if(action.get(Tags.AbstractID, "").equalsIgnoreCase(actionId)) {
416-
return action;
417-
}
418-
}
419-
return new NOP();
420-
}
421-
422-
/**
423-
* Sets TESTAR input text of compound Type and PasteText actions.
424-
* @param action CompoundAction to change.
425-
* @param inputText The characters to enter into the input field. Can be left empty if not applicable.
426-
* @return if input text changed.
427-
*/
428-
private boolean setCompoundActionInputText(Action action, String inputText) {
429-
//TODO: Create single actions in protocol so this is not necessary?
430-
if(action instanceof CompoundAction) {
431-
for(Action innerAction : ((CompoundAction)action).getActions()) {
432-
433-
if(innerAction instanceof Type) {
434-
((Type)innerAction).set(Tags.InputText, inputText);
435-
action.set(Tags.Desc, "Type '" + ((Type)innerAction).get(Tags.InputText)
436-
+ "' into '" + action.get(Tags.OriginWidget).get(Tags.Desc, "<no description>" + "'"));
437-
return true;
438-
}
439-
440-
if(innerAction instanceof PasteText) {
441-
((PasteText)innerAction).set(Tags.InputText, inputText);
442-
action.set(Tags.Desc, "PasteText '" + ((PasteText)innerAction).get(Tags.InputText)
443-
+ "' into '" + action.get(Tags.OriginWidget).get(Tags.Desc, "<no description>" + "'"));
444-
return true;
445-
}
446-
}
447-
}
448-
449-
return false;
450-
}
451-
452329
/**
453330
* Returns the amount of invalid actions (incorrect actionId, unable to parse llm response, etc.)
454331
* @return Amount of invalid actions.

0 commit comments

Comments
 (0)