diff --git a/windows_advanced_actions/pom.xml b/windows_advanced_actions/pom.xml
index 99efe260..c54a705d 100644
--- a/windows_advanced_actions/pom.xml
+++ b/windows_advanced_actions/pom.xml
@@ -6,7 +6,7 @@
4.0.0
com.testsigma.addons
windows_advanced_actions
- 1.0.23
+ 1.0.1
jar
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/util/Constants.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/Constants.java
new file mode 100644
index 00000000..d3668941
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/Constants.java
@@ -0,0 +1,12 @@
+package com.testsigma.addons.util;
+
+public class Constants {
+ public static String VISUAL_SERVER_API_END_POINT = "https://visualtesting-staging.testsigma.com/image_analysis_with_files";
+ public static String VISUAL_SERVER_FIND_IMAGE_ENDPOINT = "https://visualtesting-staging.testsigma.com/find_image_with_files";
+ public static String VISUAL_SERVER_OCR_TEXT_ENDPOINT = "https://visualtesting-staging.testsigma.com/ocr-text-points-with-files";
+
+
+
+ public static String API_TOKEN = "DHRNEYFTDCDGOEDDCOEICVYOEEUY";
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/utils/KeyboardUtils.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/KeyboardUtils.java
similarity index 71%
rename from windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/utils/KeyboardUtils.java
rename to windows_advanced_actions/src/main/java/com/testsigma/addons/util/KeyboardUtils.java
index c38c7c34..a3b286f2 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/utils/KeyboardUtils.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/KeyboardUtils.java
@@ -1,12 +1,105 @@
-package com.testsigma.addons.windowsAdvanced.utils;
+package com.testsigma.addons.util;
+import java.awt.*;
import java.awt.event.KeyEvent;
+import java.util.HashMap;
+import java.util.Map;
/**
* Utility class for keyboard operations and key code mappings
*/
public class KeyboardUtils {
+ private static final Map SPECIAL_CHAR_MAP = new HashMap<>();
+
+ static {
+ // Shifted characters: maps character -> [physicalKeyCode, needsShift (1=yes)]
+ SPECIAL_CHAR_MAP.put('!', new int[]{KeyEvent.VK_1, 1});
+ SPECIAL_CHAR_MAP.put('@', new int[]{KeyEvent.VK_2, 1});
+ SPECIAL_CHAR_MAP.put('#', new int[]{KeyEvent.VK_3, 1});
+ SPECIAL_CHAR_MAP.put('$', new int[]{KeyEvent.VK_4, 1});
+ SPECIAL_CHAR_MAP.put('%', new int[]{KeyEvent.VK_5, 1});
+ SPECIAL_CHAR_MAP.put('^', new int[]{KeyEvent.VK_6, 1});
+ SPECIAL_CHAR_MAP.put('&', new int[]{KeyEvent.VK_7, 1});
+ SPECIAL_CHAR_MAP.put('*', new int[]{KeyEvent.VK_8, 1});
+ SPECIAL_CHAR_MAP.put('(', new int[]{KeyEvent.VK_9, 1});
+ SPECIAL_CHAR_MAP.put(')', new int[]{KeyEvent.VK_0, 1});
+ SPECIAL_CHAR_MAP.put('_', new int[]{KeyEvent.VK_MINUS, 1});
+ SPECIAL_CHAR_MAP.put('+', new int[]{KeyEvent.VK_EQUALS, 1});
+ SPECIAL_CHAR_MAP.put('{', new int[]{KeyEvent.VK_OPEN_BRACKET, 1});
+ SPECIAL_CHAR_MAP.put('}', new int[]{KeyEvent.VK_CLOSE_BRACKET, 1});
+ SPECIAL_CHAR_MAP.put('|', new int[]{KeyEvent.VK_BACK_SLASH, 1});
+ SPECIAL_CHAR_MAP.put(':', new int[]{KeyEvent.VK_SEMICOLON, 1});
+ SPECIAL_CHAR_MAP.put('"', new int[]{KeyEvent.VK_QUOTE, 1});
+ SPECIAL_CHAR_MAP.put('<', new int[]{KeyEvent.VK_COMMA, 1});
+ SPECIAL_CHAR_MAP.put('>', new int[]{KeyEvent.VK_PERIOD, 1});
+ SPECIAL_CHAR_MAP.put('?', new int[]{KeyEvent.VK_SLASH, 1});
+ SPECIAL_CHAR_MAP.put('~', new int[]{KeyEvent.VK_BACK_QUOTE, 1});
+
+ // Unshifted special characters
+ SPECIAL_CHAR_MAP.put('-', new int[]{KeyEvent.VK_MINUS, 0});
+ SPECIAL_CHAR_MAP.put('=', new int[]{KeyEvent.VK_EQUALS, 0});
+ SPECIAL_CHAR_MAP.put('[', new int[]{KeyEvent.VK_OPEN_BRACKET, 0});
+ SPECIAL_CHAR_MAP.put(']', new int[]{KeyEvent.VK_CLOSE_BRACKET, 0});
+ SPECIAL_CHAR_MAP.put('\\', new int[]{KeyEvent.VK_BACK_SLASH, 0});
+ SPECIAL_CHAR_MAP.put(';', new int[]{KeyEvent.VK_SEMICOLON, 0});
+ SPECIAL_CHAR_MAP.put('\'', new int[]{KeyEvent.VK_QUOTE, 0});
+ SPECIAL_CHAR_MAP.put(',', new int[]{KeyEvent.VK_COMMA, 0});
+ SPECIAL_CHAR_MAP.put('.', new int[]{KeyEvent.VK_PERIOD, 0});
+ SPECIAL_CHAR_MAP.put('/', new int[]{KeyEvent.VK_SLASH, 0});
+ SPECIAL_CHAR_MAP.put('`', new int[]{KeyEvent.VK_BACK_QUOTE, 0});
+ SPECIAL_CHAR_MAP.put(' ', new int[]{KeyEvent.VK_SPACE, 0});
+ SPECIAL_CHAR_MAP.put('\t', new int[]{KeyEvent.VK_TAB, 0});
+ SPECIAL_CHAR_MAP.put('\n', new int[]{KeyEvent.VK_ENTER, 0});
+ }
+
+ /**
+ * Types a single character using the Robot class, correctly handling
+ * uppercase letters, digits, and all special characters (US keyboard layout).
+ */
+ public static void typeCharacter(Robot robot, char character) {
+ if (Character.isLetter(character)) {
+ int keyCode = KeyEvent.getExtendedKeyCodeForChar(Character.toUpperCase(character));
+ boolean upperCase = Character.isUpperCase(character);
+ if (upperCase) {
+ robot.keyPress(KeyEvent.VK_SHIFT);
+ }
+ robot.keyPress(keyCode);
+ sleep(10);
+ robot.keyRelease(keyCode);
+ if (upperCase) {
+ robot.keyRelease(KeyEvent.VK_SHIFT);
+ }
+ return;
+ }
+
+ if (Character.isDigit(character)) {
+ int keyCode = KeyEvent.getExtendedKeyCodeForChar(character);
+ robot.keyPress(keyCode);
+ sleep(10);
+ robot.keyRelease(keyCode);
+ return;
+ }
+
+ int[] mapping = SPECIAL_CHAR_MAP.get(character);
+ if (mapping != null) {
+ int keyCode = mapping[0];
+ boolean needsShift = mapping[1] == 1;
+ if (needsShift) {
+ robot.keyPress(KeyEvent.VK_SHIFT);
+ }
+ robot.keyPress(keyCode);
+ sleep(10);
+ robot.keyRelease(keyCode);
+ if (needsShift) {
+ robot.keyRelease(KeyEvent.VK_SHIFT);
+ }
+ return;
+ }
+
+ throw new IllegalArgumentException("Cannot type character: " + character);
+ }
+
// Allowed values for modifier keys
public static final String[] MODIFIER_KEYS = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down", "Enter",
"Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"};
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRResponse.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRResponse.java
new file mode 100644
index 00000000..fddfb41f
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRResponse.java
@@ -0,0 +1,29 @@
+package com.testsigma.addons.util;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class OCRResponse {
+ @JsonProperty("error")
+ private String error;
+
+ @JsonProperty("text")
+ private List text;
+
+ // Manual getter methods since Lombok annotation processor is not working
+ public String getError() { return error; }
+ public List getText() { return text; }
+
+ // Helper method to check if there are any errors
+ public boolean hasError() {
+ return error != null && !error.trim().isEmpty();
+ }
+
+ // Helper method to check if text was found
+ public boolean hasText() {
+ return text != null && !text.isEmpty();
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRTextPoint.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRTextPoint.java
new file mode 100644
index 00000000..1cdab69a
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRTextPoint.java
@@ -0,0 +1,45 @@
+package com.testsigma.addons.util;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+public class OCRTextPoint {
+ @JsonProperty("text")
+ private String text;
+
+ @JsonProperty("x1")
+ private double x1;
+
+ @JsonProperty("x2")
+ private double x2;
+
+ @JsonProperty("y1")
+ private double y1;
+
+ @JsonProperty("y2")
+ private double y2;
+
+ // Manual getter methods since Lombok annotation processor is not working
+ public String getText() { return text; }
+ public double getX1() { return x1; }
+ public double getX2() { return x2; }
+ public double getY1() { return y1; }
+ public double getY2() { return y2; }
+
+ // Manual setter methods
+ public void setText(String text) { this.text = text; }
+ public void setX1(double x1) { this.x1 = x1; }
+ public void setX2(double x2) { this.x2 = x2; }
+ public void setY1(double y1) { this.y1 = y1; }
+ public void setY2(double y2) { this.y2 = y2; }
+
+ // Helper method to get center coordinates for clicking
+ public double getCenterX() {
+ return (x1 + x2) / 2.0;
+ }
+
+ public double getCenterY() {
+ return (y1 + y2) / 2.0;
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRUtils.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRUtils.java
new file mode 100644
index 00000000..25ffc9eb
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/OCRUtils.java
@@ -0,0 +1,590 @@
+package com.testsigma.addons.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.testsigma.sdk.Logger;
+
+/**
+ * Utility class for OCR (Optical Character Recognition) operations
+ * This is a simplified implementation that can be enhanced with actual OCR libraries
+ * For now, it provides a basic structure that can be easily replaced with Tesseract or other OCR engines
+ */
+public class OCRUtils {
+
+ /**
+ * Finds the matching text in the list of text points with improved positioning logic
+ */
+ public OCRTextPoint findMatchingText(List textPoints, String targetText, Logger logger) {
+ logger.info("Searching for text: '" + targetText + "'");
+
+ // Create HashMap to store all exact word matches with their coordinates (multiple occurrences)
+ Map> exactWordMatches = new HashMap<>();
+
+ // First pass: collect all exact word matches in HashMap (storing all occurrences)
+ for (OCRTextPoint textPoint : textPoints) {
+ String text = textPoint.getText().trim();
+ if (!text.isEmpty()) {
+ String lowerText = text.toLowerCase();
+ exactWordMatches.computeIfAbsent(lowerText, k -> new ArrayList<>()).add(textPoint);
+ if (text.equals(targetText)) {
+ logger.info("Found exact match: " + textPoint.getText());
+ return textPoint;
+ }
+ }
+ }
+
+ // Second pass: try case-insensitive exact match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().equalsIgnoreCase(targetText)) {
+ logger.info("Found case-insensitive match: " + textPoint.getText());
+ return textPoint;
+ }
+ }
+
+ // Third pass: try contains match with improved positioning
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().toLowerCase().contains(targetText.toLowerCase())) {
+ logger.info("Found contains match in sentence: " + textPoint.getText());
+ logger.info("Text point coordinates: x1=" + textPoint.getX1() + ", x2=" + textPoint.getX2() +
+ ", y1=" + textPoint.getY1() + ", y2=" + textPoint.getY2());
+
+ // Try to find the exact position within the sentence using HashMap approach
+ OCRTextPoint positionedTextPoint = findExactPositionInSentence(textPoint, targetText, exactWordMatches, logger);
+ if (positionedTextPoint != null) {
+ return positionedTextPoint;
+ }
+
+ // Fallback to get the location based on the position by approximation
+ logger.info("Using sentence center as fallback");
+ return textPoint;
+ }
+ }
+
+ logger.info("No matching text found for: '" + targetText + "'");
+ logger.info("Available text elements:");
+ for (OCRTextPoint textPoint : textPoints) {
+ logger.info(" - '" + textPoint.getText() + "'");
+ }
+
+ return null;
+ }
+
+ /**
+ * Finds the exact position of target text within a sentence by analyzing individual words
+ */
+ private OCRTextPoint findExactPositionInSentence(OCRTextPoint sentencePoint, String targetText, Map> exactWordMatches, Logger logger) {
+ logger.info("Attempting to find exact position for: '" + targetText + "' in sentence: '" + sentencePoint.getText() + "'");
+
+ // Split target text into words
+ String[] targetWords = targetText.toLowerCase().trim().split("\\s+");
+ if (targetWords.length == 0) {
+ return null;
+ }
+
+ // Find the first and last words of target text in the exact matches
+ OCRTextPoint firstWordPoint = null;
+ OCRTextPoint lastWordPoint = null;
+ double lastWordX2 = Double.NEGATIVE_INFINITY; // Track the end position of the previous word
+
+ for (String word : targetWords) {
+ List wordPoints = exactWordMatches.get(word);
+ if (wordPoints != null && !wordPoints.isEmpty()) {
+ logger.info("Found " + wordPoints.size() + " occurrences of word '" + word + "'");
+
+ // Search through all occurrences of this word to find one within sentence boundaries and in correct sequence
+ for (OCRTextPoint wordPoint : wordPoints) {
+ // Check if this word point falls within the sentence boundaries
+ if (isPointWithinSentence(wordPoint, sentencePoint, logger)) {
+ // Check if this word comes after the previous word (sequential validation)
+ boolean isSequential = (firstWordPoint == null) || (wordPoint.getX1() > lastWordX2);
+
+ if (isSequential) {
+ if (firstWordPoint == null) {
+ firstWordPoint = wordPoint;
+ }
+ lastWordPoint = wordPoint;
+ lastWordX2 = wordPoint.getX2(); // Update the end position for next word
+ logger.info("Found word '" + word + "' at coordinates: x1=" + wordPoint.getX1() + ", x2=" + wordPoint.getX2());
+ break; // Use the first valid occurrence within sentence boundaries and in correct sequence
+ } else {
+ logger.info("Word '" + word + "' found within sentence bounds but not in correct sequence");
+ logger.info("Word x1=" + wordPoint.getX1() + " should be > previous word x2=" + lastWordX2);
+ }
+ } else {
+ logger.info("Word '" + word + "' occurrence found but outside sentence boundaries");
+ logger.info("Word point coordinates: x1=" + wordPoint.getX1() + ", x2=" + wordPoint.getX2() +
+ ", y1=" + wordPoint.getY1() + ", y2=" + wordPoint.getY2());
+ logger.info("Sentence point coordinates: x1=" + sentencePoint.getX1() + ", x2=" + sentencePoint.getX2() +
+ ", y1=" + sentencePoint.getY1() + ", y2=" + sentencePoint.getY2());
+ }
+ }
+ } else {
+ logger.info("Word '" + word + "' not found in exact matches");
+ }
+ }
+
+ // If we found both first and last words, calculate the target position
+ if (firstWordPoint != null && lastWordPoint != null) {
+ double targetX1 = Math.min(firstWordPoint.getX1(), lastWordPoint.getX1());
+ double targetX2 = Math.max(firstWordPoint.getX2(), lastWordPoint.getX2());
+ double targetY1 = Math.min(firstWordPoint.getY1(), lastWordPoint.getY1());
+ double targetY2 = Math.max(firstWordPoint.getY2(), lastWordPoint.getY2());
+ logger.info("Both first and last words found. Calculated coordinates: x1=" + targetX1 + ", x2=" + targetX2 +
+ ", y1=" + targetY1 + ", y2=" + targetY2);
+
+ // Create a new OCRTextPoint with the calculated coordinates
+ OCRTextPoint targetPoint = new OCRTextPoint();
+ targetPoint.setText(targetText);
+ targetPoint.setX1(targetX1);
+ targetPoint.setX2(targetX2);
+ targetPoint.setY1(targetY1);
+ targetPoint.setY2(targetY2);
+
+ logger.info("Calculated target position: x1=" + targetX1 + ", x2=" + targetX2 + ", y1=" + targetY1 + ", y2=" + targetY2);
+ logger.info("Target center: x=" + targetPoint.getCenterX() + ", y=" + targetPoint.getCenterY());
+
+ return targetPoint;
+ } else if (firstWordPoint != null) {
+ // If only first word found, use it as the target
+ logger.info("Using first word position as target");
+ return firstWordPoint;
+ } else if (lastWordPoint != null) {
+ // If only last word found, use it as the target
+ logger.info("Using last word position as target");
+ return lastWordPoint;
+ }
+
+ logger.info("Could not find exact word positions within sentence boundaries");
+ return null;
+ }
+
+ /**
+ * Checks if a word point falls within the boundaries of a sentence point
+ */
+ private boolean isPointWithinSentence(OCRTextPoint wordPoint, OCRTextPoint sentencePoint, Logger logger) {
+ // Check if word coordinates are within sentence boundaries with some tolerance
+ double tolerance = 15.0; // Allow some tolerance for OCR variations
+ // since in backend we are storing the center of x1,x2 i am ignoring the check for x1,x2
+ boolean withinBounds = (wordPoint.getY1() >= sentencePoint.getY1() - tolerance) &&
+ (wordPoint.getY2() <= sentencePoint.getY2() + tolerance);
+ logger.info("Word point within sentence bounds: " + withinBounds +
+ " (word: x1=" + wordPoint.getX1() + ", x2=" + wordPoint.getX2() +
+ ", sentence: x1=" + sentencePoint.getX1() + ", x2=" + sentencePoint.getX2() + ")");
+ return withinBounds;
+ }
+
+ /**
+ * Finds matching text for a sentence by breaking it into words and checking positioning constraints.
+ * This method searches for sentences by validating that consecutive words are positioned correctly
+ * relative to each other (next word's x1 should be less than previous word's x2 + 20,
+ * and y1 difference should be less than 10).
+ *
+ * @param textPoints List of OCR text points containing individual words
+ * @param targetSentence The sentence to search for
+ * @param logger The logger instance
+ * @return OCRTextPoint representing the found sentence, or null if not found
+ */
+ public OCRTextPoint findMatchingTextForSentence(List textPoints, String targetSentence, Logger logger) {
+ logger.info("Searching for sentence: '" + targetSentence + "'");
+
+ if (targetSentence == null || targetSentence.trim().isEmpty()) {
+ logger.info("Target sentence is null or empty");
+ return null;
+ }
+
+ // Split the target sentence into individual words
+ String[] targetWords = targetSentence.trim().split("\\s+");
+ if (targetWords.length == 0) {
+ logger.info("No words found in target sentence");
+ return null;
+ }
+
+ logger.info("Target sentence has " + targetWords.length + " words: "
+ + String.join(", ", targetWords));
+
+ // Filter text points to only include single words (exclude multi-word entries)
+ List singleWordPoints = new ArrayList<>();
+ for (OCRTextPoint point : textPoints) {
+ String text = point.getText().trim();
+ // Consider it a single word if it doesn't contain spaces and is not empty
+ if (!text.isEmpty() && !text.contains(" ") && !text.contains("\n")) {
+ singleWordPoints.add(point);
+ }
+ }
+
+ logger.info("Filtered to " + singleWordPoints.size() + " single word text points");
+
+ // Search for the sentence by finding consecutive words
+ for (int i = 0; i < singleWordPoints.size(); i++) {
+ OCRTextPoint firstWordPoint = singleWordPoints.get(i);
+ String firstWord = firstWordPoint.getText().trim();
+
+ // Check if this point matches the first word of our target sentence
+ if (firstWord.equalsIgnoreCase(targetWords[0])) {
+ logger.info("Found potential first word '" + firstWord + "' at index " + i +
+ " with coordinates: x1=" + firstWordPoint.getX1() + ", x2=" + firstWordPoint.getX2() +
+ ", y1=" + firstWordPoint.getY1() + ", y2=" + firstWordPoint.getY2());
+
+ // Try to find the complete sentence starting from this word
+ OCRTextPoint sentenceResult = findCompleteSentence(singleWordPoints, i, targetWords, logger);
+ if (sentenceResult != null) {
+ logger.info("Successfully found complete sentence: '" + targetSentence + "'");
+ return sentenceResult;
+ }
+ }
+ }
+
+ logger.info("Sentence '" + targetSentence + "' not found in text points");
+
+ // Fallback: Try breaking the sentence based on camelCase
+ logger.info("Attempting fallback: breaking sentence based on camelCase");
+ return findMatchingTextForSentenceWithCamelCaseFallback(textPoints, targetSentence, logger);
+ }
+
+ /**
+ * Attempts to find a complete sentence starting from a given index in the text points.
+ * Validates that consecutive words meet the positioning constraints.
+ *
+ * @param textPoints List of single word text points
+ * @param startIndex Index to start searching from
+ * @param targetWords Array of words that make up the target sentence
+ * @param logger The logger instance
+ * @return OCRTextPoint representing the complete sentence, or null if not found
+ */
+ private OCRTextPoint findCompleteSentence(List textPoints, int startIndex, String[] targetWords, Logger logger) {
+ if (targetWords.length == 1) {
+ // Single word sentence - return the first word point
+ return textPoints.get(startIndex);
+ }
+
+ List foundWords = new ArrayList<>();
+ foundWords.add(textPoints.get(startIndex));
+
+ int currentIndex = startIndex;
+
+ // Search for each subsequent word
+ for (int wordIndex = 1; wordIndex < targetWords.length; wordIndex++) {
+ String targetWord = targetWords[wordIndex];
+ OCRTextPoint previousWord = foundWords.get(foundWords.size() - 1);
+
+ logger.info("Searching for word " + (wordIndex + 1) + "/" + targetWords.length +
+ ": '" + targetWord + "' after word '" + previousWord.getText() + "'");
+
+ // Look for the next word starting from the current position
+ boolean foundNextWord = false;
+ for (int j = currentIndex + 1; j < textPoints.size(); j++) {
+ OCRTextPoint candidatePoint = textPoints.get(j);
+ String candidateText = candidatePoint.getText().trim();
+
+ if (candidateText.equalsIgnoreCase(targetWord)) {
+ // Check positioning constraints
+ if (isWordPositionValid(previousWord, candidatePoint, logger)) {
+ logger.info("Found valid word '" + targetWord + "' at index " + j +
+ " with coordinates: x1=" + candidatePoint.getX1() + ", x2=" + candidatePoint.getX2() +
+ ", y1=" + candidatePoint.getY1() + ", y2=" + candidatePoint.getY2());
+
+ foundWords.add(candidatePoint);
+ currentIndex = j;
+ foundNextWord = true;
+ break;
+ } else {
+ logger.info("Word '" + targetWord + "' found at index " + j +
+ " but positioning constraints not met");
+ }
+ }
+ }
+
+ if (!foundNextWord) {
+ logger.info("Could not find valid word '" + targetWord + "' after word '" + previousWord.getText() + "'");
+ return null;
+ }
+ }
+
+ // All words found - create a combined OCRTextPoint for the sentence
+ return createSentenceTextPoint(foundWords, targetWords, logger);
+ }
+
+ /**
+ * Validates that a word's position meets the constraints relative to the previous word.
+ * Constraints: next word's x1 should be less than previous word's x2 + 20,
+ * and y1 difference should be less than 10.
+ *
+ * @param previousWord The previous word's text point
+ * @param currentWord The current word's text point
+ * @param logger The logger instance
+ * @return true if positioning constraints are met, false otherwise
+ */
+ private boolean isWordPositionValid(OCRTextPoint previousWord, OCRTextPoint currentWord, Logger logger) {
+ // Constraint 1: next word's x1 should be less than previous word's x2 + 20
+ boolean xConstraint = currentWord.getX1() <= (previousWord.getX2() + 20);
+
+ // Constraint 2: y1 difference should be less than 10
+ boolean yConstraint = Math.abs(currentWord.getY1() - previousWord.getY1()) < 10;
+
+ logger.info("Position validation for words '" + previousWord.getText() + "' -> '" + currentWord.getText() + "':");
+ logger.info(" X constraint: " + xConstraint + " (current x1=" + currentWord.getX1() +
+ " <= previous x2+20=" + (previousWord.getX2() + 20) + ")");
+ logger.info(" Y constraint: " + yConstraint + " (y1 diff=" + Math.abs(currentWord.getY1() - previousWord.getY1()) + " < 10)");
+
+ return xConstraint && yConstraint;
+ }
+
+ /**
+ * Creates a combined OCRTextPoint representing the complete sentence from individual word points.
+ *
+ * @param wordPoints List of individual word text points
+ * @param targetWords Array of target words
+ * @param logger The logger instance
+ * @return Combined OCRTextPoint representing the sentence
+ */
+ private OCRTextPoint createSentenceTextPoint(List wordPoints, String[] targetWords, Logger logger) {
+ if (wordPoints.isEmpty()) {
+ return null;
+ }
+
+ // Calculate bounding box for the entire sentence
+ double minX1 = wordPoints.get(0).getX1();
+ double maxX2 = wordPoints.get(0).getX2();
+ double minY1 = wordPoints.get(0).getY1();
+ double maxY2 = wordPoints.get(0).getY2();
+
+ for (OCRTextPoint point : wordPoints) {
+ minX1 = Math.min(minX1, point.getX1());
+ maxX2 = Math.max(maxX2, point.getX2());
+ minY1 = Math.min(minY1, point.getY1());
+ maxY2 = Math.max(maxY2, point.getY2());
+ }
+
+ // Create the combined text point
+ OCRTextPoint sentencePoint = new OCRTextPoint();
+ sentencePoint.setText(String.join(" ", targetWords));
+ sentencePoint.setX1(minX1);
+ sentencePoint.setX2(maxX2);
+ sentencePoint.setY1(minY1);
+ sentencePoint.setY2(maxY2);
+
+ logger.info("Created sentence text point: '" + sentencePoint.getText() +
+ "' with coordinates: x1=" + minX1 + ", x2=" + maxX2 +
+ ", y1=" + minY1 + ", y2=" + maxY2);
+ logger.info("Sentence center: x=" + sentencePoint.getCenterX() + ", y=" + sentencePoint.getCenterY());
+
+ return sentencePoint;
+ }
+
+ /**
+ * Fallback method that breaks the target sentence based on camelCase and searches again.
+ * This handles cases where the sentence might be written as one word but should be treated as multiple words.
+ *
+ * @param textPoints List of OCR text points containing individual words
+ * @param targetSentence The sentence to search for
+ * @param logger The logger instance
+ * @return OCRTextPoint representing the found sentence, or null if not found
+ */
+ private OCRTextPoint findMatchingTextForSentenceWithCamelCaseFallback(List textPoints, String targetSentence, Logger logger) {
+ logger.info("Fallback: Breaking sentence '" + targetSentence + "' based on camelCase");
+
+ // Break the sentence into words based on camelCase
+ String[] camelCaseWords = breakCamelCase(targetSentence, logger);
+
+ if (camelCaseWords.length <= 1) {
+ logger.info("No camelCase breaking possible for sentence: '" + targetSentence + "'");
+ return null;
+ }
+
+ logger.info("CamelCase broken into " + camelCaseWords.length + " words: " + String.join(", ", camelCaseWords));
+
+ // Filter text points to only include single words (exclude multi-word entries)
+ List singleWordPoints = new ArrayList<>();
+ for (OCRTextPoint point : textPoints) {
+ String text = point.getText().trim();
+ // Consider it a single word if it doesn't contain spaces and is not empty
+ if (!text.isEmpty() && !text.contains(" ") && !text.contains("\n")) {
+ singleWordPoints.add(point);
+ }
+ }
+
+ logger.info("Filtered to " + singleWordPoints.size() + " single word text points for camelCase search");
+
+ // Search for the sentence by finding consecutive words using camelCase broken words
+ for (int i = 0; i < singleWordPoints.size(); i++) {
+ OCRTextPoint firstWordPoint = singleWordPoints.get(i);
+ String firstWord = firstWordPoint.getText().trim();
+
+ // Check if this point matches the first word of our camelCase broken sentence
+ if (firstWord.equalsIgnoreCase(camelCaseWords[0])) {
+ logger.info("Found potential first camelCase word '" + firstWord + "' at index " + i +
+ " with coordinates: x1=" + firstWordPoint.getX1() + ", x2=" + firstWordPoint.getX2() +
+ ", y1=" + firstWordPoint.getY1() + ", y2=" + firstWordPoint.getY2());
+
+ // Try to find the complete sentence starting from this word using camelCase words
+ OCRTextPoint sentenceResult = findCompleteSentence(singleWordPoints, i, camelCaseWords, logger);
+ if (sentenceResult != null) {
+ logger.info("Successfully found complete sentence using camelCase fallback: '" + targetSentence + "'");
+ // Update the text to match the original target sentence
+ sentenceResult.setText(targetSentence);
+ return sentenceResult;
+ }
+ }
+ }
+
+ logger.info("Sentence '" + targetSentence + "' not found even with camelCase fallback");
+ return null;
+ }
+
+ /**
+ * Breaks a string into words based on camelCase patterns.
+ * For example: "SystemData" -> ["System", "Data"]
+ *
+ * @param text The text to break into camelCase words
+ * @param logger The logger instance
+ * @return Array of words broken from camelCase
+ */
+ private String[] breakCamelCase(String text, Logger logger) {
+ if (text == null || text.trim().isEmpty()) {
+ return new String[0];
+ }
+
+ // First, try to break on existing spaces
+ String[] spaceWords = text.trim().split("\\s+");
+ if (spaceWords.length > 1) {
+ // If there are already spaces, return as is
+ return spaceWords;
+ }
+
+ // If it's a single word, try to break on camelCase
+ String singleWord = spaceWords[0];
+ List words = new ArrayList<>();
+
+ if (singleWord.length() <= 1) {
+ return new String[]{singleWord};
+ }
+
+ StringBuilder currentWord = new StringBuilder();
+ currentWord.append(singleWord.charAt(0));
+
+ for (int i = 1; i < singleWord.length(); i++) {
+ char currentChar = singleWord.charAt(i);
+ char previousChar = singleWord.charAt(i - 1);
+
+ // Check if this is a camelCase boundary
+ // Boundary conditions:
+ // 1. Current char is uppercase and previous char is lowercase
+ // 2. Current char is uppercase and previous char is uppercase but next char is lowercase (if exists)
+ // 3. Current char is digit and previous char is letter
+ // 4. Current char is letter and previous char is digit
+
+ boolean isBoundary = false;
+
+ if (Character.isUpperCase(currentChar) && Character.isLowerCase(previousChar)) {
+ // camelCase boundary: lowercase followed by uppercase
+ isBoundary = true;
+ } else if (Character.isUpperCase(currentChar) && Character.isUpperCase(previousChar) &&
+ i + 1 < singleWord.length() && Character.isLowerCase(singleWord.charAt(i + 1))) {
+ // Acronym boundary: uppercase followed by uppercase followed by lowercase
+ isBoundary = true;
+ } else if (Character.isDigit(currentChar) && Character.isLetter(previousChar)) {
+ // Letter-digit boundary
+ isBoundary = true;
+ } else if (Character.isLetter(currentChar) && Character.isDigit(previousChar)) {
+ // Digit-letter boundary
+ isBoundary = true;
+ }
+
+ if (isBoundary) {
+ // Save current word and start new one
+ if (currentWord.length() > 0) {
+ words.add(currentWord.toString());
+ currentWord = new StringBuilder();
+ }
+ }
+
+ currentWord.append(currentChar);
+ }
+
+ // Add the last word
+ if (currentWord.length() > 0) {
+ words.add(currentWord.toString());
+ }
+
+ String[] result = words.toArray(new String[0]);
+ logger.info("CamelCase breaking result for '" + text + "': " + String.join(", ", result));
+
+ return result;
+ }
+
+ /**
+ * Extracts text points from a screenshot file by calling the OCR API.
+ */
+ public static List extractTextPoints(File screenshotFile, Logger logger) throws Exception {
+ try {
+ OkHttpClient client = new OkHttpClient();
+ ObjectMapper mapper = new ObjectMapper();
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("ocrImageFile", screenshotFile.getName(),
+ RequestBody.create(screenshotFile, MediaType.parse("image/png")))
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_OCR_TEXT_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making OCR API call");
+ Response response = client.newCall(request).execute();
+
+ if (response.isSuccessful() && response.body() != null) {
+ String responseBody = response.body().string();
+ logger.info("OCR Response body: " + responseBody);
+
+ OCRResponse ocrResponse = mapper.readValue(responseBody, OCRResponse.class);
+
+ if (ocrResponse.hasError()) {
+ throw new RuntimeException("OCR API returned error: " + ocrResponse.getError());
+ }
+ if (!ocrResponse.hasText()) {
+ throw new RuntimeException("No text found in the image");
+ }
+ return ocrResponse.getText();
+ } else {
+ throw new RuntimeException("OCR API call failed with status: " + (response.code()));
+ }
+ } catch (IOException e) {
+ logger.info("Exception during OCR API call: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException("Error during OCR API call: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Searches for the target text in the list of OCR text points using
+ * exact match, case-insensitive match, contains match, and full-text concatenation.
+ */
+ public static boolean searchForText(List textPoints, String targetText, Logger logger) {
+ for (OCRTextPoint tp : textPoints) {
+ if (tp.getText().equals(targetText)) return true;
+ }
+ for (OCRTextPoint tp : textPoints) {
+ if (tp.getText().equalsIgnoreCase(targetText)) return true;
+ }
+ for (OCRTextPoint tp : textPoints) {
+ if (tp.getText().toLowerCase().contains(targetText.toLowerCase())) return true;
+ }
+ StringBuilder sb = new StringBuilder();
+ for (OCRTextPoint tp : textPoints) {
+ if (sb.length() > 0) sb.append(" ");
+ sb.append(tp.getText());
+ }
+ return sb.toString().toLowerCase().contains(targetText.toLowerCase());
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/util/ResponseObjectForFindImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/ResponseObjectForFindImage.java
new file mode 100644
index 00000000..26d05652
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/ResponseObjectForFindImage.java
@@ -0,0 +1,41 @@
+package com.testsigma.addons.util;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class ResponseObjectForFindImage {
+ @JsonProperty("isFound")
+ private Boolean isFound;
+
+ @JsonProperty("x1")
+ private int x1;
+
+ @JsonProperty("y1")
+ private int y1;
+
+ @JsonProperty("x2")
+ private int x2;
+
+ @JsonProperty("y2")
+ private int y2;
+
+ @JsonProperty("additionalData")
+ private AdditionalData additionalData;
+
+ @JsonProperty("error")
+ private String error;
+
+ @Data
+ public static class AdditionalData {
+ @JsonProperty("matchedPoints")
+ private List> matchedPoints;
+ }
+
+// private List diff_coordinates;
+// private List image_shape;
+// private double per_similar;
+// private String scalingType;
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/utils/ScreenshotUtils.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/ScreenshotUtils.java
similarity index 80%
rename from windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/utils/ScreenshotUtils.java
rename to windows_advanced_actions/src/main/java/com/testsigma/addons/util/ScreenshotUtils.java
index 0b758ace..b719ce44 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/utils/ScreenshotUtils.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/ScreenshotUtils.java
@@ -1,7 +1,7 @@
-package com.testsigma.addons.windowsAdvanced.utils;
+package com.testsigma.addons.util;
-import com.testsigma.sdk.TestStepResult;
import com.testsigma.sdk.Logger;
+import com.testsigma.sdk.TestStepResult;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -36,8 +36,17 @@ public class ScreenshotUtils {
*/
public static boolean captureAndUploadScreenshot(TestStepResult testStepResult, String screenshotName, Logger logger) {
try {
+ // Wait for 1 second before capturing screenshot to ensure UI is stable
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException interruptedException) {
+ logger.info("ignored the interrupted exception during screenshot capture wait: "
+ + ExceptionUtils.getStackTrace(interruptedException));
+ }
+
// Capture the current screen
Robot robot = new Robot();
+
Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
BufferedImage screenCapture = robot.createScreenCapture(screenRect);
@@ -114,8 +123,11 @@ public static File saveScreenshotToFile(BufferedImage screenshot, String fileNam
* @return BufferedImage of the screen capture
* @throws Exception if screenshot capture fails
*/
- public static BufferedImage captureScreenshot(Logger logger) throws Exception {
+ /* public static BufferedImage captureScreenshot(Logger logger) throws Exception {
try {
+ // Wait for 1 second before capturing screenshot to ensure UI is stable
+ Thread.sleep(1000);
+
Robot robot = new Robot();
Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
BufferedImage screenCapture = robot.createScreenCapture(screenRect);
@@ -125,7 +137,7 @@ public static BufferedImage captureScreenshot(Logger logger) throws Exception {
logger.info("Error capturing screenshot: " + e.getMessage());
throw e;
}
- }
+ }*/
/**
* Saves a screenshot to a temporary file with logging
@@ -136,7 +148,7 @@ public static BufferedImage captureScreenshot(Logger logger) throws Exception {
* @return The temporary file
* @throws Exception if file creation fails
*/
- public static File saveScreenshotToFile(BufferedImage screenshot, String fileName, Logger logger) throws Exception {
+/* public static File saveScreenshotToFile(BufferedImage screenshot, String fileName, Logger logger) throws Exception {
try {
File tempFile = File.createTempFile(fileName, ".png");
ImageIO.write(screenshot, "PNG", tempFile);
@@ -146,40 +158,40 @@ public static File saveScreenshotToFile(BufferedImage screenshot, String fileNam
logger.info("Failed to save screenshot to file: " + e.getMessage());
throw new RuntimeException("Unable to save screenshot for processing.", e);
}
- }
-
- /**
- * Captures screenshot and uploads it to S3 using S3 URL
- *
- * @param s3Url The S3 URL to upload to
- * @param fileName The base filename for the screenshot
- * @param logger The logger instance
- * @return true if successful, false otherwise
- */
- public static boolean captureAndUploadScreenshot(String s3Url, String fileName, Logger logger) {
- try {
- // Capture screenshot
- BufferedImage screenshot = captureScreenshot(logger);
-
- // Save to file
- File screenshotFile = saveScreenshotToFile(screenshot, fileName, logger);
-
- // Upload to S3
- boolean uploadResult = uploadFile(s3Url, screenshotFile.getAbsolutePath(), logger);
-
- // Clean up temporary file
- if (screenshotFile.exists()) {
- screenshotFile.delete();
- }
-
- return uploadResult;
-
- } catch (Exception e) {
- logger.info("Error in captureAndUploadScreenshot: " + ExceptionUtils.getStackTrace(e));
- return false;
- }
- }
+ }*/
+// /**
+// * Captures screenshot and uploads it to S3 using S3 URL
+// *
+// * @param s3Url The S3 URL to upload to
+// * @param fileName The base filename for the screenshot
+// * @param logger The logger instance
+// * @return true if successful, false otherwise
+// */
+// public static boolean captureAndUploadScreenshot(String s3Url, String fileName, Logger logger) {
+// try {
+// // Capture screenshot
+// BufferedImage screenshot = captureScreenshot(logger);
+//
+// // Save to file
+// File screenshotFile = saveScreenshotToFile(screenshot, fileName, logger);
+//
+// // Upload to S3
+// boolean uploadResult = uploadFile(s3Url, screenshotFile.getAbsolutePath(), logger);
+//
+// // Clean up temporary file
+// if (screenshotFile.exists()) {
+// screenshotFile.delete();
+// }
+//
+// return uploadResult;
+//
+// } catch (Exception e) {
+// logger.info("Error in captureAndUploadScreenshot: " + ExceptionUtils.getStackTrace(e));
+// return false;
+// }
+// }
+//
/**
* Uploads a file to S3 using the provided URL
*
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/util/StringCompareUtil.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/StringCompareUtil.java
new file mode 100644
index 00000000..31baf1f2
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/util/StringCompareUtil.java
@@ -0,0 +1,55 @@
+package com.testsigma.addons.util;
+
+import lombok.Data;
+
+@Data
+public class StringCompareUtil {
+ private String errorMessage;
+ private String successMessage;
+
+ public boolean performOperation(String string1, String string2, String operation){
+ boolean equalsCheck = false;
+ boolean containsCheck = false;
+ this.errorMessage = "Not a valid operator("+operation+")";
+
+ switch (operation) {
+ case "equals":
+ if (string1.equals(string2)) {
+ equalsCheck = true;
+ this.successMessage = "Both the strings match: " + string1 + " == " + string2;
+ } else {
+ this.errorMessage = "Strings do not match. Value1: " + string2 + ", Value2: " + string1;
+ }
+ break;
+ case "equals ignore-case":
+ if (string1.equalsIgnoreCase(string2)) {
+ equalsCheck = true;
+ this.successMessage = "Both the strings match (ignore case): " + string1 + " == " + string2;
+ } else {
+ this.errorMessage = "Strings do not match (ignore case). Value1: " + string2 + ", Value2: " + string1;
+ }
+ break;
+ case "contains":
+ if (string1.contains(string2)) {
+ containsCheck = true;
+ this.successMessage = string1 + " contains " + string2;
+ } else {
+ this.errorMessage = "Value1 does not contain Value2. Value1: " + string1 + ", Value2: " + string2;
+ }
+ break;
+ case "contains ignore-case":
+ if (string1.toLowerCase().contains(string2.toLowerCase())) {
+ containsCheck = true;
+ this.successMessage = string1 + " contains (ignore case) " + string2;
+ } else {
+ this.errorMessage = "Value1 does not contain Value2 (ignore case). Value1: " + string1 + ", Value2: " + string2;
+ }
+ break;
+ }
+ if(equalsCheck || containsCheck){
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/AddDataToClipboard.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/AddDataToClipboard.java
new file mode 100644
index 00000000..7a3ee2c1
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/AddDataToClipboard.java
@@ -0,0 +1,104 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.datatransfer.Clipboard;
+import java.awt.datatransfer.StringSelection;
+import java.awt.datatransfer.Transferable;
+
+@Action(actionText = "Add data data-to-copy to clipboard",
+ description = "This action copies the specified data to the system clipboard. " +
+ "The data can then be pasted using Ctrl+V or other paste operations. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Add data to clipboard",
+ useCustomScreenshot = true)
+public class AddDataToClipboard extends WindowsAdvancedAction {
+
+ @TestData(reference = "data-to-copy")
+ private com.testsigma.sdk.TestData dataToCopy;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Add Data To Clipboard: Starting Execution ===");
+
+ try {
+ String data = dataToCopy.getValue().toString();
+
+ logger.info("Adding data to clipboard: " + data);
+
+ // Validate data
+ if (data == null) {
+ setErrorMessage("Data to copy cannot be null");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_clipboard_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Get the system clipboard
+ Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
+
+ // Create a StringSelection object with the data
+ StringSelection stringSelection = new StringSelection(data);
+
+ // Set the clipboard contents
+ clipboard.setContents(stringSelection, null);
+
+ // Verify the data was copied successfully
+ Transferable clipboardContents = clipboard.getContents(null);
+ if (clipboardContents != null && clipboardContents.isDataFlavorSupported(java.awt.datatransfer.DataFlavor.stringFlavor)) {
+ try {
+ String clipboardData = (String) clipboardContents.getTransferData(java.awt.datatransfer.DataFlavor.stringFlavor);
+ if (data.equals(clipboardData)) {
+ String successMessage = String.format(
+ "Successfully copied data to clipboard: %s",
+ data
+ );
+ setSuccessMessage(successMessage);
+ logger.info("Successfully copied data to clipboard: " + data);
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_clipboard_screenshot", logger);
+
+ return Result.SUCCESS;
+ } else {
+ setErrorMessage("Data verification failed - clipboard contents do not match expected data");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_clipboard_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ } catch (Exception e) {
+ setErrorMessage("Error verifying clipboard contents: " + e.getMessage());
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_clipboard_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ } else {
+ setErrorMessage("Failed to copy data to clipboard - clipboard operation not supported");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_clipboard_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ } catch (Exception e) {
+ String errorMessage = "Error copying data to clipboard: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("Exception during clipboard operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_clipboard_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/AddDataToFile.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/AddDataToFile.java
new file mode 100644
index 00000000..2479fcfe
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/AddDataToFile.java
@@ -0,0 +1,107 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+@Action(actionText = "Add data file-content to the file file-path",
+ description = "This action adds the specified data to a file at the given path. " +
+ "If the file doesn't exist, it will be created. " +
+ "The data will be appended to the end of the file. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Add data to file",
+ useCustomScreenshot = true)
+public class AddDataToFile extends WindowsAdvancedAction {
+
+ @TestData(reference = "file-content")
+ private com.testsigma.sdk.TestData fileContents;
+
+ @TestData(reference = "file-path")
+ private com.testsigma.sdk.TestData filePath;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Add Data To File: Starting Execution ===");
+
+ try {
+ String content = fileContents.getValue().toString();
+ String path = filePath.getValue().toString();
+
+ logger.info("Adding data to file: " + path);
+ logger.info("Data to add: " + content);
+
+ // Validate file path
+ if (path == null || path.trim().isEmpty()) {
+ setErrorMessage("File path cannot be empty");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_file_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Create Path object and ensure parent directories exist
+ Path filePathObj = Paths.get(path);
+ Path parentDir = filePathObj.getParent();
+
+ if (parentDir != null && !Files.exists(parentDir)) {
+ logger.info("Creating parent directories: " + parentDir);
+ Files.createDirectories(parentDir);
+ }
+
+ // Create or append to file
+ File file = filePathObj.toFile();
+ boolean fileExisted = file.exists();
+
+ try (FileWriter writer = new FileWriter(file, true)) { // true for append mode
+ writer.write(content);
+ writer.flush();
+ }
+
+ String action = fileExisted ? "appended to" : "created and written to";
+ String successMessage = String.format(
+ "Successfully %s file: %s with data: %s",
+ action, path, content
+ );
+
+ setSuccessMessage(successMessage);
+ logger.info("Successfully " + action + " file: " + path);
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_file_screenshot", logger);
+
+ return Result.SUCCESS;
+
+ } catch (IOException e) {
+ String errorMessage = "Error writing to file: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("IOException during file write operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_file_failure_screenshot", logger);
+ return Result.FAILED;
+
+ } catch (Exception e) {
+ String errorMessage = "Unexpected error during file operation: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("Exception during file operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "add_data_to_file_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnFixedCoordinates.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnFixedCoordinates.java
new file mode 100644
index 00000000..fe4b9918
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnFixedCoordinates.java
@@ -0,0 +1,126 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.util.NoSuchElementException;
+
+@Action(actionText = "Click on fixed coordinates x-coordinate y-coordinate",
+ description = "This action clicks on the screen at the specified fixed pixel coordinates. " +
+ "The coordinates should be provided as comma-separated values (e.g., '100,200'). " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on fixed coordinates (pixel values)",
+ useCustomScreenshot = true)
+public class ClickOnFixedCoordinates extends WindowsAdvancedAction {
+
+ @TestData(reference = "x-coordinate")
+ private com.testsigma.sdk.TestData xCoordinate;
+
+ @TestData(reference = "y-coordinate")
+ private com.testsigma.sdk.TestData yCoordinate;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Click On Fixed Coordinates: Starting Execution ===");
+
+ try {
+ String xCoordinateValue = xCoordinate.getValue().toString();
+ String yCoordinateValue = yCoordinate.getValue().toString();
+
+ logger.info("Clicking on fixed coordinates: " + xCoordinateValue + ", " + yCoordinateValue);
+
+ // Validate and parse coordinates
+ if (xCoordinateValue == null || xCoordinateValue.trim().isEmpty() || yCoordinateValue == null || yCoordinateValue.trim().isEmpty()) {
+ setErrorMessage("Coordinate values cannot be empty");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_fixed_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ int x, y;
+ try {
+ x = Integer.parseInt(xCoordinateValue.trim());
+ y = Integer.parseInt(yCoordinateValue.trim());
+ } catch (NumberFormatException e) {
+ setErrorMessage("Invalid coordinate values. Coordinates must be valid integers.");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_fixed_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Validate coordinates are within screen bounds
+ Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize();
+ if (x < 0 || x >= screenSize.width || y < 0 || y >= screenSize.height) {
+ setErrorMessage(String.format(
+ "Coordinates (%d, %d) are outside screen bounds. Screen size: %dx%d",
+ x, y, screenSize.width, screenSize.height
+ ));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_fixed_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ logger.info("Clicking at coordinates: (" + x + ", " + y + ")");
+
+ // Perform the click
+ performClickWithRobot(x, y);
+
+ String successMessage = String.format(
+ "Successfully clicked at fixed coordinates: x-%d, y-%d",
+ x, y
+ );
+ setSuccessMessage(successMessage);
+ logger.info("Successfully clicked at coordinates: (" + x + ", " + y + ")");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_fixed_coordinates_screenshot", logger);
+
+ return Result.SUCCESS;
+
+ } catch (Exception e) {
+ String errorMessage = "Error clicking on fixed coordinates: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("Exception during click operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_fixed_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ */
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImage.java
new file mode 100644
index 00000000..88b0c5d6
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImage.java
@@ -0,0 +1,429 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.Constants;
+import com.testsigma.addons.util.ResponseObjectForFindImage;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.http.client.config.RequestConfig;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+
+@Action(actionText = "Click on the image image-url",
+ description = "This action takes an image URL (S3 URL or local file path), "
+ + "finds that image on the current screen, " + "and clicks on it. The action "
+ + " locates the image within the screen. " + "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on the image", useCustomScreenshot = true)
+public class ClickOnImage extends WindowsAdvancedAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ RequestConfig config = RequestConfig.custom().setSocketTimeout(10 * 60 * 1000)
+ .setConnectionRequestTimeout(60 * 1000).setConnectTimeout(60 * 1000).build();
+ ObjectMapper mapper = new ObjectMapper();
+ ResponseObjectForFindImage responseObjectForFindImage = new ResponseObjectForFindImage();
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Click On Image: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ logger.info("Looking for image from URL: " + imageUrlValue);
+ File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File baseImageFile = saveScreenshotToFile(screenCapture, "click_image_screenshot");
+ // Call visual testing API instead of OCR
+ performApiCall(baseImageFile, searchImageFile);
+ logger.info("Visual testing API call completed");
+ Thread.sleep(1000);
+ // Capture and upload screenshot on success
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_success_screenshot", logger);
+ return Result.SUCCESS;
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
+
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ *
+ * @param x X coordinate for click
+ * @param y Y coordinate for click
+ */
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+
+ public void performApiCall(File baseImageFile, File searchImageFile) {
+ try {
+ logger.info("Performing visual testing with files: " + baseImageFile + " and " + searchImageFile);
+ OkHttpClient client = new OkHttpClient();
+ logger.info("Initiating http client");
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("baseImageFile", baseImageFile.getName(),
+ RequestBody.create(baseImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("searchImageFile", searchImageFile.getName(),
+ RequestBody.create(searchImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("threshold", "0.7")
+ .addFormDataPart("scale", "40")
+ .addFormDataPart("occurance", "1")
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_FIND_IMAGE_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making api call to visual server");
+ Response response = client.newCall(request).execute();
+ if (response.isSuccessful()) {
+ logger.info("Response is successful");
+ if (response.body() != null) {
+ logger.info("Response body received");
+ String responseBody = response.body().string();
+ logger.info("Response body for testing: " + responseBody);
+ responseObjectForFindImage = mapper.readValue(responseBody, ResponseObjectForFindImage.class);
+ logger.info("Deserialized the response body");
+ JsonNode jsonNode = mapper.readTree(responseBody);
+
+ boolean isFound = jsonNode.path("isFound").asBoolean();
+ int x1 = jsonNode.path("x1").asInt();
+ int y1 = jsonNode.path("y1").asInt();
+ int x2 = jsonNode.path("x2").asInt();
+ int y2 = jsonNode.path("y2").asInt();
+
+ if (isFound) {
+ int clickLocationX = (x1 + x2) / 2;
+ int clickLocationY = (y1 + y2) / 2;
+
+ logger.info("Click Location X: " + clickLocationX);
+ logger.info("Click Location Y: " + clickLocationY);
+
+ performClickWithRobot(clickLocationX, clickLocationY);
+ setSuccessMessage(String.format(
+ "Image Found : %s Image coordinates : x1- %s, x2- %s, y1- %s, y2- %s",
+ isFound, x1, x2, y1, y2
+ ));
+ } else {
+ setErrorMessage("Image NOT Found");
+ throw new RuntimeException("Visual testing failed as image not found on the screen");
+ }
+ } else {
+ logger.info("Response body is null");
+ setErrorMessage("Visual testing failed. no response body present in the visual test response");
+ throw new RuntimeException("Visual testing failed with no response body");
+ }
+ } else {
+ setErrorMessage("Visual testing failed error occurred internally");
+ throw new RuntimeException("Visual testing failed with internal server error");
+ }
+ } catch (IOException e) {
+ logger.info("Exception occurred while performing visual test at %s : %s" + ExceptionUtils.getStackTrace(e));
+ setErrorMessage("Unable to perform visual testing for : %s");
+ throw new RuntimeException("Error occurred while performing visual test at ");
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
+ }
+ }
+
+
+ /**
+ * Converts URL to File - handles both S3 URLs and local file paths
+ *
+ * @param fileName Base filename for temporary file
+ * @param url The URL or file path
+ * @return File object
+ */
+ public File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ // Try to get extension from URL
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png"; // Default to PNG for images
+ }
+ }
+
+ File tempFile = File.createTempFile(baseName, extension);
+
+ // Download file from URL
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ logger.info("Temp file created with name for s3 file " + tempFile.getName()
+ + " at path " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+
+}
+
+
+
+
+/*
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import com.testsigma.sdk.annotation.OCR;
+
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.*;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "Click on image image-url",
+ description = "This action takes an image URL (S3 URL or local file path), finds that image on the current screen, " +
+ "and clicks on it. The action uses AI to locate the image within the screen. " +
+ "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on image",
+ useCustomScreenshot = true)
+public class ClickOnImage extends WindowsAdvancedAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Click On Image: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ logger.info("Looking for image from URL: " + imageUrlValue);
+
+ // Convert URL to file
+ File targetImageFile = urlToFileConverter("target_image", imageUrlValue);
+ logger.info("Target image file prepared: " + targetImageFile.getAbsolutePath());
+
+ // Capture the current screen
+ Robot robot = new Robot();
+
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File baseImageFile = ScreenshotUtils.saveScreenshotToFile(screenCapture, "click_image_screenshot");
+
+ String url = testStepResult.getScreenshotUrl();
+ logger.info("Amazon s3 url in which we are storing base image" + url);
+ ocr.uploadFile(url, baseImageFile);
+ logger.info("url: " + testStepResult.getScreenshotUrl());
+ FindImageResponse responseObject = ocr.findImage(imageUrl.getValue().toString());
+ if (responseObject.getIsFound()) {
+ boolean isFound = responseObject.getIsFound();
+ int x1 = responseObject.getX1();
+ int y1 = responseObject.getY1();
+ int x2 = responseObject.getX2();
+ int y2 = responseObject.getY2();
+
+ int clickLocationX = (x1 + x2) / 2;
+ int clickLocationY = (y1 + y2) / 2;
+
+ logger.info("Click Location X: " + clickLocationX);
+ logger.info("Click Location Y: " + clickLocationY);
+ // Perform the click
+ robot.mouseMove(clickLocationX, clickLocationY);
+ Thread.sleep(100); // Small delay to ensure mouse is positioned
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+
+ logger.info("Successfully clicked on image at coordinates (" +
+ clickLocationX + ", " + clickLocationY + ")");
+ setSuccessMessage("Successfully clicked on image at coordinates (" +
+ clickLocationX + ", " + clickLocationY + ")");
+ setSuccessMessage("Image Found :" + isFound +
+ " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2);
+ Thread.sleep(2000);
+ } else {
+ setErrorMessage("Unable to fetch the coordinates");
+ return Result.FAILED;
+ }
+ // Upload final screenshot to S3
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, baseImageFile, logger);
+ // Clean up temporary files
+ cleanupFile(targetImageFile);
+ cleanupFile(baseImageFile);
+
+ return Result.SUCCESS;
+
+
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
+
+ return Result.FAILED;
+ }
+ }
+
+ */
+/*
+
+ public File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ // Try to get extension from URL
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png"; // Default to PNG for images
+ }
+ }
+
+ File tempFile = File.createTempFile(baseName, extension);
+
+ // Download file from URL
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ logger.info("Temp file created with name for s3 file " + tempFile.getName()
+ + " at path " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+
+
+ */
+/*
+
+ private void cleanupFile(File file) {
+ try {
+ if (file != null && file.exists() && file.isFile()) {
+ if (file.delete()) {
+ logger.debug("Cleaned up temporary file: " + file.getAbsolutePath());
+ } else {
+ logger.debug("Failed to delete temporary file: " + file.getAbsolutePath());
+ }
+ }
+ } catch (Exception e) {
+ logger.debug("Error cleaning up file: " + e.getMessage());
+ }
+ }
+
+
+}
+*/
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImageWithFindImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImageWithFindImage.java
new file mode 100644
index 00000000..4b19fb68
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnImageWithFindImage.java
@@ -0,0 +1,248 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.Constants;
+import com.testsigma.addons.util.ResponseObjectForFindImage;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.http.client.config.RequestConfig;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+
+@Action(actionText = "Click on the image image-url with threshold threshold-value",
+ description = "This action takes an image URL (S3 URL or local file path), "
+ + "finds that image on the current screen, " + "and clicks on it. The action "
+ + " locates the image within the screen. " + "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on the image with given threshold", useCustomScreenshot = true)
+public class ClickOnImageWithFindImage extends WindowsAdvancedAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+ @TestData(reference = "threshold-value")
+ private com.testsigma.sdk.TestData thresholdValue;
+
+ RequestConfig config = RequestConfig.custom().setSocketTimeout(10 * 60 * 1000)
+ .setConnectionRequestTimeout(60 * 1000).setConnectTimeout(60 * 1000).build();
+ ObjectMapper mapper = new ObjectMapper();
+ ResponseObjectForFindImage responseObjectForFindImage = new ResponseObjectForFindImage();
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Click On Image: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ logger.info("Looking for image from URL: " + imageUrlValue);
+ File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File baseImageFile = saveScreenshotToFile(screenCapture, "click_image_screenshot");
+ // Call visual testing API instead of OCR
+ performApiCall(baseImageFile, searchImageFile);
+ logger.info("Visual testing API call completed");
+ Thread.sleep(1000);
+ // Capture and upload screenshot on success
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_success_screenshot", logger);
+ return Result.SUCCESS;
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_image_failure_screenshot", logger);
+
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ *
+ * @param x X coordinate for click
+ * @param y Y coordinate for click
+ */
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+
+ public void performApiCall(File baseImageFile, File searchImageFile) {
+ try {
+ logger.info("Performing visual testing with files: " + baseImageFile + " and " + searchImageFile);
+ OkHttpClient client = new OkHttpClient();
+ logger.info("Initiating http client");
+
+ String threshold = thresholdValue.getValue().toString();
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("baseImageFile", baseImageFile.getName(),
+ RequestBody.create(baseImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("searchImageFile", searchImageFile.getName(),
+ RequestBody.create(searchImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("threshold", threshold)
+ .addFormDataPart("scale", "40")
+ .addFormDataPart("occurance", "1")
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_FIND_IMAGE_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making api call to visual server");
+ Response response = client.newCall(request).execute();
+ if (response.isSuccessful()) {
+ logger.info("Response is successful");
+ if (response.body() != null) {
+ logger.info("Response body received");
+ String responseBody = response.body().string();
+ logger.info("Response body for testing: " + responseBody);
+ responseObjectForFindImage = mapper.readValue(responseBody, ResponseObjectForFindImage.class);
+ logger.info("Deserialized the response body");
+ JsonNode jsonNode = mapper.readTree(responseBody);
+
+ boolean isFound = jsonNode.path("isFound").asBoolean();
+ int x1 = jsonNode.path("x1").asInt();
+ int y1 = jsonNode.path("y1").asInt();
+ int x2 = jsonNode.path("x2").asInt();
+ int y2 = jsonNode.path("y2").asInt();
+
+ if (isFound) {
+ int clickLocationX = (x1 + x2) / 2;
+ int clickLocationY = (y1 + y2) / 2;
+
+ logger.info("Click Location X: " + clickLocationX);
+ logger.info("Click Location Y: " + clickLocationY);
+
+ performClickWithRobot(clickLocationX, clickLocationY);
+ setSuccessMessage(String.format(
+ "Image Found : %s Image coordinates : x1- %s, x2- %s, y1- %s, y2- %s",
+ isFound, x1, x2, y1, y2
+ ));
+ } else {
+ setErrorMessage("Image NOT Found");
+ throw new RuntimeException("Visual testing failed as image not found on the screen");
+ }
+ } else {
+ logger.info("Response body is null");
+ setErrorMessage("Visual testing failed. no response body present in the visual test response");
+ throw new RuntimeException("Visual testing failed with no response body");
+ }
+ } else {
+ setErrorMessage("Visual testing failed error occurred internally");
+ throw new RuntimeException("Visual testing failed with internal server error");
+ }
+ } catch (IOException e) {
+ logger.info("Exception occurred while performing visual test at %s : %s" + ExceptionUtils.getStackTrace(e));
+ setErrorMessage("Unable to perform visual testing for : %s");
+ throw new RuntimeException("Error occurred while performing visual test at ");
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
+ }
+ }
+
+
+ /**
+ * Converts URL to File - handles both S3 URLs and local file paths
+ *
+ * @param fileName Base filename for temporary file
+ * @param url The URL or file path
+ * @return File object
+ */
+ public File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ // Try to get extension from URL
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png"; // Default to PNG for images
+ }
+ }
+
+ File tempFile = File.createTempFile(baseName, extension);
+
+ // Download file from URL
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ logger.info("Temp file created with name for s3 file " + tempFile.getName()
+ + " at path " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+
+}
+
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnPositionRelativeToImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnPositionRelativeToImage.java
new file mode 100644
index 00000000..ced0dcb3
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnPositionRelativeToImage.java
@@ -0,0 +1,333 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.Constants;
+import com.testsigma.addons.util.ResponseObjectForFindImage;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.http.client.config.RequestConfig;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "Click on position position-type relative to the image image-url with pixel offset pixel-offset",
+ description = "This action takes an image URL (S3 URL or local file path), finds that image on the current screen, " +
+ "and clicks at a position relative to it with a pixel offset. Position can be Left, Right, Top, Bottom, or Center of the image. " +
+ "The pixel offset determines how far from the image edge to click (positive values move away from image, negative values move towards image). " +
+ "For Center position, offset is ignored. " +
+ "The action uses AI to locate the image within the screen and then performs the click at the specified relative position. " +
+ "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on position relative to image",
+ useCustomScreenshot = true)
+public class ClickOnPositionRelativeToImage extends WindowsAdvancedAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestData(reference = "position-type", allowedValues = {"Left", "Right", "Top", "Bottom", "Center"})
+ private com.testsigma.sdk.TestData position;
+
+ @TestData(reference = "pixel-offset")
+ private com.testsigma.sdk.TestData pixelOffset;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ RequestConfig config = RequestConfig.custom().setSocketTimeout(10 * 60 * 1000)
+ .setConnectionRequestTimeout(60 * 1000).setConnectTimeout(60 * 1000).build();
+ ObjectMapper mapper = new ObjectMapper();
+ ResponseObjectForFindImage responseObjectForFindImage = new ResponseObjectForFindImage();
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Click On Position Relative To Image: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ String positionValue = position.getValue().toString();
+ int offset = Integer.parseInt(pixelOffset.getValue().toString());
+
+ logger.info("Looking for image from URL: " + imageUrlValue + " to click at position: " + positionValue +
+ " with offset: " + offset + " pixels");
+
+ // Convert URL to file
+ File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
+ logger.info("Target image file prepared: " + searchImageFile.getAbsolutePath());
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File baseImageFile = saveScreenshotToFile(screenCapture, "click_relative_image_screenshot");
+
+ // Call visual testing API to find the image
+ performApiCall(baseImageFile, searchImageFile, positionValue, offset);
+ logger.info("Visual testing API call completed");
+
+ // Capture and upload screenshot on success
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_relative_image_success_screenshot", logger);
+
+ // Clean up temporary files
+ cleanupFile(searchImageFile);
+ cleanupFile(baseImageFile);
+
+ return Result.SUCCESS;
+
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_relative_image_failure_screenshot", logger);
+
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ *
+ * @param x X coordinate for click
+ * @param y Y coordinate for click
+ */
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+
+ /**
+ * Calculates the click position based on image coordinates, position, and offset
+ * @param x1 Left coordinate of image
+ * @param y1 Top coordinate of image
+ * @param x2 Right coordinate of image
+ * @param y2 Bottom coordinate of image
+ * @param position The relative position (Left, Right, Top, Bottom, Center)
+ * @param offset The pixel offset from the image
+ * @return Point with calculated click coordinates
+ */
+ private Point calculateClickPosition(int x1, int y1, int x2, int y2, String position, int offset) {
+ int centerX = (x1 + x2) / 2;
+ int centerY = (y1 + y2) / 2;
+
+ int clickX = centerX;
+ int clickY = centerY;
+
+ switch (position.toUpperCase()) {
+ case "LEFT":
+ clickX = x1 - offset;
+ clickY = centerY;
+ break;
+ case "RIGHT":
+ clickX = x2 + offset;
+ clickY = centerY;
+ break;
+ case "TOP":
+ clickX = centerX;
+ clickY = y1 - offset;
+ break;
+ case "BOTTOM":
+ clickX = centerX;
+ clickY = y2 + offset;
+ break;
+ case "CENTER":
+ // For center, offset is ignored
+ clickX = centerX;
+ clickY = centerY;
+ break;
+ default:
+ logger.debug("Unknown position: " + position + ". Using center.");
+ break;
+ }
+
+ return new Point(clickX, clickY);
+ }
+
+ public void performApiCall(File baseImageFile, File searchImageFile, String position, int offset) {
+ try {
+ logger.info("Performing visual testing with files: " + baseImageFile + " and " + searchImageFile);
+ OkHttpClient client = new OkHttpClient();
+ logger.info("Initiating http client");
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("baseImageFile", baseImageFile.getName(),
+ RequestBody.create(baseImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("searchImageFile", searchImageFile.getName(),
+ RequestBody.create(searchImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("threshold", "0.7")
+ .addFormDataPart("scale", "40")
+ .addFormDataPart("occurance", "1")
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_FIND_IMAGE_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making api call to visual server");
+ Response response = client.newCall(request).execute();
+ if (response.isSuccessful()) {
+ logger.info("Response is successful");
+ if (response.body() != null) {
+ logger.info("Response body received");
+ String responseBody = response.body().string();
+ logger.info("Response body for testing: " + responseBody);
+ responseObjectForFindImage = mapper.readValue(responseBody, ResponseObjectForFindImage.class);
+ logger.info("Deserialized the response body");
+ JsonNode jsonNode = mapper.readTree(responseBody);
+
+ boolean isFound = jsonNode.path("isFound").asBoolean();
+ int x1 = jsonNode.path("x1").asInt();
+ int y1 = jsonNode.path("y1").asInt();
+ int x2 = jsonNode.path("x2").asInt();
+ int y2 = jsonNode.path("y2").asInt();
+
+ if (isFound) {
+ // Calculate click position based on position parameter and offset
+ Point clickPoint = calculateClickPosition(x1, y1, x2, y2, position, offset);
+
+ logger.info("Image found with coordinates: x1=" + x1 + ", y1=" + y1 + ", x2=" + x2 + ", y2=" + y2);
+ logger.info("Calculated click position: (" + clickPoint.x + ", " + clickPoint.y + ") with offset: " + offset + " pixels");
+
+ performClickWithRobot(clickPoint.x, clickPoint.y);
+ setSuccessMessage(String.format(
+ "Successfully clicked %s of image with offset %d pixels at coordinates: x-%d, y-%d. Image coordinates: x1-%d, x2-%d, y1-%d, y2-%d",
+ position, offset, clickPoint.x, clickPoint.y, x1, x2, y1, y2
+ ));
+ } else {
+ setErrorMessage("Image NOT Found");
+ throw new RuntimeException("Visual testing failed as image not found on the screen");
+ }
+ } else {
+ logger.info("Response body is null");
+ setErrorMessage("Visual testing failed. no response body present in the visual test response");
+ throw new RuntimeException("Visual testing failed with no response body");
+ }
+ } else {
+ setErrorMessage("Visual testing failed error occurred internally");
+ throw new RuntimeException("Visual testing failed with internal server error");
+ }
+ } catch (IOException e) {
+ logger.info("Exception occurred while performing visual test: " + ExceptionUtils.getStackTrace(e));
+ setErrorMessage("Unable to perform visual testing");
+ throw new RuntimeException("Error occurred while performing visual test");
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
+ }
+ }
+
+ /**
+ * Converts URL to File - handles both S3 URLs and local file paths
+ *
+ * @param fileName Base filename for temporary file
+ * @param url The URL or file path
+ * @return File object
+ */
+ public File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ // Try to get extension from URL
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png"; // Default to PNG for images
+ }
+ }
+
+ File tempFile = File.createTempFile(baseName, extension);
+
+ // Download file from URL
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ logger.info("Temp file created with name for s3 file " + tempFile.getName()
+ + " at path " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+
+
+ /**
+ * Cleans up temporary file
+ * @param file File to delete
+ */
+ private void cleanupFile(File file) {
+ try {
+ if (file != null && file.exists() && file.isFile()) {
+ if (file.delete()) {
+ logger.debug("Cleaned up temporary file: " + file.getAbsolutePath());
+ } else {
+ logger.debug("Failed to delete temporary file: " + file.getAbsolutePath());
+ }
+ }
+ } catch (Exception e) {
+ logger.debug("Error cleaning up file: " + e.getMessage());
+ }
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnPositionRelativeToText.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnPositionRelativeToText.java
new file mode 100644
index 00000000..91f01132
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnPositionRelativeToText.java
@@ -0,0 +1,336 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.Constants;
+import com.testsigma.addons.util.OCRResponse;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.addons.util.OCRTextPoint;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+@Action(actionText = "Click on the position-type relative to the text text-to-find with pixel offset pixel-offset" +
+ " and maximum wait time wait-time-in-seconds seconds",
+ description = "This action finds the specified text on the screen and clicks at a position relative to it with a pixel offset. " +
+ "Position can be Left, Right, Top, Bottom, or Center of the text. " +
+ "The pixel offset determines how far from the text edge to click (positive values move away from text, negative values move towards text). " +
+ "For Center position, offset is ignored. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on position relative to text",
+ useCustomScreenshot = true)
+public class ClickOnPositionRelativeToText extends WindowsAdvancedAction {
+
+ @TestData(reference = "text-to-find")
+ private com.testsigma.sdk.TestData textToFind;
+
+ @TestData(reference = "position-type", allowedValues = {"Left", "Right", "Top", "Bottom", "Center"})
+ private com.testsigma.sdk.TestData position;
+
+ @TestData(reference = "pixel-offset")
+ private com.testsigma.sdk.TestData pixelOffset;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData maxWaitSeconds;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ ObjectMapper mapper = new ObjectMapper();
+ OCRResponse ocrResponse = new OCRResponse();
+
+ private static final int POLLING_INTERVAL_MS = 1500; // 1.5 second polling interval
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Click On Position Relative To Text: Starting Execution ===");
+
+ try {
+ String targetText = textToFind.getValue().toString();
+ String positionValue = position.getValue().toString();
+ int offset = Integer.parseInt(pixelOffset.getValue().toString());
+ int timeoutMs = Integer.parseInt(maxWaitSeconds.getValue().toString()) * 1000; // Convert seconds to milliseconds
+
+ logger.info("Looking for text: '" + targetText + "' to click " + positionValue +
+ " with offset: " + offset + " pixels, max wait time: " + maxWaitSeconds.getValue() + " seconds");
+
+ long startTime = System.currentTimeMillis();
+ long endTime = startTime + timeoutMs;
+
+ while (System.currentTimeMillis() < endTime) {
+ logger.info("Polling attempt - checking for text: '" + targetText + "'");
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "click_relative_position_screenshot");
+
+ // Extract text points using OCR
+ List textPoints = extractTextPoints(screenshotFile);
+ logger.info("Found " + textPoints.size() + " text elements");
+
+ // Find the matching text
+ OCRTextPoint textPoint = findMatchingText(textPoints, targetText);
+
+ if (textPoint != null) {
+ logger.info("Found text with coordinates: x1=" + textPoint.getX1() + ", y1=" + textPoint.getY1() +
+ ", x2=" + textPoint.getX2() + ", y2=" + textPoint.getY2());
+
+ // Calculate click position based on position and offset
+ Point clickPoint = calculateClickPosition(textPoint, positionValue, offset);
+ logger.info("Calculated click position: (" + clickPoint.x + ", " + clickPoint.y + ")");
+
+ // Perform the click
+ performClickWithRobot(clickPoint.x, clickPoint.y);
+
+ logger.info("Successfully clicked " + positionValue + " of text '" + targetText +
+ "' with offset " + offset + " pixels at coordinates (" +
+ clickPoint.x + ", " + clickPoint.y + ")");
+ setSuccessMessage("Successfully clicked " + positionValue + " of text '" + targetText +
+ "' with offset " + offset + " pixels at coordinates (" +
+ clickPoint.x + ", " + clickPoint.y + ")");
+
+ // Upload final screenshot to S3
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ return Result.SUCCESS;
+ }
+
+ // Clean up temporary file
+ if (screenshotFile.exists()) {
+ screenshotFile.delete();
+ }
+
+ // Check if we should continue polling
+ long remainingTime = endTime - System.currentTimeMillis();
+ if (remainingTime > POLLING_INTERVAL_MS) {
+ logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000)
+ + " second before next attempt. " +
+ "Remaining time: " + (remainingTime / 1000) + " seconds");
+ Thread.sleep(POLLING_INTERVAL_MS);
+ } else {
+ break; // No time left for another attempt
+ }
+ }
+
+ // If we reach here, timeout occurred
+ logger.debug("Timeout reached. Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds.");
+ setErrorMessage("Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds. Unable to perform click.");
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_relative_position_failure_screenshot", logger);
+ return Result.FAILED;
+
+ } catch (NumberFormatException e) {
+ logger.debug("Invalid numeric value: " + e.getMessage());
+ setErrorMessage("Invalid numeric value provided. Please check timeout and pixel offset values.");
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_relative_position_failure_screenshot", logger);
+ return Result.FAILED;
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_relative_position_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Calculates the click position based on text point, position, and offset
+ * @param textPoint The OCR text point
+ * @param position The relative position (Left, Right, Top, Bottom, Center)
+ * @param offset The pixel offset from the text
+ * @return Point with calculated click coordinates
+ */
+ private Point calculateClickPosition(OCRTextPoint textPoint, String position, int offset) {
+ int centerX = (int) ((textPoint.getX1() + textPoint.getX2()) / 2);
+ int centerY = (int) ((textPoint.getY1() + textPoint.getY2()) / 2);
+
+ int clickX = centerX;
+ int clickY = centerY;
+
+ switch (position.toUpperCase()) {
+ case "LEFT":
+ clickX = (int) textPoint.getX1() - offset;
+ clickY = centerY;
+ break;
+ case "RIGHT":
+ clickX = (int) textPoint.getX2() + offset;
+ clickY = centerY;
+ break;
+ case "TOP":
+ clickX = centerX;
+ clickY = (int) textPoint.getY1() - offset;
+ break;
+ case "BOTTOM":
+ clickX = centerX;
+ clickY = (int) textPoint.getY2() + offset;
+ break;
+ case "CENTER":
+ // For center, offset is ignored
+ clickX = centerX;
+ clickY = centerY;
+ break;
+ default:
+ logger.debug("Unknown position: " + position + ". Using center.");
+ break;
+ }
+
+ return new Point(clickX, clickY);
+ }
+
+ /**
+ * Extracts text points from the screenshot using OCR API
+ */
+ private List extractTextPoints(File screenshotFile) throws Exception {
+ try {
+ logger.info("Extracting text points from screenshot: " + screenshotFile.getAbsolutePath());
+ OkHttpClient client = new OkHttpClient();
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("ocrImageFile", screenshotFile.getName(),
+ RequestBody.create(screenshotFile, MediaType.parse("image/png")))
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_OCR_TEXT_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making OCR API call");
+ Response response = client.newCall(request).execute();
+
+ if (response.isSuccessful()) {
+ logger.info("OCR API response is successful");
+ if (response.body() != null) {
+ String responseBody = response.body().string();
+ logger.info("OCR Response body: " + responseBody);
+
+ ocrResponse = mapper.readValue(responseBody, OCRResponse.class);
+ logger.info("Deserialized OCR response");
+
+ if (ocrResponse.hasError()) {
+ throw new RuntimeException("OCR API returned error: " + ocrResponse.getError());
+ }
+
+ if (!ocrResponse.hasText()) {
+ throw new RuntimeException("No text found in the image");
+ }
+
+ return ocrResponse.getText();
+ } else {
+ throw new RuntimeException("OCR API returned null response body");
+ }
+ } else {
+ throw new RuntimeException("OCR API call failed with status: " + response.code());
+ }
+
+ } catch (IOException e) {
+ logger.info("Exception during OCR API call: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException("Error during OCR API call: " + e.getMessage());
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Finds the matching text in the list of text points
+ */
+ private OCRTextPoint findMatchingText(List textPoints, String targetText) {
+ logger.info("Searching for text: '" + targetText + "'");
+
+ // First try exact match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().equals(targetText)) {
+ logger.info("Found exact match: " + textPoint.getText());
+ return textPoint;
+ }
+ }
+
+ // Then try case-insensitive match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().equalsIgnoreCase(targetText)) {
+ logger.info("Found case-insensitive match: " + textPoint.getText());
+ return textPoint;
+ }
+ }
+
+ // Finally try contains match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().toLowerCase().contains(targetText.toLowerCase())) {
+ logger.info("Found contains match: " + textPoint.getText());
+ return textPoint;
+ }
+ }
+
+ logger.warn("No matching text found for: '" + targetText + "'");
+ logger.info("Available text elements:");
+ for (OCRTextPoint textPoint : textPoints) {
+ logger.info(" - '" + textPoint.getText() + "'");
+ }
+
+ return null;
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ */
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+
+ /**
+ * Saves a BufferedImage to a temporary file
+ */
+ public static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ System.out.println("Screenshot saved to: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (IOException e) {
+ System.err.println("Error saving screenshot to file: " + e.getMessage());
+ throw new Exception("Failed to save screenshot: " + e.getMessage());
+ }
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnRelativeCoordinates.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnRelativeCoordinates.java
new file mode 100644
index 00000000..2c604f5f
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnRelativeCoordinates.java
@@ -0,0 +1,125 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import java.awt.event.InputEvent;
+
+import java.awt.*;
+import java.util.NoSuchElementException;
+
+@Action(actionText = "Click on relative coordinates x-percentage y-percentage",
+ description = "This action clicks on the screen at coordinates specified as percentages of the screen dimensions. " +
+ "The coordinates should be provided as comma-separated percentage values (e.g., '50,25' for center-left). " +
+ "Values should be between 0 and 100. This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on relative coordinates (where x & y are percentage of screen width & height)",
+ useCustomScreenshot = true)
+public class ClickOnRelativeCoordinates extends WindowsAdvancedAction {
+
+ @TestData(reference = "x-percentage")
+ private com.testsigma.sdk.TestData xPercentage;
+
+ @TestData(reference = "y-percentage")
+ private com.testsigma.sdk.TestData yPercentage;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Click On Relative Coordinates: Starting Execution ===");
+
+ try {
+ // Validate and parse coordinates
+ String xPercentageValue = xPercentage.getValue().toString();
+ String yPercentageValue = yPercentage.getValue().toString();
+
+ double xPercent, yPercent;
+ try {
+ xPercent = Double.parseDouble(xPercentageValue.trim());
+ yPercent = Double.parseDouble(yPercentageValue.trim());
+ } catch (NumberFormatException e) {
+ setErrorMessage("Invalid coordinate values. Coordinates must be valid numbers.");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_relative_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Validate percentage values are within valid range
+ if (xPercent < 0 || xPercent > 100 || yPercent < 0 || yPercent > 100) {
+ setErrorMessage("Percentage values must be between 0 and 100. Provided values: x=" + xPercent + "%, y=" + yPercent + "%");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_relative_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Get screen dimensions
+ Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize();
+ int screenWidth = screenSize.width;
+ int screenHeight = screenSize.height;
+
+ // Calculate actual pixel coordinates
+ int x = (int) Math.round((xPercent / 100.0) * screenWidth);
+ int y = (int) Math.round((yPercent / 100.0) * screenHeight);
+
+ logger.info("Screen dimensions: " + screenWidth + "x" + screenHeight);
+ logger.info("Percentage coordinates: (" + xPercent + "%, " + yPercent + "%)");
+ logger.info("Calculated pixel coordinates: (" + x + ", " + y + ")");
+
+ // Perform the click
+ performClickWithRobot(x, y);
+
+ String successMessage = String.format(
+ "Successfully clicked at relative coordinates: %.1f%% width, %.1f%% height " +
+ "(pixel coordinates: x-%d, y-%d)",
+ xPercent, yPercent, x, y
+ );
+ setSuccessMessage(successMessage);
+ logger.info("Successfully clicked at relative coordinates: (" + xPercent + "%, " + yPercent + "%)");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_relative_coordinates_screenshot", logger);
+
+ return Result.SUCCESS;
+
+ } catch (Exception e) {
+ String errorMessage = "Error clicking on relative coordinates: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("Exception during click operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_relative_coordinates_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ */
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithSpacesAndWait.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithSpacesAndWait.java
new file mode 100644
index 00000000..d003dab9
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithSpacesAndWait.java
@@ -0,0 +1,236 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.*;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+@Action(actionText = "Click on the sentence sentence-to-click with maximum wait time wait-time-in-seconds seconds",
+ description = "This action waits for the specified text to appear on the screen and then clicks on it. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on sentence with wait",
+ useCustomScreenshot = true)
+public class ClickOnTextWithSpacesAndWait extends WindowsAdvancedAction {
+
+ @TestData(reference = "sentence-to-click")
+ private com.testsigma.sdk.TestData textToClick;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData maxWaitSeconds;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ ObjectMapper mapper = new ObjectMapper();
+ OCRResponse ocrResponse = new OCRResponse();
+
+ private static final int POLLING_INTERVAL_MS = 1500;
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Click On Text With Wait: Starting Execution ===");
+
+ try {
+ String targetText = textToClick.getValue().toString();
+ // Convert seconds to milliseconds
+ int timeoutMs = Integer.parseInt(maxWaitSeconds.getValue().toString()) * 1000;
+
+ logger.info("Looking for text to click: '" + targetText + "' with max wait time: "
+ + maxWaitSeconds.getValue() + " seconds");
+
+ long startTime = System.currentTimeMillis();
+ long endTime = startTime + timeoutMs;
+ OCRUtils ocrUtils = new OCRUtils();
+ Robot robot = new Robot();
+ while (System.currentTimeMillis() < endTime) {
+ logger.info("Polling attempt - checking for text: '" + targetText + "'");
+
+ // Capture the current screen
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x"
+ + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "click_text_screenshot");
+
+ // Extract text points using OCR
+ List textPoints = extractTextPoints(screenshotFile);
+ logger.info("Found " + textPoints.size() + " text elements");
+
+ // Find the matching text
+ OCRTextPoint targetTextPoint = ocrUtils.findMatchingTextForSentence(textPoints, targetText, logger);
+ if (targetTextPoint != null) {
+ // Text found - perform click and return success
+ logger.info("Found Text point with text = " + targetTextPoint.getText()
+ + ", x1 = " + targetTextPoint.getX1() + ", y1 = " + targetTextPoint.getY1() +
+ ", x2 = " + targetTextPoint.getX2() + ", y2 = " + targetTextPoint.getY2());
+
+ int clickX = (int) targetTextPoint.getCenterX();
+ int clickY = (int) targetTextPoint.getCenterY();
+ logger.info("Clicking on text at coordinates: (" + clickX + ", " + clickY + ")");
+
+ performClickWithRobot(robot, clickX, clickY);
+ logger.info("Successfully clicked on text: '" + targetText +
+ "' at coordinates (" + clickX + ", " + clickY + ")");
+
+ setSuccessMessage(String.format(
+ "Successfully clicked on text: %s at coordinates: x-%d, y-%d",
+ targetText, clickX, clickY
+ ));
+ // wait for one second before taking screenshot
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {
+ // Ignore
+ }
+ // Upload final screenshot to S3
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+ return Result.SUCCESS;
+ }
+
+ // Text not found - check if we should continue polling
+ long remainingTime = endTime - System.currentTimeMillis();
+ if (remainingTime > POLLING_INTERVAL_MS) {
+ logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000.0)
+ + " seconds before next attempt. " +
+ "Remaining time: " + (remainingTime / 1000) + " seconds");
+ Thread.sleep(POLLING_INTERVAL_MS);
+ } else {
+ break; // No time left for another attempt
+ }
+ }
+ // If we reach here, timeout occurred
+ logger.debug("Timeout reached. Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds.");
+ setErrorMessage("Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds. Unable to perform click.");
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_text_wait_failure_screenshot", logger);
+ return Result.FAILED;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "click_text_wait_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+ // i have not moved these three methods to utility class as i am facing some issue with files being passed as
+ // argument to utility classes.
+ /**
+ * Extracts text points from the screenshot using OCR API
+ */
+ private List extractTextPoints(File screenshotFile) throws Exception {
+ try {
+ logger.info("Extracting text points from screenshot: " + screenshotFile.getAbsolutePath());
+ OkHttpClient client = new OkHttpClient();
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("ocrImageFile", screenshotFile.getName(),
+ RequestBody.create(screenshotFile, MediaType.parse("image/png")))
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_OCR_TEXT_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making OCR API call");
+ Response response = client.newCall(request).execute();
+
+ if (response.isSuccessful()) {
+ logger.info("OCR API response is successful");
+ if (response.body() != null) {
+ String responseBody = response.body().string();
+ logger.info("OCR Response body: " + responseBody);
+
+ ocrResponse = mapper.readValue(responseBody, OCRResponse.class);
+ logger.info("Deserialized OCR response");
+
+ if (ocrResponse.hasError()) {
+ throw new RuntimeException("OCR API returned error: " + ocrResponse.getError());
+ }
+
+ if (!ocrResponse.hasText()) {
+ throw new RuntimeException("No text found in the image");
+ }
+
+ return ocrResponse.getText();
+ } else {
+ throw new RuntimeException("OCR API returned null response body");
+ }
+ } else {
+ throw new RuntimeException("OCR API call failed with status: " + response.code());
+ }
+
+ } catch (IOException e) {
+ logger.info("Exception during OCR API call: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException("Error during OCR API call: " + e.getMessage());
+ } catch (Exception e) {
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Performs click using Robot with appropriate delays
+ */
+ private void performClickWithRobot(Robot robot, int x, int y) throws Exception {
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
+
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+
+
+ /**
+ * Saves a BufferedImage to a temporary file
+ */
+ public static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ System.out.println("Screenshot saved to: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (IOException e) {
+ System.err.println("Error saving screenshot to file: " + e.getMessage());
+ throw new Exception("Failed to save screenshot: " + e.getMessage());
+ }
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithWait.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithWait.java
index aff3a1a4..8f8afea6 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithWait.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ClickOnTextWithWait.java
@@ -1,154 +1,124 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.sdk.AIRequest;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.Constants;
+import com.testsigma.addons.util.OCRResponse;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.addons.util.OCRTextPoint;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
-import com.testsigma.sdk.annotation.AI;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
-import lombok.Data;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import javax.imageio.ImageIO;
import java.awt.*;
-import java.awt.event.InputEvent;
import java.awt.image.BufferedImage;
import java.io.File;
-import java.util.ArrayList;
+import java.io.IOException;
+import java.util.List;
import java.util.NoSuchElementException;
-@Action(actionText = "Click on text test-data with maximum wait time test-data2 seconds",
+@Action(actionText = "Click on text text-to-click with maximum wait time wait-time-in-seconds seconds",
description = "This action waits for the specified text to appear on the screen and then clicks on it. " +
- "It uses AI to locate the text and performs a mouse click at the center of the text area. " +
+ "It uses OCR to locate the text within the screen and performs a mouse click at the center of the text area. " +
"This works only for local executions",
- applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "ClickOnTextWithWait",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Click on text with wait",
useCustomScreenshot = true)
public class ClickOnTextWithWait extends WindowsAdvancedAction {
- @TestData(reference = "test-data", description = "The text to search for and click on")
+ @TestData(reference = "text-to-click")
private com.testsigma.sdk.TestData textToClick;
- @TestData(reference = "test-data2", description = "Maximum wait time in seconds")
+ @TestData(reference = "wait-time-in-seconds")
private com.testsigma.sdk.TestData maxWaitSeconds;
- @AI
- private com.testsigma.sdk.AI ai;
-
@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;
-
- private final String prompt = "You are provided with a screenshot of a computer application with dimensions " +
- "WIDTHxHEIGHT pixels. Your task is to analyze this screenshot and determine if the specified text is present anywhere in " +
- "the image. If the text is found, you must also provide the coordinates (x,y) of the center of the text area, " +
- "where x and y are pixel coordinates within the image dimensions (0,0 is top-left corner). " +
- "Look for the text in any form - it could be in buttons, labels, text fields, menus, " +
- "or any other UI element. " +
- "Return 'YES,x,y' if the text is found (where x,y are the pixel coordinates), or 'NO' if the text is not found. " +
- "The text to search for is: ";
-
- private static final int POLLING_INTERVAL_MS = 1500; // 1.5 second polling interval
+
+ ObjectMapper mapper = new ObjectMapper();
+ OCRResponse ocrResponse = new OCRResponse();
+
+ private static final int POLLING_INTERVAL_MS = 1500;
@Override
protected Result execute() throws NoSuchElementException {
logger.info("=== Click On Text With Wait: Starting Execution ===");
- try {
+ try {
String targetText = textToClick.getValue().toString();
int timeoutMs = Integer.parseInt(maxWaitSeconds.getValue().toString()) * 1000; // Convert seconds to milliseconds
-
+
logger.info("Looking for text to click: '" + targetText + "' with max wait time: " + maxWaitSeconds.getValue() + " seconds");
long startTime = System.currentTimeMillis();
long endTime = startTime + timeoutMs;
-
+
while (System.currentTimeMillis() < endTime) {
logger.info("Polling attempt - checking for text: '" + targetText + "'");
-
+
// Capture the current screen
Robot robot = new Robot();
Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
BufferedImage screenCapture = robot.createScreenCapture(screenRect);
logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
-
+
// Save the screenshot to a temporary file
File screenshotFile = saveScreenshotToFile(screenCapture, "click_text_screenshot");
-
- // Create AI request with screen dimensions
- AIRequest aiRequest = new AIRequest();
- String fullPrompt = prompt.replace("WIDTHxHEIGHT",
- screenCapture.getWidth() + "x" + screenCapture.getHeight()) +
- "'" + targetText + "'. ";
- aiRequest.setPrompt(fullPrompt);
- aiRequest.setModel("gpt-4o");
-
- // Add the screenshot file
- ArrayList files = new ArrayList<>();
- files.add(screenshotFile);
- aiRequest.setFiles(files);
-
- // Invoke AI
- String aiResponse = ai.invokeAI(aiRequest);
- logger.info("AI response: " + aiResponse);
-
- // Parse AI response for text location
- ClickLocation clickLocation = parseAIResponseForClick(aiResponse);
-
- if (clickLocation != null && clickLocation.isFound()) {
- logger.info("Text found at coordinates: (" + clickLocation.getX() + ", " + clickLocation.getY() + ")");
-
- // Perform the click
- robot.mouseMove(clickLocation.getX(), clickLocation.getY());
- Thread.sleep(100); // Small delay to ensure mouse is positioned
- robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
- Thread.sleep(50);
- robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
-
- logger.info("Successfully clicked on text: '" + targetText + "' at coordinates (" +
- clickLocation.getX() + ", " + clickLocation.getY() + ")");
- setSuccessMessage("Successfully clicked on text '" + targetText + "' at coordinates (" +
- clickLocation.getX() + ", " + clickLocation.getY() + ")");
-
+
+ // Extract text points using OCR
+ List textPoints = extractTextPoints(screenshotFile);
+ logger.info("Found " + textPoints.size() + " text elements");
+
+ // Find the matching text
+ OCRTextPoint targetTextPoint = findMatchingText(textPoints, targetText);
+ if (targetTextPoint != null) {
+ // Text found - perform click and return success
+ logger.info("Found Textpoint with text = " + targetTextPoint.getText() + ", x1 = " + targetTextPoint.getX1() +
+ ", y1 = " + targetTextPoint.getY1() + ", x2 = " + targetTextPoint.getX2() + ", y2 = " + targetTextPoint.getY2());
+
+ int clickX = (int) targetTextPoint.getCenterX();
+ int clickY = (int) targetTextPoint.getCenterY();
+ logger.info("Clicking on text at coordinates: (" + clickX + ", " + clickY + ")");
+
+ performClickWithRobot(clickX, clickY);
+ logger.info("Successfully clicked on text: '" + targetText + "' at coordinates (" + clickX + ", " + clickY + ")");
+
+ setSuccessMessage(String.format(
+ "Successfully clicked on text: %s at coordinates: x-%d, y-%d",
+ targetText, clickX, clickY
+ ));
+
// Upload final screenshot to S3
ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
-
return Result.SUCCESS;
}
-
- // Clean up temporary file
- if (screenshotFile.exists()) {
- screenshotFile.delete();
- }
-
- // Check if we should continue polling
+
+ // Text not found - check if we should continue polling
long remainingTime = endTime - System.currentTimeMillis();
if (remainingTime > POLLING_INTERVAL_MS) {
- logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000)
- + " second before next attempt. " +
+ logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000.0)
+ + " seconds before next attempt. " +
"Remaining time: " + (remainingTime / 1000) + " seconds");
Thread.sleep(POLLING_INTERVAL_MS);
} else {
break; // No time left for another attempt
}
}
-
// If we reach here, timeout occurred
- logger.debug("Timeout reached. Text '" + targetText + "' was not found on the screen within " +
+ logger.debug("Timeout reached. Text '" + targetText + "' was not found on the screen within " +
maxWaitSeconds.getValue() + " seconds.");
- setErrorMessage("Text '" + targetText + "' was not found on the screen within " +
+ setErrorMessage("Text '" + targetText + "' was not found on the screen within " +
maxWaitSeconds.getValue() + " seconds. Unable to perform click.");
// Capture and upload screenshot even on failure
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_text_wait_failure_screenshot", logger);
return Result.FAILED;
-
- } catch (NumberFormatException e) {
- logger.debug("Invalid timeout value: " + maxWaitSeconds.getValue());
- setErrorMessage("Invalid timeout value: " + maxWaitSeconds.getValue() +
- ". Please provide a valid number of seconds.");
- // Capture and upload screenshot even on failure
- ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "click_text_wait_failure_screenshot", logger);
- return Result.FAILED;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
} catch (Exception e) {
logger.debug("Exception during click operation: " + e.getMessage());
setErrorMessage("Error during click operation: " + e.getMessage());
@@ -158,82 +128,141 @@ protected Result execute() throws NoSuchElementException {
}
}
-
-
/**
- * Saves the screenshot to a temporary file
- * @param screenshot The captured screenshot
- * @param fileName The base filename
- * @return The temporary file
- * @throws Exception if file creation fails
+ * Extracts text points from the screenshot using OCR API
*/
- private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ private List extractTextPoints(File screenshotFile) throws Exception {
try {
- File tempFile = File.createTempFile(fileName, ".png");
- ImageIO.write(screenshot, "PNG", tempFile);
- return tempFile;
+ logger.info("Extracting text points from screenshot: " + screenshotFile.getAbsolutePath());
+ OkHttpClient client = new OkHttpClient();
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("ocrImageFile", screenshotFile.getName(),
+ RequestBody.create(screenshotFile, MediaType.parse("image/png")))
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_OCR_TEXT_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ logger.info("Making OCR API call");
+ Response response = client.newCall(request).execute();
+
+ if (response.isSuccessful()) {
+ logger.info("OCR API response is successful");
+ if (response.body() != null) {
+ String responseBody = response.body().string();
+ logger.info("OCR Response body: " + responseBody);
+
+ ocrResponse = mapper.readValue(responseBody, OCRResponse.class);
+ logger.info("Deserialized OCR response");
+
+ if (ocrResponse.hasError()) {
+ throw new RuntimeException("OCR API returned error: " + ocrResponse.getError());
+ }
+
+ if (!ocrResponse.hasText()) {
+ throw new RuntimeException("No text found in the image");
+ }
+
+ return ocrResponse.getText();
+ } else {
+ throw new RuntimeException("OCR API returned null response body");
+ }
+ } else {
+ throw new RuntimeException("OCR API call failed with status: " + response.code());
+ }
+
+ } catch (IOException e) {
+ logger.info("Exception during OCR API call: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException("Error during OCR API call: " + e.getMessage());
} catch (Exception e) {
- logger.debug("Failed to save screenshot to file: " + e.getMessage());
- throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ logger.info("Exception: " + ExceptionUtils.getStackTrace(e));
+ throw new RuntimeException(e);
}
}
/**
- * Parses the AI response to determine if text was found and get click coordinates
- * @param aiResponse The response from AI
- * @return ClickLocation object with coordinates if found, null otherwise
+ * Finds the matching text in the list of text points
*/
- private ClickLocation parseAIResponseForClick(String aiResponse) {
- if (aiResponse == null || aiResponse.trim().isEmpty()) {
- logger.debug("AI response is null or empty");
- return null;
+ private OCRTextPoint findMatchingText(List textPoints, String targetText) {
+ logger.info("Searching for text: '" + targetText + "'");
+
+ // First try exact match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().equals(targetText)) {
+ logger.info("Found exact match: " + textPoint.getText());
+ return textPoint;
+ }
}
- String response = aiResponse.trim().toUpperCase();
- logger.debug("Parsing AI response for click: " + response);
-
- // Check for positive response with coordinates (format: YES,x,y)
- if (response.startsWith("YES,")) {
- try {
- String[] parts = response.split(",");
- if (parts.length >= 3) {
- int x = Integer.parseInt(parts[1].trim());
- int y = Integer.parseInt(parts[2].trim());
- logger.info("Parsed coordinates: x=" + x + ", y=" + y);
- return new ClickLocation(x, y, true);
- }
- } catch (NumberFormatException e) {
- logger.debug("Failed to parse coordinates from AI response: " + aiResponse);
+ // Then try case-insensitive match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().equalsIgnoreCase(targetText)) {
+ logger.info("Found case-insensitive match: " + textPoint.getText());
+ return textPoint;
}
}
- // Check for various negative responses
- if (response.contains("NO") || response.contains("FALSE") || response.contains("NOT FOUND") ||
- response.contains("ABSENT") || response.contains("NOT PRESENT")) {
- return new ClickLocation(0, 0, false);
+ // Finally try contains match
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textPoint.getText().toLowerCase().contains(targetText.toLowerCase())) {
+ logger.info("Found contains match: " + textPoint.getText());
+ return textPoint;
+ }
+ }
+
+ logger.warn("No matching text found for: '" + targetText + "'");
+ logger.info("Available text elements:");
+ for (OCRTextPoint textPoint : textPoints) {
+ logger.info(" - '" + textPoint.getText() + "'");
}
- // If response is unclear, log it and return null
- logger.debug("Unclear AI response: " + aiResponse + ". Treating as 'not found'.");
return null;
}
/**
- * Inner class to hold click location information
+ * Performs click using Robot with appropriate delays
*/
- private static class ClickLocation {
- private final int x;
- private final int y;
- private final boolean found;
-
- public ClickLocation(int x, int y, boolean found) {
- this.x = x;
- this.y = y;
- this.found = found;
- }
+ private void performClickWithRobot(int x, int y) throws Exception {
+ Robot robot = new Robot();
+
+ // Move mouse to the target location
+ logger.info("Moving mouse to coordinates (" + x + ", " + y + ")");
+ robot.mouseMove(x, y);
+ Thread.sleep(200); // Delay to ensure mouse is positioned
- public int getX() { return x; }
- public int getY() { return y; }
- public boolean isFound() { return found; }
+ // Press mouse button
+ logger.info("Pressing mouse button");
+ robot.mousePress(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(100); // Delay between press and release
+
+ // Release mouse button
+ logger.info("Releasing mouse button");
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(200); // Delay after click completion
+
+ logger.info("Click completed successfully");
+ }
+
+
+
+ /**
+ * Saves a BufferedImage to a temporary file
+ */
+ public static File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ System.out.println("Screenshot saved to: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (IOException e) {
+ System.err.println("Error saving screenshot to file: " + e.getMessage());
+ throw new Exception("Failed to save screenshot: " + e.getMessage());
+ }
}
+
}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/CopyAndPasteTestDataOnScreen.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/CopyAndPasteTestDataOnScreen.java
new file mode 100644
index 00000000..fe915922
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/CopyAndPasteTestDataOnScreen.java
@@ -0,0 +1,67 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.datatransfer.Clipboard;
+import java.awt.datatransfer.StringSelection;
+import java.awt.datatransfer.Transferable;
+import java.awt.event.KeyEvent;
+
+@Data
+@Action(actionText = "Copy and paste given data on screen text-to-copy-paste",
+ description = "This action allows you to copy and paste data on the screen using the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Copy and paste given data on screen",
+ useCustomScreenshot = true)
+public class CopyAndPasteTestDataOnScreen extends WindowsAdvancedAction {
+
+ @TestData(reference = "text-to-copy-paste")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public com.testsigma.sdk.Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ StringSelection stringSelection = new StringSelection(testData.getValue().toString());
+ Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
+ Transferable data = clipboard.getContents(null);
+ clipboard.setContents(stringSelection, null);
+ robot.keyPress(KeyEvent.VK_CONTROL);
+ robot.keyPress(KeyEvent.VK_V);
+ KeyboardUtils.sleep(100);
+ robot.keyRelease(KeyEvent.VK_V);
+ robot.keyRelease(KeyEvent.VK_CONTROL);
+ Thread.sleep(500);
+ clipboard.setContents(data, null);
+ setSuccessMessage("Given data copied and pasted successfully on the screen");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "copy_paste_data_screenshot", logger);
+
+ } catch (Exception e) {
+ result = Result.FAILED;
+ setErrorMessage("An error occurred while copying and pasting data: " + e.getMessage());
+ logger.debug("Error copying and pasting data: " + ExceptionUtils.getStackTrace(e));
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "copy_paste_data_failure_screenshot", logger);
+ return result;
+ }
+ return result;
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboard.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboard.java
index 94ced113..1b6c1a17 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboard.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboard.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
import com.testsigma.sdk.ApplicationType;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
@@ -12,10 +12,6 @@
import org.apache.commons.lang3.exception.ExceptionUtils;
import java.awt.*;
-import java.awt.datatransfer.Clipboard;
-import java.awt.datatransfer.StringSelection;
-import java.awt.datatransfer.Transferable;
-import java.awt.event.KeyEvent;
@Data
@@ -23,7 +19,7 @@
description = "This action allows you to enter data into a field using the keyboard. " +
"This works only for local executions",
applicationType = ApplicationType.WINDOWS_ADVANCED,
- displayName = "EnterData",
+ displayName = "Enter data on screen",
useCustomScreenshot = true
)
public class EnterDataUsingKeyboard extends WindowsAdvancedAction {
@@ -39,17 +35,15 @@ public com.testsigma.sdk.Result execute() {
try {
// Instantiate the Robot Class
Robot robot = new Robot();
- StringSelection stringSelection = new StringSelection(testData.getValue().toString());
- Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
- Transferable data = clipboard.getContents(null);
- clipboard.setContents(stringSelection, null);
- robot.keyPress(KeyEvent.VK_CONTROL);
- robot.keyPress(KeyEvent.VK_V);
- KeyboardUtils.sleep(30);
- robot.keyRelease(KeyEvent.VK_V);
- robot.keyRelease(KeyEvent.VK_CONTROL);
- Thread.sleep(500);
- clipboard.setContents(data, null);
+ String text = testData.getValue().toString();
+
+ Thread.sleep(1000); // Wait 1 seconds to focus the target window
+
+ for (char c : text.toCharArray()) {
+ KeyboardUtils.typeCharacter(robot, c);
+ Thread.sleep(100); // Delay between keystrokes (optional)
+ }
+
setSuccessMessage("Given data entered successfully on the given image");
// Capture and upload screenshot
@@ -60,11 +54,11 @@ public com.testsigma.sdk.Result execute() {
setErrorMessage("An error occurred while initializing the Robot class: " + e.getMessage());
logger.debug("Error initializing Robot class: " + ExceptionUtils.getStackTrace(e));
// Capture and upload screenshot even on failure
- ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "enter_data_failure_screenshot", logger);
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "enter_data_failure_screenshot", logger);
return result;
}
return result;
}
-
}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboardAndPressEnter.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboardAndPressEnter.java
new file mode 100644
index 00000000..270493cf
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EnterDataUsingKeyboardAndPressEnter.java
@@ -0,0 +1,69 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.event.KeyEvent;
+
+
+@Data
+@Action(actionText = "Enter data test-data using keyboard",
+ description = "This action allows you to enter data into a field using the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Enter Data on screen and press Enter",
+ useCustomScreenshot = true
+)
+public class EnterDataUsingKeyboardAndPressEnter extends WindowsAdvancedAction {
+ @TestData(reference = "test-data")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public com.testsigma.sdk.Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String text = testData.getValue().toString();
+
+ Thread.sleep(1000); // Wait 1 second to focus the target window
+
+ for (char c : text.toCharArray()) {
+ KeyboardUtils.typeCharacter(robot, c);
+ Thread.sleep(50); // Delay between keystrokes (optional)
+ }
+ logger.info("Pressing Enter key");
+ robot.keyPress(KeyEvent.VK_ENTER);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(KeyEvent.VK_ENTER);
+
+ setSuccessMessage("Given data entered successfully on the given image and Enter key pressed");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "enter_data_screenshot", logger);
+
+ } catch (Exception e) {
+ result = Result.FAILED;
+ setErrorMessage("An error occurred while initializing the Robot class: " + e.getMessage());
+ logger.debug("Error initializing Robot class: " + ExceptionUtils.getStackTrace(e));
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "enter_data_failure_screenshot", logger);
+ return result;
+ }
+ return result;
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EraseDataInFile.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EraseDataInFile.java
new file mode 100644
index 00000000..c90f4687
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/EraseDataInFile.java
@@ -0,0 +1,121 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+@Action(actionText = "Erase data in file file-path",
+ description = "This action erases all data from the specified file. " +
+ "The file will be truncated to zero length, effectively removing all content. " +
+ "If the file doesn't exist, an error will be returned. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Erase data in file",
+ useCustomScreenshot = true)
+public class EraseDataInFile extends WindowsAdvancedAction {
+
+ @TestData(reference = "file-path")
+ private com.testsigma.sdk.TestData filePath;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Erase Data In File: Starting Execution ===");
+
+ try {
+ String path = filePath.getValue().toString();
+
+ logger.info("Erasing data from file: " + path);
+
+ // Validate file path
+ if (path == null || path.trim().isEmpty()) {
+ setErrorMessage("File path cannot be empty");
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "erase_data_in_file_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Check if file exists
+ Path filePathObj = Paths.get(path);
+ File file = filePathObj.toFile();
+
+ if (!file.exists()) {
+ String errorMessage = "File does not exist: " + path;
+ setErrorMessage(errorMessage);
+ logger.info(errorMessage);
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "erase_data_in_file_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Check if file is writable
+ if (!file.canWrite()) {
+ String errorMessage = "File is not writable: " + path;
+ setErrorMessage(errorMessage);
+ logger.info(errorMessage);
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "erase_data_in_file_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+
+ // Get file size before erasing for logging
+ long fileSizeBefore = file.length();
+ logger.info("File size before erasing: " + fileSizeBefore + " bytes");
+
+ // Erase file content by truncating it
+ try (FileWriter writer = new FileWriter(file, false)) { // false for overwrite mode
+ // Write nothing, effectively truncating the file
+ writer.write("");
+ writer.flush();
+ }
+
+ // Verify file is now empty
+ long fileSizeAfter = file.length();
+ logger.info("File size after erasing: " + fileSizeAfter + " bytes");
+
+ String successMessage = String.format(
+ "Successfully erased all data from file: %s. " +
+ "File size changed from %d bytes to %d bytes",
+ path, fileSizeBefore, fileSizeAfter
+ );
+
+ setSuccessMessage(successMessage);
+ logger.info("Successfully erased data from file: " + path);
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "erase_data_in_file_screenshot", logger);
+
+ return Result.SUCCESS;
+
+ } catch (IOException e) {
+ String errorMessage = "Error erasing file content: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("IOException during file erase operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "erase_data_in_file_failure_screenshot", logger);
+ return Result.FAILED;
+
+ } catch (Exception e) {
+ String errorMessage = "Unexpected error during file erase operation: " + e.getMessage();
+ setErrorMessage(errorMessage);
+ logger.debug("Exception during file erase operation: " + ExceptionUtils.getStackTrace(e));
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "erase_data_in_file_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ExtractDataFromScreen.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ExtractDataFromScreen.java
new file mode 100644
index 00000000..4381eb29
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/ExtractDataFromScreen.java
@@ -0,0 +1,169 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.AIRequest;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.annotation.*;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.List;
+
+
+@Action(actionText = "AI: Extract the data and store in a variable Query-to-extract-data Variable-Name",
+ description = "Store data from screen based on the query",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Extract the data and store in a variable",
+ useCustomScreenshot = true)
+public class ExtractDataFromScreen extends WindowsAdvancedAction {
+
+
+ @TestData(reference = "Query-to-extract-data")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestData(reference = "Variable-Name", isRuntimeVariable = true)
+ private com.testsigma.sdk.TestData runtimeVariable;
+
+ @RunTimeData
+ private com.testsigma.sdk.RunTimeData runTimeData;
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+ @AI
+ private com.testsigma.sdk.AI ai;
+
+ final String prompt = "You are given with a screenshot of a desktop, you have to extract data based on user queries." +
+ " You MUST respond in the following JSON format only:" +
+ " {\"isQueryRelated\": boolean, \"extracted Text\": \"string\", \"additional data\": \"string\"}" +
+ " Rules:" +
+ " - Set 'isQueryRelated' to true if the query is related to the screenshot content, false otherwise." +
+ " - 'extracted Text' should contain ONLY the raw data requested by the user, no additional context or explanation." +
+ " - For example, - If user asks for a number, give only the number. If for text, give only that text." +
+ " - Use 'additional data' for any contextual information, explanations, or additional details." +
+ " - If the query is not related to the screenshot, set 'isQueryRelated' to false, 'extracted Text' to empty string, and explain in 'additional data'." +
+ " - If the requested data is not present in the screenshot, set 'isQueryRelated' to true, 'extracted Text' to empty string, and explain in 'additional data'." +
+ " - Ensure your response is valid JSON format only, no additional text.";
+
+ @Override
+ protected com.testsigma.sdk.Result execute() {
+ Result result = Result.SUCCESS;
+ logger.info("=== Store Data From Screen: Starting Execution ===");
+
+ String userQuery = testData.getValue().toString();
+ String fullPrompt = prompt + " The query is: " + testData.getValue().toString();
+
+ Robot robot = null;
+ try {
+ robot = new Robot();
+
+
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = ScreenshotUtils.saveScreenshotToFile(screenCapture, "click_text_screenshot");
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile , logger);
+
+ // Add the screenshot file
+ AIRequest aiRequest = new AIRequest();
+ aiRequest.setPrompt(fullPrompt);
+ aiRequest.setFiles(List.of(screenshotFile));
+ aiRequest.setModel("gpt-4.1");
+
+ // Invoke AI
+ String aiResponse = ai.invokeAI(aiRequest);
+ logger.info("AI response: " + aiResponse);
+
+ // Parse the JSON response
+ AIResponse parsedResponse = parseAIResponse(aiResponse);
+
+ if (parsedResponse != null) {
+ if (parsedResponse.isQueryRelated) {
+ if (!parsedResponse.extractedText.trim().isEmpty()) {
+ // Store the extracted text in the runtime variable
+ logger.info("additional data : " + parsedResponse.additionalData);
+ runTimeData.setKey(runtimeVariable.getValue().toString());
+ runTimeData.setValue(parsedResponse.extractedText);
+ setSuccessMessage("Data extracted and stored in variable " + runtimeVariable.getValue().toString() +
+ ": '" + parsedResponse.extractedText + "'" +
+ (parsedResponse.additionalData.isEmpty() ? "" : " (Additional info: " + parsedResponse.additionalData + ")"));
+ } else {
+ setErrorMessage("Requested data not found in screenshot" +
+ (parsedResponse.additionalData.isEmpty() ? "" : ": " + parsedResponse.additionalData));
+ return Result.FAILED;
+ }
+ } else {
+ setErrorMessage("Query not related to screenshot" +
+ (parsedResponse.additionalData.isEmpty() ? "" : ": " + parsedResponse.additionalData));
+ return Result.FAILED;
+ }
+ } else {
+ setErrorMessage("Failed to get the response from AI");
+ return Result.FAILED;
+ }
+
+ } catch (Exception e) {
+ logger.info("Error during execution: " + e.getMessage());
+ setErrorMessage("Error during execution: " + e.getMessage());
+ return Result.FAILED;
+ }
+
+ return Result.SUCCESS;
+ }
+
+ /**
+ * Parses the AI response JSON
+ * @param aiResponse The raw AI response
+ * @return Parsed AIResponse object or null if parsing fails
+ */
+ private AIResponse parseAIResponse(String aiResponse) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode jsonNode = mapper.readTree(aiResponse);
+
+ AIResponse response = new AIResponse();
+ response.isQueryRelated = jsonNode.get("isQueryRelated").asBoolean();
+ response.extractedText = jsonNode.get("extracted Text").asText("");
+ response.additionalData = jsonNode.get("additional data").asText("");
+
+ return response;
+ } catch (Exception e) {
+ logger.warn("Failed to parse AI response JSON: " + e.getMessage());
+ logger.warn("Raw AI response: " + aiResponse);
+ return null;
+ }
+ }
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Inner class to hold parsed AI response
+ */
+ private static class AIResponse {
+ boolean isQueryRelated;
+ String extractedText;
+ String additionalData;
+ }
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PerformMultipleActions.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PerformMultipleActions.java
new file mode 100644
index 00000000..97818189
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PerformMultipleActions.java
@@ -0,0 +1,412 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import com.testsigma.sdk.Result;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.datatransfer.Clipboard;
+import java.awt.datatransfer.StringSelection;
+import java.awt.datatransfer.Transferable;
+import java.awt.event.KeyEvent;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Data
+@lombok.EqualsAndHashCode(callSuper = false)
+@Action(actionText = "Perform Keyboard actions actions-to-perform",
+ description = "This action performs multiple keyboard actions in sequence. " +
+ "Supports text copy-paste, special keys like {TAB}, {ENTER}, individual character typing with {KEY(xyz)}, and wait delays with {WAIT(n)}. " +
+ "Example: {WAIT(1)}test123{TAB}{WAIT(1)}abc{ENTER}{KEY(xyz)}{WAIT(2)}",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Perform Keyboard actions",
+ useCustomScreenshot = true)
+public class PerformMultipleActions extends WindowsAdvancedAction {
+
+ @TestData(reference = "actions-to-perform")
+ private com.testsigma.sdk.TestData actionsToPerform;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ // Pattern to match any delimiter: {KEY(xyz)}, {WAIT(n)}, or {SPECIALKEY}
+ private static final Pattern DELIMITER_PATTERN = Pattern.compile("\\{(?:KEY\\([^)]+\\)|WAIT\\([^)]+\\)|[^}]+)\\}");
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ logger.info("=== Perform Multiple Actions: Starting Execution ===");
+
+ String actionsToPerformValue = actionsToPerform.getValue().toString();
+ logger.info("Actions to perform: " + actionsToPerformValue);
+
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+
+ // Wait 100ms to ensure focus
+ Thread.sleep(100);
+
+ // Parse and execute actions in sequential order
+ parseAndExecuteActions(robot, actionsToPerformValue);
+
+ setSuccessMessage("All keyboard actions performed successfully");
+ logger.info("Successfully completed all keyboard actions");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "perform_multiple_actions_screenshot", logger);
+
+ } catch (Exception e) {
+ result = Result.FAILED;
+ setErrorMessage("An error occurred while performing keyboard actions: " + e.getMessage());
+ logger.debug("Error performing keyboard actions: " + ExceptionUtils.getStackTrace(e));
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "perform_multiple_actions_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+ /**
+ * Parse the input string and execute the corresponding actions in sequence
+ */
+ private void parseAndExecuteActions(Robot robot, String input) throws Exception {
+ Matcher matcher = DELIMITER_PATTERN.matcher(input);
+ int lastEnd = 0;
+
+ logger.info("=== Starting Sequential Execution ===");
+
+ while (matcher.find()) {
+ ;
+ // Process text segment before this delimiter
+ if (matcher.start() > lastEnd) {
+ String textSegment = input.substring(lastEnd, matcher.start());
+ if (!textSegment.isEmpty()) {
+ logger.info("Copy-pasting text: '" + textSegment + "'");
+ copyAndPasteText(robot, textSegment);
+ }
+ }
+
+ // Process the delimiter
+ String delimiter = matcher.group();
+ logger.info("Processing delimiter: " + delimiter);
+ processDelimiter(robot, delimiter);
+
+ lastEnd = matcher.end();
+ }
+
+ // Process any remaining text after the last delimiter
+ if (lastEnd < input.length()) {
+ String textSegment = input.substring(lastEnd);
+ if (!textSegment.isEmpty()) {
+ logger.info("Copy-pasting remaining text: '" + textSegment + "'");
+ copyAndPasteText(robot, textSegment);
+ }
+ }
+
+ logger.info("=== Sequential Execution Complete ===");
+ }
+
+ /**
+ * Process a delimiter (special key, KEY pattern, or WAIT pattern)
+ */
+ private void processDelimiter(Robot robot, String delimiter) throws Exception {
+ // Handle {WAIT(n)} pattern
+ if (delimiter.matches("\\{WAIT\\([^)]+\\)\\}")) {
+ String waitTime = delimiter.substring(6, delimiter.length() - 2); // Extract n from {WAIT(n)}
+ logger.info("Waiting for: " + waitTime + " seconds");
+ handleWait(waitTime);
+ }
+ // Handle {KEY(xyz)} pattern
+ else if (delimiter.matches("\\{KEY\\([^)]+\\)\\}")) {
+ String characters = delimiter.substring(5, delimiter.length() - 2); // Extract xyz from {KEY(xyz)}
+ logger.info("Typing individually: '" + characters + "'");
+ typeCharactersIndividually(robot, characters);
+ }
+ // Handle special keys like {TAB}, {ENTER} (but not {WAIT(n)} or {KEY(xyz)})
+ else if (delimiter.matches("\\{[^}()]+\\}")) {
+ String keyName = delimiter.substring(1, delimiter.length() - 1);
+ logger.info("Pressing special key: " + keyName);
+ pressSpecialKey(robot, keyName);
+ Thread.sleep(300);
+ }
+ }
+
+ /**
+ * Handle wait functionality for {WAIT(n)} pattern
+ */
+ private void handleWait(String waitTime) throws Exception {
+ try {
+ // Parse the wait time as integer (seconds)
+ int seconds = Integer.parseInt(waitTime.trim());
+
+ if (seconds < 0) {
+ throw new IllegalArgumentException("Wait time cannot be negative: " + seconds);
+ }
+
+ // Convert seconds to milliseconds and sleep
+ long milliseconds = seconds * 1000L;
+ logger.info("Sleeping for " + seconds + " seconds (" + milliseconds + " ms)");
+ Thread.sleep(milliseconds);
+
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid wait time format: " + waitTime + ". Expected integer value.");
+ }
+ }
+
+ /**
+ * Copy and paste text using clipboard (Windows: Ctrl+V)
+ */
+ private void copyAndPasteText(Robot robot, String text) throws Exception {
+ Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
+ Transferable originalContent = null;
+
+ try {
+ // Save current clipboard (optional - ignore if fails)
+ try {
+ originalContent = clipboard.getContents(null);
+ } catch (Exception e) {
+ logger.debug("Could not save original clipboard content: " + e.getMessage());
+ }
+
+ // Set text to clipboard
+ StringSelection stringSelection = new StringSelection(text);
+ clipboard.setContents(stringSelection, null);
+ Thread.sleep(100);
+
+ // Paste using Ctrl+V (Windows)
+ robot.keyPress(KeyEvent.VK_CONTROL);
+ KeyboardUtils.sleep(50);
+ robot.keyPress(KeyEvent.VK_V);
+ KeyboardUtils.sleep(50);
+ robot.keyRelease(KeyEvent.VK_V);
+ KeyboardUtils.sleep(50);
+ robot.keyRelease(KeyEvent.VK_CONTROL);
+ Thread.sleep(100);
+
+ } finally {
+ // Restore original clipboard content
+ if (originalContent != null) {
+ try {
+ clipboard.setContents(originalContent, null);
+ } catch (Exception e) {
+ logger.debug("Could not restore original clipboard content: " + e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
+ * Type characters individually
+ */
+ private void typeCharactersIndividually(Robot robot, String characters) throws Exception {
+ for (char c : characters.toCharArray()) {
+ typeCharacter(robot, c);
+ Thread.sleep(50);
+ }
+ }
+
+ /**
+ * Type a single character
+ */
+ private void typeCharacter(Robot robot, char character) throws Exception {
+ boolean upperCase = Character.isUpperCase(character);
+ boolean needsShift = false;
+ int keyCode;
+
+ // Handle special characters that need shift
+ switch (character) {
+ case '!':
+ keyCode = KeyEvent.VK_1;
+ needsShift = true;
+ break;
+ case '@':
+ keyCode = KeyEvent.VK_2;
+ needsShift = true;
+ break;
+ case '#':
+ keyCode = KeyEvent.VK_3;
+ needsShift = true;
+ break;
+ case '$':
+ keyCode = KeyEvent.VK_4;
+ needsShift = true;
+ break;
+ case '%':
+ keyCode = KeyEvent.VK_5;
+ needsShift = true;
+ break;
+ case '^':
+ keyCode = KeyEvent.VK_6;
+ needsShift = true;
+ break;
+ case '&':
+ keyCode = KeyEvent.VK_7;
+ needsShift = true;
+ break;
+ case '*':
+ keyCode = KeyEvent.VK_8;
+ needsShift = true;
+ break;
+ case '(':
+ keyCode = KeyEvent.VK_9;
+ needsShift = true;
+ break;
+ case ')':
+ keyCode = KeyEvent.VK_0;
+ needsShift = true;
+ break;
+ case '_':
+ keyCode = KeyEvent.VK_MINUS;
+ needsShift = true;
+ break;
+ case '+':
+ keyCode = KeyEvent.VK_EQUALS;
+ needsShift = true;
+ break;
+ case '{':
+ keyCode = KeyEvent.VK_OPEN_BRACKET;
+ needsShift = true;
+ break;
+ case '}':
+ keyCode = KeyEvent.VK_CLOSE_BRACKET;
+ needsShift = true;
+ break;
+ case '|':
+ keyCode = KeyEvent.VK_BACK_SLASH;
+ needsShift = true;
+ break;
+ case ':':
+ keyCode = KeyEvent.VK_SEMICOLON;
+ needsShift = true;
+ break;
+ case '"':
+ keyCode = KeyEvent.VK_QUOTE;
+ needsShift = true;
+ break;
+ case '<':
+ keyCode = KeyEvent.VK_COMMA;
+ needsShift = true;
+ break;
+ case '>':
+ keyCode = KeyEvent.VK_PERIOD;
+ needsShift = true;
+ break;
+ case '?':
+ keyCode = KeyEvent.VK_SLASH;
+ needsShift = true;
+ break;
+ case '~':
+ keyCode = KeyEvent.VK_BACK_QUOTE;
+ needsShift = true;
+ break;
+ default:
+ keyCode = KeyEvent.getExtendedKeyCodeForChar(character);
+ if (keyCode == KeyEvent.VK_UNDEFINED) {
+ throw new IllegalArgumentException("Cannot type character: " + character);
+ }
+ needsShift = upperCase;
+ }
+
+ if (needsShift || upperCase) {
+ robot.keyPress(KeyEvent.VK_SHIFT);
+ KeyboardUtils.sleep(10);
+ }
+
+ robot.keyPress(keyCode);
+ KeyboardUtils.sleep(10);
+ robot.keyRelease(keyCode);
+
+ if (needsShift || upperCase) {
+ KeyboardUtils.sleep(10);
+ robot.keyRelease(KeyEvent.VK_SHIFT);
+ }
+ }
+
+ /**
+ * Press a special key
+ */
+ private void pressSpecialKey(Robot robot, String keyName) throws Exception {
+ int keyCode = getSpecialKeyCode(keyName);
+
+ robot.keyPress(keyCode);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(keyCode);
+ Thread.sleep(50);
+ }
+
+ /**
+ * Get the key code for special keys
+ */
+ private int getSpecialKeyCode(String keyName) {
+ switch (keyName.toUpperCase()) {
+ case "TAB":
+ return KeyEvent.VK_TAB;
+ case "ENTER":
+ return KeyEvent.VK_ENTER;
+ case "SPACE":
+ return KeyEvent.VK_SPACE;
+ case "BACKSPACE":
+ return KeyEvent.VK_BACK_SPACE;
+ case "DELETE":
+ return KeyEvent.VK_DELETE;
+ case "WINDOWS":
+ case "WIN":
+ case "WINDOW":
+ return KeyEvent.VK_WINDOWS;
+ case "ESC":
+ case "ESCAPE":
+ return KeyEvent.VK_ESCAPE;
+ case "UP":
+ return KeyEvent.VK_UP;
+ case "DOWN":
+ return KeyEvent.VK_DOWN;
+ case "LEFT":
+ return KeyEvent.VK_LEFT;
+ case "RIGHT":
+ return KeyEvent.VK_RIGHT;
+ case "HOME":
+ return KeyEvent.VK_HOME;
+ case "END":
+ return KeyEvent.VK_END;
+ case "PAGE_UP":
+ return KeyEvent.VK_PAGE_UP;
+ case "PAGE_DOWN":
+ return KeyEvent.VK_PAGE_DOWN;
+ case "INSERT":
+ return KeyEvent.VK_INSERT;
+ case "F1":
+ return KeyEvent.VK_F1;
+ case "F2":
+ return KeyEvent.VK_F2;
+ case "F3":
+ return KeyEvent.VK_F3;
+ case "F4":
+ return KeyEvent.VK_F4;
+ case "F5":
+ return KeyEvent.VK_F5;
+ case "F6":
+ return KeyEvent.VK_F6;
+ case "F7":
+ return KeyEvent.VK_F7;
+ case "F8":
+ return KeyEvent.VK_F8;
+ case "F9":
+ return KeyEvent.VK_F9;
+ case "F10":
+ return KeyEvent.VK_F10;
+ case "F11":
+ return KeyEvent.VK_F11;
+ case "F12":
+ return KeyEvent.VK_F12;
+ default:
+ throw new IllegalArgumentException("Unsupported special key: " + keyName);
+ }
+ }
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressFunctionKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressFunctionKey.java
index 3349476f..3cec1d88 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressFunctionKey.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressFunctionKey.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
import com.testsigma.sdk.annotation.Action;
@@ -19,7 +19,7 @@
description = "This action allows you to press a function key on the keyboard. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "PressFunctionKey",
+ displayName = "Press a Function Key",
useCustomScreenshot = true
)
public class PressFunctionKey extends WindowsAdvancedAction {
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKey.java
index 3224d3e6..139ee056 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKey.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKey.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
import com.testsigma.sdk.annotation.Action;
@@ -17,7 +17,7 @@
description = "This action allows you to press a modifier key on the keyboard. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "PressModifierKey",
+ displayName = "Press a Modifier Key",
useCustomScreenshot = true)
public class PressModifierKey extends WindowsAdvancedAction {
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithBasicKeyCombination.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithBasicKeyCombination.java
index 08e141e3..16e6f515 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithBasicKeyCombination.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithBasicKeyCombination.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
import com.testsigma.sdk.annotation.Action;
@@ -17,7 +17,7 @@
description = "This action allows you to press a modifier key with an alphanumeric key on the keyboard. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "PressModifierKeyWithAlphanumericKey",
+ displayName = "Press a Modifier Key with a basic key",
useCustomScreenshot = true)
public class PressModifierKeyWithBasicKeyCombination extends WindowsAdvancedAction {
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithGivenKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithGivenKey.java
index 2a4ffa89..5431248f 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithGivenKey.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressModifierKeyWithGivenKey.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
import com.testsigma.sdk.annotation.Action;
@@ -17,7 +17,7 @@
description = "This action allows you to press a modifier key with a specific key on the keyboard. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "PressModifierKeyWithSpecificKey",
+ displayName = "Press a Modifier Key with a given Key",
useCustomScreenshot = true)
public class PressModifierKeyWithGivenKey extends WindowsAdvancedAction {
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeys.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeys.java
index e1b3f1f7..c0a6a2e2 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeys.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeys.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
import com.testsigma.sdk.annotation.Action;
@@ -17,7 +17,7 @@
description = "This action allows you to press two modifier keys simultaneously on the keyboard. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "PressTwoModifierKeys",
+ displayName = "Press two Modifier Keys",
useCustomScreenshot = true)
public class PressTwoModifierKeys extends WindowsAdvancedAction {
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeysWithBasicKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeysWithBasicKey.java
index 979eee7f..fbf445b0 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeysWithBasicKey.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/PressTwoModifierKeysWithBasicKey.java
@@ -1,7 +1,7 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.addons.windowsAdvanced.utils.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.addons.util.KeyboardUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
import com.testsigma.sdk.annotation.Action;
@@ -17,7 +17,7 @@
description = "This action allows you to press two modifier keys together with a specific key on the keyboard. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "PressTwoModifierKeysWithSpecificKey",
+ displayName = "Press two Modifier Keys with a basic Key",
useCustomScreenshot = true)
public class PressTwoModifierKeysWithBasicKey extends WindowsAdvancedAction {
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/StorePresenceOfTextInRuntimeVariable.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/StorePresenceOfTextInRuntimeVariable.java
new file mode 100644
index 00000000..35fdb20a
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/StorePresenceOfTextInRuntimeVariable.java
@@ -0,0 +1,95 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.OCRTextPoint;
+import com.testsigma.addons.util.OCRUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.*;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+
+@Action(actionText = "verify that the text text-to-verify is present in opened application and" +
+ " store result in runtime variable variable-to-store-result",
+ description = "This action stores true if text is present in the screen else it stores false in the variable " +
+ "using OCR API capabilities. " +
+ "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Store the presence of text in runtime variable",
+ useCustomScreenshot = true)
+public class StorePresenceOfTextInRuntimeVariable extends WindowsAdvancedAction {
+
+ @TestData(reference = "text-to-verify")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestData(reference = "variable-to-store-result", isRuntimeVariable = true)
+ private com.testsigma.sdk.TestData testData1;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @RunTimeData
+ private com.testsigma.sdk.RunTimeData runTimeData;
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== OCR Text Verification: Starting Execution ===");
+
+ try {
+ String expectedText = testData.getValue().toString();
+ logger.info("Looking for text: " + expectedText);
+
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ File screenshotFile = saveScreenshotToFile(screenCapture, "application_screenshot");
+ logger.info("Screenshot saved to: " + screenshotFile.getAbsolutePath());
+
+ List textPoints = OCRUtils.extractTextPoints(screenshotFile, logger);
+ logger.info("Found " + textPoints.size() + " text elements via OCR");
+
+ boolean textFound = OCRUtils.searchForText(textPoints, expectedText, logger);
+
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ if (textFound) {
+ logger.info("Text found in application. Step passed.");
+ runTimeData.setValue("true");
+ runTimeData.setKey(testData1.getValue().toString());
+ } else {
+ logger.debug("Text not found in application. Step failed.");
+ runTimeData.setValue("false");
+ runTimeData.setKey(testData1.getValue().toString());
+ }
+ } catch (Exception e) {
+ logger.debug("Exception during OCR text verification: " + e.getMessage());
+ setErrorMessage("Error during text verification: " + e.getMessage());
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "verify_text_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ setSuccessMessage("Successfully stored the presence of the text " + testData.getValue().toString() +
+ " in the variable " + testData1.getValue().toString());
+ return Result.SUCCESS;
+ }
+
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ logger.info("Screenshot saved to temporary file: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/StringCompareWindowsAdvanced.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/StringCompareWindowsAdvanced.java
new file mode 100644
index 00000000..5c94072d
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/StringCompareWindowsAdvanced.java
@@ -0,0 +1,56 @@
+package com.testsigma.addons.web;
+
+import com.testsigma.addons.util.StringCompareUtil;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.WebAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import lombok.Data;
+import org.openqa.selenium.NoSuchElementException;
+
+
+@Data
+@Action(actionText = "Verify if input-string1 match-condition with input-string2",
+ description = "Verify if both the string equals/contains with and without ignore-case",
+ applicationType = ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Verify if strings match the condition")
+public class StringCompareWindowsAdvanced extends WebAction {
+
+ @TestData(reference = "input-string1")
+ private com.testsigma.sdk.TestData Actual_Value;
+ @TestData(reference = "input-string2")
+ private com.testsigma.sdk.TestData Expected_Value;
+ @TestData(reference = "match-condition",
+ allowedValues = {"equals", "equals ignore-case", "contains", "contains ignore-case"})
+ private com.testsigma.sdk.TestData Compared_Value;
+
+
+ @Override
+ public com.testsigma.sdk.Result execute() throws NoSuchElementException {
+ //Your Awesome code starts here
+ logger.info("Initiating execution");
+ logger.info(" ActualValue: " + this.Actual_Value.getValue() + " " +
+ " ExpectedValue: " + this.Expected_Value.getValue() + " " +
+ " Operation: " + this.Compared_Value.getValue());
+ com.testsigma.sdk.Result result;
+
+ String str1 = String.valueOf(Actual_Value.getValue());
+ String str2 = String.valueOf(Expected_Value.getValue());
+ String operation = String.valueOf(Compared_Value.getValue());
+
+
+ StringCompareUtil util = new StringCompareUtil();
+ boolean operationResult = util.performOperation(str1, str2, operation);
+
+ if (operationResult) {
+ logger.info("Operation success: " + getSuccessMessage());
+ setSuccessMessage(getSuccessMessage());
+ result = com.testsigma.sdk.Result.SUCCESS;
+ } else {
+ logger.info("Operation failed: " + getErrorMessage());
+ setErrorMessage(getErrorMessage());
+ result = com.testsigma.sdk.Result.FAILED;
+ }
+ return result;
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplication.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplication.java
index 5b6c3f38..f49b7499 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplication.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplication.java
@@ -1,88 +1,61 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.sdk.AIRequest;
+import com.testsigma.addons.util.OCRTextPoint;
+import com.testsigma.addons.util.OCRUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
-import com.testsigma.sdk.annotation.AI;
-import com.testsigma.sdk.annotation.Action;
-import com.testsigma.sdk.annotation.TestData;
-import com.testsigma.sdk.annotation.TestStepResult;
-import lombok.Data;
+import com.testsigma.sdk.annotation.*;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.File;
-import java.util.ArrayList;
+import java.util.List;
import java.util.NoSuchElementException;
-@Data
-@Action(actionText = "verify that the text test-data is present in opened application",
+@Action(actionText = "verify that the text text-to-verify is present in opened application " +
+ "and store result in runtime variable result-variable-name",
description = "This action verifies that the specified text is present in the opened application" +
- " using AI capabilities. " +
+ " using OCR API capabilities. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "Verify if text is present in application",
+ displayName = "Verify if text is present in the application and store result",
useCustomScreenshot = true)
public class VerifyTextInApplication extends WindowsAdvancedAction {
- @TestData(reference = "test-data")
+ @TestData(reference = "text-to-verify")
private com.testsigma.sdk.TestData testData;
- @AI
- private com.testsigma.sdk.AI ai;
-
+ @TestData(reference = "result-variable-name", isRuntimeVariable = true)
+ private com.testsigma.sdk.TestData testData1;
+
@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;
-
- private final String prompt = "You are provided with a screenshot of a computer application." +
- " Your task is to analyze this screenshot and determine if the specified text is present anywhere in " +
- "the image.Look for the text in any form - it could be in buttons, labels, text fields, menus, " +
- "or any other UI element. Return only 'YES' if the text is found, or 'NO' if the text is not found. " +
- "The text to search for is: ";
@Override
protected Result execute() throws NoSuchElementException {
- logger.info("=== AI Text Verification: Starting Execution ===");
+ logger.info("=== OCR Text Verification: Starting Execution ===");
try {
String expectedText = testData.getValue().toString();
logger.info("Looking for text: " + expectedText);
- // Capture the current screen
Robot robot = new Robot();
Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
BufferedImage screenCapture = robot.createScreenCapture(screenRect);
logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
- // Save the screenshot to a temporary file
File screenshotFile = saveScreenshotToFile(screenCapture, "application_screenshot");
logger.info("Screenshot saved to: " + screenshotFile.getAbsolutePath());
- // Create AI request
- AIRequest aiRequest = new AIRequest();
- String fullPrompt = prompt + "'" + expectedText + "'. ";
- aiRequest.setPrompt(fullPrompt);
- aiRequest.setModel("gpt-4o");
-
- logger.info("Sending AI prompt: " + fullPrompt);
+ List textPoints = OCRUtils.extractTextPoints(screenshotFile, logger);
+ logger.info("Found " + textPoints.size() + " text elements via OCR");
- // Add the screenshot file
- ArrayList files = new ArrayList<>();
- files.add(screenshotFile);
- aiRequest.setFiles(files);
+ boolean textFound = OCRUtils.searchForText(textPoints, expectedText, logger);
- // Invoke AI
- String aiResponse = ai.invokeAI(aiRequest);
- logger.info("AI response: " + aiResponse);
-
- // Parse AI response
- boolean textFound = parseAIResponse(aiResponse);
-
- // Upload screenshot to S3
ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
-
+
if (textFound) {
logger.info("Text found in application. Step passed.");
setSuccessMessage("Text '" + expectedText + "' was found in the application.");
@@ -94,23 +67,13 @@ protected Result execute() throws NoSuchElementException {
}
} catch (Exception e) {
- logger.debug("Exception during AI text verification: " + e.getMessage());
+ logger.debug("Exception during OCR text verification: " + e.getMessage());
setErrorMessage("Error during text verification: " + e.getMessage());
- // Capture and upload screenshot even on failure
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "verify_text_failure_screenshot", logger);
return Result.FAILED;
}
}
-
-
- /**
- * Saves the screenshot to a temporary file
- * @param screenshot The captured screenshot
- * @param fileName The base filename
- * @return The temporary file
- * @throws Exception if file creation fails
- */
private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
try {
File tempFile = File.createTempFile(fileName, ".png");
@@ -119,38 +82,7 @@ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) thr
return tempFile;
} catch (Exception e) {
logger.debug("Failed to save screenshot to file: " + e.getMessage());
- throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
}
}
-
- /**
- * Parses the AI response to determine if text was found
- * @param aiResponse The response from AI
- * @return true if text was found, false otherwise
- */
- private boolean parseAIResponse(String aiResponse) {
- if (aiResponse == null || aiResponse.trim().isEmpty()) {
- logger.debug("AI response is null or empty");
- return false;
- }
-
- String response = aiResponse.trim().toUpperCase();
- logger.info("Parsing AI response: " + response);
-
- // Check for various positive responses
- if (response.contains("YES") || response.contains("TRUE") || response.contains("FOUND") ||
- response.contains("PRESENT") || response.contains("EXISTS")) {
- return true;
- }
-
- // Check for various negative responses
- if (response.contains("NO") || response.contains("FALSE") || response.contains("NOT FOUND") ||
- response.contains("ABSENT") || response.contains("NOT PRESENT")) {
- return false;
- }
-
- // If response is unclear, log it and return false
- logger.debug("Unclear AI response: " + aiResponse + ". Treating as 'not found'.");
- return false;
- }
-}
\ No newline at end of file
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplicationWithAI.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplicationWithAI.java
new file mode 100644
index 00000000..22b193e1
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/VerifyTextInApplicationWithAI.java
@@ -0,0 +1,84 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.testsigma.addons.util.OCRTextPoint;
+import com.testsigma.addons.util.OCRUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.*;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+
+@Action(actionText = "verify that the text text-to-verify is present in the screen",
+ description = "This action verifies that the specified text is present in the opened application" +
+ "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Verify if text is present in application",
+ useCustomScreenshot = true)
+public class VerifyTextInApplicationWithAI extends WindowsAdvancedAction {
+
+ @TestData(reference = "text-to-verify")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== OCR Text Verification: Starting Execution ===");
+
+ try {
+ String expectedText = testData.getValue().toString();
+ logger.info("Looking for text: " + expectedText);
+
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ File screenshotFile = saveScreenshotToFile(screenCapture, "application_screenshot");
+ logger.info("Screenshot saved to: " + screenshotFile.getAbsolutePath());
+
+ List textPoints = OCRUtils.extractTextPoints(screenshotFile, logger);
+ logger.info("Found " + textPoints.size() + " text elements via OCR");
+
+ boolean textFound = OCRUtils.searchForText(textPoints, expectedText, logger);
+
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ if (textFound) {
+ logger.info("Text found in application. Step passed.");
+ setSuccessMessage("Text '" + expectedText + "' was found in the application.");
+ return Result.SUCCESS;
+ } else {
+ logger.debug("Text not found in application. Step failed.");
+ setErrorMessage("Text '" + expectedText + "' was not found in the application.");
+ return Result.FAILED;
+ }
+
+ } catch (Exception e) {
+ logger.debug("Exception during OCR text verification: " + e.getMessage());
+ setErrorMessage("Error during text verification: " + e.getMessage());
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "verify_text_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ logger.info("Screenshot saved to temporary file: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresent.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresent.java
index 1c61f637..4f44d5d8 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresent.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresent.java
@@ -70,6 +70,7 @@ protected Result execute() {
long endTime = startTime + timeoutMs;
while (System.currentTimeMillis() < endTime) {
+
logger.info("Polling attempt - checking for image on screen");
Robot robot = new Robot();
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresentWithFindImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresentWithFindImage.java
new file mode 100644
index 00000000..667bb6a0
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilImagePresentWithFindImage.java
@@ -0,0 +1,208 @@
+package com.testsigma.addons.windowsAdvanced;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.testsigma.addons.util.Constants;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAdvancedAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import okhttp3.*;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "Wait until image image-url is present on screen with timeout wait-time-in-seconds seconds with threshold threshold-value",
+ description = "This action waits until the specified image appears on the screen within the given timeout. "
+ + "It does not click the image; it only verifies that the image is present. "
+ + "It takes an image URL (S3 URL or local file path), polls the screen every 1.5 seconds. "
+ + "Threshold (0 to 1) controls match sensitivity. Uses visual testing API for image detection.",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
+ displayName = "Wait until image is present (Find Image API)",
+ useCustomScreenshot = true)
+public class WaitUntilImagePresentWithFindImage extends WindowsAdvancedAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData timeoutSeconds;
+
+ @TestData(reference = "threshold-value")
+ private com.testsigma.sdk.TestData thresholdValue;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private static final int POLLING_INTERVAL_MS = 1500;
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Wait Until Image Present (Find Image API): Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ int timeoutMs = Integer.parseInt(timeoutSeconds.getValue().toString()) * 1000;
+ String thresholdStr = thresholdValue.getValue().toString().trim();
+ double threshold = Double.parseDouble(thresholdStr);
+ if (threshold < 0 || threshold > 1) {
+ setErrorMessage("Threshold must be between 0 and 1. Got: " + thresholdStr);
+ return Result.FAILED;
+ }
+
+ logger.info("Waiting for image: " + imageUrlValue + " | timeout: "
+ + timeoutSeconds.getValue() + "s | threshold: " + thresholdStr);
+
+ File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
+ Robot robot = new Robot();
+ long endTime = System.currentTimeMillis() + timeoutMs;
+
+ while (System.currentTimeMillis() < endTime) {
+ logger.info("Polling attempt - capturing fresh screenshot");
+
+ Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenSize);
+ File screenshotFile = new File(System.getProperty("java.io.tmpdir"),
+ "screenshot" + System.currentTimeMillis() + ".png");
+ ImageIO.write(screenCapture, "png", screenshotFile);
+ logger.info("Screenshot saved to: " + screenshotFile.getAbsolutePath());
+
+ boolean isFound = callFindImageApi(screenshotFile, searchImageFile, thresholdStr);
+
+ if (isFound) {
+ logger.info("Image found on screen. Wait successful.");
+ return Result.SUCCESS;
+ }
+
+ long remainingTime = endTime - System.currentTimeMillis();
+ if (remainingTime > 0) {
+ long sleepTime = Math.min(POLLING_INTERVAL_MS, remainingTime);
+ logger.info("Image not found yet. Waiting " + sleepTime + "ms. Remaining: " + remainingTime + "ms");
+ Thread.sleep(sleepTime);
+ }
+ }
+
+ logger.debug("Timeout reached. Image was not found within " + timeoutSeconds.getValue() + " seconds.");
+ setErrorMessage("Image was not found on the screen within " + timeoutSeconds.getValue() + " seconds.");
+ return Result.FAILED;
+
+ } catch (NumberFormatException e) {
+ logger.debug("Invalid number format: " + e.getMessage());
+ setErrorMessage("Invalid input. Timeout must be a number (seconds). Threshold must be a number between 0 and 1.");
+ return Result.FAILED;
+ } catch (Exception e) {
+ logger.debug("Exception during wait operation: " + e.getMessage());
+ setErrorMessage("Error during wait operation: " + e.getMessage());
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Calls the visual testing API to check whether the search image is present
+ * in the base screenshot. Returns true if found, false otherwise.
+ * Sets success/error message accordingly.
+ */
+ private boolean callFindImageApi(File baseImageFile, File searchImageFile, String threshold) {
+ try {
+ logger.info("Calling visual testing API | base: " + baseImageFile + " | search: " + searchImageFile);
+ OkHttpClient client = new OkHttpClient();
+
+ RequestBody requestBody = new MultipartBody.Builder()
+ .setType(MultipartBody.FORM)
+ .addFormDataPart("baseImageFile", baseImageFile.getName(),
+ RequestBody.create(baseImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("searchImageFile", searchImageFile.getName(),
+ RequestBody.create(searchImageFile, MediaType.parse("image/png")))
+ .addFormDataPart("threshold", threshold)
+ .addFormDataPart("scale", "40")
+ .addFormDataPart("occurance", "1")
+ .build();
+
+ Request request = new Request.Builder()
+ .url(Constants.VISUAL_SERVER_FIND_IMAGE_ENDPOINT)
+ .post(requestBody)
+ .addHeader("Authorization", "Bearer " + Constants.API_TOKEN)
+ .build();
+
+ Response response = client.newCall(request).execute();
+
+ if (response.isSuccessful() && response.body() != null) {
+ String responseBody = response.body().string();
+ logger.info("API response: " + responseBody);
+ JsonNode jsonNode = mapper.readTree(responseBody);
+
+ boolean isFound = jsonNode.path("isFound").asBoolean();
+ int x1 = jsonNode.path("x1").asInt();
+ int y1 = jsonNode.path("y1").asInt();
+ int x2 = jsonNode.path("x2").asInt();
+ int y2 = jsonNode.path("y2").asInt();
+
+ if (isFound) {
+ int centerX = x1 + (x2 - x1) / 2;
+ int centerY = y1 + (y2 - y1) / 2;
+ logger.info("Image found at center (" + centerX + ", " + centerY + ")");
+ setSuccessMessage(String.format(
+ "Image found on screen. Coordinates: x1-%s, x2-%s, y1-%s, y2-%s",
+ x1, x2, y1, y2));
+ } else {
+ logger.info("Image not found in this poll attempt");
+ }
+ return isFound;
+
+ } else {
+ logger.info("API call failed or returned empty body. Code: "
+ + (response.body() != null ? response.code() : "no body"));
+ return false;
+ }
+
+ } catch (Exception e) {
+ logger.info("Exception during API call: " + ExceptionUtils.getStackTrace(e));
+ return false;
+ }
+ }
+
+ private File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Downloading image from URL: " + url);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png";
+ }
+ }
+ File tempFile = File.createTempFile(baseName, extension);
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.info("Temp file created: " + tempFile.getName() + " at " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Using local file path: " + url);
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilTextPresentInScreen.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilTextPresentInScreen.java
index 6d19839f..42acd030 100644
--- a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilTextPresentInScreen.java
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsAdvanced/WaitUntilTextPresentInScreen.java
@@ -1,10 +1,10 @@
package com.testsigma.addons.windowsAdvanced;
-import com.testsigma.addons.windowsAdvanced.utils.ScreenshotUtils;
-import com.testsigma.sdk.AIRequest;
+import com.testsigma.addons.util.OCRTextPoint;
+import com.testsigma.addons.util.OCRUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.WindowsAdvancedAction;
-import com.testsigma.sdk.annotation.AI;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
@@ -14,98 +14,70 @@
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.File;
-import java.util.ArrayList;
+import java.util.List;
import java.util.NoSuchElementException;
@Data
-@Action(actionText = "Wait until text test-data is present in screen with timeout test-data2 seconds",
- description = "This action waits until the specified text is present on the screen using AI capabilities. " +
- "It polls every 1 second until the text is found or timeout is reached. " +
+@lombok.EqualsAndHashCode(callSuper = false)
+@Action(actionText = "Wait until text text-to-verify is present in screen with timeout wait-time-in-seconds seconds",
+ description = "This action waits until the specified text is present on the screen using OCR API capabilities. " +
+ "It polls every 1.5 seconds until the text is found or timeout is reached. " +
"This works only for local executions",
applicationType = com.testsigma.sdk.ApplicationType.WINDOWS_ADVANCED,
- displayName = "WaitUntilTextPresentInScreen",
+ displayName = "Wait until the text is present on the screen",
useCustomScreenshot = true)
public class WaitUntilTextPresentInScreen extends WindowsAdvancedAction {
- @TestData(reference = "test-data", description = "The text to search for on the screen")
+ @TestData(reference = "text-to-verify")
private com.testsigma.sdk.TestData textToSearch;
- @TestData(reference = "test-data2", description = "Timeout in seconds")
+ @TestData(reference = "wait-time-in-seconds")
private com.testsigma.sdk.TestData timeoutSeconds;
- @AI
- private com.testsigma.sdk.AI ai;
-
@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;
-
- private final String prompt = "You are provided with a screenshot of a computer application." +
- " Your task is to analyze this screenshot and determine if the specified text is present anywhere in " +
- "the image. Look for the text in any form - it could be in buttons, labels, text fields, menus, " +
- "or any other UI element. Return only 'YES' if the text is found, or 'NO' if the text is not found. " +
- "The text to search for is: ";
- private static final int POLLING_INTERVAL_MS = 1500; // 1 second polling interval
+ private static final int POLLING_INTERVAL_MS = 1500;
@Override
protected Result execute() throws NoSuchElementException {
- logger.info("=== Wait Until Text Present: Starting Execution ===");
+ logger.info("=== Wait Until Text Present (OCR): Starting Execution ===");
try {
String expectedText = textToSearch.getValue().toString();
int timeoutMs = Integer.parseInt(timeoutSeconds.getValue().toString()) * 1000;
-
+
logger.info("Looking for text: '" + expectedText + "' with timeout: " +
timeoutSeconds.getValue() + " seconds");
long startTime = System.currentTimeMillis();
long endTime = startTime + timeoutMs;
-
+
while (System.currentTimeMillis() < endTime) {
logger.info("Polling attempt - checking for text: '" + expectedText + "'");
-
- // Capture the current screen
+
Robot robot = new Robot();
Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
BufferedImage screenCapture = robot.createScreenCapture(screenRect);
-
- // Save the screenshot to a temporary file
+
File screenshotFile = saveScreenshotToFile(screenCapture, "wait_text_screenshot");
-
- // Create AI request
- AIRequest aiRequest = new AIRequest();
- String fullPrompt = prompt + "'" + expectedText + "'. ";
- aiRequest.setPrompt(fullPrompt);
- aiRequest.setModel("gpt-4o");
-
- // Add the screenshot file
- ArrayList files = new ArrayList<>();
- files.add(screenshotFile);
- aiRequest.setFiles(files);
-
- // Invoke AI
- String aiResponse = ai.invokeAI(aiRequest);
- logger.info("AI response: " + aiResponse);
-
- // Parse AI response
- boolean textFound = parseAIResponse(aiResponse);
-
+
+ List textPoints = OCRUtils.extractTextPoints(screenshotFile, logger);
+ logger.info("Found " + textPoints.size() + " text elements via OCR");
+
+ boolean textFound = OCRUtils.searchForText(textPoints, expectedText, logger);
+
if (textFound) {
logger.info("Text found in application. Wait successful.");
setSuccessMessage("Text '" + expectedText + "' was found on the screen after waiting.");
-
- // Upload final screenshot to S3
ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
-
return Result.SUCCESS;
}
-
- // Clean up temporary file
+
if (screenshotFile.exists()) {
screenshotFile.delete();
}
-
- // Check if we should continue polling
+
long remainingTime = endTime - System.currentTimeMillis();
if (remainingTime > POLLING_INTERVAL_MS) {
logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000)
@@ -113,16 +85,14 @@ protected Result execute() throws NoSuchElementException {
"Remaining time: " + (remainingTime / 1000) + " seconds");
Thread.sleep(POLLING_INTERVAL_MS);
} else {
- break; // No time left for another attempt
+ break;
}
}
-
- // If we reach here, timeout occurred
- logger.debug("Timeout reached. Text '" + expectedText + "' was not found on the screen within " +
+
+ logger.debug("Timeout reached. Text '" + expectedText + "' was not found on the screen within " +
timeoutSeconds.getValue() + " seconds.");
- setErrorMessage("Text '" + expectedText + "' was not found on the screen within " +
+ setErrorMessage("Text '" + expectedText + "' was not found on the screen within " +
timeoutSeconds.getValue() + " seconds.");
- // Capture and upload screenshot even on failure
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "wait_text_failure_screenshot", logger);
return Result.FAILED;
@@ -130,25 +100,16 @@ protected Result execute() throws NoSuchElementException {
logger.debug("Invalid timeout value: " + timeoutSeconds.getValue());
setErrorMessage("Invalid timeout value: " + timeoutSeconds.getValue() +
". Please provide a valid number of seconds.");
- // Capture and upload screenshot even on failure
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "wait_text_failure_screenshot", logger);
return Result.FAILED;
} catch (Exception e) {
logger.debug("Exception during wait operation: " + e.getMessage());
setErrorMessage("Error during wait operation: " + e.getMessage());
- // Capture and upload screenshot even on failure
ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "wait_text_failure_screenshot", logger);
return Result.FAILED;
}
}
- /**
- * Saves the screenshot to a temporary file
- * @param screenshot The captured screenshot
- * @param fileName The base filename
- * @return The temporary file
- * @throws Exception if file creation fails
- */
private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
try {
File tempFile = File.createTempFile(fileName, ".png");
@@ -156,38 +117,7 @@ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) thr
return tempFile;
} catch (Exception e) {
logger.debug("Failed to save screenshot to file: " + e.getMessage());
- throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ throw new RuntimeException("Unable to save screenshot for processing.", e);
}
}
-
- /**
- * Parses the AI response to determine if text was found
- * @param aiResponse The response from AI
- * @return true if text was found, false otherwise
- */
- private boolean parseAIResponse(String aiResponse) {
- if (aiResponse == null || aiResponse.trim().isEmpty()) {
- logger.debug("AI response is null or empty");
- return false;
- }
-
- String response = aiResponse.trim().toUpperCase();
- logger.debug("Parsing AI response: " + response);
-
- // Check for various positive responses
- if (response.contains("YES") || response.contains("TRUE") || response.contains("FOUND") ||
- response.contains("PRESENT") || response.contains("EXISTS")) {
- return true;
- }
-
- // Check for various negative responses
- if (response.contains("NO") || response.contains("FALSE") || response.contains("NOT FOUND") ||
- response.contains("ABSENT") || response.contains("NOT PRESENT")) {
- return false;
- }
-
- // If response is unclear, log it and return false
- logger.debug("Unclear AI response: " + aiResponse + ". Treating as 'not found'.");
- return false;
- }
}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnImage.java
new file mode 100644
index 00000000..6f4bc0db
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnImage.java
@@ -0,0 +1,191 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.ApplicationType;
+
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.*;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "lite: Click on image image-url",
+ description = "This action takes an image URL (S3 URL or local file path), finds that image on the current screen, " +
+ "and clicks on it. The action uses AI to locate the image within the screen. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_UFT,
+ displayName = "Click on image")
+public class ClickOnImage extends WindowsAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Click On Image: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ logger.info("Looking for image from URL: " + imageUrlValue);
+
+ // Convert URL to file
+ File targetImageFile = urlToFileConverter("target_image", imageUrlValue);
+ logger.info("Target image file prepared: " + targetImageFile.getAbsolutePath());
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File baseImageFile = saveScreenshotToFile(screenCapture, "click_image_screenshot");
+
+ String url = testStepResult.getScreenshotUrl();
+ logger.info("Amazon s3 url in which we are storing base image" + url);
+ ocr.uploadFile(url, baseImageFile);
+ logger.info("url: " + testStepResult.getScreenshotUrl());
+ FindImageResponse responseObject = ocr.findImage(imageUrl.getValue().toString());
+ if (responseObject.getIsFound()) {
+ boolean isFound = responseObject.getIsFound();
+ int x1 = responseObject.getX1();
+ int y1 = responseObject.getY1();
+ int x2 = responseObject.getX2();
+ int y2 = responseObject.getY2();
+
+ int clickLocationX = (x1 + x2) / 2;
+ int clickLocationY = (y1 + y2) / 2;
+
+ logger.info("Click Location X: " + clickLocationX);
+ logger.info("Click Location Y: " + clickLocationY);
+ // Perform the click
+ robot.mouseMove(clickLocationX, clickLocationY);
+ Thread.sleep(100); // Small delay to ensure mouse is positioned
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+
+ logger.info("Successfully clicked on image at coordinates (" +
+ clickLocationX + ", " + clickLocationY + ")");
+ setSuccessMessage("Successfully clicked on image at coordinates (" +
+ clickLocationX + ", " + clickLocationY + ")");
+ setSuccessMessage("Image Found :" + isFound +
+ " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2);
+ Thread.sleep(2000);
+ } else {
+ setErrorMessage("Unable to fetch the coordinates");
+ return Result.FAILED;
+ }
+ // Clean up temporary files
+ cleanupFile(targetImageFile);
+ cleanupFile(baseImageFile);
+
+ return Result.SUCCESS;
+
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ javax.imageio.ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Converts URL to File - handles both S3 URLs and local file paths
+ *
+ * @param fileName Base filename for temporary file
+ * @param url The URL or file path
+ * @return File object
+ */
+ public File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ // Try to get extension from URL
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png"; // Default to PNG for images
+ }
+ }
+
+ File tempFile = File.createTempFile(baseName, extension);
+
+ // Download file from URL
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ logger.info("Temp file created with name for s3 file " + tempFile.getName()
+ + " at path " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+
+ /**
+ * Cleans up temporary file
+ *
+ * @param file File to delete
+ */
+ private void cleanupFile(File file) {
+ try {
+ if (file != null && file.exists() && file.isFile()) {
+ if (file.delete()) {
+ logger.debug("Cleaned up temporary file: " + file.getAbsolutePath());
+ } else {
+ logger.debug("Failed to delete temporary file: " + file.getAbsolutePath());
+ }
+ }
+ } catch (Exception e) {
+ logger.debug("Error cleaning up file: " + e.getMessage());
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnPositionRelativeToImage.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnPositionRelativeToImage.java
new file mode 100644
index 00000000..1d5cbceb
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnPositionRelativeToImage.java
@@ -0,0 +1,276 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import com.testsigma.sdk.ApplicationType;
+
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.*;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "lite: Click on position position-type relative to the image image-url",
+ description = "This action takes an image URL (S3 URL or local file path), finds that image on the current screen, " +
+ "and clicks at a position relative to it. Position can be Left, Right, Top, Bottom, or Center of the image. " +
+ "The action uses AI to locate the image within the screen and then performs the click at the specified relative position. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_UFT,
+ displayName = "Click on position relative to image")
+public class ClickOnPositionRelativeToImage extends WindowsAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestData(reference = "position-type", allowedValues = {"Left", "Right", "Top", "Bottom", "Center"})
+ private com.testsigma.sdk.TestData position;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Click On Position Relative To Image: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ String positionValue = position.getValue().toString();
+
+ logger.info("Looking for image from URL: " + imageUrlValue + " to click at position: " + positionValue);
+
+ // Convert URL to file
+ File targetImageFile = urlToFileConverter("target_image", imageUrlValue);
+ logger.info("Target image file prepared: " + targetImageFile.getAbsolutePath());
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File baseImageFile = saveScreenshotToFile(screenCapture, "click_relative_image_screenshot");
+
+ // Upload base image to S3 and use OCR to find target image
+ String url = testStepResult.getScreenshotUrl();
+ logger.info("Amazon s3 url in which we are storing base image: " + url);
+ ocr.uploadFile(url, baseImageFile);
+ logger.info("url: " + testStepResult.getScreenshotUrl());
+
+ // Find the target image using OCR
+ FindImageResponse responseObject = ocr.findImage(imageUrl.getValue().toString());
+ ImageBoundingBox boundingBox = null;
+
+ if (responseObject.getIsFound()) {
+ // Extract coordinates from FindImageResponse
+ int x1 = responseObject.getX1();
+ int y1 = responseObject.getY1();
+ int x2 = responseObject.getX2();
+ int y2 = responseObject.getY2();
+
+ // Create bounding box from OCR response
+ boundingBox = new ImageBoundingBox(x1, y1, x2, y2, true);
+ logger.info("Image found with bounding box: (" + boundingBox.getX1() + ", " + boundingBox.getY1() +
+ ") to (" + boundingBox.getX2() + ", " + boundingBox.getY2() + ")");
+
+ // Calculate click position based on bounding box and position parameter
+ Point clickPoint = calculateClickPosition(boundingBox, positionValue);
+ logger.info("Calculated click position: (" + clickPoint.x + ", " + clickPoint.y + ")");
+
+ // Perform the click
+ robot.mouseMove(clickPoint.x, clickPoint.y);
+ Thread.sleep(100); // Small delay to ensure mouse is positioned
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+
+ logger.info("Successfully clicked " + positionValue + " of image at coordinates (" +
+ clickPoint.x + ", " + clickPoint.y + ")");
+ setSuccessMessage("Successfully clicked " + positionValue + " of image at coordinates (" +
+ clickPoint.x + ", " + clickPoint.y + ")");
+
+ // Clean up temporary files
+ cleanupFile(targetImageFile);
+ cleanupFile(baseImageFile);
+
+ return Result.SUCCESS;
+ } else {
+ logger.debug("Image not found on the screen");
+ setErrorMessage("The specified image was not found on the screen. Unable to perform click.");
+
+ // Clean up temporary files
+ cleanupFile(targetImageFile);
+ cleanupFile(baseImageFile);
+
+ return Result.FAILED;
+ }
+
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Calculates the click position based on image bounding box and position
+ * @param boundingBox The image bounding box
+ * @param position The relative position (Left, Right, Top, Bottom, Center)
+ * @return Point with calculated click coordinates
+ */
+ private Point calculateClickPosition(ImageBoundingBox boundingBox, String position) {
+ int centerX = (boundingBox.getX1() + boundingBox.getX2()) / 2;
+ int centerY = (boundingBox.getY1() + boundingBox.getY2()) / 2;
+
+ int clickX = centerX;
+ int clickY = centerY;
+
+ // Calculate position with a small offset from the edge (10 pixels)
+ int edgeOffset = 10;
+
+ switch (position.toUpperCase()) {
+ case "LEFT":
+ clickX = boundingBox.getX1() - edgeOffset;
+ clickY = centerY;
+ break;
+ case "RIGHT":
+ clickX = boundingBox.getX2() + edgeOffset;
+ clickY = centerY;
+ break;
+ case "TOP":
+ clickX = centerX;
+ clickY = boundingBox.getY1() - edgeOffset;
+ break;
+ case "BOTTOM":
+ clickX = centerX;
+ clickY = boundingBox.getY2() + edgeOffset;
+ break;
+ case "CENTER":
+ // For center, no offset needed
+ clickX = centerX;
+ clickY = centerY;
+ break;
+ default:
+ logger.debug("Unknown position: " + position + ". Using center.");
+ break;
+ }
+
+ return new Point(clickX, clickY);
+ }
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ javax.imageio.ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Converts URL to File - handles both S3 URLs and local file paths
+ * @param fileName Base filename for temporary file
+ * @param url The URL or file path
+ * @return File object
+ */
+ public File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ // Try to get extension from URL
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png"; // Default to PNG for images
+ }
+ }
+
+ File tempFile = File.createTempFile(baseName, extension);
+
+ // Download file from URL
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ logger.info("Temp file created with name for s3 file " + tempFile.getName()
+ + " at path " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+
+ /**
+ * Cleans up temporary file
+ * @param file File to delete
+ */
+ private void cleanupFile(File file) {
+ try {
+ if (file != null && file.exists() && file.isFile()) {
+ if (file.delete()) {
+ logger.debug("Cleaned up temporary file: " + file.getAbsolutePath());
+ } else {
+ logger.debug("Failed to delete temporary file: " + file.getAbsolutePath());
+ }
+ }
+ } catch (Exception e) {
+ logger.debug("Error cleaning up file: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Inner class to hold image bounding box information
+ */
+ private static class ImageBoundingBox {
+ private final int x1;
+ private final int y1;
+ private final int x2;
+ private final int y2;
+
+ public ImageBoundingBox(int x1, int y1, int x2, int y2, boolean found) {
+ this.x1 = x1;
+ this.y1 = y1;
+ this.x2 = x2;
+ this.y2 = y2;
+ }
+
+ public int getX1() { return x1; }
+ public int getY1() { return y1; }
+ public int getX2() { return x2; }
+ public int getY2() { return y2; }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnPositionRelativeToText.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnPositionRelativeToText.java
new file mode 100644
index 00000000..4cbb164f
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnPositionRelativeToText.java
@@ -0,0 +1,222 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.sdk.*;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import com.testsigma.sdk.ApplicationType;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.NoSuchElementException;
+import java.util.List;
+
+@Action(actionText = "lite: Click on position position-type relative to the text text-to-find with pixel offset pixel-offset and maximum wait time wait-time-in-seconds seconds",
+ description = "This action finds the specified text on the screen and clicks at a position relative to it with a pixel offset. " +
+ "Position can be Left, Right, Top, Bottom, or Center of the text. " +
+ "The pixel offset determines how far from the text edge to click (positive values move away from text, negative values move towards text). " +
+ "For Center position, offset is ignored. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_UFT,
+ displayName = "Click on position relative to text")
+public class ClickOnPositionRelativeToText extends WindowsAction {
+
+ @TestData(reference = "text-to-find")
+ private com.testsigma.sdk.TestData textToFind;
+
+ @TestData(reference = "position-type", allowedValues = {"Left", "Right", "Top", "Bottom", "Center"})
+ private com.testsigma.sdk.TestData position;
+
+ @TestData(reference = "pixel-offset")
+ private com.testsigma.sdk.TestData pixelOffset;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData maxWaitSeconds;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Click On Position Relative To Text: Starting Execution ===");
+
+ try {
+ String targetText = textToFind.getValue().toString();
+ String positionValue = position.getValue().toString();
+ int offset = Integer.parseInt(pixelOffset.getValue().toString());
+
+ logger.info("Looking for text: '" + targetText + "' to click " + positionValue +
+ " with offset: " + offset + " pixels, max wait time: " + maxWaitSeconds.getValue() + " seconds");
+
+ logger.info("Polling attempt - checking for text: '" + targetText + "'");
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "click_relative_position_screenshot");
+
+ // Use OCR to find text
+ OCRImage ocrImage = new OCRImage();
+ ocrImage.setOcrImageFile(screenshotFile);
+ List textPoints = ocr.extractTextFromImage(ocrImage);
+ printAllCoordinates(textPoints);
+ OCRTextPoint textPoint = getTextPointFromText(textPoints, targetText);
+
+ if (textPoint != null) {
+ logger.info("Found text with coordinates: x1=" + textPoint.getX1() + ", y1=" + textPoint.getY1() +
+ ", x2=" + textPoint.getX2() + ", y2=" + textPoint.getY2());
+
+ // Calculate click position based on position and offset
+ Point clickPoint = calculateClickPosition(textPoint, positionValue, offset);
+ logger.info("Calculated click position: (" + clickPoint.x + ", " + clickPoint.y + ")");
+
+ // Perform the click
+ robot.mouseMove(clickPoint.x, clickPoint.y);
+ Thread.sleep(100); // Small delay to ensure mouse is positioned
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+
+ logger.info("Successfully clicked " + positionValue + " of text '" + targetText +
+ "' with offset " + offset + " pixels at coordinates (" +
+ clickPoint.x + ", " + clickPoint.y + ")");
+ setSuccessMessage("Successfully clicked " + positionValue + " of text '" + targetText +
+ "' with offset " + offset + " pixels at coordinates (" +
+ clickPoint.x + ", " + clickPoint.y + ")");
+
+ // Clean up temporary file
+ if (screenshotFile.exists()) {
+ screenshotFile.delete();
+ }
+
+ return Result.SUCCESS;
+ }
+
+ // Clean up temporary file
+ if (screenshotFile.exists()) {
+ screenshotFile.delete();
+ }
+
+
+
+ // If we reach here, timeout occurred
+ logger.debug("Timeout reached. Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds.");
+ setErrorMessage("Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds. Unable to perform click.");
+ return Result.FAILED;
+
+ } catch (NumberFormatException e) {
+ logger.debug("Invalid numeric value: " + e.getMessage());
+ setErrorMessage("Invalid numeric value provided. Please check timeout and pixel offset values.");
+ return Result.FAILED;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Calculates the click position based on text point, position, and offset
+ * @param textPoint The OCR text point
+ * @param position The relative position (Left, Right, Top, Bottom, Center)
+ * @param offset The pixel offset from the text
+ * @return Point with calculated click coordinates
+ */
+ private Point calculateClickPosition(OCRTextPoint textPoint, String position, int offset) {
+ int centerX = (textPoint.getX1() + textPoint.getX2()) / 2;
+ int centerY = (textPoint.getY1() + textPoint.getY2()) / 2;
+
+ int clickX = centerX;
+ int clickY = centerY;
+
+ switch (position.toUpperCase()) {
+ case "LEFT":
+ clickX = textPoint.getX1() - offset;
+ clickY = centerY;
+ break;
+ case "RIGHT":
+ clickX = textPoint.getX2() + offset;
+ clickY = centerY;
+ break;
+ case "TOP":
+ clickX = centerX;
+ clickY = textPoint.getY1() - offset;
+ break;
+ case "BOTTOM":
+ clickX = centerX;
+ clickY = textPoint.getY2() + offset;
+ break;
+ case "CENTER":
+ // For center, offset is ignored
+ clickX = centerX;
+ clickY = centerY;
+ break;
+ default:
+ logger.debug("Unknown position: " + position + ". Using center.");
+ break;
+ }
+
+ return new Point(clickX, clickY);
+ }
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Gets the OCRTextPoint for the target text
+ * @param textPoints List of OCR text points
+ * @param targetText The text to find
+ * @return OCRTextPoint if found, null otherwise
+ */
+ private OCRTextPoint getTextPointFromText(List textPoints, String targetText) {
+ if (textPoints == null) {
+ return null;
+ }
+ for (OCRTextPoint textPoint : textPoints) {
+ if (targetText.equals(textPoint.getText())) {
+ return textPoint;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Prints all OCR text coordinates for debugging
+ * @param textPoints List of OCR text points
+ */
+ private void printAllCoordinates(List textPoints) {
+ for (OCRTextPoint textPoint : textPoints) {
+ logger.info("text =" + textPoint.getText() + " x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() + "\n\n\n\n");
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnTextWithWait.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnTextWithWait.java
new file mode 100644
index 00000000..d7b42793
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/ClickOnTextWithWait.java
@@ -0,0 +1,155 @@
+package com.testsigma.addons.windowsLite;
+
+
+import com.testsigma.sdk.*;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import com.testsigma.sdk.ApplicationType;
+
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.NoSuchElementException;
+import java.util.List;
+
+@Action(actionText = "Click on text text-to-click with maximum wait time wait-time-in-seconds seconds",
+ description = "This action waits for the specified text to appear on the screen and then clicks on it. " +
+ "it performs a mouse click at the center of the text area. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS_UFT,
+ displayName = "Click on text with wait")
+public class ClickOnTextWithWait extends WindowsAction {
+
+ @TestData(reference = "text-to-click")
+ private com.testsigma.sdk.TestData textToClick;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData maxWaitSeconds;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private static final int POLLING_INTERVAL_MS = 1500; // 1.5 second polling interval
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Click On Text With Wait: Starting Execution ===");
+
+ try {
+ String targetText = textToClick.getValue().toString();
+ int timeoutMs = Integer.parseInt(maxWaitSeconds.getValue().toString()) * 1000; // Convert seconds to milliseconds
+
+ logger.info("Looking for text to click: '" + targetText + "' with max wait time: " + maxWaitSeconds.getValue() + " seconds");
+
+ long startTime = System.currentTimeMillis();
+ long endTime = startTime + timeoutMs;
+
+ while (System.currentTimeMillis() < endTime) {
+ logger.info("Polling attempt - checking for text: '" + targetText + "'");
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "click_text_screenshot");
+
+ OCRImage ocrImage = new OCRImage();
+ ocrImage.setOcrImageFile(screenshotFile);
+ List textPoints = ocr.extractTextFromImage(ocrImage);
+ printAllCoordinates(textPoints);
+ OCRTextPoint textPoint = getTextPointFromText(textPoints);
+ if (textPoint != null) {
+ // Text found - perform click and return success
+ logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() +
+ ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2());
+ int x = (textPoint.getX1() + textPoint.getX2()) / 2;
+ int y = (textPoint.getY1() + textPoint.getY2()) / 2;
+ robot.mouseMove(x, y);
+ Thread.sleep(100); // Small delay to ensure mouse is positioned
+ robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(50);
+ robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+ Thread.sleep(1000); // Wait a moment after click
+ logger.info("Successfully clicked on text: '" + targetText + "' at coordinates (" + x + ", " + y + ")");
+ setSuccessMessage("Successfully clicked on text '" + targetText + "' at coordinates (" + x + ", " + y + ")");
+ return Result.SUCCESS;
+ }
+
+ // Text not found - check if we should continue polling
+ long remainingTime = endTime - System.currentTimeMillis();
+ if (remainingTime > POLLING_INTERVAL_MS) {
+ logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000.0)
+ + " seconds before next attempt. " +
+ "Remaining time: " + (remainingTime / 1000) + " seconds");
+ Thread.sleep(POLLING_INTERVAL_MS);
+ } else {
+ break; // No time left for another attempt
+ }
+ }
+ // If we reach here, timeout occurred
+ logger.debug("Timeout reached. Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds.");
+ setErrorMessage("Text '" + targetText + "' was not found on the screen within " +
+ maxWaitSeconds.getValue() + " seconds. Unable to perform click.");
+ // Capture and upload screenshot even on failure
+ return Result.FAILED;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ } catch (Exception e) {
+ logger.debug("Exception during click operation: " + e.getMessage());
+ setErrorMessage("Error during click operation: " + e.getMessage());
+ // Capture and upload screenshot even on failure
+ return Result.FAILED;
+ }
+ }
+
+
+ /**
+ * Saves the screenshot to a temporary file
+ *
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ private OCRTextPoint getTextPointFromText(List textPoints) {
+ if (textPoints == null) {
+ return null;
+ }
+ for (OCRTextPoint textPoint : textPoints) {
+ if (textToClick.getValue().equals(textPoint.getText())) {
+ return textPoint;
+
+ }
+ }
+ return null;
+ }
+
+ private void printAllCoordinates(List textPoints) {
+ for (OCRTextPoint textPoint : textPoints) {
+ logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() + "\n\n\n\n");
+ }
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/CopyAndPasteTestDataOnScreen.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/CopyAndPasteTestDataOnScreen.java
new file mode 100644
index 00000000..45a8ca55
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/CopyAndPasteTestDataOnScreen.java
@@ -0,0 +1,67 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.datatransfer.Clipboard;
+import java.awt.datatransfer.StringSelection;
+import java.awt.datatransfer.Transferable;
+import java.awt.event.KeyEvent;
+
+@Data
+@Action(actionText = "Copy and paste given data on screen text-to-copy-paste",
+ description = "This action allows you to copy and paste data on the screen using the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Copy and paste given data on screen",
+ useCustomScreenshot = true)
+public class CopyAndPasteTestDataOnScreen extends WindowsAction {
+
+ @TestData(reference = "text-to-copy-paste")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ StringSelection stringSelection = new StringSelection(testData.getValue().toString());
+ Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
+ Transferable data = clipboard.getContents(null);
+ clipboard.setContents(stringSelection, null);
+ robot.keyPress(KeyEvent.VK_CONTROL);
+ robot.keyPress(KeyEvent.VK_V);
+ KeyboardUtils.sleep(100);
+ robot.keyRelease(KeyEvent.VK_V);
+ robot.keyRelease(KeyEvent.VK_CONTROL);
+ Thread.sleep(500);
+ clipboard.setContents(data, null);
+ setSuccessMessage("Given data copied and pasted successfully on the screen");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "copy_paste_data_screenshot", logger);
+
+ } catch (Exception e) {
+ result = Result.FAILED;
+ setErrorMessage("An error occurred while copying and pasting data: " + e.getMessage());
+ logger.debug("Error copying and pasting data: " + ExceptionUtils.getStackTrace(e));
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "copy_paste_data_failure_screenshot", logger);
+ return result;
+ }
+ return result;
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/EnterDataUsingKeyboard.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/EnterDataUsingKeyboard.java
new file mode 100644
index 00000000..fb4ff283
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/EnterDataUsingKeyboard.java
@@ -0,0 +1,66 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+
+
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+
+
+@Data
+@Action(actionText = "Enter data test-data using keyboard",
+ description = "This action allows you to enter data into a field using the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Enter data on screen",
+ useCustomScreenshot = true
+)
+public class EnterDataUsingKeyboard extends WindowsAction {
+ @TestData(reference = "test-data")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String text = testData.getValue().toString();
+
+ Thread.sleep(2000); // Wait 2 seconds to focus the target window
+
+ for (char c : text.toCharArray()) {
+ KeyboardUtils.typeCharacter(robot, c);
+ Thread.sleep(100); // Delay between keystrokes (optional)
+ }
+
+ setSuccessMessage("Given data entered successfully on the given image");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "enter_data_screenshot", logger);
+
+ } catch (Exception e) {
+ result = Result.FAILED;
+ setErrorMessage("An error occurred while initializing the Robot class: " + e.getMessage());
+ logger.debug("Error initializing Robot class: " + ExceptionUtils.getStackTrace(e));
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "enter_data_failure_screenshot", logger);
+ return result;
+ }
+ return result;
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/EnterDataUsingKeyboardAndPressEnter.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/EnterDataUsingKeyboardAndPressEnter.java
new file mode 100644
index 00000000..ea4baa01
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/EnterDataUsingKeyboardAndPressEnter.java
@@ -0,0 +1,69 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.event.KeyEvent;
+
+
+@Data
+@Action(actionText = "Enter data test-data using keyboard",
+ description = "This action allows you to enter data into a field using the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Enter Data on screen and press Enter",
+ useCustomScreenshot = true
+)
+public class EnterDataUsingKeyboardAndPressEnter extends WindowsAction {
+ @TestData(reference = "test-data")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String text = testData.getValue().toString();
+
+ Thread.sleep(2000); // Wait 2 seconds to focus the target window
+
+ for (char c : text.toCharArray()) {
+ KeyboardUtils.typeCharacter(robot, c);
+ Thread.sleep(50); // Delay between keystrokes (optional)
+ }
+ logger.info("Pressing Enter key");
+ robot.keyPress(KeyEvent.VK_ENTER);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(KeyEvent.VK_ENTER);
+
+ setSuccessMessage("Given data entered successfully on the given image and Enter key pressed");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "enter_data_screenshot", logger);
+
+ } catch (Exception e) {
+ result = Result.FAILED;
+ setErrorMessage("An error occurred while initializing the Robot class: " + e.getMessage());
+ logger.debug("Error initializing Robot class: " + ExceptionUtils.getStackTrace(e));
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "enter_data_failure_screenshot", logger);
+ return result;
+ }
+ return result;
+ }
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressFunctionKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressFunctionKey.java
new file mode 100644
index 00000000..f08becc0
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressFunctionKey.java
@@ -0,0 +1,69 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+import java.awt.event.KeyEvent;
+
+
+@Data
+@Action(actionText = "Press Function Key key-type",
+ description = "This action allows you to press a function key on the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Press a Function Key",
+ useCustomScreenshot = true
+ )
+public class PressFunctionKey extends WindowsAction {
+
+ @TestData(reference = "key-type",allowedValues = {"F1", "F2", "F3", "F4", "F5", "F6",
+ "F7", "F8", "F9", "F10", "F11", "F12"})
+ private com.testsigma.sdk.TestData keyType;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String key = keyType.getValue().toString();
+
+ // Convert the function key to its corresponding KeyEvent constant
+ int keyCode = KeyEvent.class.getField("VK_" + key).getInt(null);
+ logger.info("Key Code: " + keyCode);
+
+ // Press and release the function key
+ robot.keyPress(keyCode);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(keyCode);
+ logger.info("Key Released: " + keyCode);
+
+ setSuccessMessage("Successfully pressed the " + key + " key.");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_function_key_screenshot", logger);
+
+ } catch (Exception e) {
+ setErrorMessage("An error occurred while pressing the function key: " + e.getMessage());
+ logger.debug("Error pressing function key: " + ExceptionUtils.getStackTrace(e));
+ result = Result.FAILED;
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_function_key_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKey.java
new file mode 100644
index 00000000..1b0e7e2b
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKey.java
@@ -0,0 +1,65 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+
+@Data
+@Action(actionText = "Press a modifier key key-type",
+ description = "This action allows you to press a modifier key on the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Press a Modifier Key",
+ useCustomScreenshot = true)
+public class PressModifierKey extends WindowsAction {
+
+ @TestData(reference = "key-type", allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String key = keyType.getValue().toString();
+
+ // Convert the modifier key to its corresponding KeyEvent constant
+ int keyCode = KeyboardUtils.getModifierKeyCode(key);
+ logger.info("key code " + keyCode);
+
+ // Press and release the modifier key
+ robot.keyPress(keyCode);
+ // Adding a small delay to ensure the key press is registered
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(keyCode);
+ setSuccessMessage("Successfully pressed the " + key + " key.");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_modifier_key_screenshot", logger);
+
+ } catch (Exception e) {
+ setErrorMessage("An error occurred while pressing the modifier key: " + e.getMessage());
+ logger.debug("Error pressing modifier key: " + ExceptionUtils.getStackTrace(e));
+ result = Result.FAILED;
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_modifier_key_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKeyWithBasicKeyCombination.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKeyWithBasicKeyCombination.java
new file mode 100644
index 00000000..c821b53e
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKeyWithBasicKeyCombination.java
@@ -0,0 +1,87 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+
+@Data
+@Action(actionText = "Press a modifier key key-type with an alphanumeric key alphanumeric-key",
+ description = "This action allows you to press a modifier key with an alphanumeric key on the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Press a Modifier Key with a basic key",
+ useCustomScreenshot = true)
+public class PressModifierKeyWithBasicKeyCombination extends WindowsAction {
+
+ @TestData(reference = "key-type", allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType;
+
+ @TestData(reference = "alphanumeric-key", allowedValues = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J",
+ "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T",
+ "U", "V", "W", "X", "Y", "Z",
+ "Space", "Comma", "Period", "Semicolon", "Colon", "Exclamation", "Question",
+ "At", "Hash", "Dollar", "Percent", "Caret", "Ampersand", "Asterisk",
+ "Left_Parenthesis", "Right_Parenthesis", "Minus", "Plus", "Equals",
+ "Left_Bracket", "Right_Bracket", "Backslash", "Forward_Slash", "Pipe",
+ "Left_Brace", "Right_Brace", "Tilde", "Backtick", "Quote", "Double_Quote"})
+ private com.testsigma.sdk.TestData alphanumericKey;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String modifierKey = keyType.getValue().toString();
+ String alphanumericKeyValue = alphanumericKey.getValue().toString();
+
+ logger.info("Modifier Key: " + modifierKey);
+ logger.info("Alphanumeric Key: " + alphanumericKeyValue);
+ // Convert the modifier key to its corresponding KeyEvent constant
+ int modifierKeyCode = KeyboardUtils.getModifierKeyCode(modifierKey);
+ int alphanumericKeyCode = KeyboardUtils.getAlphanumericKeyCode(alphanumericKeyValue);
+
+ logger.info("Modifier Key Code: " + modifierKeyCode);
+ logger.info("Alphanumeric Key Code: " + alphanumericKeyCode);
+
+ // Press modifier key, then alphanumeric key
+ robot.keyPress(modifierKeyCode);
+ KeyboardUtils.sleep(30);
+ robot.keyPress(alphanumericKeyCode);
+ KeyboardUtils.sleep(30);
+ // Release keys in reverse order
+ robot.keyRelease(alphanumericKeyCode);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(modifierKeyCode);
+
+ setSuccessMessage("Successfully pressed " + modifierKey + " + " + alphanumericKeyValue + " keys.");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_modifier_key_with_basic_key_screenshot", logger);
+
+ } catch (Exception e) {
+ setErrorMessage("An error occurred while pressing the modifier key with alphanumeric key: " + e.getMessage());
+ logger.debug("Error pressing modifier key with alphanumeric key: " + ExceptionUtils.getStackTrace(e));
+ result = Result.FAILED;
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_modifier_key_with_basic_key_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKeyWithGivenKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKeyWithGivenKey.java
new file mode 100644
index 00000000..918a9be0
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressModifierKeyWithGivenKey.java
@@ -0,0 +1,75 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+
+@Data
+@Action(actionText = "Press a modifier key key-type with a specific key test-data",
+ description = "This action allows you to press a modifier key with a specific key on the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Press a Modifier Key with a given Key",
+ useCustomScreenshot = true)
+public class PressModifierKeyWithGivenKey extends WindowsAction {
+
+ @TestData(reference = "key-type", allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType;
+
+ @TestData(reference = "test-data")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String modifierKey = keyType.getValue().toString();
+ String specificKey = testData.getValue().toString();
+
+ // Convert the modifier key to its corresponding KeyEvent constant
+ int modifierKeyCode = KeyboardUtils.getModifierKeyCode(modifierKey);
+ int specificKeyCode = KeyboardUtils.getSpecificKeyCode(specificKey);
+
+ // Press modifier key, then specific key
+ robot.keyPress(modifierKeyCode);
+ robot.keyPress(specificKeyCode);
+
+ // Small delay
+ Thread.sleep(100);
+
+ // Release keys in reverse order
+ robot.keyRelease(specificKeyCode);
+ robot.keyRelease(modifierKeyCode);
+
+ setSuccessMessage("Successfully pressed " + modifierKey + " + " + specificKey + " keys.");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_modifier_key_with_given_key_screenshot", logger);
+
+ } catch (Exception e) {
+ setErrorMessage("An error occurred while pressing the modifier key with specific key: " + e.getMessage());
+ logger.debug("Error pressing modifier key with specific key: " + ExceptionUtils.getStackTrace(e));
+ result = Result.FAILED;
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_modifier_key_with_given_key_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressTwoModifierKeys.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressTwoModifierKeys.java
new file mode 100644
index 00000000..3d19d85a
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressTwoModifierKeys.java
@@ -0,0 +1,77 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+
+@Data
+@Action(actionText = "Press two modifier keys simultaneously: Modifier Key 1: key-type-1, Modifier Key 2: key-type-2",
+ description = "This action allows you to press two modifier keys simultaneously on the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Press two Modifier Keys",
+ useCustomScreenshot = true)
+public class PressTwoModifierKeys extends WindowsAction {
+
+ @TestData(reference = "key-type-1", allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType1;
+
+ @TestData(reference = "key-type-2", allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType2;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String key1 = keyType1.getValue().toString();
+ String key2 = keyType2.getValue().toString();
+
+ // Convert the modifier keys to their corresponding KeyEvent constants
+ int keyCode1 = KeyboardUtils.getModifierKeyCode(key1);
+ int keyCode2 = KeyboardUtils.getModifierKeyCode(key2);
+
+ // Press both modifier keys simultaneously
+ robot.keyPress(keyCode1);
+ robot.keyPress(keyCode2);
+
+ // Small delay to ensure both keys are pressed
+ Thread.sleep(100);
+
+ // Release both modifier keys
+ robot.keyRelease(keyCode2);
+ robot.keyRelease(keyCode1);
+
+ setSuccessMessage("Successfully pressed the " + key1 + " and " + key2 + " keys simultaneously.");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_two_modifier_keys_screenshot", logger);
+
+ } catch (Exception e) {
+ setErrorMessage("An error occurred while pressing the two modifier keys: " + e.getMessage());
+ logger.debug("Error pressing two modifier keys: " + ExceptionUtils.getStackTrace(e));
+ result = Result.FAILED;
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_two_modifier_keys_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressTwoModifierKeysWithBasicKey.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressTwoModifierKeysWithBasicKey.java
new file mode 100644
index 00000000..705c8b20
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/PressTwoModifierKeysWithBasicKey.java
@@ -0,0 +1,101 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.KeyboardUtils;
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import java.awt.*;
+
+@Data
+@Action(actionText = "Press two modifier keys together with a specific key: Modifier Key 1: key-type-1, Modifier Key 2: key-type-2, Key: test-data",
+ description = "This action allows you to press two modifier keys together with a specific key on the keyboard. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Press two Modifier Keys with a basic Key",
+ useCustomScreenshot = true)
+public class PressTwoModifierKeysWithBasicKey extends WindowsAction {
+
+ @TestData(reference = "key-type-1",
+ allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType1;
+
+ @TestData(reference = "key-type-2",
+ allowedValues = {"Alt", "BackSpace", "CapsLock", "Ctrl", "Delete", "Down",
+ "Enter", "Esc", "Left", "Right", "Shift", "Tab", "Up", "WINDOW"})
+ private com.testsigma.sdk.TestData keyType2;
+
+ @TestData(reference = "test-data", allowedValues = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T",
+ "U", "V", "W", "X", "Y", "Z", "Space", "Comma", "Period", "Semicolon", "Colon", "Exclamation", "Question",
+ "At", "Hash", "Dollar", "Percent", "Caret", "Ampersand", "Asterisk", "Left_Parenthesis", "Right_Parenthesis",
+ "Minus", "Plus", "Equals", "Left_Bracket", "Right_Bracket", "Backslash", "Forward_Slash", "Pipe",
+ "Left_Brace", "Right_Brace", "Tilde", "Backtick", "Quote", "Double_Quote"})
+ private com.testsigma.sdk.TestData testData;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @Override
+ public Result execute() {
+ Result result = Result.SUCCESS;
+ try {
+ // Instantiate the Robot Class
+ Robot robot = new Robot();
+ String modifierKey1 = keyType1.getValue().toString();
+ String modifierKey2 = keyType2.getValue().toString();
+ String specificKey = testData.getValue().toString();
+
+ logger.info("Modifier Key 1: " + modifierKey1);
+ logger.info("Modifier Key 2: " + modifierKey2);
+ logger.info("Specific Key: " + specificKey);
+
+ // Convert the modifier keys to their corresponding KeyEvent constants
+ int modifierKeyCode1 = KeyboardUtils.getModifierKeyCode(modifierKey1);
+ int modifierKeyCode2 = KeyboardUtils.getModifierKeyCode(modifierKey2);
+ int specificKeyCode = KeyboardUtils.getSpecificKeyCode(specificKey);
+
+ logger.info("Modifier Key Code 1: " + modifierKeyCode1);
+ logger.info("Modifier Key Code 2: " + modifierKeyCode2);
+ logger.info("Specific Key Code: " + specificKeyCode);
+
+ // Press both modifier keys first, then the specific key
+ robot.keyPress(modifierKeyCode1);
+ KeyboardUtils.sleep(30);
+ robot.keyPress(modifierKeyCode2);
+ KeyboardUtils.sleep(30);
+ robot.keyPress(specificKeyCode);
+ KeyboardUtils.sleep(30);
+
+ // Release keys in reverse order
+ robot.keyRelease(specificKeyCode);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(modifierKeyCode2);
+ KeyboardUtils.sleep(30);
+ robot.keyRelease(modifierKeyCode1);
+
+
+ setSuccessMessage("Successfully pressed " + modifierKey1 + " + " + modifierKey2 + " + " + specificKey + " keys.");
+
+ // Capture and upload screenshot
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_two_modifier_keys_with_basic_key_screenshot", logger);
+
+ } catch (Exception e) {
+ setErrorMessage("An error occurred while pressing the two modifier keys with specific key: " + e.getMessage());
+ logger.debug("Error pressing two modifier keys with specific key: " + ExceptionUtils.getStackTrace(e));
+ result = Result.FAILED;
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "press_two_modifier_keys_with_basic_key_failure_screenshot", logger);
+ }
+ return result;
+ }
+
+
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/RightClick.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/RightClick.java
new file mode 100644
index 00000000..f8f891c1
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/RightClick.java
@@ -0,0 +1,42 @@
+package com.testsigma.addons.windowsLite;
+
+
+
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+
+
+@Action(actionText = "lite: Right Click on coordinates x: \"x-coordinate\" y: \"y-coordinate\"",
+ description = "Right Click on coordinates",
+ applicationType = ApplicationType.WINDOWS_UFT)
+public class RightClick extends WindowsAction {
+
+ @TestData(reference = "x-coordinate")
+ private com.testsigma.sdk.TestData xCoordinate;
+
+ @TestData(reference = "y-coordinate")
+ private com.testsigma.sdk.TestData yCoordinate;
+
+ @Override
+ public com.testsigma.sdk.Result execute() {
+ com.testsigma.sdk.Result result = com.testsigma.sdk.Result.SUCCESS;
+ try {
+ int x = Integer.parseInt(xCoordinate.getValue().toString());
+ int y = Integer.parseInt(yCoordinate.getValue().toString());
+
+ java.awt.Robot robot = new java.awt.Robot();
+ robot.mouseMove(x, y);
+ robot.mousePress(java.awt.event.InputEvent.BUTTON3_DOWN_MASK);
+ robot.mouseRelease(java.awt.event.InputEvent.BUTTON3_DOWN_MASK);
+
+ setSuccessMessage(String.format("Successfully performed right click at coordinates (%d, %d)", x, y));
+ } catch (Exception e) {
+ result = com.testsigma.sdk.Result.FAILED;
+ setErrorMessage("Failed to perform right click at the specified coordinates: " + e.getMessage());
+ logger.debug("Error performing right click at coordinates" + e);
+ }
+ return result;
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/StorePresenceOfTextInRuntimeVariable.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/StorePresenceOfTextInRuntimeVariable.java
new file mode 100644
index 00000000..75fe1f25
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/StorePresenceOfTextInRuntimeVariable.java
@@ -0,0 +1,166 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.AIRequest;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.*;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+
+@Action(actionText = "AI: verify that the text text-to-verify is present in opened application and" +
+ " store result in runtime variable variable-to-store-result",
+ description = "This action stores true if text is present in the screen else it stores false in the variable " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Store the presence of text in runtime variable",
+ useCustomScreenshot = true)
+public class StorePresenceOfTextInRuntimeVariable extends WindowsAction {
+
+ @TestData(reference = "text-to-verify")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestData(reference = "variable-to-store-result", isRuntimeVariable = true)
+ private com.testsigma.sdk.TestData testData1;
+
+ @AI
+ private com.testsigma.sdk.AI ai;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ @RunTimeData
+ private com.testsigma.sdk.RunTimeData runTimeData;
+
+ private final String prompt = "You are provided with a screenshot of a computer application." +
+ " Your task is to analyze this screenshot and determine if the specified text is present anywhere in " +
+ "the image.Look for the text in any form - it could be in buttons, labels, text fields, menus, " +
+ "or any other UI element. Return only 'YES' if the text is found, or 'NO' if the text is not found. " +
+ "The text to search for is: ";
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== AI Text Verification: Starting Execution ===");
+
+ try {
+ String expectedText = testData.getValue().toString();
+ logger.info("Looking for text: " + expectedText);
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "application_screenshot");
+ logger.info("Screenshot saved to: " + screenshotFile.getAbsolutePath());
+
+ // Create AI request
+ AIRequest aiRequest = new AIRequest();
+ String fullPrompt = prompt + "'" + expectedText + "'. ";
+ aiRequest.setPrompt(fullPrompt);
+ aiRequest.setModel("gpt-4o");
+
+ logger.info("Sending AI prompt: " + fullPrompt);
+
+ // Add the screenshot file
+// ArrayList files = new ArrayList<>();
+ List files = new ArrayList<>();
+ files.add(screenshotFile);
+ aiRequest.setFiles(files);
+
+ // Invoke AI
+ String aiResponse = ai.invokeAI(aiRequest);
+ logger.info("AI response: " + aiResponse);
+
+ // Parse AI response
+ boolean textFound = parseAIResponse(aiResponse);
+
+ // Upload screenshot to S3
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ // this step will pass irrespective of the presence of the text, it will store the result which can be used
+ // in the if condition step.
+ if (textFound) {
+ logger.info("Text found in application. Step passed.");
+ runTimeData.setValue("true");
+ runTimeData.setKey(testData1.getValue().toString());
+ } else {
+ logger.debug("Text not found in application. Step failed.");
+ runTimeData.setValue("false");
+ runTimeData.setKey(testData1.getValue().toString());
+ }
+ } catch (Exception e) {
+ logger.debug("Exception during AI text verification: " + e.getMessage());
+ setErrorMessage("Error during text verification: " + e.getMessage());
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult,
+ "verify_text_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ setSuccessMessage("Successfully stored the presence of the text " + testData.getValue().toString() +
+ " in the variable " + testData1.getValue().toString());
+ return Result.SUCCESS;
+ }
+
+
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ logger.info("Screenshot saved to temporary file: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Parses the AI response to determine if text was found
+ * @param aiResponse The response from AI
+ * @return true if text was found, false otherwise
+ */
+ private boolean parseAIResponse(String aiResponse) {
+ if (aiResponse == null || aiResponse.trim().isEmpty()) {
+ logger.debug("AI response is null or empty");
+ return false;
+ }
+
+ String response = aiResponse.trim().toUpperCase();
+ logger.info("Parsing AI response: " + response);
+
+ // Check for various positive responses
+ if (response.contains("YES") || response.contains("TRUE") || response.contains("FOUND") ||
+ response.contains("PRESENT") || response.contains("EXISTS")) {
+ return true;
+ }
+
+ // Check for various negative responses
+ if (response.contains("NO") || response.contains("FALSE") || response.contains("NOT FOUND") ||
+ response.contains("ABSENT") || response.contains("NOT PRESENT")) {
+ return false;
+ }
+
+ // If response is unclear, log it and return false
+ logger.debug("Unclear AI response: " + aiResponse + ". Treating as 'not found'.");
+ return false;
+ }
+}
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/VerifyTextInApplication.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/VerifyTextInApplication.java
new file mode 100644
index 00000000..a2dbcff9
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/VerifyTextInApplication.java
@@ -0,0 +1,163 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+
+
+import com.testsigma.sdk.AIRequest;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.AI;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+@Data
+@Action(actionText = "AI: verify that the text text-to-verify is present in opened application and store result in runtime variable test-data1",
+ description = "This action verifies that the specified text is present in the opened application" +
+ " using AI capabilities. " +
+ "This works only for local executions",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS,
+ displayName = "Verify if text is present in application",
+ useCustomScreenshot = true)
+public class VerifyTextInApplication extends WindowsAction {
+
+ @TestData(reference = "text-to-verify")
+ private com.testsigma.sdk.TestData testData;
+
+ @TestData(reference = "test-data1", isRuntimeVariable = true)
+ private com.testsigma.sdk.TestData testData1;
+
+ @AI
+ private com.testsigma.sdk.AI ai;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private final String prompt = "You are provided with a screenshot of a computer application." +
+ " Your task is to analyze this screenshot and determine if the specified text is present anywhere in " +
+ "the image.Look for the text in any form - it could be in buttons, labels, text fields, menus, " +
+ "or any other UI element. Return only 'YES' if the text is found, or 'NO' if the text is not found. " +
+ "The text to search for is: ";
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== AI Text Verification: Starting Execution ===");
+
+ try {
+ String expectedText = testData.getValue().toString();
+ logger.info("Looking for text: " + expectedText);
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+ logger.info("Screen capture dimensions: " + screenCapture.getWidth() + "x" + screenCapture.getHeight());
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "application_screenshot");
+ logger.info("Screenshot saved to: " + screenshotFile.getAbsolutePath());
+
+ // Create AI request
+ AIRequest aiRequest = new AIRequest();
+ String fullPrompt = prompt + "'" + expectedText + "'. ";
+ aiRequest.setPrompt(fullPrompt);
+ aiRequest.setModel("gpt-4o");
+
+ logger.info("Sending AI prompt: " + fullPrompt);
+
+ // Add the screenshot file
+// ArrayList files = new ArrayList<>();
+ List files = new ArrayList<>();
+ files.add(screenshotFile);
+ aiRequest.setFiles(files);
+
+ // Invoke AI
+ String aiResponse = ai.invokeAI(aiRequest);
+ logger.info("AI response: " + aiResponse);
+
+ // Parse AI response
+ boolean textFound = parseAIResponse(aiResponse);
+
+ // Upload screenshot to S3
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ if (textFound) {
+ logger.info("Text found in application. Step passed.");
+ setSuccessMessage("Text '" + expectedText + "' was found in the application.");
+ return Result.SUCCESS;
+ } else {
+ logger.debug("Text not found in application. Step failed.");
+ setErrorMessage("Text '" + expectedText + "' was not found in the application.");
+ return Result.FAILED;
+ }
+
+ } catch (Exception e) {
+ logger.debug("Exception during AI text verification: " + e.getMessage());
+ setErrorMessage("Error during text verification: " + e.getMessage());
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "verify_text_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ logger.info("Screenshot saved to temporary file: " + tempFile.getAbsolutePath());
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Parses the AI response to determine if text was found
+ * @param aiResponse The response from AI
+ * @return true if text was found, false otherwise
+ */
+ private boolean parseAIResponse(String aiResponse) {
+ if (aiResponse == null || aiResponse.trim().isEmpty()) {
+ logger.debug("AI response is null or empty");
+ return false;
+ }
+
+ String response = aiResponse.trim().toUpperCase();
+ logger.info("Parsing AI response: " + response);
+
+ // Check for various positive responses
+ if (response.contains("YES") || response.contains("TRUE") || response.contains("FOUND") ||
+ response.contains("PRESENT") || response.contains("EXISTS")) {
+ return true;
+ }
+
+ // Check for various negative responses
+ if (response.contains("NO") || response.contains("FALSE") || response.contains("NOT FOUND") ||
+ response.contains("ABSENT") || response.contains("NOT PRESENT")) {
+ return false;
+ }
+
+ // If response is unclear, log it and return false
+ logger.debug("Unclear AI response: " + aiResponse + ". Treating as 'not found'.");
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/WaitUntilImagePresent.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/WaitUntilImagePresent.java
new file mode 100644
index 00000000..b102dbc1
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/WaitUntilImagePresent.java
@@ -0,0 +1,332 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "wait until image image-url is present on screen with timeout wait-time-in-seconds seconds " +
+ "with threshold threshold-value (Ex: 0.9 , means 90% match)",
+ description = "This action waits until the specified image appears on the screen within the given timeout. "
+ + "It does not click the image; it only verifies that the image is present. "
+ + "It takes an image URL (S3 URL or local file path), polls the screen every 1.5 seconds. "
+ + "Threshold (0 to 1) controls match sensitivity",
+ applicationType = ApplicationType.WINDOWS)
+public class WaitUntilImagePresent extends WindowsAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData timeoutSeconds;
+
+ @TestData(reference = "threshold-value")
+ private com.testsigma.sdk.TestData threshold;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private static final int POLLING_INTERVAL_MS = 1500;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Wait Until Image Present: Starting Execution ===");
+
+ try {
+ Result result = Result.SUCCESS;
+ String imageUrlValue = imageUrl.getValue().toString();
+ int timeoutMs = Integer.parseInt(timeoutSeconds.getValue().toString()) * 1000;
+ String thresholdStr = threshold.getValue().toString().trim();
+
+ logger.info("Waiting for image from URL: " + imageUrlValue + " with timeout: "
+ + timeoutSeconds.getValue() + " seconds, threshold: " + thresholdStr);
+
+ File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
+ Robot robot = new Robot();
+ long startTime = System.currentTimeMillis();
+ long endTime = startTime + timeoutMs;
+
+ while (System.currentTimeMillis() < endTime) {
+ try {
+ // Fetch the Details of the Screen Size
+ Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+
+ // Take the Snapshot of the Screen
+ BufferedImage tmp = robot.createScreenCapture(screenSize);
+
+ // Provide the destination details to copy the screenshot
+ String tempDir = System.getProperty("java.io.tmpdir");
+ String filename = "screenshot"+System.currentTimeMillis()+".jpg";
+ String path = tempDir + filename;
+
+ // To copy source image in to destination path
+ ImageIO.write(tmp, "jpg",new File(path));
+ int width = tmp.getWidth();
+ int height = tmp.getHeight();
+ logger.info("Width of image: " + width);
+ logger.info("Height of image: " + height);
+
+ File baseImageFile = new File(path);
+ String url = testStepResult.getScreenshotUrl();
+ ocr.uploadFile(url, baseImageFile);
+ logger.info("url: "+ testStepResult.getScreenshotUrl());
+ FindImageResponse responseObject = ocr.findImage(imageUrl.getValue().toString(),
+ Float.valueOf(threshold.getValue().toString()));
+ if (responseObject.getIsFound()){
+ boolean isFound = responseObject.getIsFound();
+ int x1 = responseObject.getX1();
+ int y1 = responseObject.getY1();
+ int x2 = responseObject.getX2();
+ int y2 = responseObject.getY2();
+
+ int clickLocationX = (x1 + x2) / 2;
+ int clickLocationY = (y1 + y2) / 2;
+
+ logger.info("Click Location X: " + clickLocationX);
+ logger.info("Click Location Y: " + clickLocationY);
+
+
+ setSuccessMessage("Image Found :" + isFound +
+ " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2);
+ Thread.sleep(1000);
+ return Result.SUCCESS;
+ } else {
+ setErrorMessage("Unable to fetch the coordinates");
+ result = Result.FAILED;
+ return result;
+ }
+ }
+ catch (Exception e){
+ logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
+ setErrorMessage("Exception occurred while performing click action");
+ result = Result.FAILED;
+ return result;
+ }
+ }
+
+ logger.debug("Timeout reached. Image was not found on the screen within "
+ + timeoutSeconds.getValue() + " seconds.");
+ setErrorMessage("Image was not found on the screen within " + timeoutSeconds.getValue() + " seconds.");
+ return Result.FAILED;
+ } catch (NumberFormatException e) {
+ logger.debug("Invalid number format: " + e.getMessage());
+ setErrorMessage("Invalid input. Timeout must be a number (seconds). Threshold must be a number between 0 and 1.");
+ return Result.FAILED;
+ } catch (Exception e) {
+ logger.debug("Exception during wait operation: " + e.getMessage());
+ setErrorMessage("Error during wait operation: " + e.getMessage());
+ return Result.FAILED;
+ }
+ }
+
+ private File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png";
+ }
+ }
+ File tempFile = File.createTempFile(baseName, extension);
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.info("Temp file created: " + tempFile.getName() + " at " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+}
+
+/*
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.sdk.FindImageResponse;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.OCR;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+@Action(actionText = "Wait until image image-url is present on screen with timeout wait-time-in-seconds seconds with threshold threshold-value",
+ description = "This action waits until the specified image appears on the screen within the given timeout. "
+ + "It does not click the image; it only verifies that the image is present. "
+ + "It takes an image URL (S3 URL or local file path), polls the screen every 1.5 seconds. "
+ + "Threshold (0 to 1) controls match sensitivity",
+ applicationType = com.testsigma.sdk.ApplicationType.WINDOWS,
+ useCustomScreenshot = true)
+public class WaitUntilImagePresent extends WindowsAction {
+
+ @TestData(reference = "image-url")
+ private com.testsigma.sdk.TestData imageUrl;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData timeoutSeconds;
+
+ @TestData(reference = "threshold-value")
+ private com.testsigma.sdk.TestData threshold;
+
+ @OCR
+ private com.testsigma.sdk.OCR ocr;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private static final int POLLING_INTERVAL_MS = 1500;
+
+ @Override
+ protected Result execute() {
+ logger.info("=== Wait Until Image Present: Starting Execution ===");
+
+ try {
+ String imageUrlValue = imageUrl.getValue().toString();
+ int timeoutMs = Integer.parseInt(timeoutSeconds.getValue().toString()) * 1000;
+ String thresholdStr = threshold.getValue().toString().trim();
+ double thresholdValue = Double.parseDouble(thresholdStr);
+ if (thresholdValue < 0 || thresholdValue > 1) {
+ setErrorMessage("Threshold must be between 0 and 1. Got: " + thresholdStr);
+ return Result.FAILED;
+ }
+
+ logger.info("Waiting for image from URL: " + imageUrlValue + " with timeout: "
+ + timeoutSeconds.getValue() + " seconds, threshold: " + thresholdStr);
+
+ File searchImageFile = urlToFileConverter("target_image", imageUrlValue);
+ Robot robot = new Robot();
+ long startTime = System.currentTimeMillis();
+ long endTime = startTime + timeoutMs;
+
+ while (System.currentTimeMillis() < endTime) {
+ logger.info("Polling attempt - capturing fresh screenshot and checking for image on screen");
+
+ Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenSize);
+ File screenshotFile = new File(System.getProperty("java.io.tmpdir"),
+ "screenshot" + System.currentTimeMillis() + ".png");
+ String screenshotPath = screenshotFile.getAbsolutePath();
+ ImageIO.write(screenCapture, "png", screenshotFile);
+ logger.info("Screenshot saved to: " + screenshotPath);
+
+ ocr.uploadFile(testStepResult.getScreenshotUrl(), screenshotFile);
+ logger.info("Screenshot uploaded to: " + testStepResult.getScreenshotUrl());
+
+ float thresholdPercentage = (float) thresholdValue;
+ logger.info("Threshold percentage: " + thresholdPercentage);
+ FindImageResponse findImageResponse = ocr.findImage(searchImageFile.getAbsolutePath(), thresholdPercentage);
+
+ if (findImageResponse != null && findImageResponse.getIsFound()) {
+ int centerX = findImageResponse.getX1() + (findImageResponse.getX2() - findImageResponse.getX1()) / 2;
+ int centerY = findImageResponse.getY1() + (findImageResponse.getY2() - findImageResponse.getY1()) / 2;
+ logger.info("Image found at center (" + centerX + ", " + centerY + "). Wait successful.");
+ setSuccessMessage("Image found on screen at coordinates (" + centerX + ", " + centerY + ").");
+ return Result.SUCCESS;
+ }
+
+ long remainingTime = endTime - System.currentTimeMillis();
+ if (remainingTime > 0) {
+ long sleepTime = Math.min(POLLING_INTERVAL_MS, remainingTime);
+ logger.info("Image not found yet. Waiting " + sleepTime + "ms before next attempt. Remaining time: " + remainingTime + "ms");
+ Thread.sleep(sleepTime);
+ }
+ }
+
+ logger.debug("Timeout reached. Image was not found on the screen within "
+ + timeoutSeconds.getValue() + " seconds.");
+ setErrorMessage("Image was not found on the screen within " + timeoutSeconds.getValue() + " seconds.");
+ return Result.FAILED;
+
+ } catch (NumberFormatException e) {
+ logger.debug("Invalid number format: " + e.getMessage());
+ setErrorMessage("Invalid input. Timeout must be a number (seconds). Threshold must be a number between 0 and 1.");
+ return Result.FAILED;
+ } catch (Exception e) {
+ logger.debug("Exception during wait operation: " + e.getMessage());
+ setErrorMessage("Error during wait operation: " + e.getMessage());
+ return Result.FAILED;
+ }
+ }
+
+ private File urlToFileConverter(String fileName, String url) {
+ try {
+ if (url.startsWith("https://") || url.startsWith("http://")) {
+ logger.info("Given is s3 url ...File name:" + fileName);
+ URL urlObject = new URL(url);
+ String baseName = fileName;
+ String extension = "";
+ int lastDotIndex = fileName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ baseName = fileName.substring(0, lastDotIndex);
+ extension = fileName.substring(lastDotIndex);
+ } else {
+ String urlPath = urlObject.getPath();
+ int urlLastDotIndex = urlPath.lastIndexOf('.');
+ if (urlLastDotIndex > 0) {
+ extension = urlPath.substring(urlLastDotIndex);
+ } else {
+ extension = ".png";
+ }
+ }
+ File tempFile = File.createTempFile(baseName, extension);
+ try (InputStream in = urlObject.openStream()) {
+ Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.info("Temp file created: " + tempFile.getName() + " at " + tempFile.getAbsolutePath());
+ return tempFile;
+ } else {
+ logger.info("Given is local file path..");
+ return new File(url);
+ }
+ } catch (Exception e) {
+ logger.info("Error while accessing: " + url);
+ throw new RuntimeException("Unable to access the given file, please check the given inputs.");
+ }
+ }
+}
+*/
+
diff --git a/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/WaitUntilTextPresentInScreen.java b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/WaitUntilTextPresentInScreen.java
new file mode 100644
index 00000000..2a096216
--- /dev/null
+++ b/windows_advanced_actions/src/main/java/com/testsigma/addons/windowsLite/WaitUntilTextPresentInScreen.java
@@ -0,0 +1,196 @@
+package com.testsigma.addons.windowsLite;
+
+import com.testsigma.addons.util.ScreenshotUtils;
+import com.testsigma.sdk.AIRequest;
+import com.testsigma.sdk.ApplicationType;
+import com.testsigma.sdk.Result;
+import com.testsigma.sdk.WindowsAction;
+import com.testsigma.sdk.annotation.AI;
+import com.testsigma.sdk.annotation.Action;
+import com.testsigma.sdk.annotation.TestData;
+import com.testsigma.sdk.annotation.TestStepResult;
+import lombok.Data;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+@Data
+@Action(actionText = "AI: Wait until text text-to-verify is present in screen with timeout wait-time-in-seconds seconds",
+ description = "This action waits until the specified text is present on the screen using AI capabilities. " +
+ "It polls every 1 second until the text is found or timeout is reached. " +
+ "This works only for local executions",
+ applicationType = ApplicationType.WINDOWS,
+ displayName = "Wait until the text is present on the screen",
+ useCustomScreenshot = true)
+public class WaitUntilTextPresentInScreen extends WindowsAction {
+
+ @TestData(reference = "text-to-verify")
+ private com.testsigma.sdk.TestData textToSearch;
+
+ @TestData(reference = "wait-time-in-seconds")
+ private com.testsigma.sdk.TestData timeoutSeconds;
+
+ @AI
+ private com.testsigma.sdk.AI ai;
+
+ @TestStepResult
+ private com.testsigma.sdk.TestStepResult testStepResult;
+
+ private final String prompt = "You are provided with a screenshot of a computer application." +
+ " Your task is to analyze this screenshot and determine if the specified text is present anywhere in " +
+ "the image. Look for the text in any form - it could be in buttons, labels, text fields, menus, " +
+ "or any other UI element. Return only 'YES' if the text is found, or 'NO' if the text is not found. " +
+ "The text to search for is: ";
+
+ private static final int POLLING_INTERVAL_MS = 1500; // 1 second polling interval
+
+ @Override
+ protected Result execute() throws NoSuchElementException {
+ logger.info("=== Wait Until Text Present: Starting Execution ===");
+
+ try {
+ String expectedText = textToSearch.getValue().toString();
+ int timeoutMs = Integer.parseInt(timeoutSeconds.getValue().toString()) * 1000;
+
+ logger.info("Looking for text: '" + expectedText + "' with timeout: " +
+ timeoutSeconds.getValue() + " seconds");
+
+ long startTime = System.currentTimeMillis();
+ long endTime = startTime + timeoutMs;
+
+ while (System.currentTimeMillis() < endTime) {
+ logger.info("Polling attempt - checking for text: '" + expectedText + "'");
+
+ // Capture the current screen
+ Robot robot = new Robot();
+ Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
+ BufferedImage screenCapture = robot.createScreenCapture(screenRect);
+
+ // Save the screenshot to a temporary file
+ File screenshotFile = saveScreenshotToFile(screenCapture, "wait_text_screenshot");
+
+ // Create AI request
+ AIRequest aiRequest = new AIRequest();
+ String fullPrompt = prompt + "'" + expectedText + "'. ";
+ aiRequest.setPrompt(fullPrompt);
+ aiRequest.setModel("gpt-4o");
+
+ // Add the screenshot file
+ List files = new ArrayList<>();
+// ArrayList files = new ArrayList<>();
+ files.add(screenshotFile);
+ aiRequest.setFiles(files);
+
+ // Invoke AI
+ String aiResponse = ai.invokeAI(aiRequest);
+ logger.info("AI response: " + aiResponse);
+
+ // Parse AI response
+ boolean textFound = parseAIResponse(aiResponse);
+
+ if (textFound) {
+ logger.info("Text found in application. Wait successful.");
+ setSuccessMessage("Text '" + expectedText + "' was found on the screen after waiting.");
+
+ // Upload final screenshot to S3
+ ScreenshotUtils.uploadScreenshotToS3(testStepResult, screenshotFile, logger);
+
+ return Result.SUCCESS;
+ }
+
+ // Clean up temporary file
+ if (screenshotFile.exists()) {
+ screenshotFile.delete();
+ }
+
+ // Check if we should continue polling
+ long remainingTime = endTime - System.currentTimeMillis();
+ if (remainingTime > POLLING_INTERVAL_MS) {
+ logger.info("Text not found yet. Waiting " + (POLLING_INTERVAL_MS / 1000)
+ + " second before next attempt. " +
+ "Remaining time: " + (remainingTime / 1000) + " seconds");
+ Thread.sleep(POLLING_INTERVAL_MS);
+ } else {
+ break; // No time left for another attempt
+ }
+ }
+
+ // If we reach here, timeout occurred
+ logger.debug("Timeout reached. Text '" + expectedText + "' was not found on the screen within " +
+ timeoutSeconds.getValue() + " seconds.");
+ setErrorMessage("Text '" + expectedText + "' was not found on the screen within " +
+ timeoutSeconds.getValue() + " seconds.");
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "wait_text_failure_screenshot", logger);
+ return Result.FAILED;
+
+ } catch (NumberFormatException e) {
+ logger.debug("Invalid timeout value: " + timeoutSeconds.getValue());
+ setErrorMessage("Invalid timeout value: " + timeoutSeconds.getValue() +
+ ". Please provide a valid number of seconds.");
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "wait_text_failure_screenshot", logger);
+ return Result.FAILED;
+ } catch (Exception e) {
+ logger.debug("Exception during wait operation: " + e.getMessage());
+ setErrorMessage("Error during wait operation: " + e.getMessage());
+ // Capture and upload screenshot even on failure
+ ScreenshotUtils.captureAndUploadScreenshot(testStepResult, "wait_text_failure_screenshot", logger);
+ return Result.FAILED;
+ }
+ }
+
+ /**
+ * Saves the screenshot to a temporary file
+ * @param screenshot The captured screenshot
+ * @param fileName The base filename
+ * @return The temporary file
+ * @throws Exception if file creation fails
+ */
+ private File saveScreenshotToFile(BufferedImage screenshot, String fileName) throws Exception {
+ try {
+ File tempFile = File.createTempFile(fileName, ".png");
+ ImageIO.write(screenshot, "PNG", tempFile);
+ return tempFile;
+ } catch (Exception e) {
+ logger.debug("Failed to save screenshot to file: " + e.getMessage());
+ throw new RuntimeException("Unable to save screenshot for AI processing.", e);
+ }
+ }
+
+ /**
+ * Parses the AI response to determine if text was found
+ * @param aiResponse The response from AI
+ * @return true if text was found, false otherwise
+ */
+ private boolean parseAIResponse(String aiResponse) {
+ if (aiResponse == null || aiResponse.trim().isEmpty()) {
+ logger.debug("AI response is null or empty");
+ return false;
+ }
+
+ String response = aiResponse.trim().toUpperCase();
+ logger.debug("Parsing AI response: " + response);
+
+ // Check for various positive responses
+ if (response.contains("YES") || response.contains("TRUE") || response.contains("FOUND") ||
+ response.contains("PRESENT") || response.contains("EXISTS")) {
+ return true;
+ }
+
+ // Check for various negative responses
+ if (response.contains("NO") || response.contains("FALSE") || response.contains("NOT FOUND") ||
+ response.contains("ABSENT") || response.contains("NOT PRESENT")) {
+ return false;
+ }
+
+ // If response is unclear, log it and return false
+ logger.debug("Unclear AI response: " + aiResponse + ". Treating as 'not found'.");
+ return false;
+ }
+}