diff --git a/DocTest/VisualTest.py b/DocTest/VisualTest.py index d6dd546..d4d3429 100644 --- a/DocTest/VisualTest.py +++ b/DocTest/VisualTest.py @@ -4,6 +4,7 @@ import uuid from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union +import re import cv2 import imutils @@ -108,6 +109,7 @@ def __init__( document_page_cache_size: int = 2, verbose_movement_logging: bool = False, character_replacements: Optional[Dict[str, str]] = None, + run_keyword_on_warn_threshold: str = None, **kwargs, ): """ @@ -134,11 +136,13 @@ def __init__( | ``document_page_cache_size`` | Maximum number of rendered pages to keep in memory per document when streaming. Default is ``2``. | | ``verbose_movement_logging`` | When ``True``, emit detailed warnings from movement detection helpers (template, ORB, SIFT). Disabled by default to reduce noise. | | ``character_replacements`` | Dict mapping characters to replacements, applied to text extraction results. Example: ``{'\u00A0': ' '}`` to normalize non-breaking spaces. | + | ``run_keyword_on_warn_threshold`` | Robot Framework keyword (with optional arguments) to execute when differences exceed warning threshold but are within acceptable threshold. The keyword can include arguments separated by spaces (e.g., ``"Log WARN \ level=WARN"``). Default is ``None``. | | ``**kwargs`` | Everything else. | """ self.threshold = threshold + self.run_keyword_on_warn_threshold = run_keyword_on_warn_threshold self.dpi = dpi self.take_screenshots = take_screenshots self.show_diff = show_diff @@ -235,6 +239,7 @@ def compare_images( resize_candidate: bool = False, blur: bool = False, threshold: float = None, + threshold_warn: float = None, mask: Union[str, dict, list] = None, get_pdf_content: bool = False, block_based_ssim: bool = False, @@ -263,6 +268,7 @@ def compare_images( | ``resize_candidate`` | Allow visual comparison, even of documents have different sizes | | ``blur`` | Blur the image before comparison to reduce visual difference caused by noise | | ``threshold`` | Threshold for visual comparison between 0.0000 and 1.0000 . Default is 0.0000. Higher values mean more tolerance for visual differences. | + | ``threshold_warn`` | Warning threshold for visual comparison between 0.0000 and 1.0000 . If differences exceed this but are within ``threshold``, the test passes but executes ``run_keyword_on_warn_threshold``. Default is ``None`` (disabled). | | ``block_based_ssim`` | Uses additional block based block-based comparison, to catch differences in smaller areas. Makes only sense, for ``threshold`` > 0 . Default is `False` | | ``block_size`` | Size of the blocks for block-based comparison. Default is 32. Only relevant for ``block_based_ssim`` | | ``llm`` / ``llm_enabled`` | When ``${True}``, summarise differences and forward them to the configured LLM. Default ``${False}``. | @@ -354,6 +360,22 @@ def compare_images( dpi = DPI if DPI else self.dpi threshold = threshold if threshold is not None else self.threshold + # Validate threshold_warn parameter + if threshold_warn is not None: + if not isinstance(threshold_warn, (int, float)): + raise ValueError( + f"threshold_warn must be a number, got {type(threshold_warn).__name__}" + ) + if threshold_warn < 0.0 or threshold_warn > 1.0: + raise ValueError( + f"threshold_warn must be between 0.0 and 1.0, got {threshold_warn}" + ) + if threshold_warn >= threshold: + raise ValueError( + f"threshold_warn ({threshold_warn}) must be less than threshold ({threshold}). " + f"The warning threshold should be more strict (lower) than the acceptable threshold." + ) + # Set OCR engine if provided ocr_engine = ocr_engine if ocr_engine else self.ocr_engine @@ -437,9 +459,14 @@ def compare_images( ) continue + # Use threshold_warn for comparison if it's set, so images in warning zone + # are still detected as different (and added to detected_differences) + comparison_threshold = ( + threshold_warn if threshold_warn is not None else threshold + ) similar, diff, thresh, absolute_diff, score = ref_page.compare_with( cand_page, - threshold=threshold, + threshold=comparison_threshold, blur=blur, block_based_ssim=block_based_ssim, block_size=block_size, @@ -865,7 +892,59 @@ def compare_images( elif not llm_decision.is_positive and llm_override_result: print("LLM rejected differences. Baseline failure will be raised.") - for diff in detected_differences: + # Classify differences into warning zone vs actual failures + differences_in_warn_zone = [] + actual_failures = [] + + if threshold_warn is not None: + for diff in detected_differences: + score = diff.get("score") + if score is not None and threshold_warn < score <= threshold: + differences_in_warn_zone.append(diff) + else: + actual_failures.append(diff) + else: + # If no warning threshold is set, all differences are failures + actual_failures = detected_differences + + # Handle differences in warning zone (PASS with warning) + if differences_in_warn_zone: + for diff in differences_in_warn_zone: + score = diff.get("score") + print( + f"Visual differences detected in warning zone. " + f"SSIM score: {score:.6f}, " + f"Warning threshold: {threshold_warn:.6f}, " + f"Acceptable threshold: {threshold:.6f}" + ) + + # Execute the warning keyword if configured + if self.run_keyword_on_warn_threshold: + try: + built_in = BuiltIn() + # Split keyword name and its arguments + parts = re.split(r'\s{2,}', self.run_keyword_on_warn_threshold) + keyword_name = ( + parts[0] + if parts + else self.run_keyword_on_warn_threshold + ) + all_args = parts[1:] if len(parts) > 1 else [] + + built_in.run_keyword(keyword_name, *all_args) + print( + f"Executed warning keyword: {self.run_keyword_on_warn_threshold}" + ) + except Exception as e: + print( + f"Failed to execute keyword '{self.run_keyword_on_warn_threshold}': {str(e)}" + ) + + print("Images/Document comparison passed with warnings.") + # Return early - test passes despite warnings + return + + for diff in actual_failures: print(diff["message"]) self._raise_comparison_failure() diff --git a/atest/Compare.robot b/atest/Compare.robot index f449fc6..821a1a9 100644 --- a/atest/Compare.robot +++ b/atest/Compare.robot @@ -1,5 +1,6 @@ *** Settings *** -Library DocTest.VisualTest show_diff=true take_screenshots=true screenshot_format=png #pdf_rendering_engine=ghostscript +Library DocTest.VisualTest show_diff=true take_screenshots=true screenshot_format=png +... run_keyword_on_warn_threshold=Set Tags \ test:warn #pdf_rendering_engine=ghostscript Library Collections Library String @@ -129,3 +130,22 @@ Compare Images with Combined Watermarks Log Testing combined watermarks feature - parameter acceptance @{single_watermark} Create List testdata/Beach_date_mask_full.png Compare Images testdata/Beach_date.png testdata/Beach_left.png watermark_file=${single_watermark} + + +Compare Two Similar Map Images + Compare Images testdata/map_orig.png testdata/map_orig.png + +Compare Two Map Images With Differences + Run Keyword And Expect Error The compared images are different. + ... Compare Images testdata/map_orig.png testdata/map_3_diff.png + +Compare Two Map Images With Differences And Passing Warn Threshold + Compare Images testdata/map_orig.png testdata/map_3_diff.png + ... threshold_warn=0.001 threshold=0.002 + + +Compare Two Map Images With Differences And Exceeding Warn Threshold + Run Keyword And Expect Error The compared images are different. + ... Compare Images testdata/map_orig.png testdata/map_29pct_diff.png + ... threshold_warn=0.001 threshold=0.002 + diff --git a/testdata/map_1_diff.png b/testdata/map_1_diff.png new file mode 100644 index 0000000..f0ab775 Binary files /dev/null and b/testdata/map_1_diff.png differ diff --git a/testdata/map_29pct_diff.png b/testdata/map_29pct_diff.png new file mode 100644 index 0000000..0bc47c8 Binary files /dev/null and b/testdata/map_29pct_diff.png differ diff --git a/testdata/map_2_diff.png b/testdata/map_2_diff.png new file mode 100644 index 0000000..b2d0a13 Binary files /dev/null and b/testdata/map_2_diff.png differ diff --git a/testdata/map_3_diff.png b/testdata/map_3_diff.png new file mode 100644 index 0000000..feb879d Binary files /dev/null and b/testdata/map_3_diff.png differ diff --git a/testdata/map_orig.png b/testdata/map_orig.png new file mode 100644 index 0000000..2f35794 Binary files /dev/null and b/testdata/map_orig.png differ diff --git a/utest/test_visualtest.py b/utest/test_visualtest.py index 3096315..54d8d0f 100644 --- a/utest/test_visualtest.py +++ b/utest/test_visualtest.py @@ -244,3 +244,261 @@ def test_block_based_ssim(testdata_dir): ref_image=str(testdata_dir / "birthday_left.png") cand_image=str(testdata_dir / "birthday_right.png") ssim = visual_tester.compare_images(ref_image, cand_image, block_based_ssim=True, block_size=32) +# ============================================================================ +# Threshold Warning Tests +# ============================================================================ + + +def test_threshold_warn_validation_equal_to_threshold(testdata_dir): + """Test that threshold_warn equal to threshold raises ValueError.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_right.png") + + with pytest.raises(ValueError, match="must be less than threshold"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn=0.1 + ) + + +def test_threshold_warn_validation_greater_than_threshold(testdata_dir): + """Test that threshold_warn greater than threshold raises ValueError.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_right.png") + + with pytest.raises(ValueError, match="must be less than threshold"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn=0.2 + ) + + +def test_threshold_warn_validation_negative(testdata_dir): + """Test that negative threshold_warn raises ValueError.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_right.png") + + with pytest.raises(ValueError, match="must be between 0.0 and 1.0"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn=-0.1 + ) + + +def test_threshold_warn_validation_above_one(testdata_dir): + """Test that threshold_warn > 1.0 raises ValueError.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_right.png") + + with pytest.raises(ValueError, match="must be between 0.0 and 1.0"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.5, threshold_warn=1.5 + ) + + +def test_threshold_warn_validation_non_numeric(testdata_dir): + """Test that non-numeric threshold_warn raises ValueError.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_right.png") + + with pytest.raises(ValueError, match="must be a number"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn="invalid" + ) + + +def test_threshold_warn_passes_without_warning(testdata_dir): + """Test that images within threshold_warn pass without warning.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_left_copy.png") + + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn=0.05 + ) + + +def test_threshold_warn_in_warning_zone_without_keyword(testdata_dir, capsys): + """Test that differences in warning zone pass with warnings logged.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "map_orig.png") + cand_image = str(testdata_dir / "map_3_diff.png") + + visual_tester.compare_images( + ref_image, cand_image, threshold=0.002, threshold_warn=0.001 + ) + + captured = capsys.readouterr() + assert "Visual differences detected in warning zone" in captured.out + assert "Warning threshold:" in captured.out + assert "passed with warnings" in captured.out + + +def test_threshold_warn_in_warning_zone_with_keyword(testdata_dir, capsys, monkeypatch): + """Test that differences in warning zone execute the warning keyword.""" + from unittest.mock import MagicMock + from robot.libraries.BuiltIn import BuiltIn + + mock_builtin = MagicMock() + mock_run_keyword = MagicMock() + mock_builtin.run_keyword = mock_run_keyword + + def mock_builtin_init(): + return mock_builtin + + monkeypatch.setattr("DocTest.VisualTest.BuiltIn", mock_builtin_init) + + visual_tester = VisualTest(run_keyword_on_warn_threshold="Log") + ref_image = str(testdata_dir / "map_orig.png") + cand_image = str(testdata_dir / "map_3_diff.png") + + visual_tester.compare_images( + ref_image, cand_image, threshold=0.002, threshold_warn=0.001 + ) + + assert mock_run_keyword.called + call_args = mock_run_keyword.call_args + # Verify keyword name is split correctly (first part) + assert call_args[0][0] == "Log" + # Verify all arguments are passed (3 required: score, threshold_warn, threshold) + assert len(call_args[0]) == 4 + score = call_args[0][1] + warn_threshold = call_args[0][2] + threshold = call_args[0][3] + + assert isinstance(score, float) + assert 0.0017 < score < 0.0020 + + captured = capsys.readouterr() + assert "passed with warnings" in captured.out + assert "Executed warning keyword: Log" in captured.out + + +def test_threshold_warn_keyword_with_arguments(testdata_dir, capsys, monkeypatch): + """Test that keyword with pre-defined arguments splits correctly.""" + from unittest.mock import MagicMock + from robot.libraries.BuiltIn import BuiltIn + + mock_builtin = MagicMock() + mock_run_keyword = MagicMock() + mock_builtin.run_keyword = mock_run_keyword + + def mock_builtin_init(): + return mock_builtin + + monkeypatch.setattr("DocTest.VisualTest.BuiltIn", mock_builtin_init) + + # Use a keyword with pre-defined arguments: "Log WARN level=WARN" + visual_tester = VisualTest(run_keyword_on_warn_threshold="Log WARN level=WARN") + ref_image = str(testdata_dir / "map_orig.png") + cand_image = str(testdata_dir / "map_3_diff.png") + + visual_tester.compare_images( + ref_image, cand_image, threshold=0.002, threshold_warn=0.001 + ) + + assert mock_run_keyword.called + call_args = mock_run_keyword.call_args + # Verify keyword name is split correctly (first part) + assert call_args[0][0] == "Log" + # Verify all arguments: 2 pre-defined + 3 required = 5 total + assert len(call_args[0]) == 6 + # Pre-defined arguments come first + assert call_args[0][1] == "WARN" + assert call_args[0][2] == "level=WARN" + # Then the three required parameters + score = call_args[0][3] + warn_threshold = call_args[0][4] + threshold = call_args[0][5] + + assert isinstance(score, float) + assert 0.0017 < score < 0.0020 + + captured = capsys.readouterr() + assert "passed with warnings" in captured.out + assert "Executed warning keyword: Log WARN level=WARN" in captured.out + + +def test_threshold_warn_keyword_execution_failure(testdata_dir, capsys, monkeypatch): + """Test that keyword execution failure is handled gracefully.""" + from unittest.mock import MagicMock + from robot.libraries.BuiltIn import BuiltIn + + mock_builtin = MagicMock() + mock_builtin.run_keyword.side_effect = Exception("Keyword execution failed") + + def mock_builtin_init(): + return mock_builtin + + monkeypatch.setattr("DocTest.VisualTest.BuiltIn", mock_builtin_init) + + visual_tester = VisualTest(run_keyword_on_warn_threshold="Failing Keyword") + ref_image = str(testdata_dir / "map_orig.png") + cand_image = str(testdata_dir / "map_3_diff.png") + + visual_tester.compare_images( + ref_image, cand_image, threshold=0.002, threshold_warn=0.001 + ) + + captured = capsys.readouterr() + assert "Failed to execute keyword" in captured.out + assert "passed with warnings" in captured.out + + +def test_threshold_warn_actual_failure(testdata_dir): + """Test that differences exceeding threshold still fail.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "map_orig.png") + cand_image = str(testdata_dir / "map_3_diff.png") + + with pytest.raises(AssertionError, match="The compared images are different"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.001, threshold_warn=0.0005 + ) + + +def test_threshold_warn_none_backward_compatibility(testdata_dir): + """Test that threshold_warn=None maintains backward compatibility.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_1080.png") + cand_image = str(testdata_dir / "birthday_1080_noise_001.png") + + with pytest.raises(AssertionError, match="The compared images are different"): + visual_tester.compare_images( + ref_image, cand_image, threshold=0.01, threshold_warn=None + ) + + +def test_threshold_warn_valid_range(testdata_dir): + """Test that valid threshold_warn values are accepted.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_left.png") + cand_image = str(testdata_dir / "birthday_left_copy.png") + + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn=0.05 + ) + visual_tester.compare_images( + ref_image, cand_image, threshold=0.1, threshold_warn=0.0 + ) + visual_tester.compare_images( + ref_image, cand_image, threshold=1.0, threshold_warn=0.99 + ) + + +def test_threshold_warn_without_score(testdata_dir): + """Test that differences without score (e.g., dimension mismatch) are treated as failures.""" + visual_tester = VisualTest() + ref_image = str(testdata_dir / "birthday_1080.png") + cand_image = str(testdata_dir / "birthday_partial_banana.png") + + with pytest.raises(AssertionError): + visual_tester.compare_images( + ref_image, + cand_image, + threshold=0.9, + threshold_warn=0.5, + resize_candidate=False, + ) diff --git a/utest/testdata/map_1_diff.png b/utest/testdata/map_1_diff.png new file mode 100644 index 0000000..f0ab775 Binary files /dev/null and b/utest/testdata/map_1_diff.png differ diff --git a/utest/testdata/map_29pct_diff.png b/utest/testdata/map_29pct_diff.png new file mode 100644 index 0000000..0bc47c8 Binary files /dev/null and b/utest/testdata/map_29pct_diff.png differ diff --git a/utest/testdata/map_2_diff.png b/utest/testdata/map_2_diff.png new file mode 100644 index 0000000..b2d0a13 Binary files /dev/null and b/utest/testdata/map_2_diff.png differ diff --git a/utest/testdata/map_3_diff.png b/utest/testdata/map_3_diff.png new file mode 100644 index 0000000..feb879d Binary files /dev/null and b/utest/testdata/map_3_diff.png differ diff --git a/utest/testdata/map_orig.png b/utest/testdata/map_orig.png new file mode 100644 index 0000000..2f35794 Binary files /dev/null and b/utest/testdata/map_orig.png differ