Added acceptable pixel tolerances feature for testing.

openeo/testing/results.py

@@ -20,11 +20,13 @@
 
 _DEFAULT_RTOL = 1e-6
 _DEFAULT_ATOL = 1e-6
+_DEFAULT_PXTOL = 0.0
 
 # https://paulbourke.net/dataformats/asciiart
 DEFAULT_GRAYSCALE_70_CHARACTERS = "$@B%8&WM#*oahkbdpqwmZO0QLCJUYXzcvunxrjft/\|()1{}[]?-_+~<>i!lI;:,\"^`'. "[::-1]
 DEFAULT_GRAYSCALE_10_CHARACTERS = " .:-=+*#%@"
 
+
 def _load_xarray_netcdf(path: Union[str, Path], **kwargs) -> xarray.Dataset:
     """
     Load a netCDF file as Xarray Dataset
@@ -194,10 +196,11 @@ def _compare_xarray_dataarray(
     *,
     rtol: float = _DEFAULT_RTOL,
     atol: float = _DEFAULT_ATOL,
+    pxtol: Optional[float] = _DEFAULT_PXTOL,
     name: str = None,
 ) -> List[str]:
     """
-    Compare two xarray DataArrays with tolerance and report mismatch issues (as strings)
+    Compare two xarray DataArrays either with tolerance or with allowable pixels and report mismatch issues (as strings)
 
     Checks that are done (with tolerance):
     - (optional) Check fraction of mismatching pixels (difference exceeding some tolerance).
@@ -230,7 +233,12 @@ def _compare_xarray_dataarray(
         issues.append(f"Shape mismatch: {actual.shape} != {expected.shape}")
     compatible = len(issues) == 0
     try:
-        xarray.testing.assert_allclose(a=actual, b=expected, rtol=rtol, atol=atol)
+        if pxtol > _DEFAULT_PXTOL:
+            assert (
+                actual != expected
+            ).mean().item() <= pxtol, "Percentage number of pixels that are different is above the threshold"
+        else:
+            xarray.testing.assert_allclose(a=actual, b=expected, rtol=rtol, atol=atol)
     except AssertionError as e:
         # TODO: message of `assert_allclose` is typically multiline, split it again or make it one line?
         issues.append(str(e).strip())
@@ -266,12 +274,14 @@ def assert_xarray_dataarray_allclose(
     if issues:
         raise AssertionError("\n".join(issues))
 
+
 def _compare_xarray_datasets(
     actual: Union[xarray.Dataset, str, Path],
     expected: Union[xarray.Dataset, str, Path],
     *,
     rtol: float = _DEFAULT_RTOL,
     atol: float = _DEFAULT_ATOL,
+    pxtol: Optional[float] = _DEFAULT_PXTOL,
 ) -> List[str]:
     """
     Compare two xarray ``DataSet``s with tolerance and report mismatch issues (as strings)
@@ -290,7 +300,7 @@ def _compare_xarray_datasets(
         all_issues.append(f"Xarray DataSet variables mismatch: {actual_vars} != {expected_vars}")
     for var in expected_vars.intersection(actual_vars):
         _log.debug(f"_compare_xarray_datasets: comparing variable {var!r}")
-        issues = _compare_xarray_dataarray(actual[var], expected[var], rtol=rtol, atol=atol, name=var)
+        issues = _compare_xarray_dataarray(actual[var], expected[var], rtol=rtol, atol=atol, pxtol=pxtol, name=var)
         if issues:
             all_issues.append(f"Issues for variable {var!r}:")
             all_issues.extend(issues)
@@ -387,6 +397,7 @@ def _compare_job_results(
     *,
     rtol: float = _DEFAULT_RTOL,
     atol: float = _DEFAULT_ATOL,
+    pxtol: Optional[float] = _DEFAULT_PXTOL,
     tmp_path: Optional[Path] = None,
 ) -> List[str]:
     """
@@ -415,12 +426,16 @@ def _compare_job_results(
                 all_issues.append(f"Issues for metadata file {filename!r}:")
                 all_issues.extend(issues)
         elif expected_path.suffix.lower() in {".nc", ".netcdf"}:
-            issues = _compare_xarray_datasets(actual=actual_path, expected=expected_path, rtol=rtol, atol=atol)
+            issues = _compare_xarray_datasets(
+                actual=actual_path, expected=expected_path, rtol=rtol, atol=atol, pxtol=pxtol
+            )
             if issues:
                 all_issues.append(f"Issues for file {filename!r}:")
                 all_issues.extend(issues)
         elif expected_path.suffix.lower() in {".tif", ".tiff", ".gtiff", ".geotiff"}:
-            issues = _compare_xarray_dataarray(actual=actual_path, expected=expected_path, rtol=rtol, atol=atol)
+            issues = _compare_xarray_dataarray(
+                actual=actual_path, expected=expected_path, rtol=rtol, atol=atol, pxtol=pxtol
+            )
             if issues:
                 all_issues.append(f"Issues for file {filename!r}:")
                 all_issues.extend(issues)
@@ -463,6 +478,7 @@ def assert_job_results_allclose(
     *,
     rtol: float = _DEFAULT_RTOL,
     atol: float = _DEFAULT_ATOL,
+    pxtol: Optional[float] = _DEFAULT_PXTOL,
     tmp_path: Optional[Path] = None,
 ):
     """
@@ -474,6 +490,7 @@ def assert_job_results_allclose(
         :py:meth:`~openeo.rest.job.JobResults` object or path to directory with downloaded assets.
     :param rtol: relative tolerance
     :param atol: absolute tolerance
+    :param pxtol: allowable tolerance for the number of pixels in percentage
     :param tmp_path: root temp path to download results if needed.
         It's recommended to pass pytest's `tmp_path` fixture here
     :raises AssertionError: if not equal within the given tolerance
@@ -483,6 +500,6 @@ def assert_job_results_allclose(
     .. warning::
         This function is experimental and subject to change.
     """
-    issues = _compare_job_results(actual, expected, rtol=rtol, atol=atol, tmp_path=tmp_path)
+    issues = _compare_job_results(actual, expected, rtol=rtol, atol=atol, pxtol=pxtol, tmp_path=tmp_path)
     if issues:
         raise AssertionError("\n".join(issues))