python(feat): Add report_outcome(result, reason) functionality to test step context managers.

Author: ian-sift

python/lib/sift_client/_tests/util/test_test_results_utils.py

@@ -152,6 +152,19 @@ def test_measurement_update(self, report_context):
         assert test_step.measurements[2].boolean_value == True
         assert test_step.measurements[2].measurement_type == TestMeasurementType.BOOLEAN
 
+    def test_report_outcome(self, report_context, step):
+        # Capture current state of report context's failures so we can keep things passed at a high level if the test's induced failures happen as expected.
+        current_step_path = step.current_step.step_path
+        initial_open_step_result = report_context.open_step_results.get(current_step_path, True)
+        initial_any_failures = report_context.any_failures
+        assert step.report_outcome("Test Pass Outcome", True, "Test Pass Description") == True
+        assert step.report_outcome("Test Fail Outcome", False, "Test Failure Description") == False
+        # If this test was successful, mark that at a high level.
+        if initial_open_step_result:
+            report_context.open_step_results[current_step_path] = True
+        if not initial_any_failures:
+            report_context.any_failures = False
+
     def test_bad_assert(self, report_context, step):
         # Capture current state of report context's failures so we can keep things passed at a high level if the test's induced failures happen as expected.
         current_step_path = step.current_step.step_path

python/lib/sift_client/util/test_results/context_manager.py

@@ -138,10 +138,10 @@ def create_step(self, name: str, description: str | None = None) -> TestStep:
 
         return step
 
-    def report_measurement(self, measurement: TestMeasurement, step: TestStep):
+    def report_outcome(self, outcome: bool, step: TestStep):
         """Report a failure to the report context."""
         # Failures will be propogated when the step exits.
-        if not measurement.passed:
+        if not outcome:
             self.open_step_results[step.step_path] = False
             self.any_failures = True
 
@@ -303,10 +303,24 @@ def measure(
         )
         evaluate_measurement_bounds(create, value, bounds)
         measurement = self.client.test_results.create_measurement(create)
-        self.report_context.report_measurement(measurement, self.current_step)
+        self.report_context.report_outcome(measurement.passed, self.current_step)
 
         return measurement.passed
 
+    def report_outcome(self, name: str, result: bool, reason: str | None = None) -> bool:
+        """Report an outcome from some action or measurement. Creates a substep that is pass/fail with the optional reason as the description.
+
+        Args:
+            name: The name of the substep.
+            result: True if the action or measurement passed, False otherwise.
+            reason: [Optional] The context to include in the description of the substep.
+
+        returns: The given result so the function can be used in line.
+        """
+        with self.substep(name=name, description=reason) as substep:
+            self.report_context.report_outcome(result, substep.current_step)
+        return result
+
     def substep(self, name: str, description: str | None = None) -> NewStep:
         """Alias to return a new step context manager from the current step. The ReportContext will manage nesting of steps."""
         return self.report_context.new_step(name=name, description=description)