Validate result dicts produced by handlers

Author: Tattoo

src/oxygen/base_handler.py

@@ -2,10 +2,10 @@
 
 from inspect import signature, Parameter
 
-from oxygen.errors import MismatchArgumentException
+from .errors import MismatchArgumentException
 from .robot_interface import (RobotInterface, get_keywords_from,
                               set_special_keyword)
-
+from .utils import validate_with_deprecation_warning
 
 class BaseHandler(object):
     DEFAULT_CLI = {tuple(['result_file']): {}}
@@ -132,6 +132,8 @@ def _build_results(self, keyword, setup_keyword, teardown_keyword):
                 f'parse_results expects at least {accepted_params_min} '
                 'arguments but got 1')
 
+        self._validate(test_results)
+
         _, result_suite = self._interface.result.build_suite(
             100000, test_results)
 
@@ -149,6 +151,9 @@ def _build_results(self, keyword, setup_keyword, teardown_keyword):
 
         self._inject_suite_report(test, result_suite)
 
+    def _validate(self, oxygen_result_dict):
+        validate_with_deprecation_warning(oxygen_result_dict, self)
+
     def _inject_suite_report(self, test, result_suite):
         '''Add the given suite to the parent suite of the test case.
 

src/oxygen/oxygen.py

@@ -18,6 +18,7 @@
                      InvalidConfigurationException,
                      ResultFileNotFoundException)
 from .robot_interface import RobotInterface
+from .utils import validate_with_deprecation_warning
 from .version import VERSION
 
 
@@ -301,6 +302,7 @@ def convert_to_robot_result(self, args):
         output_filename = self.get_output_filename(args.get('result_file'))
         parsed_results = args['func'](
             **{k: v for (k, v) in args.items() if not callable(v)})
+        validate_with_deprecation_warning(parsed_results, args['func'])
         robot_suite = RobotInterface().running.build_suite(parsed_results)
         robot_suite.run(output=output_filename,
                         log=None,

src/oxygen/utils.py

@@ -4,10 +4,11 @@
 
 from pathlib import Path
 
-from .errors import (SubprocessException,
+from .errors import (InvalidOxygenResultException,
                      ResultFileIsNotAFileException,
-                     ResultFileNotFoundException)
-
+                     ResultFileNotFoundException,
+                     SubprocessException)
+from .oxygen_handler_result import validate_oxygen_suite
 
 def run_command_line(command, check_return_code=True, **env):
     new_env = os.environ.copy()
@@ -27,7 +28,6 @@ def run_command_line(command, check_return_code=True, **env):
                                   f'code {proc.returncode}:\n"{proc.stdout.decode("utf-8")}"')
     return proc.stdout
 
-
 def validate_path(filepath):
     try:
         path = Path(filepath)
@@ -39,3 +39,17 @@ def validate_path(filepath):
         raise ResultFileIsNotAFileException(f'File "{path}" is not a file, '
                                             'but a directory')
     return path
+
+def validate_with_deprecation_warning(oxygen_result_dict, handler):
+    try:
+        validate_oxygen_suite(oxygen_result_dict)
+    except InvalidOxygenResultException as e:
+        import warnings
+        # this is not done with triple quotes intentionally
+        # to get sensible formatting to output
+        msg = (f'\n{handler.__module__} is producing invalid results:\n'
+               f'{e}\n\n'
+               'In Oxygen 1.0, handlers will need to produce valid '
+               'results.\nSee: '
+               'https://github.com/eficode/robotframework-oxygen/blob/master/parser_specification.md')
+        warnings.warn(msg)

tests/utest/helpers.py

@@ -59,6 +59,11 @@ def example_robot_output():
 
 MINIMAL_KEYWORD_DICT = { 'name': 'someKeyword', 'pass': True }
 MINIMAL_TC_DICT = { 'name': 'Minimal TC', 'keywords': [MINIMAL_KEYWORD_DICT] }
+MINIMAL_SUITE_DICT = {'name': 'Minimal Suite',
+                      'suites': [{
+                          'name': 'Minimal Subsuite',
+                          'tests': [ MINIMAL_TC_DICT ]}]}
+
 
 class _ListSubclass(list):
     '''Used in test cases'''

tests/utest/oxygen_handler_result/test_OxygenSuiteDict.py

@@ -3,7 +3,10 @@
 from oxygen.errors import InvalidOxygenResultException
 from oxygen.oxygen_handler_result import OxygenSuiteDict, validate_oxygen_suite
 
-from ..helpers import MINIMAL_TC_DICT, _ListSubclass, _TCSubclass
+from ..helpers import (MINIMAL_TC_DICT,
+                       MINIMAL_SUITE_DICT,
+                       _ListSubclass,
+                       _TCSubclass)
 from .shared_tests import (SharedTestsForName,
                            SharedTestsForKeywordField,
                            SharedTestsForTags)
@@ -13,13 +16,7 @@ class TestOxygenSuiteDict(TestCase,
                           SharedTestsForKeywordField,
                           SharedTestsForTags):
     def setUp(self):
-        self.minimal = {
-            'name': 'Minimal Suite',
-            'suites': [{
-                'name': 'Minimal Subsuite',
-                'tests': [ MINIMAL_TC_DICT ]
-            }]
-        }
+        self.minimal = MINIMAL_SUITE_DICT
 
     def test_validate_oxygen_suite_validates_correctly(self):
         with self.assertRaises(InvalidOxygenResultException):

tests/utest/oxygen_handler_result/test_deprecation_warning.py

@@ -0,0 +1,47 @@
+'''
+Currently, the introduction of pydantic[1] to validate result dictionaries that
+handlers return is planned to just raise a deprecation warning.
+
+After 1.0 -- ie. a backwards-incompatible release -- we should turn deprecation
+warning to actually start failing. See issue [2].
+
+[1| https://github.com/eficode/robotframework-oxygen/issues/43
+[2] https://github.com/eficode/robotframework-oxygen/issues/45
+'''
+from unittest import TestCase
+from unittest.mock import patch
+from oxygen.base_handler import BaseHandler
+from oxygen.oxygen import OxygenCLI
+
+from ..helpers import get_config, MINIMAL_SUITE_DICT
+
+class TestDeprecationWarningWhenValidating(TestCase):
+    def setUp(self):
+        self.cli = OxygenCLI()
+
+    def test_warning_about_invalid_result(self):
+        handler = BaseHandler(get_config()['oxygen.junit'])
+
+        with self.assertWarns(UserWarning) as warning:
+            handler._validate({})
+
+        warning_message = str(warning.warning)
+        self.assertIn('oxygen.base_handler', warning_message)
+        self.assertIn('validation error for typed-dict', warning_message)
+        self.assertIn('In Oxygen 1.0, handlers will'
+                      ' need to produce valid results.', warning_message)
+
+    @patch('oxygen.oxygen.RobotInterface')
+    def test_warning_about_invalid_result_in_CLI(self, mock_iface):
+        with self.assertWarns(UserWarning) as warning:
+            self.cli.convert_to_robot_result({
+                'result_file': 'doesentmatter',
+                'func': lambda **_: {**MINIMAL_SUITE_DICT, 'setup': []}
+            })
+
+            mock_iface.assert_any_call()
+
+    def test_deprecation_was_removed(self):
+        '''Remove this test once deprecation warning has been removed'''
+        if self.cli.__version__.startswith('1'):
+            self.fail('Deprecation warning should have been removed in 1.0')