From d8741602f22d6e9713ebd9269141dbad3c819aa3 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 23 Jun 2023 17:32:21 +0100 Subject: [PATCH 1/5] Specify disable_auto in test suite definition --- nmostesting/NMOSTesting.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index d5340423..9d04dbee 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -221,6 +221,7 @@ "api_key": "query", "disable_fields": ["host", "port"] }], + "disable_auto": True, "class": IS0404Test.IS0404Test }, "IS-05-01": { @@ -256,6 +257,7 @@ "api_key": "connection", "disable_fields": ["host", "port"] }], + "disable_auto": True, "class": IS0503Test.IS0503Test }, "IS-06-01": { @@ -329,6 +331,7 @@ "api_key": "system", "disable_fields": ["host", "port"] }], + "disable_auto": True, "class": IS0902Test.IS0902Test }, # IS-10 testing is disabled until testing can be refactored to deal with commercial servers @@ -346,6 +349,7 @@ "spec_key": "bcp-003-01", "api_key": "secure" }], + "disable_auto": True, "class": BCP00301Test.BCP00301Test }, "BCP-006-01-01": { @@ -382,12 +386,12 @@ } -def enumerate_tests(class_def, describe=False): - if describe: - tests = ["all: Runs all tests in the suite", - "auto: Basic API tests derived directly from the specification RAML"] - else: - tests = ["all", "auto"] +def enumerate_tests(class_def, describe=False, disable_auto=False): + + tests = ["all: Runs all tests in the suite" if describe else "all"] + if not disable_auto: + tests.append("auto: Basic API tests derived directly from the specification RAML" if describe else "auto") + for method_name in dir(class_def): if method_name.startswith("test_"): method = getattr(class_def, method_name) @@ -453,8 +457,12 @@ class DataForm(Form): for test_id in TEST_DEFINITIONS: test_data[test_id] = copy.deepcopy(TEST_DEFINITIONS[test_id]) test_data[test_id].pop("class") - test_data[test_id]["test_methods"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"]) - test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True) + test_data[test_id]["test_methods"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], + disable_auto=TEST_DEFINITIONS[test_id] + .get("disable_auto")) + test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True, + disable_auto=TEST_DEFINITIONS[test_id] + .get("disable_auto")) hidden_options = HiddenField(default=max_endpoints) hidden_tests = HiddenField(default=json.dumps(test_data)) @@ -617,7 +625,8 @@ def run_tests(test, endpoints, test_selection=["all"]): registries=REGISTRIES, node=NODE, dns_server=DNS_SERVER, - auths=[PRIMARY_AUTH, SECONDARY_AUTH]) + auths=[PRIMARY_AUTH, SECONDARY_AUTH], + disable_auto=test_def.get("disable_auto")) core_app.config['TEST_ACTIVE'] = time.time() try: @@ -839,11 +848,14 @@ def validate_args(args, access_type="cli"): msg = "ERROR: The requested test suite '{}' does not exist".format(args.suite) return_type = ExitCodes.ERROR elif args.list_tests: - tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"]) + tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], + disable_auto=TEST_DEFINITIONS[args.suite].get("disable_auto")) for test_name in tests: msg += test_name + '\n' elif args.describe_tests: - tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], describe=True) + tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], + disable_auto=TEST_DEFINITIONS[args.suite].get("disable_auto"), + describe=True) for test_description in tests: msg += test_description + '\n' elif getattr(args, "selection", "all") not in enumerate_tests(TEST_DEFINITIONS[args.suite]["class"]): From e2fe61389ad584d5ef0d017973aba6627c530b92 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 23 Jun 2023 17:32:45 +0100 Subject: [PATCH 2/5] Pass through arguments to test suites --- nmostesting/ControllerTest.py | 4 ++-- nmostesting/suites/BCP00301Test.py | 2 +- nmostesting/suites/IS0404Test.py | 4 ++-- nmostesting/suites/IS0503Test.py | 4 ++-- nmostesting/suites/IS0902Test.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nmostesting/ControllerTest.py b/nmostesting/ControllerTest.py index 0c7b4bc1..b479b067 100644 --- a/nmostesting/ControllerTest.py +++ b/nmostesting/ControllerTest.py @@ -76,7 +76,7 @@ class ControllerTest(GenericTest): """ Testing initial set up of new test suite for controller testing """ - def __init__(self, apis, registries, node, dns_server, disable_auto=True): + def __init__(self, apis, registries, node, dns_server, **kwargs): # Remove the spec_path as there are no corresponding GitHub repos for Controller Tests apis[CONTROLLER_TEST_API_KEY].pop("spec_path", None) if CONFIG.ENABLE_HTTPS: @@ -84,7 +84,7 @@ def __init__(self, apis, registries, node, dns_server, disable_auto=True): apis[CONTROLLER_TEST_API_KEY]["base_url"] \ = apis[CONTROLLER_TEST_API_KEY]["base_url"].replace("https", "http") apis[CONTROLLER_TEST_API_KEY]["url"] = apis[CONTROLLER_TEST_API_KEY]["url"].replace("https", "http") - GenericTest.__init__(self, apis, disable_auto=disable_auto) + GenericTest.__init__(self, apis, **kwargs) self.authorization = False self.primary_registry = registries[1] self.node = node diff --git a/nmostesting/suites/BCP00301Test.py b/nmostesting/suites/BCP00301Test.py index 08fa1da9..596c328b 100644 --- a/nmostesting/suites/BCP00301Test.py +++ b/nmostesting/suites/BCP00301Test.py @@ -31,7 +31,7 @@ class BCP00301Test(GenericTest): Runs BCP-003-01-Test """ def __init__(self, apis, **kwargs): - GenericTest.__init__(self, apis) + GenericTest.__init__(self, apis, **kwargs) if not CONFIG.ENABLE_HTTPS: raise NMOSInitException("BCP-003-01 can only be tested when ENABLE_HTTPS is set to True in UserConfig.py") self.authorization = False # Don't send tokens in every request diff --git a/nmostesting/suites/IS0404Test.py b/nmostesting/suites/IS0404Test.py index db6494f3..75a8b9e6 100644 --- a/nmostesting/suites/IS0404Test.py +++ b/nmostesting/suites/IS0404Test.py @@ -29,8 +29,8 @@ class IS0404Test(ControllerTest): """ Testing initial set up of new test suite for controller testing """ - def __init__(self, apis, registries, node, dns_server, **kwargs): - ControllerTest.__init__(self, apis, registries, node, dns_server) + def __init__(self, apis, **kwargs): + ControllerTest.__init__(self, apis, **kwargs) def set_up_tests(self): # Sender initial details diff --git a/nmostesting/suites/IS0503Test.py b/nmostesting/suites/IS0503Test.py index 454de7f9..77359055 100644 --- a/nmostesting/suites/IS0503Test.py +++ b/nmostesting/suites/IS0503Test.py @@ -28,8 +28,8 @@ class IS0503Test(ControllerTest): """ Testing initial set up of new test suite for controller testing """ - def __init__(self, apis, registries, node, dns_server, **kwargs): - ControllerTest.__init__(self, apis, registries, node, dns_server) + def __init__(self, apis, **kwargs): + ControllerTest.__init__(self, apis, **kwargs) def set_up_tests(self): # Sender initial details diff --git a/nmostesting/suites/IS0902Test.py b/nmostesting/suites/IS0902Test.py index 40a984d8..49252e40 100644 --- a/nmostesting/suites/IS0902Test.py +++ b/nmostesting/suites/IS0902Test.py @@ -30,7 +30,7 @@ class IS0902Test(GenericTest): Runs IS-09-02-Test """ def __init__(self, apis, systems, dns_server, **kwargs): - GenericTest.__init__(self, apis, disable_auto=True) + GenericTest.__init__(self, apis, **kwargs) self.authorization = False # System API doesn't use auth, so don't send tokens in every request self.invalid_system = systems[0] self.primary_system = systems[1] From 7b1b2a71dd900b525c8d8c9aacd26827d28198e0 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 23 Jun 2023 17:41:57 +0100 Subject: [PATCH 3/5] flake8 FTW! --- nmostesting/NMOSTesting.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index 9d04dbee..fcae1c8a 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -387,7 +387,6 @@ def enumerate_tests(class_def, describe=False, disable_auto=False): - tests = ["all: Runs all tests in the suite" if describe else "all"] if not disable_auto: tests.append("auto: Basic API tests derived directly from the specification RAML" if describe else "auto") @@ -461,8 +460,8 @@ class DataForm(Form): disable_auto=TEST_DEFINITIONS[test_id] .get("disable_auto")) test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True, - disable_auto=TEST_DEFINITIONS[test_id] - .get("disable_auto")) + disable_auto=TEST_DEFINITIONS[test_id] + .get("disable_auto")) hidden_options = HiddenField(default=max_endpoints) hidden_tests = HiddenField(default=json.dumps(test_data)) From a0be6898f256da925e7b6876d8290509f0905f7a Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 24 Jul 2023 16:21:23 +0100 Subject: [PATCH 4/5] Don't pull repo or parse RAML if disable_auto --- nmostesting/GenericTest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nmostesting/GenericTest.py b/nmostesting/GenericTest.py index 951eb1fa..c18865ff 100644 --- a/nmostesting/GenericTest.py +++ b/nmostesting/GenericTest.py @@ -86,7 +86,7 @@ def __init__(self, apis, omit_paths=None, disable_auto=False, auths=None, **kwar test = Test("Test initialisation") for api_name, api_data in self.apis.items(): - if "spec_path" not in api_data or api_data["version"] is None: + if self.disable_auto or "spec_path" not in api_data or api_data["version"] is None: continue repo = git.Repo(api_data["spec_path"]) @@ -122,7 +122,7 @@ def __init__(self, apis, omit_paths=None, disable_auto=False, auths=None, **kwar def parse_RAML(self): """Create a Specification object for each API defined in this object""" for api in self.apis: - if "raml" not in self.apis[api]: + if self.disable_auto or "raml" not in self.apis[api]: continue raml_path = os.path.join(self.apis[api]["spec_path"] + '/APIs/' + self.apis[api]["raml"]) self.apis[api]["spec"] = Specification(raml_path) From 06848ccf0e6a7aa21492fb95adee1c1d8b2a905a Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 24 Jul 2023 16:22:10 +0100 Subject: [PATCH 5/5] Disable_auto now implemented in GenericTest --- nmostesting/ControllerTest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nmostesting/ControllerTest.py b/nmostesting/ControllerTest.py index b479b067..a5739e12 100644 --- a/nmostesting/ControllerTest.py +++ b/nmostesting/ControllerTest.py @@ -77,8 +77,6 @@ class ControllerTest(GenericTest): Testing initial set up of new test suite for controller testing """ def __init__(self, apis, registries, node, dns_server, **kwargs): - # Remove the spec_path as there are no corresponding GitHub repos for Controller Tests - apis[CONTROLLER_TEST_API_KEY].pop("spec_path", None) if CONFIG.ENABLE_HTTPS: # Comms with Testing Facade are http only apis[CONTROLLER_TEST_API_KEY]["base_url"] \