diff --git a/nmostesting/ControllerTest.py b/nmostesting/ControllerTest.py index 0c7b4bc1..a5739e12 100644 --- a/nmostesting/ControllerTest.py +++ b/nmostesting/ControllerTest.py @@ -76,15 +76,13 @@ class ControllerTest(GenericTest): """ Testing initial set up of new test suite for controller testing """ - def __init__(self, apis, registries, node, dns_server, disable_auto=True): - # Remove the spec_path as there are no corresponding GitHub repos for Controller Tests - apis[CONTROLLER_TEST_API_KEY].pop("spec_path", None) + def __init__(self, apis, registries, node, dns_server, **kwargs): if CONFIG.ENABLE_HTTPS: # Comms with Testing Facade are http only apis[CONTROLLER_TEST_API_KEY]["base_url"] \ = apis[CONTROLLER_TEST_API_KEY]["base_url"].replace("https", "http") apis[CONTROLLER_TEST_API_KEY]["url"] = apis[CONTROLLER_TEST_API_KEY]["url"].replace("https", "http") - GenericTest.__init__(self, apis, disable_auto=disable_auto) + GenericTest.__init__(self, apis, **kwargs) self.authorization = False self.primary_registry = registries[1] self.node = node diff --git a/nmostesting/GenericTest.py b/nmostesting/GenericTest.py index 951eb1fa..c18865ff 100644 --- a/nmostesting/GenericTest.py +++ b/nmostesting/GenericTest.py @@ -86,7 +86,7 @@ def __init__(self, apis, omit_paths=None, disable_auto=False, auths=None, **kwar test = Test("Test initialisation") for api_name, api_data in self.apis.items(): - if "spec_path" not in api_data or api_data["version"] is None: + if self.disable_auto or "spec_path" not in api_data or api_data["version"] is None: continue repo = git.Repo(api_data["spec_path"]) @@ -122,7 +122,7 @@ def __init__(self, apis, omit_paths=None, disable_auto=False, auths=None, **kwar def parse_RAML(self): """Create a Specification object for each API defined in this object""" for api in self.apis: - if "raml" not in self.apis[api]: + if self.disable_auto or "raml" not in self.apis[api]: continue raml_path = os.path.join(self.apis[api]["spec_path"] + '/APIs/' + self.apis[api]["raml"]) self.apis[api]["spec"] = Specification(raml_path) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index d5340423..fcae1c8a 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -221,6 +221,7 @@ "api_key": "query", "disable_fields": ["host", "port"] }], + "disable_auto": True, "class": IS0404Test.IS0404Test }, "IS-05-01": { @@ -256,6 +257,7 @@ "api_key": "connection", "disable_fields": ["host", "port"] }], + "disable_auto": True, "class": IS0503Test.IS0503Test }, "IS-06-01": { @@ -329,6 +331,7 @@ "api_key": "system", "disable_fields": ["host", "port"] }], + "disable_auto": True, "class": IS0902Test.IS0902Test }, # IS-10 testing is disabled until testing can be refactored to deal with commercial servers @@ -346,6 +349,7 @@ "spec_key": "bcp-003-01", "api_key": "secure" }], + "disable_auto": True, "class": BCP00301Test.BCP00301Test }, "BCP-006-01-01": { @@ -382,12 +386,11 @@ } -def enumerate_tests(class_def, describe=False): - if describe: - tests = ["all: Runs all tests in the suite", - "auto: Basic API tests derived directly from the specification RAML"] - else: - tests = ["all", "auto"] +def enumerate_tests(class_def, describe=False, disable_auto=False): + tests = ["all: Runs all tests in the suite" if describe else "all"] + if not disable_auto: + tests.append("auto: Basic API tests derived directly from the specification RAML" if describe else "auto") + for method_name in dir(class_def): if method_name.startswith("test_"): method = getattr(class_def, method_name) @@ -453,8 +456,12 @@ class DataForm(Form): for test_id in TEST_DEFINITIONS: test_data[test_id] = copy.deepcopy(TEST_DEFINITIONS[test_id]) test_data[test_id].pop("class") - test_data[test_id]["test_methods"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"]) - test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True) + test_data[test_id]["test_methods"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], + disable_auto=TEST_DEFINITIONS[test_id] + .get("disable_auto")) + test_data[test_id]["test_descriptions"] = enumerate_tests(TEST_DEFINITIONS[test_id]["class"], describe=True, + disable_auto=TEST_DEFINITIONS[test_id] + .get("disable_auto")) hidden_options = HiddenField(default=max_endpoints) hidden_tests = HiddenField(default=json.dumps(test_data)) @@ -617,7 +624,8 @@ def run_tests(test, endpoints, test_selection=["all"]): registries=REGISTRIES, node=NODE, dns_server=DNS_SERVER, - auths=[PRIMARY_AUTH, SECONDARY_AUTH]) + auths=[PRIMARY_AUTH, SECONDARY_AUTH], + disable_auto=test_def.get("disable_auto")) core_app.config['TEST_ACTIVE'] = time.time() try: @@ -839,11 +847,14 @@ def validate_args(args, access_type="cli"): msg = "ERROR: The requested test suite '{}' does not exist".format(args.suite) return_type = ExitCodes.ERROR elif args.list_tests: - tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"]) + tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], + disable_auto=TEST_DEFINITIONS[args.suite].get("disable_auto")) for test_name in tests: msg += test_name + '\n' elif args.describe_tests: - tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], describe=True) + tests = enumerate_tests(TEST_DEFINITIONS[args.suite]["class"], + disable_auto=TEST_DEFINITIONS[args.suite].get("disable_auto"), + describe=True) for test_description in tests: msg += test_description + '\n' elif getattr(args, "selection", "all") not in enumerate_tests(TEST_DEFINITIONS[args.suite]["class"]): diff --git a/nmostesting/suites/BCP00301Test.py b/nmostesting/suites/BCP00301Test.py index 08fa1da9..596c328b 100644 --- a/nmostesting/suites/BCP00301Test.py +++ b/nmostesting/suites/BCP00301Test.py @@ -31,7 +31,7 @@ class BCP00301Test(GenericTest): Runs BCP-003-01-Test """ def __init__(self, apis, **kwargs): - GenericTest.__init__(self, apis) + GenericTest.__init__(self, apis, **kwargs) if not CONFIG.ENABLE_HTTPS: raise NMOSInitException("BCP-003-01 can only be tested when ENABLE_HTTPS is set to True in UserConfig.py") self.authorization = False # Don't send tokens in every request diff --git a/nmostesting/suites/IS0404Test.py b/nmostesting/suites/IS0404Test.py index db6494f3..75a8b9e6 100644 --- a/nmostesting/suites/IS0404Test.py +++ b/nmostesting/suites/IS0404Test.py @@ -29,8 +29,8 @@ class IS0404Test(ControllerTest): """ Testing initial set up of new test suite for controller testing """ - def __init__(self, apis, registries, node, dns_server, **kwargs): - ControllerTest.__init__(self, apis, registries, node, dns_server) + def __init__(self, apis, **kwargs): + ControllerTest.__init__(self, apis, **kwargs) def set_up_tests(self): # Sender initial details diff --git a/nmostesting/suites/IS0503Test.py b/nmostesting/suites/IS0503Test.py index 454de7f9..77359055 100644 --- a/nmostesting/suites/IS0503Test.py +++ b/nmostesting/suites/IS0503Test.py @@ -28,8 +28,8 @@ class IS0503Test(ControllerTest): """ Testing initial set up of new test suite for controller testing """ - def __init__(self, apis, registries, node, dns_server, **kwargs): - ControllerTest.__init__(self, apis, registries, node, dns_server) + def __init__(self, apis, **kwargs): + ControllerTest.__init__(self, apis, **kwargs) def set_up_tests(self): # Sender initial details diff --git a/nmostesting/suites/IS0902Test.py b/nmostesting/suites/IS0902Test.py index 40a984d8..49252e40 100644 --- a/nmostesting/suites/IS0902Test.py +++ b/nmostesting/suites/IS0902Test.py @@ -30,7 +30,7 @@ class IS0902Test(GenericTest): Runs IS-09-02-Test """ def __init__(self, apis, systems, dns_server, **kwargs): - GenericTest.__init__(self, apis, disable_auto=True) + GenericTest.__init__(self, apis, **kwargs) self.authorization = False # System API doesn't use auth, so don't send tokens in every request self.invalid_system = systems[0] self.primary_system = systems[1]