From 26b63f52efb02f15b5d218b044c243d68bba3c69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 15 Aug 2025 12:37:25 +0200 Subject: [PATCH 01/31] record aborted testcases, part 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- .../flavor-naming/flavor-names-openstack.py | 297 ++++++++++-------- 1 file changed, 174 insertions(+), 123 deletions(-) diff --git a/Tests/iaas/flavor-naming/flavor-names-openstack.py b/Tests/iaas/flavor-naming/flavor-names-openstack.py index 7fa921b33..80b589406 100755 --- a/Tests/iaas/flavor-naming/flavor-names-openstack.py +++ b/Tests/iaas/flavor-naming/flavor-names-openstack.py @@ -16,15 +16,19 @@ SPDX-License-Identifier: CC-BY-SA 4.0 """ +import logging import os import sys +import typing import getopt -import yaml import openstack import flavor_names +logger = logging.getLogger(__name__) + + def usage(rcode=1): "help output" print("Usage: flavor-names-openstack.py [options]", file=sys.stderr) @@ -41,11 +45,160 @@ def usage(rcode=1): sys.exit(rcode) +TESTCASES = ('scs-0100-syntax-check', 'scs-0100-semantics-check', 'flavor-name-check') +STRATEGY = flavor_names.ParsingStrategy( + vstr='v3', + parsers=(flavor_names.parser_v3, ), + tolerated_parsers=(flavor_names.parser_v2, flavor_names.parser_v1), +) +ACC_DISK = (0, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000) + + +def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor], parser=STRATEGY) -> list: + result = [] + for flv in flavors: + if not flv.name or flv.name[:4] != 'SCS-': + continue # not an SCS flavor; none of our business + try: + flavorname = parser(flv.name) + except ValueError as exc: + logger.info(f"error parsing {flv.name}: {exc}") + flavorname = None + result.append((flv, flavorname)) + return result + + +def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: + problems = [flv.name for flv, flavorname in scs_flavors if not flavorname] + if problems: + logger.error(f"scs-100-syntax-check: flavor(s) failed: {', '.join(sorted(problems))}") + return not problems + + +def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: + problems = set() + for flv, flavorname in scs_flavors: + if not flavorname: + continue # this case is handled by syntax check + cpuram = flavorname.cpuram + if flv.vcpus < cpuram.cpus: + logger.info(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}") + problems.add(flv.name) + elif flv.vcpus > cpuram.cpus: + logger.info(f"Flavor {flv.name} CPU underpromise: {flv.vcpus} > {cpuram.cpus}") + # RAM + flvram = int((flv.ram + 51) / 102.4) / 10 + # Warn for strange sizes (want integer numbers, half allowed for < 10GiB) + if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2): + logger.info(f"Flavor {flv.name} uses discouraged uneven size of memory {flvram:.1f} GiB") + if flvram < cpuram.ram: + logger.info(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}") + problems.add(flv.name) + elif flvram > cpuram.ram: + logger.info(f"Flavor {flv.name} RAM underpromise {flvram:.1f} > {cpuram.ram:.1f}") + # Disk could have been omitted + disksize = flavorname.disk.disksize if flavorname.disk else 0 + # We have a recommendation for disk size steps + if disksize not in ACC_DISK: + logger.info(f"Flavor {flv.name} non-standard disk size {disksize}, should have (5, 10, 20, 50, 100, 200, ...)") + if flv.disk < disksize: + logger.info(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}") + problems.add(flv.name) + elif flv.disk > disksize: + logger.info(f"Flavor {flv.name} disk underpromise {flv.disk} > {disksize}") + if problems: + logger.error(f"scs-100-semantics-check: flavor(s) failed: {', '.join(sorted(problems))}") + return not problems + + +def compute_flavor_name_check(syntax_check_result, semantics_check_result): + return syntax_check_result and semantics_check_result + + +# TODO see comment in main function about moving to another module + +class Container: + """ + This class does lazy evaluation and memoization. You register any potential value either + by giving the value directly, using `add_value`, or + by specifying how it is computed using `add_function`, + which expects a function that takes a container (so other values may be referred to). + In each case, you have to give the value a name. + + The value will be available as a normal member variable under this name. + If given via a function, this function will only be evaluated when the value is accessed, + and the value will be memoized, so the function won't be called twice. + If the function raises an exception, then this will be memoized just as well. + + For instance, + + >>>> container = Container() + >>>> container.add_function('pi', lambda _: 22/7) + >>>> container.add_function('pi_squared', lambda c: c.pi * c.pi) + >>>> assert c.pi_squared == 22/7 * 22/7 + """ + def __init__(self): + self._values = {} + self._functions = {} + + def __getattr__(self, key): + val = self._values.get(key) + if val is None: + try: + ret = self._functions[key](self) + except BaseException as e: + val = (True, e) + else: + val = (False, ret) + self._values[key] = val + error, ret = val + if error: + raise ret + return ret + + def add_function(self, name, fn): + if name in self._functions: + raise RuntimeError(f"fn {name} already registered") + self._functions[name] = fn + + def add_value(self, name, value): + if name in self._values: + raise RuntimeError(f"value {name} already registered") + self._values[name] = value + + +# TODO see comment in main function about moving to another module + +def harness(name, *check_fns): + """Harness for evaluating testcase `name`. + + Logs beginning of computation. + Calls each fn in `check_fns`. + Prints (to stdout) 'name: RESULT', where RESULT is one of + + - 'ABORT' if an exception occurs during the function calls + - 'FAIL' if one of the functions has a falsy result + - 'PASS' otherwise + """ + logger.debug(f'** {name}') + try: + result = all(check_fn() for check_fn in check_fns) + except BaseException: + logger.debug('exception during check', exc_info=True) + result = 'ABORT' + else: + result = ['FAIL', 'PASS'][min(1, result)] + # this is quite redundant + # logger.debug(f'** computation end for {name}') + print(f"{name}: {result}") + + def main(argv): """Entry point -- main loop going over flavors""" - fnmck = flavor_names.CompatLayer() + # configure logging, disable verbose library logging + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) + openstack.enable_logging(debug=False) cloud = None - verbose = False try: cloud = os.environ["OS_CLOUD"] @@ -70,15 +223,15 @@ def main(argv): # fnmck.disallow_old = True print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) elif opt[0] == "-2" or opt[0] == "--v2plus": - fnmck.disallow_old = True + print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) elif opt[0] == "-1" or opt[0] == "--v1prefer": print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) elif opt[0] == "-o" or opt[0] == "--accept-old-mandatory": print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) elif opt[0] == "-v" or opt[0] == "--verbose": - verbose = True + print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) elif opt[0] == "-q" or opt[0] == "--quiet": - fnmck.quiet = True + print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) else: usage(2) if len(args) > 0: @@ -88,123 +241,21 @@ def main(argv): if not cloud: print("CRITICAL: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr) sys.exit(1) - conn = openstack.connect(cloud=cloud, timeout=32) - flavors = conn.compute.flavors() - - # Lists of flavors: mandatory, good-SCS, bad-SCS, non-SCS, with-warnings - SCSFlv = [] - wrongFlv = [] - nonSCSFlv = [] - warnFlv = [] - errors = 0 - for flv in flavors: - # Skip non-SCS flavors - if flv.name and flv.name[:4] != "SCS-": # and flv.name[:4] != "SCSx" - nonSCSFlv.append(flv.name) - continue - try: - ret = fnmck.parsename(flv.name) - assert ret - # Parser error - except ValueError as exc: - errors += 1 - wrongFlv.append(flv.name) - print(f"ERROR: Wrong flavor \"{flv.name}\": {exc}", file=sys.stderr) - continue - # We have a successfully parsed SCS- name now - # See if the OpenStack provided data fulfills what we - # expect from the flavor based on its name - err = 0 - warn = 0 - # Split list for readability - cpuram = ret.cpuram - # next qwould be hype, hwvirt, cpubrand, gpu, ib - # see flavor-name-check.py: parsename() - # vCPUS - if flv.vcpus < cpuram.cpus: - print(f"ERROR: Flavor {flv.name} has only {flv.vcpus} vCPUs, " - f"should have >= {cpuram.cpus}", file=sys.stderr) - err += 1 - elif flv.vcpus > cpuram.cpus: - print(f"WARNING: Flavor {flv.name} has {flv.vcpus} vCPUs, " - f"only needs {cpuram.cpus}", file=sys.stderr) - warn += 1 - # RAM - flvram = int((flv.ram + 51) / 102.4) / 10 - # Warn for strange sizes (want integer numbers, half allowed for < 10GiB) - if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2): - print(f"WARNING: Flavor {flv.name} uses discouraged uneven size " - f"of memory {flvram:.1f} GiB", file=sys.stderr) - if flvram < cpuram.ram: - print(f"ERROR: Flavor {flv.name} has only {flvram:.1f} GiB RAM, " - f"should have >= {cpuram.ram:.1f} GiB", file=sys.stderr) - err += 1 - elif flvram > cpuram.ram: - print(f"WARNING: Flavor {flv.name} has {flvram:.1f} GiB RAM, " - f"only needs {cpuram.ram:.1f} GiB", file=sys.stderr) - warn += 1 - # DISK - accdisk = (0, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000) - # Disk could have been omitted - disksize = ret.disk.disksize if ret.disk else 0 - # We have a recommendation for disk size steps - if disksize not in accdisk: - print(f"WARNING: Flavor {flv.name} advertizes disk size {disksize}, " - f"should have (5, 10, 20, 50, 100, 200, ...)", file=sys.stderr) - warn += 1 - if flv.disk < disksize: - print(f"ERROR: Flavor {flv.name} has only {flv.disk} GB root disk, " - f"should have >= {disksize} GB", file=sys.stderr) - err += 1 - elif flv.disk > disksize: - print(f"WARNING: Flavor {flv.name} has {flv.disk} GB root disk, " - f"only needs {disksize} GB", file=sys.stderr) - warn += 1 - # Ev'thing checked, react to errors by putting the bad flavors in the bad bucket - if err: - wrongFlv.append(flv.name) - errors += 1 - else: - SCSFlv.append(flv.name) - if warn: - warnFlv.append(flv.name) - # This makes the output more readable - SCSFlv.sort() - nonSCSFlv.sort() - wrongFlv.sort() - warnFlv.sort() - # Produce dicts for YAML reporting - flvSCSList = { - "SCSFlavorsValid": SCSFlv, - "SCSFlavorsWrong": wrongFlv, - "FlavorsWithWarnings": warnFlv, - } - flvOthList = { - "OtherFlavors": nonSCSFlv - } - flvSCSRep = { - "TotalAmount": len(SCSFlv) + len(wrongFlv), - "FlavorsValid": len(SCSFlv), - "FlavorsWrong": len(wrongFlv), - "FlavorsWithWarnings": len(warnFlv), - } - flvOthRep = { - "TotalAmount": len(nonSCSFlv), - } - totSummary = { - "Errors": errors, - "Warnings": len(warnFlv), - } - Report = {cloud: {"TotalSummary": totSummary}} - if not fnmck.quiet: - Report[cloud]["SCSFlavorSummary"] = flvSCSRep - Report[cloud]["OtherFlavorSummary"] = flvOthRep - if verbose: - Report[cloud]["SCSFlavorReport"] = flvSCSList - Report[cloud]["OtherFlavorReport"] = flvOthList - print(f"{yaml.dump(Report, default_flow_style=False)}") - print("flavor-name-check: " + ('PASS', 'FAIL')[min(1, errors)]) - return errors + + # TODO in the future, the remainder should be moved to a central module `scs_compatible_iaas.py`, + # which would import the test logic (i.e., the functions called compute_XYZ) from here. + # Then this module wouldn't need to know about containers, and the central module can handle + # information sharing as well as running precisely the requested set of testcases. + c = Container() + c.add_function('conn', lambda _: openstack.connect(cloud=cloud, timeout=32)) + c.add_function('flavors', lambda c: list(c.conn.compute.flavors())) + c.add_function('scs_flavors', lambda c: compute_scs_flavors(c.flavors)) + c.add_function('scs_0100_syntax_check', lambda c: compute_scs_0100_syntax_check(c.scs_flavors)) + c.add_function('scs_0100_semantics_check', lambda c: compute_scs_0100_semantics_check(c.scs_flavors)) + c.add_function('flavor_name_check', lambda c: compute_flavor_name_check(c.scs_0100_syntax_check, c.scs_0100_semantics_check)) + for testcase in TESTCASES: + harness(testcase, lambda: getattr(c, testcase.replace('-', '_'))) + return 0 if __name__ == "__main__": From 6ac57e625b2bddf884be0fb62231116aecb375f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 15 Aug 2025 14:13:09 +0200 Subject: [PATCH 02/31] fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/flavor-naming/flavor-names-openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/iaas/flavor-naming/flavor-names-openstack.py b/Tests/iaas/flavor-naming/flavor-names-openstack.py index 80b589406..5cde9deef 100755 --- a/Tests/iaas/flavor-naming/flavor-names-openstack.py +++ b/Tests/iaas/flavor-naming/flavor-names-openstack.py @@ -135,7 +135,7 @@ class Container: >>>> container = Container() >>>> container.add_function('pi', lambda _: 22/7) >>>> container.add_function('pi_squared', lambda c: c.pi * c.pi) - >>>> assert c.pi_squared == 22/7 * 22/7 + >>>> assert container.pi_squared == 22/7 * 22/7 """ def __init__(self): self._values = {} From 1073fbd72fec92841e526a8c6f1d676cab0db93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 15 Aug 2025 14:14:24 +0200 Subject: [PATCH 03/31] removed help on obsolete command-line switches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/flavor-naming/flavor-names-openstack.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Tests/iaas/flavor-naming/flavor-names-openstack.py b/Tests/iaas/flavor-naming/flavor-names-openstack.py index 5cde9deef..3c0d82ce7 100755 --- a/Tests/iaas/flavor-naming/flavor-names-openstack.py +++ b/Tests/iaas/flavor-naming/flavor-names-openstack.py @@ -33,14 +33,7 @@ def usage(rcode=1): "help output" print("Usage: flavor-names-openstack.py [options]", file=sys.stderr) print("Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env)", file=sys.stderr) - print(" [-C/--mand mand.yaml] overrides the list of mandatory flavor names", file=sys.stderr) - print(" [-1/--v1prefer] prefer v1 flavor names (but still tolerates v2", file=sys.stderr) - print(" [-o/--accept-old-mandatory] prefer v2 flavor names, but v1 ones can fulfill mand list", file=sys.stderr) - print(" [-2/--v2plus] only accepts v2 flavor names, old ones result in errors", file=sys.stderr) - print(" [-3/--v3] differentiate b/w mand and recommended flavors", file=sys.stderr) - print(" [-v/--verbose] [-q/--quiet] control verbosity of output", file=sys.stderr) print("This tool retrieves the list of flavors from the OpenStack cloud OS_CLOUD", file=sys.stderr) - print(" and checks for the presence of the mandatory SCS flavors (read from mand.yaml)", file=sys.stderr) print(" and reports inconsistencies, errors etc. It returns 0 on success.", file=sys.stderr) sys.exit(rcode) From a3f562aba017b7d3bfd8f46c65a9815def86f684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 15:48:19 +0200 Subject: [PATCH 04/31] factor out test logic into separate module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- .../flavor-naming/flavor-names-openstack.py | 174 ++---------------- .../iaas/flavor-naming/flavor_names_check.py | 82 +++++++++ 2 files changed, 93 insertions(+), 163 deletions(-) create mode 100644 Tests/iaas/flavor-naming/flavor_names_check.py diff --git a/Tests/iaas/flavor-naming/flavor-names-openstack.py b/Tests/iaas/flavor-naming/flavor-names-openstack.py index 3c0d82ce7..23e93bc43 100755 --- a/Tests/iaas/flavor-naming/flavor-names-openstack.py +++ b/Tests/iaas/flavor-naming/flavor-names-openstack.py @@ -21,16 +21,18 @@ import sys import typing import getopt + import openstack -import flavor_names +from flavor_names_check import \ + compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check logger = logging.getLogger(__name__) def usage(rcode=1): - "help output" + """help output""" print("Usage: flavor-names-openstack.py [options]", file=sys.stderr) print("Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env)", file=sys.stderr) print("This tool retrieves the list of flavors from the OpenStack cloud OS_CLOUD", file=sys.stderr) @@ -38,154 +40,6 @@ def usage(rcode=1): sys.exit(rcode) -TESTCASES = ('scs-0100-syntax-check', 'scs-0100-semantics-check', 'flavor-name-check') -STRATEGY = flavor_names.ParsingStrategy( - vstr='v3', - parsers=(flavor_names.parser_v3, ), - tolerated_parsers=(flavor_names.parser_v2, flavor_names.parser_v1), -) -ACC_DISK = (0, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000) - - -def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor], parser=STRATEGY) -> list: - result = [] - for flv in flavors: - if not flv.name or flv.name[:4] != 'SCS-': - continue # not an SCS flavor; none of our business - try: - flavorname = parser(flv.name) - except ValueError as exc: - logger.info(f"error parsing {flv.name}: {exc}") - flavorname = None - result.append((flv, flavorname)) - return result - - -def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: - problems = [flv.name for flv, flavorname in scs_flavors if not flavorname] - if problems: - logger.error(f"scs-100-syntax-check: flavor(s) failed: {', '.join(sorted(problems))}") - return not problems - - -def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: - problems = set() - for flv, flavorname in scs_flavors: - if not flavorname: - continue # this case is handled by syntax check - cpuram = flavorname.cpuram - if flv.vcpus < cpuram.cpus: - logger.info(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}") - problems.add(flv.name) - elif flv.vcpus > cpuram.cpus: - logger.info(f"Flavor {flv.name} CPU underpromise: {flv.vcpus} > {cpuram.cpus}") - # RAM - flvram = int((flv.ram + 51) / 102.4) / 10 - # Warn for strange sizes (want integer numbers, half allowed for < 10GiB) - if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2): - logger.info(f"Flavor {flv.name} uses discouraged uneven size of memory {flvram:.1f} GiB") - if flvram < cpuram.ram: - logger.info(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}") - problems.add(flv.name) - elif flvram > cpuram.ram: - logger.info(f"Flavor {flv.name} RAM underpromise {flvram:.1f} > {cpuram.ram:.1f}") - # Disk could have been omitted - disksize = flavorname.disk.disksize if flavorname.disk else 0 - # We have a recommendation for disk size steps - if disksize not in ACC_DISK: - logger.info(f"Flavor {flv.name} non-standard disk size {disksize}, should have (5, 10, 20, 50, 100, 200, ...)") - if flv.disk < disksize: - logger.info(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}") - problems.add(flv.name) - elif flv.disk > disksize: - logger.info(f"Flavor {flv.name} disk underpromise {flv.disk} > {disksize}") - if problems: - logger.error(f"scs-100-semantics-check: flavor(s) failed: {', '.join(sorted(problems))}") - return not problems - - -def compute_flavor_name_check(syntax_check_result, semantics_check_result): - return syntax_check_result and semantics_check_result - - -# TODO see comment in main function about moving to another module - -class Container: - """ - This class does lazy evaluation and memoization. You register any potential value either - by giving the value directly, using `add_value`, or - by specifying how it is computed using `add_function`, - which expects a function that takes a container (so other values may be referred to). - In each case, you have to give the value a name. - - The value will be available as a normal member variable under this name. - If given via a function, this function will only be evaluated when the value is accessed, - and the value will be memoized, so the function won't be called twice. - If the function raises an exception, then this will be memoized just as well. - - For instance, - - >>>> container = Container() - >>>> container.add_function('pi', lambda _: 22/7) - >>>> container.add_function('pi_squared', lambda c: c.pi * c.pi) - >>>> assert container.pi_squared == 22/7 * 22/7 - """ - def __init__(self): - self._values = {} - self._functions = {} - - def __getattr__(self, key): - val = self._values.get(key) - if val is None: - try: - ret = self._functions[key](self) - except BaseException as e: - val = (True, e) - else: - val = (False, ret) - self._values[key] = val - error, ret = val - if error: - raise ret - return ret - - def add_function(self, name, fn): - if name in self._functions: - raise RuntimeError(f"fn {name} already registered") - self._functions[name] = fn - - def add_value(self, name, value): - if name in self._values: - raise RuntimeError(f"value {name} already registered") - self._values[name] = value - - -# TODO see comment in main function about moving to another module - -def harness(name, *check_fns): - """Harness for evaluating testcase `name`. - - Logs beginning of computation. - Calls each fn in `check_fns`. - Prints (to stdout) 'name: RESULT', where RESULT is one of - - - 'ABORT' if an exception occurs during the function calls - - 'FAIL' if one of the functions has a falsy result - - 'PASS' otherwise - """ - logger.debug(f'** {name}') - try: - result = all(check_fn() for check_fn in check_fns) - except BaseException: - logger.debug('exception during check', exc_info=True) - result = 'ABORT' - else: - result = ['FAIL', 'PASS'][min(1, result)] - # this is quite redundant - # logger.debug(f'** computation end for {name}') - print(f"{name}: {result}") - - def main(argv): """Entry point -- main loop going over flavors""" # configure logging, disable verbose library logging @@ -235,19 +89,13 @@ def main(argv): print("CRITICAL: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr) sys.exit(1) - # TODO in the future, the remainder should be moved to a central module `scs_compatible_iaas.py`, - # which would import the test logic (i.e., the functions called compute_XYZ) from here. - # Then this module wouldn't need to know about containers, and the central module can handle - # information sharing as well as running precisely the requested set of testcases. - c = Container() - c.add_function('conn', lambda _: openstack.connect(cloud=cloud, timeout=32)) - c.add_function('flavors', lambda c: list(c.conn.compute.flavors())) - c.add_function('scs_flavors', lambda c: compute_scs_flavors(c.flavors)) - c.add_function('scs_0100_syntax_check', lambda c: compute_scs_0100_syntax_check(c.scs_flavors)) - c.add_function('scs_0100_semantics_check', lambda c: compute_scs_0100_semantics_check(c.scs_flavors)) - c.add_function('flavor_name_check', lambda c: compute_flavor_name_check(c.scs_0100_syntax_check, c.scs_0100_semantics_check)) - for testcase in TESTCASES: - harness(testcase, lambda: getattr(c, testcase.replace('-', '_'))) + with openstack.connect(cloud=cloud, timeout=32) as conn: + scs_flavors = compute_scs_flavors(conn.compute.flavors()) + result = compute_flavor_name_check( + compute_scs_0100_syntax_check(scs_flavors), + compute_scs_0100_semantics_check(scs_flavors), + ) + print("flavor-name-check: " + ('FAIL', 'PASS')[min(1, result)]) return 0 diff --git a/Tests/iaas/flavor-naming/flavor_names_check.py b/Tests/iaas/flavor-naming/flavor_names_check.py new file mode 100644 index 000000000..73decfb8a --- /dev/null +++ b/Tests/iaas/flavor-naming/flavor_names_check.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: + +import logging +import typing + +import openstack + +import flavor_names + + +logger = logging.getLogger(__name__) + + +TESTCASES = ('scs-0100-syntax-check', 'scs-0100-semantics-check', 'flavor-name-check') +STRATEGY = flavor_names.ParsingStrategy( + vstr='v3', + parsers=(flavor_names.parser_v3, ), + tolerated_parsers=(flavor_names.parser_v2, flavor_names.parser_v1), +) +ACC_DISK = (0, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000) + + +def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor], parser=STRATEGY) -> list: + result = [] + for flv in flavors: + if not flv.name or flv.name[:4] != 'SCS-': + continue # not an SCS flavor; none of our business + try: + flavorname = parser(flv.name) + except ValueError as exc: + logger.info(f"error parsing {flv.name}: {exc}") + flavorname = None + result.append((flv, flavorname)) + return result + + +def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: + problems = [flv.name for flv, flavorname in scs_flavors if not flavorname] + if problems: + logger.error(f"scs-100-syntax-check: flavor(s) failed: {', '.join(sorted(problems))}") + return not problems + + +def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: + problems = set() + for flv, flavorname in scs_flavors: + if not flavorname: + continue # this case is handled by syntax check + cpuram = flavorname.cpuram + if flv.vcpus < cpuram.cpus: + logger.info(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}") + problems.add(flv.name) + elif flv.vcpus > cpuram.cpus: + logger.info(f"Flavor {flv.name} CPU underpromise: {flv.vcpus} > {cpuram.cpus}") + # RAM + flvram = int((flv.ram + 51) / 102.4) / 10 + # Warn for strange sizes (want integer numbers, half allowed for < 10GiB) + if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2): + logger.info(f"Flavor {flv.name} uses discouraged uneven size of memory {flvram:.1f} GiB") + if flvram < cpuram.ram: + logger.info(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}") + problems.add(flv.name) + elif flvram > cpuram.ram: + logger.info(f"Flavor {flv.name} RAM underpromise {flvram:.1f} > {cpuram.ram:.1f}") + # Disk could have been omitted + disksize = flavorname.disk.disksize if flavorname.disk else 0 + # We have a recommendation for disk size steps + if disksize not in ACC_DISK: + logger.info(f"Flavor {flv.name} non-standard disk size {disksize}, should have (5, 10, 20, 50, 100, 200, ...)") + if flv.disk < disksize: + logger.info(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}") + problems.add(flv.name) + elif flv.disk > disksize: + logger.info(f"Flavor {flv.name} disk underpromise {flv.disk} > {disksize}") + if problems: + logger.error(f"scs-100-semantics-check: flavor(s) failed: {', '.join(sorted(problems))}") + return not problems + + +def compute_flavor_name_check(syntax_check_result, semantics_check_result): + return syntax_check_result and semantics_check_result From 127118abcab5587861727b65010b4ff8c5cd13d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 15:49:30 +0200 Subject: [PATCH 05/31] rename subdirectory to make imports possible, also more telling name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/README.md | 0 .../iaas/{flavor-naming => scs_0100_flavor_naming}/check_yaml.py | 0 .../{flavor-naming => scs_0100_flavor_naming}/check_yaml_test.py | 0 Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/cli.py | 0 .../flavor-add-extra-specs.py | 0 .../flavor-form-server.py | 0 .../iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-form.py | 0 .../flavor-manager-input.py | 0 .../flavor-name-check.py | 0 .../flavor-name-describe.py | 0 .../flavor-names-openstack.py | 0 .../{flavor-naming => scs_0100_flavor_naming}/flavor_names.py | 0 .../flavor_names_check.py | 0 .../{flavor-naming => scs_0100_flavor_naming}/page/index.html | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/README.md (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/check_yaml.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/check_yaml_test.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/cli.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-add-extra-specs.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-form-server.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-form.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-manager-input.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-name-check.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-name-describe.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor-names-openstack.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor_names.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/flavor_names_check.py (100%) rename Tests/iaas/{flavor-naming => scs_0100_flavor_naming}/page/index.html (100%) diff --git a/Tests/iaas/flavor-naming/README.md b/Tests/iaas/scs_0100_flavor_naming/README.md similarity index 100% rename from Tests/iaas/flavor-naming/README.md rename to Tests/iaas/scs_0100_flavor_naming/README.md diff --git a/Tests/iaas/flavor-naming/check_yaml.py b/Tests/iaas/scs_0100_flavor_naming/check_yaml.py similarity index 100% rename from Tests/iaas/flavor-naming/check_yaml.py rename to Tests/iaas/scs_0100_flavor_naming/check_yaml.py diff --git a/Tests/iaas/flavor-naming/check_yaml_test.py b/Tests/iaas/scs_0100_flavor_naming/check_yaml_test.py similarity index 100% rename from Tests/iaas/flavor-naming/check_yaml_test.py rename to Tests/iaas/scs_0100_flavor_naming/check_yaml_test.py diff --git a/Tests/iaas/flavor-naming/cli.py b/Tests/iaas/scs_0100_flavor_naming/cli.py similarity index 100% rename from Tests/iaas/flavor-naming/cli.py rename to Tests/iaas/scs_0100_flavor_naming/cli.py diff --git a/Tests/iaas/flavor-naming/flavor-add-extra-specs.py b/Tests/iaas/scs_0100_flavor_naming/flavor-add-extra-specs.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-add-extra-specs.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-add-extra-specs.py diff --git a/Tests/iaas/flavor-naming/flavor-form-server.py b/Tests/iaas/scs_0100_flavor_naming/flavor-form-server.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-form-server.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-form-server.py diff --git a/Tests/iaas/flavor-naming/flavor-form.py b/Tests/iaas/scs_0100_flavor_naming/flavor-form.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-form.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-form.py diff --git a/Tests/iaas/flavor-naming/flavor-manager-input.py b/Tests/iaas/scs_0100_flavor_naming/flavor-manager-input.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-manager-input.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-manager-input.py diff --git a/Tests/iaas/flavor-naming/flavor-name-check.py b/Tests/iaas/scs_0100_flavor_naming/flavor-name-check.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-name-check.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-name-check.py diff --git a/Tests/iaas/flavor-naming/flavor-name-describe.py b/Tests/iaas/scs_0100_flavor_naming/flavor-name-describe.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-name-describe.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-name-describe.py diff --git a/Tests/iaas/flavor-naming/flavor-names-openstack.py b/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor-names-openstack.py rename to Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py diff --git a/Tests/iaas/flavor-naming/flavor_names.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor_names.py rename to Tests/iaas/scs_0100_flavor_naming/flavor_names.py diff --git a/Tests/iaas/flavor-naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py similarity index 100% rename from Tests/iaas/flavor-naming/flavor_names_check.py rename to Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py diff --git a/Tests/iaas/flavor-naming/page/index.html b/Tests/iaas/scs_0100_flavor_naming/page/index.html similarity index 100% rename from Tests/iaas/flavor-naming/page/index.html rename to Tests/iaas/scs_0100_flavor_naming/page/index.html From 01d4dbeee3c0415e87a83cbd95b130b77bdf1c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 15:50:40 +0200 Subject: [PATCH 06/31] fixup: adapt scs-compatible-iaas.yaml after rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/scs-compatible-iaas.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index ada4a68be..c44635019 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -15,7 +15,7 @@ modules: name: Flavor naming v3.1 url: https://docs.scs.community/standards/scs-0100-v3-flavor-naming run: - - executable: ./iaas/flavor-naming/flavor-names-openstack.py + - executable: ./iaas/scs_0100_flavor_naming/flavor-names-openstack.py args: -c {os_cloud} --mand=./iaas/scs-0100-v3-flavors.yaml # Note: --v2plus would outlaw the v1 flavor names. Don't do this yet. testcases: From fbf1bc56d9371f8651c6f739b9d815c5ebc4a56e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 15:53:20 +0200 Subject: [PATCH 07/31] fixup: adapt adr_syntax.yaml after rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- playbooks/adr_syntax.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/adr_syntax.yaml b/playbooks/adr_syntax.yaml index 3d10a0292..ff8b26053 100644 --- a/playbooks/adr_syntax.yaml +++ b/playbooks/adr_syntax.yaml @@ -12,7 +12,7 @@ - name: Run test script consistency check script ansible.builtin.shell: - cmd: python3 Tests/iaas/flavor-naming/check_yaml.py Tests/iaas + cmd: python3 Tests/iaas/scs_0100_flavor_naming/check_yaml.py Tests/iaas chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" register: result changed_when: true From 17b92b4e2367394a3b41e6db5812b52cc62c7efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 16:12:31 +0200 Subject: [PATCH 08/31] add openstack_test.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 171 ++++++++++++++++++ .../flavor_names_check.py | 2 +- Tests/scs-compatible-iaas.yaml | 5 +- 3 files changed, 174 insertions(+), 4 deletions(-) create mode 100755 Tests/iaas/openstack_test.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py new file mode 100755 index 000000000..c4fbdb3ec --- /dev/null +++ b/Tests/iaas/openstack_test.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: + +"""Openstack testcase runner + +(c) Matthias Büchse , 8/2025 +SPDX-License-Identifier: CC-BY-SA 4.0 +""" + +import logging +import os +import sys +import typing +import getopt +import openstack + +from scs_0100_flavor_naming.flavor_names_check import \ + TESTCASES as SCS_0100_TESTCASES, \ + compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check + + +logger = logging.getLogger(__name__) + + +def usage(rcode=1): + """help output""" + print("Usage: openstack_test.py [options] testcase-id1 ... testcase-idN", file=sys.stderr) + print("Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env)", file=sys.stderr) + print("This tool retrieves the list of flavors from the OpenStack cloud OS_CLOUD", file=sys.stderr) + print(" and reports inconsistencies, errors etc. It returns 0 on success.", file=sys.stderr) + sys.exit(rcode) + + +TESTCASES = set( + SCS_0100_TESTCASES +) + + +def make_container(cloud): + c = Container() + # scs_0100_flavor_naming + c.add_function('conn', lambda _: openstack.connect(cloud=cloud, timeout=32)) + c.add_function('flavors', lambda c: list(c.conn.compute.flavors())) + c.add_function('scs_flavors', lambda c: compute_scs_flavors(c.flavors)) + c.add_function('scs_0100_syntax_check', lambda c: compute_scs_0100_syntax_check(c.scs_flavors)) + c.add_function('scs_0100_semantics_check', lambda c: compute_scs_0100_semantics_check(c.scs_flavors)) + c.add_function('flavor_name_check', lambda c: compute_flavor_name_check(c.scs_0100_syntax_check, c.scs_0100_semantics_check)) + return c + + +class Container: + """ + This class does lazy evaluation and memoization. You register any potential value either + by giving the value directly, using `add_value`, or + by specifying how it is computed using `add_function`, + which expects a function that takes a container (so other values may be referred to). + In each case, you have to give the value a name. + + The value will be available as a normal member variable under this name. + If given via a function, this function will only be evaluated when the value is accessed, + and the value will be memoized, so the function won't be called twice. + If the function raises an exception, then this will be memoized just as well. + + For instance, + + >>>> container = Container() + >>>> container.add_function('pi', lambda _: 22/7) + >>>> container.add_function('pi_squared', lambda c: c.pi * c.pi) + >>>> assert container.pi_squared == 22/7 * 22/7 + """ + def __init__(self): + self._values = {} + self._functions = {} + + def __getattr__(self, key): + val = self._values.get(key) + if val is None: + try: + ret = self._functions[key](self) + except BaseException as e: + val = (True, e) + else: + val = (False, ret) + self._values[key] = val + error, ret = val + if error: + raise ret + return ret + + def add_function(self, name, fn): + if name in self._functions: + raise RuntimeError(f"fn {name} already registered") + self._functions[name] = fn + + def add_value(self, name, value): + if name in self._values: + raise RuntimeError(f"value {name} already registered") + self._values[name] = value + + +def harness(name, *check_fns): + """Harness for evaluating testcase `name`. + + Logs beginning of computation. + Calls each fn in `check_fns`. + Prints (to stdout) 'name: RESULT', where RESULT is one of + + - 'ABORT' if an exception occurs during the function calls + - 'FAIL' if one of the functions has a falsy result + - 'PASS' otherwise + """ + logger.debug(f'** {name}') + try: + result = all(check_fn() for check_fn in check_fns) + except BaseException: + logger.debug('exception during check', exc_info=True) + result = 'ABORT' + else: + result = ['FAIL', 'PASS'][min(1, result)] + # this is quite redundant + # logger.debug(f'** computation end for {name}') + print(f"{name}: {result}") + + +def main(argv): + """Entry point -- main loop going over flavors""" + # configure logging, disable verbose library logging + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) + openstack.enable_logging(debug=False) + cloud = None + + try: + cloud = os.environ["OS_CLOUD"] + except KeyError: + pass + try: + opts, args = getopt.gnu_getopt(argv, "c:C:", ("os-cloud=", )) + except getopt.GetoptError as exc: + print(f"CRITICAL: {exc!r}", file=sys.stderr) + usage(1) + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + usage(0) + elif opt[0] == "-c" or opt[0] == "--os-cloud": + cloud = opt[1] + else: + usage(2) + + testcases = [t for t in args if t in TESTCASES] + if len(testcases) != len(args): + unknown = [a for a in args if a not in TESTCASES] + logger.warning(f"ignoring unknown testcases: {','.join(unknown)}") + + if not cloud: + print("CRITICAL: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr) + sys.exit(1) + + c = make_container(cloud) + for testcase in testcases: + harness(testcase, lambda: getattr(c, testcase.replace('-', '_'))) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except SystemExit: + raise + except BaseException as exc: + print(f"CRITICAL: {exc!r}", file=sys.stderr) + sys.exit(1) diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py index 73decfb8a..743d48dd8 100644 --- a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py @@ -6,7 +6,7 @@ import openstack -import flavor_names +from . import flavor_names logger = logging.getLogger(__name__) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index c44635019..958de2613 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -15,9 +15,8 @@ modules: name: Flavor naming v3.1 url: https://docs.scs.community/standards/scs-0100-v3-flavor-naming run: - - executable: ./iaas/scs_0100_flavor_naming/flavor-names-openstack.py - args: -c {os_cloud} --mand=./iaas/scs-0100-v3-flavors.yaml - # Note: --v2plus would outlaw the v1 flavor names. Don't do this yet. + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} flavor-name-check testcases: - id: flavor-name-check tags: [mandatory] From 7de15b167f64295789483b9e23dd9a079530d2bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 16:15:58 +0200 Subject: [PATCH 09/31] fixup: acquiesce flake8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 6 +++--- Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index c4fbdb3ec..45fd1ef0f 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -7,16 +7,16 @@ SPDX-License-Identifier: CC-BY-SA 4.0 """ +import getopt import logging import os import sys -import typing -import getopt + import openstack from scs_0100_flavor_naming.flavor_names_check import \ TESTCASES as SCS_0100_TESTCASES, \ - compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check + compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check logger = logging.getLogger(__name__) diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py b/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py index 23e93bc43..373829473 100755 --- a/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py @@ -16,16 +16,15 @@ SPDX-License-Identifier: CC-BY-SA 4.0 """ +import getopt import logging import os import sys -import typing -import getopt import openstack from flavor_names_check import \ - compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check + compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check logger = logging.getLogger(__name__) From 38d78ac050c427746e259a7a797a334e4f5ed3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 22:48:45 +0200 Subject: [PATCH 10/31] adapt entropy-check parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 25 +- Tests/iaas/scs_0101_entropy/entropy-check.py | 127 +++++++++ .../entropy_check.py} | 267 +++++------------- Tests/scs-compatible-iaas.yaml | 30 +- 4 files changed, 218 insertions(+), 231 deletions(-) create mode 100755 Tests/iaas/scs_0101_entropy/entropy-check.py rename Tests/iaas/{entropy/entropy-check.py => scs_0101_entropy/entropy_check.py} (56%) mode change 100755 => 100644 diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 45fd1ef0f..ea046cdd5 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -15,8 +15,11 @@ import openstack from scs_0100_flavor_naming.flavor_names_check import \ - TESTCASES as SCS_0100_TESTCASES, \ compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check +from scs_0101_entropy.entropy_check import \ + compute_scs_0101_image_property, compute_scs_0101_flavor_property, compute_canonical_image, \ + compute_collected_vm_output, compute_scs_0101_entropy_avail, compute_scs_0101_rngd, \ + compute_scs_0101_fips_test, compute_scs_0101_entropy_check logger = logging.getLogger(__name__) @@ -31,20 +34,24 @@ def usage(rcode=1): sys.exit(rcode) -TESTCASES = set( - SCS_0100_TESTCASES -) - - def make_container(cloud): c = Container() # scs_0100_flavor_naming c.add_function('conn', lambda _: openstack.connect(cloud=cloud, timeout=32)) - c.add_function('flavors', lambda c: list(c.conn.compute.flavors())) + c.add_function('flavors', lambda c: list(c.conn.list_flavors(get_extra=True))) + c.add_function('images', lambda c: [img for img in c.conn.list_images() if img.visibility in ('public', 'community')]) c.add_function('scs_flavors', lambda c: compute_scs_flavors(c.flavors)) c.add_function('scs_0100_syntax_check', lambda c: compute_scs_0100_syntax_check(c.scs_flavors)) c.add_function('scs_0100_semantics_check', lambda c: compute_scs_0100_semantics_check(c.scs_flavors)) c.add_function('flavor_name_check', lambda c: compute_flavor_name_check(c.scs_0100_syntax_check, c.scs_0100_semantics_check)) + c.add_function('scs_0101_image_property', lambda c: compute_scs_0101_image_property(c.images)) + c.add_function('scs_0101_flavor_property', lambda c: compute_scs_0101_flavor_property(c.flavors)) + c.add_function('canonical_image', lambda c: compute_canonical_image(c.images)) + c.add_function('collected_vm_output', lambda c: compute_collected_vm_output(c.conn, c.flavors, c.canonical_image)) + c.add_function('scs_0101_entropy_avail', lambda c: compute_scs_0101_entropy_avail(c.collected_vm_output, c.canonical_image.name)) + c.add_function('scs_0101_rngd', lambda c: compute_scs_0101_rngd(c.collected_vm_output, c.canonical_image.name)) + c.add_function('scs_0101_fips_test', lambda c: compute_scs_0101_fips_test(c.collected_vm_output, c.canonical_image.name)) + c.add_function('entropy_check', lambda c: compute_scs_0101_entropy_check(c.scs_0101_entropy_avail, c.scs_0101_fips_test)) return c @@ -146,9 +153,9 @@ def main(argv): else: usage(2) - testcases = [t for t in args if t in TESTCASES] + testcases = [t for t in args if t.endswith('check') or t.startswith('scs-')] if len(testcases) != len(args): - unknown = [a for a in args if a not in TESTCASES] + unknown = [a for a in args if a not in testcases] logger.warning(f"ignoring unknown testcases: {','.join(unknown)}") if not cloud: diff --git a/Tests/iaas/scs_0101_entropy/entropy-check.py b/Tests/iaas/scs_0101_entropy/entropy-check.py new file mode 100755 index 000000000..03f333311 --- /dev/null +++ b/Tests/iaas/scs_0101_entropy/entropy-check.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Entropy checker + +Check given cloud for conformance with SCS standard regarding +entropy, to be found under /Standards/scs-0101-v1-entropy.md + +Return code is 0 precisely when it could be verified that the standard is satisfied. +Otherwise the return code is the number of errors that occurred (up to 127 due to OS +restrictions); for further information, see the log messages on various channels: + CRITICAL for problems preventing the test to complete, + ERROR for violations of requirements, + WARNING for violations of recommendations, + DEBUG for background information and problems that don't hinder the test. +""" +import getopt +import logging +import os +import sys +import warnings + +import openstack +import openstack.cloud + + +try: + from . import entropy_check +except ImportError: + import entropy_check + + +logger = logging.getLogger(__name__) + + +def print_usage(file=sys.stderr): + """Help output""" + print("""Usage: entropy-check.py [options] +This tool checks the requested images and flavors according to the SCS Standard 0101 "Entropy". +Options: + [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) + [-d/--debug] enables DEBUG logging channel + [-i/--images IMAGE_LIST] sets images to be tested, separated by comma. + [-V/--image-visibility VIS_LIST] filters images by visibility + (default: 'public,community'; use '*' to disable) +""", end='', file=file) + + +def print_result(check_id, passed): + print(check_id + ": " + ('FAIL', 'PASS')[bool(passed)]) + + +def main(argv): + # configure logging, disable verbose library logging + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + openstack.enable_logging(debug=False) + warnings.filterwarnings("ignore", "search_floating_ips") + + try: + opts, args = getopt.gnu_getopt(argv, "c:i:hdV:", ["os-cloud=", "images=", "help", "debug", "image-visibility="]) + except getopt.GetoptError as exc: + logger.critical(f"{exc}") + print_usage() + return 1 + + cloud = os.environ.get("OS_CLOUD") + image_visibility = set() + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + print_usage() + return 0 + if opt[0] == "-i" or opt[0] == "--images": + logger.info("ignoring obsolete option -i") + if opt[0] == "-c" or opt[0] == "--os-cloud": + cloud = opt[1] + if opt[0] == "-d" or opt[0] == "--debug": + logging.getLogger().setLevel(logging.DEBUG) + if opt[0] == "-V" or opt[0] == "--image-visibility": + image_visibility.update([v.strip() for v in opt[1].split(',')]) + + if not cloud: + logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") + return 1 + + if not image_visibility: + image_visibility.update(("public", "community")) + + try: + logger.debug(f"Connecting to cloud '{cloud}'") + with openstack.connect(cloud=cloud, timeout=32) as conn: + all_images = conn.list_images() + all_flavors = conn.list_flavors(get_extra=True) + + if '*' not in image_visibility: + logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") + all_images = [img for img in all_images if img.visibility in image_visibility] + all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] + logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") + + if not all_images: + logger.critical("Can't run this test without image") + return 1 + + logger.debug("Checking images and flavors for recommended attributes") + print_result('entropy-check-image-properties', entropy_check.compute_scs_0101_image_property(all_images)) + print_result('entropy-check-flavor-properties', entropy_check.compute_scs_0101_flavor_property(all_flavors)) + + logger.debug("Checking dynamic instance properties") + canonical_image = entropy_check.compute_canonical_image(all_images) + collected_vm_output = entropy_check.compute_collected_vm_output(conn, all_flavors, canonical_image) + print_result('entropy-check-rngd', entropy_check.compute_scs_0101_rngd(collected_vm_output, canonical_image.name)) + scs_0101_entropy_avail_result = entropy_check.compute_scs_0101_entropy_avail(collected_vm_output, canonical_image.name) + scs_0101_fips_test_result = entropy_check.compute_scs_0101_fips_test(collected_vm_output, canonical_image.name) + entropy_check_result = entropy_check.compute_scs_0101_entropy_check( + scs_0101_entropy_avail_result, + scs_0101_fips_test_result, + ) + print_result('entropy-check-entropy-avail', scs_0101_entropy_avail_result) + print_result('entropy-check-fips-test', scs_0101_fips_test_result) + print_result('entropy-check', entropy_check_result) + return not entropy_check_result + except BaseException as e: + logger.critical(f"{e!r}") + logger.debug("Exception info", exc_info=True) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/Tests/iaas/entropy/entropy-check.py b/Tests/iaas/scs_0101_entropy/entropy_check.py old mode 100755 new mode 100644 similarity index 56% rename from Tests/iaas/entropy/entropy-check.py rename to Tests/iaas/scs_0101_entropy/entropy_check.py index e6fbb7898..62c832d8d --- a/Tests/iaas/entropy/entropy-check.py +++ b/Tests/iaas/scs_0101_entropy/entropy_check.py @@ -1,28 +1,10 @@ #!/usr/bin/env python3 -"""Entropy checker - -Check given cloud for conformance with SCS standard regarding -entropy, to be found under /Standards/scs-0101-v1-entropy.md - -Return code is 0 precisely when it could be verified that the standard is satisfied. -Otherwise the return code is the number of errors that occurred (up to 127 due to OS -restrictions); for further information, see the log messages on various channels: - CRITICAL for problems preventing the test to complete, - ERROR for violations of requirements, - WARNING for violations of recommendations, - DEBUG for background information and problems that don't hinder the test. -""" from collections import Counter, defaultdict -import getopt import logging import os import re import sys import time -import warnings - -import openstack -import openstack.cloud logger = logging.getLogger(__name__) @@ -77,20 +59,7 @@ } -def print_usage(file=sys.stderr): - """Help output""" - print("""Usage: entropy-check.py [options] -This tool checks the requested images and flavors according to the SCS Standard 0101 "Entropy". -Options: - [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) - [-d/--debug] enables DEBUG logging channel - [-i/--images IMAGE_LIST] sets images to be tested, separated by comma. - [-V/--image-visibility VIS_LIST] filters images by visibility - (default: 'public,community'; use '*' to disable) -""", end='', file=file) - - -def check_image_attributes(images, attributes=IMAGE_ATTRIBUTES): +def compute_scs_0101_image_property(images, attributes=IMAGE_ATTRIBUTES): candidates = [ (image.name, [f"{key}={value}" for key, value in attributes.items() if image.get(key) != value]) for image in images @@ -98,11 +67,11 @@ def check_image_attributes(images, attributes=IMAGE_ATTRIBUTES): # drop those candidates that are fine offenders = [candidate for candidate in candidates if candidate[1]] for name, wrong in offenders: - logger.warning(f"Image '{name}' missing recommended attributes: {', '.join(wrong)}") + logger.info(f"Image '{name}' missing recommended attributes: {', '.join(wrong)}") return not offenders -def check_flavor_attributes(flavors, attributes=FLAVOR_ATTRIBUTES, optional=FLAVOR_OPTIONAL): +def compute_scs_0101_flavor_property(flavors, attributes=FLAVOR_ATTRIBUTES, optional=FLAVOR_OPTIONAL): offenses = 0 for flavor in flavors: extra_specs = flavor['extra_specs'] @@ -116,14 +85,15 @@ def check_flavor_attributes(flavors, attributes=FLAVOR_ATTRIBUTES, optional=FLAV # and if the recommended attributes are present, we assume that implementers have done their job already if miss_opt: message += f"; additionally, missing optional attributes: {', '.join(miss_opt)}" - logger.warning(message) + logger.info(message) return not offenses -def check_entropy_avail(lines, image_name): +def compute_scs_0101_entropy_avail(collected_vm_output, image_name): + lines = collected_vm_output['entropy-avail'] entropy_avail = lines[0].strip() if entropy_avail != "256": - logger.error( + logger.info( f"VM '{image_name}' didn't have a fixed amount of entropy available. " f"Expected 256, got {entropy_avail}." ) @@ -131,14 +101,16 @@ def check_entropy_avail(lines, image_name): return True -def check_rngd(lines, image_name): +def compute_scs_0101_rngd(collected_vm_output, image_name): + lines = collected_vm_output['rngd'] if "could not be found" in '\n'.join(lines): - logger.warning(f"VM '{image_name}' doesn't provide the recommended service rngd") + logger.info(f"VM '{image_name}' doesn't provide the recommended service rngd") return False return True -def check_fips_test(lines, image_name): +def compute_scs_0101_fips_test(collected_vm_output, image_name): + lines = collected_vm_output['fips-test'] try: fips_data = '\n'.join(lines) failure_re = re.search(r'failures:\s\d+', fips_data, flags=re.MULTILINE) @@ -146,39 +118,41 @@ def check_fips_test(lines, image_name): fips_failures = failure_re.string[failure_re.regs[0][0]:failure_re.regs[0][1]].split(" ")[1] if int(fips_failures) <= 3: return True # strict test passed - logger.warning( + logger.info( f"VM '{image_name}' didn't pass the strict FIPS 140-2 testing. " f"Expected a maximum of 3 failures, got {fips_failures}." ) if int(fips_failures) <= 5: return True # lenient test passed - logger.error( + logger.info( f"VM '{image_name}' didn't pass the FIPS 140-2 testing. " f"Expected a maximum of 5 failures, got {fips_failures}." ) else: - logger.error(f"VM '{image_name}': failed to determine fips failures") + logger.info(f"VM '{image_name}': failed to determine fips failures") logger.debug(f"stderr following:\n{fips_data}") except BaseException: logger.critical(f"Couldn't check VM '{image_name}' requirements", exc_info=True) return False # any unsuccessful path should end up here -def check_virtio_rng(lines, image, flavor): +# FIXME this is not actually being used AFAICT -- mbuechse +def compute_scs_0101_virtio_rng(collected_vm_output, image_name): + lines = collected_vm_output['virtio-rng'] try: - # Check the existence of the HRNG -- can actually be skipped if the flavor - # or the image doesn't have the corresponding attributes anyway! - if image.hw_rng_model != "virtio" or flavor.extra_specs.get("hw_rng:allowed") != "True": - logger.debug("Not looking for virtio-rng because required attributes are missing") - return False # `cat` can fail with return code 1 if special file does not exist hw_device = lines[0] if not hw_device.strip() or "No such device" in lines[1]: - logger.warning(f"VM '{image.name}' doesn't provide a hardware device.") + logger.info(f"VM '{image_name}' doesn't provide a hardware device.") return False return True except BaseException: - logger.critical(f"Couldn't check VM '{image.name}' recommends", exc_info=True) + logger.critical(f"Couldn't check VM '{image_name}' recommends", exc_info=True) + + +def compute_scs_0101_entropy_check(scs_0101_entropy_avail_result, scs_0101_fips_test_result): + # note: this is about the mandatory things + return scs_0101_entropy_avail_result and scs_0101_fips_test_result class TestEnvironment: @@ -263,7 +237,7 @@ def __exit__(self, exc_type, exc_value, traceback): self.clean() -def create_vm(env, all_flavors, image, server_name=SERVER_NAME): +def select_flavor_for_image(all_flavors, image): # Pick a flavor matching the image flavors = [flv for flv in all_flavors if flv.ram >= image.min_ram] # if at all possible, prefer a flavor that provides hw_rng:allowed! @@ -273,13 +247,13 @@ def create_vm(env, all_flavors, image, server_name=SERVER_NAME): elif flavors: logger.debug(f"Unable to pick flavor with hw_rng:allowed=true for image '{image.name}'") else: - logger.critical(f"No flavor could be found for the image '{image.name}'") - return + raise RuntimeError(f"No flavor could be found for the image '{image.name}'") # try to pick a frugal flavor - flavor = min(flavors, key=lambda flv: flv.vcpus + flv.ram / 3.0 + flv.disk / 10.0) - userdata = SERVER_USERDATA.get(image.os_distro, SERVER_USERDATA_GENERIC) - logger.debug(f"Using userdata:\n{userdata}") + return min(flavors, key=lambda flv: flv.vcpus + flv.ram / 3.0 + flv.disk / 10.0) + + +def create_vm(conn, flavor, image, network, userdata=None, server_name=SERVER_NAME): volume_size = max(image.min_disk, 8) # sometimes, the min_disk property is not set correctly # create a server with the image and the flavor as well as # the previously created keys and security group @@ -287,10 +261,11 @@ def create_vm(env, all_flavors, image, server_name=SERVER_NAME): f"Creating instance of image '{image.name}' using flavor '{flavor.name}' and " f"{volume_size} GiB ephemeral boot volume" ) + logger.debug(f"Using userdata:\n{userdata}") # explicitly set auto_ip=False, we may still get a (totally unnecessary) floating IP assigned - server = env.conn.create_server( + server = conn.create_server( server_name, image=image, flavor=flavor, userdata=userdata, wait=True, timeout=500, auto_ip=False, - boot_from_volume=True, terminate_volume=True, volume_size=volume_size, network=env.network, + boot_from_volume=True, terminate_volume=True, volume_size=volume_size, network=network, ) logger.debug(f"Server '{server_name}' ('{server.id}') has been created") # next, do an explicit get_server because, beginning with version 3.2.0, the openstacksdk no longer @@ -298,7 +273,7 @@ def create_vm(env, all_flavors, image, server_name=SERVER_NAME): # I (mbuechse) consider this a bug in openstacksdk; it was introduced with # https://opendev.org/openstack/openstacksdk/commit/a8adbadf0c4cdf1539019177fb1be08e04d98e82 # I also consider openstacksdk architecture with the Mixins etc. smelly to say the least - return env.conn.get_server(server.id) + return conn.get_server(server.id) def delete_vm(conn, server_name=SERVER_NAME): @@ -309,15 +284,6 @@ def delete_vm(conn, server_name=SERVER_NAME): logger.debug(f"The server '{server_name}' couldn't be deleted.", exc_info=True) -class CountingHandler(logging.Handler): - def __init__(self, level=logging.NOTSET): - super().__init__(level=level) - self.bylevel = Counter() - - def handle(self, record): - self.bylevel[record.levelno] += 1 - - # the following functions are used to map any OpenStack Image to a pair of integers # used for sorting the images according to fitness for our test # - debian take precedence over ubuntu @@ -328,6 +294,7 @@ def handle(self, record): "buster": 10, "bullseye": 11, "bookworm": 12, + "trixie": 13, } @@ -359,16 +326,12 @@ def _deduce_sort(img): return deducer(img.os_version.strip().lower()) -def select_deb_image(images): +def compute_canonical_image(all_images): """From a list of OpenStack image objects, select a recent Debian derivative.""" - return max(images, key=_deduce_sort, default=None) - + return max(all_images, key=_deduce_sort, default=None) -def print_result(check_id, passed): - print(check_id + ": " + ('FAIL', 'PASS')[bool(passed)]) - -def evaluate_output(lines, image, flavor, marker=MARKER): +def _convert_to_collected(lines, marker=MARKER): # parse lines from console output # removing any "indent", stuff that looks like '[ 70.439502] cloud-init[513]: ' section = None @@ -384,130 +347,32 @@ def evaluate_output(lines, image, flavor, marker=MARKER): continue if section: collected[section].append(line[indent:]) - # always check if we have something, because print_result won't do MISS, only PASS/FAIL - if collected['virtio-rng']: - # virtio-rng is not an official test case according to testing notes, - # but for some reason we check it nonetheless (call it informative) - check_virtio_rng(collected['virtio-rng'], image, flavor) - if collected['entropy-avail']: - print_result('entropy-check-entropy-avail', check_entropy_avail(collected['entropy-avail'], image.name)) - if collected['rngd']: - print_result('entropy-check-rngd', check_rngd(collected['rngd'], image.name)) - if collected['fips-test']: - print_result('entropy-check-fips-test', check_fips_test(collected['fips-test'], image.name)) - - -def main(argv): - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - warnings.filterwarnings("ignore", "search_floating_ips") - # count the number of log records per level (used for summary and return code) - counting_handler = CountingHandler(level=logging.INFO) - logger.addHandler(counting_handler) - - try: - opts, args = getopt.gnu_getopt(argv, "c:i:hdV:", ["os-cloud=", "images=", "help", "debug", "image-visibility="]) - except getopt.GetoptError as exc: - logger.critical(f"{exc}") - print_usage() - return 1 - - cloud = os.environ.get("OS_CLOUD") - image_names = set() - image_visibility = set() - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - print_usage() - return 0 - if opt[0] == "-i" or opt[0] == "--images": - image_names.update([img.strip() for img in opt[1].split(',')]) - if opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-d" or opt[0] == "--debug": - logging.getLogger().setLevel(logging.DEBUG) - if opt[0] == "-V" or opt[0] == "--image-visibility": - image_visibility.update([v.strip() for v in opt[1].split(',')]) - - if not cloud: - logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") - return 1 - - if not image_visibility: - image_visibility.update(("public", "community")) + return collected - try: - logger.debug(f"Connecting to cloud '{cloud}'") - with openstack.connect(cloud=cloud, timeout=32) as conn: - all_images = conn.list_images() - all_flavors = conn.list_flavors(get_extra=True) - - if '*' not in image_visibility: - logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") - all_images = [img for img in all_images if img.visibility in image_visibility] - all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] - logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") - - if not all_images: - logger.critical("Can't run this test without image") - return 1 - - if image_names: - # find images by the names given, BAIL out if some image is missing - images = sorted([img for img in all_images if img.name in image_names], key=lambda img: img.name) - names = [img.name for img in images] - logger.debug(f"Selected images: {', '.join(names)}") - missing_names = image_names - set(names) - if missing_names: - logger.critical(f"Missing images: {', '.join(missing_names)}") - return 1 - else: - images = [select_deb_image(all_images) or all_images[0]] - logger.debug(f"Selected image: {images[0].name} ({images[0].id})") - - logger.debug("Checking images and flavors for recommended attributes") - print_result('entropy-check-image-properties', check_image_attributes(all_images)) - print_result('entropy-check-flavor-properties', check_flavor_attributes(all_flavors)) - - logger.debug("Checking dynamic instance properties") - # Check a VM for services and requirements - with TestEnvironment(conn) as env: - for image in images: - try: - # ugly: create the server inside the try-block because the call - # can be interrupted via Ctrl-C, and then the instance will be - # started without us knowing its id - server = create_vm(env, all_flavors, image) - remainder = TIMEOUT - console = conn.compute.get_server_console_output(server) - while remainder > 0: - if "Failed to run module scripts-user" in console['output']: - raise RuntimeError(f"Failed tests for {server.id}") - if "_scs-test-end" in console['output']: - break - time.sleep(1.0) - remainder -= 1 - console = conn.compute.get_server_console_output(server) - # if the timeout was hit, maybe we won't find everything, but that's okay -- - # these testcases will count as missing, rightly so - logger.debug(f'Finished waiting with timeout remainder: {remainder} s') - evaluate_output(console['output'].splitlines(), image, server.flavor) - finally: - delete_vm(conn) - except BaseException as e: - logger.critical(f"{e!r}") - logger.debug("Exception info", exc_info=True) - - c = counting_handler.bylevel - logger.debug( - "Total critical / error / warning: " - f"{c[logging.CRITICAL]} / {c[logging.ERROR]} / {c[logging.WARNING]}" - ) - # include this one for backwards compatibility - if not c[logging.CRITICAL]: - print("entropy-check: " + ('PASS', 'FAIL')[min(1, c[logging.ERROR])]) - return min(127, c[logging.CRITICAL] + c[logging.ERROR]) # cap at 127 due to OS restrictions - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) +def compute_collected_vm_output(conn, all_flavors, image): + # Check a VM for services and requirements + logger.debug(f"Selected image: {image.name} ({image.id})") + flavor = select_flavor_for_image(all_flavors, image) + userdata = SERVER_USERDATA.get(image.os_distro, SERVER_USERDATA_GENERIC) + with TestEnvironment(conn) as env: + try: + # ugly: create the server inside the try-block because the call + # can be interrupted via Ctrl-C, and then the instance will be + # started without us knowing its id + server = create_vm(env.conn, flavor, image, env.network, userdata) + remainder = TIMEOUT + console = conn.compute.get_server_console_output(server) + while True: + if "_scs-test-end" in console['output']: + break + if remainder <= 0: + raise RuntimeError("timeout while waiting for VM to complete computation") + if "Failed to run module scripts-user" in console['output']: + raise RuntimeError(f"Failed tests for {server.id}") + time.sleep(1.0) + remainder -= 1 + console = conn.compute.get_server_console_output(server) + return _convert_to_collected(console['output'].splitlines()) + finally: + delete_vm(conn) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 958de2613..9e488bf69 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -27,19 +27,9 @@ modules: name: Entropy v1 url: https://docs.scs.community/standards/scs-0101-v1-entropy run: - - executable: ./iaas/entropy/entropy-check.py - args: -c {os_cloud} -d + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} entropy-check testcases: - - id: entropy-check-flavor-properties - tags: [] # don't use this testcase, but list it anyway because the script will output a result - - id: entropy-check-image-properties - tags: [] # don't use this testcase, but list it anyway because the script will output a result - - id: entropy-check-rngd - tags: [] # don't use this testcase, but list it anyway because the script will output a result - - id: entropy-check-entropy-avail - tags: [] # don't use this testcase, but list it anyway because the script will output a result - - id: entropy-check-fips-test - tags: [] # don't use this testcase, but list it anyway because the script will output a result - id: entropy-check tags: [mandatory] description: > @@ -49,36 +39,34 @@ modules: name: Entropy v1 url: https://docs.scs.community/standards/scs-0101-v1-entropy run: - - executable: ./iaas/entropy/entropy-check.py - args: -c {os_cloud} -d + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} scs-0101-flavor-property scs-0101-image-property scs-0101-rngd scs-0101-entropy-avail scs-0101-fips-test testcases: - - id: entropy-check-flavor-properties + - id: scs-0101-flavor-property tags: [recommended] description: > Must have all flavor properties recommended in - - id: entropy-check-image-properties + - id: scs-0101-image-property tags: [recommended] description: > Must have all image properties recommended in - - id: entropy-check-rngd + - id: scs-0101-rngd tags: [mandatory] description: > Images of the test sample must have the service `rngd`; see - - id: entropy-check-entropy-avail + - id: scs-0101-entropy-avail tags: [mandatory] description: > A test instance must have the correct `entropy_avail`; see - - id: entropy-check-fips-test + - id: scs-0101-fips-test tags: [mandatory] description: > A test instance must pass the "FIPS test"; see - - id: entropy-check - tags: [] # don't use this testcase, but list it anyway because the script will output a result - id: scs-0102-v1 name: Image metadata v1 url: https://docs.scs.community/standards/scs-0102-v1-image-metadata From ba10feb05f25c190dbb6f14c91420e2e5a18bb0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 19 Aug 2025 22:55:10 +0200 Subject: [PATCH 11/31] fixup: satisfy flake8 (actually good points) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/scs_0101_entropy/entropy_check.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Tests/iaas/scs_0101_entropy/entropy_check.py b/Tests/iaas/scs_0101_entropy/entropy_check.py index 62c832d8d..36ddc80fa 100644 --- a/Tests/iaas/scs_0101_entropy/entropy_check.py +++ b/Tests/iaas/scs_0101_entropy/entropy_check.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -from collections import Counter, defaultdict import logging -import os import re -import sys import time +import openstack + logger = logging.getLogger(__name__) @@ -336,7 +335,7 @@ def _convert_to_collected(lines, marker=MARKER): # removing any "indent", stuff that looks like '[ 70.439502] cloud-init[513]: ' section = None indent = 0 - collected = defaultdict(list) + collected = {} for line in lines: idx = line.find(marker) if idx != -1: @@ -346,7 +345,7 @@ def _convert_to_collected(lines, marker=MARKER): indent = idx continue if section: - collected[section].append(line[indent:]) + collected.setdefault(section, []).append(line[indent:]) return collected From 9ed22151e60206f08f38650e0a6cb8775e86387c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 20 Aug 2025 18:48:27 +0200 Subject: [PATCH 12/31] revise outdated usage text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index ea046cdd5..62c19a5f6 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -25,12 +25,12 @@ logger = logging.getLogger(__name__) -def usage(rcode=1): +def usage(rcode=1, file=sys.stderr): """help output""" - print("Usage: openstack_test.py [options] testcase-id1 ... testcase-idN", file=sys.stderr) - print("Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env)", file=sys.stderr) - print("This tool retrieves the list of flavors from the OpenStack cloud OS_CLOUD", file=sys.stderr) - print(" and reports inconsistencies, errors etc. It returns 0 on success.", file=sys.stderr) + print("Usage: openstack_test.py [options] testcase-id1 ... testcase-idN", file=file) + print("Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env)", file=file) + print("Runs specified testcases against the OpenStack cloud OS_CLOUD", file=file) + print("and reports inconsistencies, errors etc. It returns 0 on success.", file=file) sys.exit(rcode) @@ -130,7 +130,6 @@ def harness(name, *check_fns): def main(argv): - """Entry point -- main loop going over flavors""" # configure logging, disable verbose library logging logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) openstack.enable_logging(debug=False) @@ -153,7 +152,7 @@ def main(argv): else: usage(2) - testcases = [t for t in args if t.endswith('check') or t.startswith('scs-')] + testcases = [t for t in args if t.endswith('-check') or t.startswith('scs-')] if len(testcases) != len(args): unknown = [a for a in args if a not in testcases] logger.warning(f"ignoring unknown testcases: {','.join(unknown)}") From 9287559ab3fe2bcefafdbdc512a5d5a0635ea11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 20 Aug 2025 18:52:22 +0200 Subject: [PATCH 13/31] change channel back to error when testcase is not passed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- .../scs_0100_flavor_naming/flavor_names_check.py | 6 +++--- Tests/iaas/scs_0101_entropy/entropy_check.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py index 743d48dd8..a8b476b71 100644 --- a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py @@ -49,7 +49,7 @@ def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: continue # this case is handled by syntax check cpuram = flavorname.cpuram if flv.vcpus < cpuram.cpus: - logger.info(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}") + logger.error(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}") problems.add(flv.name) elif flv.vcpus > cpuram.cpus: logger.info(f"Flavor {flv.name} CPU underpromise: {flv.vcpus} > {cpuram.cpus}") @@ -59,7 +59,7 @@ def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2): logger.info(f"Flavor {flv.name} uses discouraged uneven size of memory {flvram:.1f} GiB") if flvram < cpuram.ram: - logger.info(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}") + logger.error(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}") problems.add(flv.name) elif flvram > cpuram.ram: logger.info(f"Flavor {flv.name} RAM underpromise {flvram:.1f} > {cpuram.ram:.1f}") @@ -69,7 +69,7 @@ def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: if disksize not in ACC_DISK: logger.info(f"Flavor {flv.name} non-standard disk size {disksize}, should have (5, 10, 20, 50, 100, 200, ...)") if flv.disk < disksize: - logger.info(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}") + logger.error(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}") problems.add(flv.name) elif flv.disk > disksize: logger.info(f"Flavor {flv.name} disk underpromise {flv.disk} > {disksize}") diff --git a/Tests/iaas/scs_0101_entropy/entropy_check.py b/Tests/iaas/scs_0101_entropy/entropy_check.py index 36ddc80fa..f2e694635 100644 --- a/Tests/iaas/scs_0101_entropy/entropy_check.py +++ b/Tests/iaas/scs_0101_entropy/entropy_check.py @@ -66,7 +66,7 @@ def compute_scs_0101_image_property(images, attributes=IMAGE_ATTRIBUTES): # drop those candidates that are fine offenders = [candidate for candidate in candidates if candidate[1]] for name, wrong in offenders: - logger.info(f"Image '{name}' missing recommended attributes: {', '.join(wrong)}") + logger.error(f"Image '{name}' missing recommended attributes: {', '.join(wrong)}") return not offenders @@ -84,7 +84,7 @@ def compute_scs_0101_flavor_property(flavors, attributes=FLAVOR_ATTRIBUTES, opti # and if the recommended attributes are present, we assume that implementers have done their job already if miss_opt: message += f"; additionally, missing optional attributes: {', '.join(miss_opt)}" - logger.info(message) + logger.error(message) return not offenses @@ -92,7 +92,7 @@ def compute_scs_0101_entropy_avail(collected_vm_output, image_name): lines = collected_vm_output['entropy-avail'] entropy_avail = lines[0].strip() if entropy_avail != "256": - logger.info( + logger.error( f"VM '{image_name}' didn't have a fixed amount of entropy available. " f"Expected 256, got {entropy_avail}." ) @@ -103,7 +103,7 @@ def compute_scs_0101_entropy_avail(collected_vm_output, image_name): def compute_scs_0101_rngd(collected_vm_output, image_name): lines = collected_vm_output['rngd'] if "could not be found" in '\n'.join(lines): - logger.info(f"VM '{image_name}' doesn't provide the recommended service rngd") + logger.error(f"VM '{image_name}' doesn't provide the recommended service rngd") return False return True @@ -123,12 +123,12 @@ def compute_scs_0101_fips_test(collected_vm_output, image_name): ) if int(fips_failures) <= 5: return True # lenient test passed - logger.info( + logger.error( f"VM '{image_name}' didn't pass the FIPS 140-2 testing. " f"Expected a maximum of 5 failures, got {fips_failures}." ) else: - logger.info(f"VM '{image_name}': failed to determine fips failures") + logger.error(f"VM '{image_name}': failed to determine fips failures") logger.debug(f"stderr following:\n{fips_data}") except BaseException: logger.critical(f"Couldn't check VM '{image_name}' requirements", exc_info=True) @@ -142,7 +142,7 @@ def compute_scs_0101_virtio_rng(collected_vm_output, image_name): # `cat` can fail with return code 1 if special file does not exist hw_device = lines[0] if not hw_device.strip() or "No such device" in lines[1]: - logger.info(f"VM '{image_name}' doesn't provide a hardware device.") + logger.error(f"VM '{image_name}' doesn't provide a hardware device.") return False return True except BaseException: From aa17fbb74f06d1de245baf1cfcd6ad668af33a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 26 Aug 2025 16:11:43 +0200 Subject: [PATCH 14/31] adapt check pertaining to scs-0102-image-metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/image-metadata/image-md-check.py | 424 ------------------ Tests/iaas/openstack_test.py | 56 ++- .../flavor_names_check.py | 4 - Tests/iaas/scs_0101_entropy/entropy-check.py | 5 +- Tests/iaas/scs_0101_entropy/entropy_check.py | 5 - .../scs_0102_image_metadata/image-md-check.py | 138 ++++++ .../scs_0102_image_metadata/image_metadata.py | 303 +++++++++++++ Tests/scs-compatible-iaas.yaml | 4 +- 8 files changed, 493 insertions(+), 446 deletions(-) delete mode 100755 Tests/iaas/image-metadata/image-md-check.py create mode 100755 Tests/iaas/scs_0102_image_metadata/image-md-check.py create mode 100644 Tests/iaas/scs_0102_image_metadata/image_metadata.py diff --git a/Tests/iaas/image-metadata/image-md-check.py b/Tests/iaas/image-metadata/image-md-check.py deleted file mode 100755 index a73bb5807..000000000 --- a/Tests/iaas/image-metadata/image-md-check.py +++ /dev/null @@ -1,424 +0,0 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: -# -# SCS/Docs/tools/image-md-check.py - -""" -Retrieve metadata from (public) images and check for compliance -with SCS specifications. - -(c) Kurt Garloff , 09/2022 -SPDX-License-Identifier: CC-BY-SA-4.0 -""" - -import calendar -from collections import Counter -import getopt -import logging -import os -import sys -import time - -import openstack - - -logger = logging.getLogger(__name__) - - -def usage(ret): - "Usage information" - print("Usage: image-md-check.py [options] [images]") - print("image-md-check.py will create a report on public images by retrieving") - print(" the image metadata (properties) and comparing this against the image") - print(" metadata spec from SCS.") - print("Options: --os-cloud CLOUDNAME: Use this cloud config, default is $OS_CLOUD") - print(" -p/--private : Also consider private images") - print(" -v/--verbose : Be more verbose") - print(" -s/--skip-completeness: Don't check whether we have all mandatory images") - print(" -h/--help : Print this usage information") - print(" [-V/--image-visibility VIS_LIST] : filters images by visibility") - print(" (default: 'public,community'; use '*' to disable)") - print("If you pass images, only these will be validated, otherwise all images") - print("(filtered according to -p, -V) from the catalog will be processed.") - sys.exit(ret) - - -# global options -verbose = False - -# Image list -mand_images = ["Ubuntu 22.04", "Ubuntu 20.04", "Debian 11"] -rec1_images = ["CentOS 8", "Rocky 8", "AlmaLinux 8", "Debian 10", "Fedora 36"] -rec2_images = ["SLES 15SP4", "RHEL 9", "RHEL 8", "Windows Server 2022", "Windows Server 2019"] -sugg_images = ["openSUSE Leap 15.4", "Cirros 0.5.2", "Alpine", "Arch"] - -# Just for nice formatting of image naming hints -- otherwise we capitalize the 1st letter -OS_LIST = ("CentOS", "AlmaLinux", "Windows Server", "RHEL", "SLES", "openSUSE") -# Auxiliary mapping for `freq2secs` (note that values are rounded up a bit on purpose) -FREQ_TO_SEC = { - "never": 0, - "critical_bug": 0, - "yearly": 365 * 24 * 3600, - "quarterly": 92 * 24 * 3600, - "monthly": 31 * 24 * 3600, - "weekly": 7 * 25 * 3600, - "daily": 25 * 3600, -} -STRICT_FORMATS = ("%Y-%m-%dT%H:%M:%SZ", ) -DATE_FORMATS = STRICT_FORMATS + ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d") -MARKER_DATE_FORMATS = ("%Y-%m-%d", "%Y%m%d") -OUTDATED_MARKERS = ("old", "prev") -KIB, MIB, GIB = (1024 ** n for n in (1, 2, 3)) - - -def recommended_name(nm, os_list=OS_LIST): - """Return capitalized name""" - for osnm in os_list: - osln = len(osnm) - if nm[:osln].casefold() == osnm.casefold(): - return osnm + nm[osln:] - return nm[0].upper() + nm[1:] - - -class Property: - "Class to specify properties, allowed values, ..." - def __init__(self, name, mand, values, desc = ""): - self.name = name - self.ismand = mand - self.values = values - if desc: - self.desc = desc - else: - self.desc = name - - def is_ok(self, props, warn = ""): - "Check validity of properties" - if self.name in props: - if self.values and not props[self.name] in self.values: - if warn: - print(f'ERROR: Image "{warn}": value "{props[self.name]}" for property ' - f'"{self.name}" not allowed', file=sys.stderr) - return False - if not props[self.name] and not self.values: - err = "ERROR" - ret = False - else: - err = "WARNING" - ret = True - if not props[self.name] and (verbose or not self.values) and warn: - print(f'{err}: Image "{warn}": empty value for property "{self.name}" not recommended', - file=sys.stderr) - return ret - if self.ismand: - if warn: - print(f'ERROR: Image "{warn}": Mandatory property "{self.name}" is missing', - file=sys.stderr) - return False - if warn and verbose: - print(f'INFO: Image "{warn}": Optional property "{self.name}" is missing') # , file=sys.stderr) - return True - - -# From the image metadata specification -# https://github.com/SovereignCloudStack/Docs/blob/main/Design-Docs/Image-Properties-Spec.md -os_props = (Property("os_distro", True, ()), - Property("os_version", True, ())) -arch_props = (Property("architecture", True, ("x86_64", "aarch64", "risc-v"), "CPU architecture"), - Property("hypervisor_type", True, ("qemu", "kvm", "xen", "hyper-v", "esxi", None))) -hw_props = (Property("hw_rng_model", False, ("virtio", None), "Random Number Generator"), - Property("hw_disk_bus", True, ("virtio", "scsi", None), "Disk bus: vda or sda")) -build_props = (Property("image_build_date", True, ()), - Property("patchlevel", False, (),), - Property("image_original_user", True, (), "Default ssh username"), - Property("image_source", True, ()), - Property("image_description", True, ())) -maint_props = (Property("replace_frequency", True, ("yearly", "quarterly", "monthly", "weekly", - "daily", "critical_bug", "never")), - Property("provided_until", True, ()), - Property("uuid_validity", True, ()), - Property("hotfix_hours", False, ())) - - -def is_url(stg): - """Is string stg a URL?""" - idx = stg.find("://") - return idx >= 0 and stg[:idx] in ("http", "https", "ftp", "ftps") - - -def parse_date(stg, formats=DATE_FORMATS): - """ - Return time in Unix seconds or 0 if stg is not a valid date. - We recognize: %Y-%m-%dT%H:%M:%SZ, %Y-%m-%d %H:%M[:%S], and %Y-%m-%d - """ - bdate = 0 - for fmt in formats: - try: - tmdate = time.strptime(stg, fmt) - bdate = calendar.timegm(tmdate) - break - except ValueError: # as exc: - # print(f'date {stg} does not match {fmt}\n{exc}', file=sys.stderr) - pass - return bdate - - -def freq2secs(stg): - """Convert frequency to seconds (round up a bit), return 0 if not applicable""" - secs = FREQ_TO_SEC.get(stg) - if secs is None: - print(f'ERROR: replace frequency {stg}?', file=sys.stderr) - secs = 0 - return secs - - -def is_outdated(img, bdate): - """return 1 if img (with build/regdate bdate) is outdated, - 2 if it's not hidden or marked, 3 if error""" - max_age = 0 - if "replace_frequency" in img.properties: - max_age = 1.1 * (freq2secs(img.properties["replace_frequency"])) - if not max_age or time.time() <= max_age + bdate: - return 0 - # So we found an outdated image that should have been updated - # (5a1) Check whether we are past the provided_until date - until_str = img.properties["provided_until"] - if until_str in ("none", "notice"): - return 0 - until = parse_date(until_str) - if not until: - return 3 - if time.time() > until: - return 0 - if img.is_hidden: - return 1 - parts = img.name.rsplit(" ", 1) - marker = parts[1] if len(parts) >= 2 else "" - if marker in OUTDATED_MARKERS or parse_date(marker, formats=MARKER_DATE_FORMATS): - return 1 - return 2 - - -def validate_imageMD(img, outd_list): - """Retrieve image properties and test for compliance with spec""" - imgnm = img.name - # Now the hard work: Look at properties .... - errors = 0 - warnings = 0 - # (1) recommended os_* and hw_* - # (4) image_build_date, image_original_user, image_source (opt image_description) - # (5) maintained_until, provided_until, uuid_validity, replace_frequency - for prop in (*os_props, *arch_props, *hw_props): - if not prop.is_ok(img, imgnm): - errors += 1 - for prop in (*build_props, *maint_props): - if not prop.is_ok(img.properties, imgnm): - errors += 1 - constr_name = f"{img.os_distro} {img.os_version}" - # (3) os_hash - if img.hash_algo not in ('sha256', 'sha512'): - print(f'WARNING: Image "{imgnm}": no valid hash algorithm {img.hash_algo}', file=sys.stderr) - # errors += 1 - warnings += 1 - # Some more sanity checks: - # - Dateformat for image_build_date - rdate = parse_date(img.created_at, formats=STRICT_FORMATS) - bdate = rdate - if "image_build_date" in img.properties: - bdate = parse_date(img.properties["image_build_date"]) - if not bdate: - print(f'ERROR: Image "{imgnm}": no valid image_build_date ' - f'{img.properties["image_build_date"]}', file=sys.stderr) - errors += 1 - bdate = rdate - elif bdate > rdate: - print(f'ERROR: Image "{imgnm}" with build date {img.properties["image_build_date"]} after registration date {img.created_at}', - file=sys.stderr) - errors += 1 - if bdate > time.time(): - print(f'ERROR: Image "{imgnm}" has build time in the future: {bdate}') - errors += 1 - # - image_source should be a URL - if "image_source" not in img.properties: - pass # we have already noted this as error, no need to do it again - elif img.properties["image_source"] == "private": - if verbose: - print(f'INFO: Image {imgnm} has image_source set to private', file=sys.stderr) - elif not is_url(img.properties["image_source"]): - print(f'ERROR: Image "{imgnm}": image_source should be a URL or "private"', file=sys.stderr) - errors += 1 - # - uuid_validity has a distinct set of options (none, last-X, DATE, notice, forever) - img_uuid_val = img.properties.get("uuid_validity") - if img_uuid_val in (None, "none", "notice", "forever"): - pass - elif img_uuid_val[:5] == "last-" and img_uuid_val[5:].isdecimal(): - pass - elif parse_date(img_uuid_val): - pass - else: - print(f'ERROR: Image "{imgnm}": invalid uuid_validity {img_uuid_val}', file=sys.stderr) - errors += 1 - # - hotfix hours (if set!) should be numeric - if "hotfix_hours" in img.properties: - if not img.properties["hotfix_hours"].isdecimal(): - print(f'ERROR: Image "{imgnm}" has non-numeric hotfix_hours set', file=sys.stderr) - errors += 1 - # (5a) Sanity: Are we actually in violation of replace_frequency? - # This is a bit tricky: We need to disregard images that have been rotated out - # - os_hidden = True is a safe sign for this - # - A name with a date stamp or old or prev (and a newer exists) - outd = is_outdated(img, bdate) - if outd == 3: - print(f'ERROR: Image "{imgnm}" does not provide a valid provided until date', - file=sys.stderr) - errors += 1 - elif outd == 2: - print(f'WARNING: Image "{imgnm}" seems outdated (acc. to its repl freq) but is not hidden or otherwise marked', - file=sys.stderr) - warnings += 1 - outd_list.append(imgnm) - elif outd: - outd_list.append(imgnm) - # (2) sanity min_ram (>=64), min_disk (>= size) - if img.min_ram < 64: - print(f'WARNING: Image "{imgnm}": min_ram == {img.min_ram} MiB < 64 MiB', file=sys.stderr) - warnings += 1 - if not img.min_ram: - print(f'ERROR: Image "{imgnm}": min_ram == 0', file=sys.stderr) - errors += 1 - if not img.min_disk: - print(f'ERROR: Image "{imgnm}": min_disk == 0', file=sys.stderr) - errors += 1 - elif img.min_disk * GIB < img.size: - print(f'ERROR: Image "{imgnm}": img size == {img.size / MIB:.0f} MiB, but min_disk == {img.min_disk * GIB / MIB:.0f} MiB', - file=sys.stderr) - errors += 1 - # (6) tags os:*, managed_by_* - # Nothing to do here ... we could do a warning if those are missing ... - - # (7) Recommended naming - if imgnm[:len(constr_name)].casefold() != constr_name.casefold(): # and verbose - rec_name = recommended_name(constr_name) - print(f'WARNING: Image "{imgnm}" does not start with recommended name "{rec_name}"', - file=sys.stderr) - warnings += 1 - - if not errors and verbose: - print(f'Image "{imgnm}": All good ({warnings} warnings)') - return errors - - -def report_stdimage_coverage(imgs): - "Have we covered all standard images?" - err = 0 - for inm in mand_images: - if inm not in imgs: - err += 1 - print(f'WARNING: Mandatory image "{inm}" is missing', file=sys.stderr) - for inm in (*rec1_images, *rec2_images): - if inm not in imgs: - print(f'INFO: Recommended image "{inm}" is missing', file=sys.stderr) - # Ignore sugg_images for now - return err - - -def miss_replacement_images(by_name, outd_list): - """Go over list of images to find replacement imgs for outd_list, return the ones that are left missing""" - rem_list = [] - for outd in outd_list: - img = None - shortnm = outd.rsplit(" ", 1)[0].rstrip() - if shortnm != outd: - img = by_name.get(shortnm) - if img is not None: - bdate = 0 - if "build_date" in img.properties: - bdate = parse_date(img.properties["build_date"]) - if not bdate: - bdate = parse_date(img.created_at, formats=STRICT_FORMATS) - if is_outdated(img, bdate): - img = None - if img is None: - rem_list.append(outd) - elif verbose: - print(f'INFO: Image "{img.name}" is a valid replacement for outdated "{outd}"', file=sys.stderr) - return rem_list - - -def main(argv): - "Main entry point" - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - # Option parsing - global verbose - image_visibility = set() - private = False - skip = False - cloud = os.environ.get("OS_CLOUD") - err = 0 - try: - opts, args = getopt.gnu_getopt(argv[1:], "phvc:sV:", - ("private", "help", "os-cloud=", "verbose", "skip-completeness", "image-visibility=")) - except getopt.GetoptError: # as exc: - print("CRITICAL: Command-line syntax error", file=sys.stderr) - usage(1) - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - usage(0) - elif opt[0] == "-p" or opt[0] == "--private": - private = True # only keep this for backwards compatibility (we have -V now) - elif opt[0] == "-v" or opt[0] == "--verbose": - verbose = True - logging.getLogger().setLevel(logging.DEBUG) - elif opt[0] == "-s" or opt[0] == "--skip-completeness": - skip = True - elif opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-V" or opt[0] == "--image-visibility": - image_visibility.update([v.strip() for v in opt[1].split(',')]) - images = args - if not cloud: - print("CRITICAL: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr) - usage(1) - if not image_visibility: - image_visibility.update(("public", "community")) - if private: - image_visibility.add("private") - try: - conn = openstack.connect(cloud=cloud, timeout=24) - all_images = list(conn.image.images()) - if '*' not in image_visibility: - logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") - all_images = [img for img in all_images if img.visibility in image_visibility] - all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] - logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") - by_name = {img.name: img for img in all_images} - if len(by_name) != len(all_images): - counter = Counter([img.name for img in all_images]) - duplicates = [name for name, count in counter.items() if count > 1] - print(f'WARNING: duplicate names detected: {", ".join(duplicates)}', file=sys.stderr) - if not images: - images = [img.name for img in all_images] - # Analyse image metadata - outdated_images = [] - for imgnm in images: - err += validate_imageMD(by_name[imgnm], outdated_images) - if not skip: - err += report_stdimage_coverage(images) - if outdated_images: - if verbose: - print(f'INFO: The following outdated images have been detected: {outdated_images}', - file=sys.stderr) - rem_list = miss_replacement_images(by_name, outdated_images) - if rem_list: - print(f'ERROR: Outdated images without replacement: {rem_list}', file=sys.stderr) - err += len(rem_list) - except BaseException as exc: - print(f"CRITICAL: {exc!r}", file=sys.stderr) - return 1 + err - print("image-metadata-check: " + ('PASS', 'FAIL')[min(1, err)]) - return err - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 62c19a5f6..0f146ddd9 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -15,11 +15,20 @@ import openstack from scs_0100_flavor_naming.flavor_names_check import \ - compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check + compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check from scs_0101_entropy.entropy_check import \ compute_scs_0101_image_property, compute_scs_0101_flavor_property, compute_canonical_image, \ compute_collected_vm_output, compute_scs_0101_entropy_avail, compute_scs_0101_rngd, \ - compute_scs_0101_fips_test, compute_scs_0101_entropy_check + compute_scs_0101_fips_test +from scs_0102_image_metadata.image_metadata import \ + compute_scs_0102_prop_architecture, compute_scs_0102_prop_hash_algo, compute_scs_0102_prop_min_disk, \ + compute_scs_0102_prop_min_ram, compute_scs_0102_prop_os_version, compute_scs_0102_prop_os_distro, \ + compute_scs_0102_prop_hw_disk_bus, compute_scs_0102_prop_hypervisor_type, compute_scs_0102_prop_hw_rng_model, \ + compute_scs_0102_prop_image_build_date, compute_scs_0102_prop_image_original_user, \ + compute_scs_0102_prop_image_source, compute_scs_0102_prop_image_description, \ + compute_scs_0102_prop_replace_frequency, compute_scs_0102_prop_provided_until, \ + compute_scs_0102_prop_uuid_validity, compute_scs_0102_prop_hotfix_hours, \ + compute_scs_0102_image_recency logger = logging.getLogger(__name__) @@ -36,22 +45,54 @@ def usage(rcode=1, file=sys.stderr): def make_container(cloud): c = Container() - # scs_0100_flavor_naming + # basic support attributes shared by multiple testcases c.add_function('conn', lambda _: openstack.connect(cloud=cloud, timeout=32)) c.add_function('flavors', lambda c: list(c.conn.list_flavors(get_extra=True))) c.add_function('images', lambda c: [img for img in c.conn.list_images() if img.visibility in ('public', 'community')]) + # scs_0100_flavor_naming c.add_function('scs_flavors', lambda c: compute_scs_flavors(c.flavors)) c.add_function('scs_0100_syntax_check', lambda c: compute_scs_0100_syntax_check(c.scs_flavors)) c.add_function('scs_0100_semantics_check', lambda c: compute_scs_0100_semantics_check(c.scs_flavors)) - c.add_function('flavor_name_check', lambda c: compute_flavor_name_check(c.scs_0100_syntax_check, c.scs_0100_semantics_check)) - c.add_function('scs_0101_image_property', lambda c: compute_scs_0101_image_property(c.images)) - c.add_function('scs_0101_flavor_property', lambda c: compute_scs_0101_flavor_property(c.flavors)) + c.add_function('flavor_name_check', lambda c: all(( + c.scs_0100_syntax_check, c.scs_0100_semantics_check, + ))) + # scs_0101_entropy c.add_function('canonical_image', lambda c: compute_canonical_image(c.images)) c.add_function('collected_vm_output', lambda c: compute_collected_vm_output(c.conn, c.flavors, c.canonical_image)) + c.add_function('scs_0101_image_property', lambda c: compute_scs_0101_image_property(c.images)) + c.add_function('scs_0101_flavor_property', lambda c: compute_scs_0101_flavor_property(c.flavors)) c.add_function('scs_0101_entropy_avail', lambda c: compute_scs_0101_entropy_avail(c.collected_vm_output, c.canonical_image.name)) c.add_function('scs_0101_rngd', lambda c: compute_scs_0101_rngd(c.collected_vm_output, c.canonical_image.name)) c.add_function('scs_0101_fips_test', lambda c: compute_scs_0101_fips_test(c.collected_vm_output, c.canonical_image.name)) - c.add_function('entropy_check', lambda c: compute_scs_0101_entropy_check(c.scs_0101_entropy_avail, c.scs_0101_fips_test)) + c.add_function('entropy_check', lambda c: all(( + c.scs_0101_entropy_avail, c.scs_0101_fips_test, + ))) + # scs_0102_image_metadata + c.add_function('scs_0102_prop_architecture', lambda c: compute_scs_0102_prop_architecture(c.images)) + c.add_function('scs_0102_prop_hash_algo', lambda c: compute_scs_0102_prop_hash_algo(c.images)) + c.add_function('scs_0102_prop_min_disk', lambda c: compute_scs_0102_prop_min_disk(c.images)) + c.add_function('scs_0102_prop_min_ram', lambda c: compute_scs_0102_prop_min_ram(c.images)) + c.add_function('scs_0102_prop_os_version', lambda c: compute_scs_0102_prop_os_version(c.images)) + c.add_function('scs_0102_prop_os_distro', lambda c: compute_scs_0102_prop_os_distro(c.images)) + c.add_function('scs_0102_prop_hw_disk_bus', lambda c: compute_scs_0102_prop_hw_disk_bus(c.images)) + c.add_function('scs_0102_prop_hypervisor_type', lambda c: compute_scs_0102_prop_hypervisor_type(c.images)) + c.add_function('scs_0102_prop_hw_rng_model', lambda c: compute_scs_0102_prop_hw_rng_model(c.images)) + c.add_function('scs_0102_prop_image_build_date', lambda c: compute_scs_0102_prop_image_build_date(c.images)) + c.add_function('scs_0102_prop_image_original_user', lambda c: compute_scs_0102_prop_image_original_user(c.images)) + c.add_function('scs_0102_prop_image_source', lambda c: compute_scs_0102_prop_image_source(c.images)) + c.add_function('scs_0102_prop_image_description', lambda c: compute_scs_0102_prop_image_description(c.images)) + c.add_function('scs_0102_prop_replace_frequency', lambda c: compute_scs_0102_prop_replace_frequency(c.images)) + c.add_function('scs_0102_prop_provided_until', lambda c: compute_scs_0102_prop_provided_until(c.images)) + c.add_function('scs_0102_prop_uuid_validity', lambda c: compute_scs_0102_prop_uuid_validity(c.images)) + c.add_function('scs_0102_prop_hotfix_hours', lambda c: compute_scs_0102_prop_hotfix_hours(c.images)) + c.add_function('scs_0102_image_recency', lambda c: compute_scs_0102_image_recency(c.images)) + c.add_function('image_metadata_check', lambda c: all(( + c.scs_0102_prop_architecture, c.scs_0102_prop_min_disk, c.scs_0102_prop_min_ram, c.scs_0102_prop_os_version, + c.scs_0102_prop_os_distro, c.scs_0102_prop_hw_disk_bus, c.scs_0102_prop_image_build_date, + c.scs_0102_prop_image_original_user, c.scs_0102_prop_image_source, c.scs_0102_prop_image_description, + c.scs_0102_prop_replace_frequency, c.scs_0102_prop_provided_until, c.scs_0102_prop_uuid_validity, + c.scs_0102_image_recency, + ))) return c @@ -82,6 +123,7 @@ def __init__(self): def __getattr__(self, key): val = self._values.get(key) if val is None: + logger.debug(f'... {key}') try: ret = self._functions[key](self) except BaseException as e: diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py index a8b476b71..ae3f661ae 100644 --- a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py @@ -76,7 +76,3 @@ def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: if problems: logger.error(f"scs-100-semantics-check: flavor(s) failed: {', '.join(sorted(problems))}") return not problems - - -def compute_flavor_name_check(syntax_check_result, semantics_check_result): - return syntax_check_result and semantics_check_result diff --git a/Tests/iaas/scs_0101_entropy/entropy-check.py b/Tests/iaas/scs_0101_entropy/entropy-check.py index 03f333311..22c107193 100755 --- a/Tests/iaas/scs_0101_entropy/entropy-check.py +++ b/Tests/iaas/scs_0101_entropy/entropy-check.py @@ -22,10 +22,7 @@ import openstack.cloud -try: - from . import entropy_check -except ImportError: - import entropy_check +import entropy_check logger = logging.getLogger(__name__) diff --git a/Tests/iaas/scs_0101_entropy/entropy_check.py b/Tests/iaas/scs_0101_entropy/entropy_check.py index f2e694635..593127d02 100644 --- a/Tests/iaas/scs_0101_entropy/entropy_check.py +++ b/Tests/iaas/scs_0101_entropy/entropy_check.py @@ -149,11 +149,6 @@ def compute_scs_0101_virtio_rng(collected_vm_output, image_name): logger.critical(f"Couldn't check VM '{image_name}' recommends", exc_info=True) -def compute_scs_0101_entropy_check(scs_0101_entropy_avail_result, scs_0101_fips_test_result): - # note: this is about the mandatory things - return scs_0101_entropy_avail_result and scs_0101_fips_test_result - - class TestEnvironment: def __init__(self, conn): self.conn = conn diff --git a/Tests/iaas/scs_0102_image_metadata/image-md-check.py b/Tests/iaas/scs_0102_image_metadata/image-md-check.py new file mode 100755 index 000000000..54d8bd6af --- /dev/null +++ b/Tests/iaas/scs_0102_image_metadata/image-md-check.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# +# SCS/Docs/tools/image-md-check.py + +""" +Retrieve metadata from (public) images and check for compliance +with SCS specifications. + +(c) Kurt Garloff , 09/2022 +SPDX-License-Identifier: CC-BY-SA-4.0 +""" + +from collections import Counter +import getopt +import logging +import os +import sys + +import openstack + + +from image_metadata import \ + compute_scs_0102_prop_architecture, compute_scs_0102_prop_hash_algo, compute_scs_0102_prop_min_disk, \ + compute_scs_0102_prop_min_ram, compute_scs_0102_prop_os_version, compute_scs_0102_prop_os_distro, \ + compute_scs_0102_prop_hw_disk_bus, compute_scs_0102_prop_hypervisor_type, compute_scs_0102_prop_hw_rng_model, \ + compute_scs_0102_prop_image_build_date, compute_scs_0102_prop_image_original_user, \ + compute_scs_0102_prop_image_source, compute_scs_0102_prop_image_description, \ + compute_scs_0102_prop_replace_frequency, compute_scs_0102_prop_provided_until, \ + compute_scs_0102_prop_uuid_validity, compute_scs_0102_prop_hotfix_hours, \ + compute_scs_0102_image_recency + + +logger = logging.getLogger(__name__) + + +def usage(ret): + "Usage information" + print("Usage: image-md-check.py [options] [images]") + print("image-md-check.py will create a report on public images by retrieving") + print(" the image metadata (properties) and comparing this against the image") + print(" metadata spec from SCS.") + print("Options: --os-cloud CLOUDNAME: Use this cloud config, default is $OS_CLOUD") + print(" -p/--private : Also consider private images") + print(" -v/--verbose : Be more verbose") + print(" -h/--help : Print this usage information") + print(" [-V/--image-visibility VIS_LIST] : filters images by visibility") + print(" (default: 'public,community'; use '*' to disable)") + print("If you pass images, only these will be validated, otherwise all images") + print("(filtered according to -p, -V) from the catalog will be processed.") + sys.exit(ret) + + +def main(argv): + "Main entry point" + # configure logging, disable verbose library logging + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + openstack.enable_logging(debug=False) + image_visibility = set() + private = False + cloud = os.environ.get("OS_CLOUD") + err = 0 + try: + opts, args = getopt.gnu_getopt(argv[1:], "phvc:sV:", + ("private", "help", "os-cloud=", "verbose", "skip-completeness", "image-visibility=")) + except getopt.GetoptError: # as exc: + print("CRITICAL: Command-line syntax error", file=sys.stderr) + usage(1) + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + usage(0) + elif opt[0] == "-p" or opt[0] == "--private": + private = True # only keep this for backwards compatibility (we have -V now) + elif opt[0] == "-v" or opt[0] == "--verbose": + logging.getLogger().setLevel(logging.DEBUG) + elif opt[0] == "-s" or opt[0] == "--skip-completeness": + logger.info("ignoring obsolete command-line option -s") + elif opt[0] == "-c" or opt[0] == "--os-cloud": + cloud = opt[1] + if opt[0] == "-V" or opt[0] == "--image-visibility": + image_visibility.update([v.strip() for v in opt[1].split(',')]) + image_names = args + if not cloud: + print("CRITICAL: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr) + usage(1) + if not image_visibility: + image_visibility.update(("public", "community")) + if private: + image_visibility.add("private") + try: + conn = openstack.connect(cloud=cloud, timeout=24) + all_images = list(conn.image.images()) + if '*' not in image_visibility: + logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") + all_images = [img for img in all_images if img.visibility in image_visibility] + all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] + logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") + by_name = {img.name: img for img in all_images} + if len(by_name) != len(all_images): + counter = Counter([img.name for img in all_images]) + duplicates = [name for name, count in counter.items() if count > 1] + print(f'WARNING: duplicate names detected: {", ".join(duplicates)}', file=sys.stderr) + if image_names: + images = [by_name[nm] for nm in image_names] + else: + images = all_images + result = all(( + compute_scs_0102_prop_architecture(images), + compute_scs_0102_prop_min_disk(images), + compute_scs_0102_prop_min_ram(images), + compute_scs_0102_prop_os_version(images), + compute_scs_0102_prop_os_distro(images), + compute_scs_0102_prop_hw_disk_bus(images), + compute_scs_0102_prop_image_build_date(images), + compute_scs_0102_prop_image_original_user(images), + compute_scs_0102_prop_image_source(images), + compute_scs_0102_prop_image_description(images), + compute_scs_0102_prop_replace_frequency(images), + compute_scs_0102_prop_provided_until(images), + compute_scs_0102_prop_uuid_validity(images), + compute_scs_0102_prop_hotfix_hours(images), + compute_scs_0102_image_recency(images), + )) + print("image-metadata-check: " + ('FAIL', 'PASS')[min(1, result)]) + # recommended stuff + _ = all(( + compute_scs_0102_prop_hash_algo(images), + compute_scs_0102_prop_hypervisor_type(images), + compute_scs_0102_prop_hw_rng_model(images), + )) + except BaseException as exc: + print(f"CRITICAL: {exc!r}", file=sys.stderr) + return 1 + err + return err + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/Tests/iaas/scs_0102_image_metadata/image_metadata.py b/Tests/iaas/scs_0102_image_metadata/image_metadata.py new file mode 100644 index 000000000..ac1e92c1a --- /dev/null +++ b/Tests/iaas/scs_0102_image_metadata/image_metadata.py @@ -0,0 +1,303 @@ +import calendar +from collections import Counter +import logging +import sys +import time + + +logger = logging.getLogger(__name__) + + +ARCHITECTURES = ("x86_64", "aarch64", "risc-v") +HW_DISK_BUSES = ("virtio", "scsi", None) # FIXME why None? +HYPERVISOR_TYPES = ("qemu", "kvm", "xen", "hyper-v", "esxi", None) +HW_RNG_MODELS = ("virtio", None) +# Just for nice formatting of image naming hints -- otherwise we capitalize the 1st letter +OS_LIST = ("CentOS", "AlmaLinux", "Windows Server", "RHEL", "SLES", "openSUSE") +# Auxiliary mapping for `freq2secs` (note that values are rounded up a bit on purpose) +FREQ_TO_SEC = { + "never": 0, + "critical_bug": 0, + "yearly": 365 * 24 * 3600, + "quarterly": 92 * 24 * 3600, + "monthly": 31 * 24 * 3600, + "weekly": 7 * 25 * 3600, + "daily": 25 * 3600, +} +STRICT_FORMATS = ("%Y-%m-%dT%H:%M:%SZ", ) +DATE_FORMATS = STRICT_FORMATS + ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d") +MARKER_DATE_FORMATS = ("%Y-%m-%d", "%Y%m%d") +OUTDATED_MARKERS = ("old", "prev") +KIB, MIB, GIB = (1024 ** n for n in (1, 2, 3)) + + +def recommended_name(nm, os_list=OS_LIST): + """Return capitalized name""" + for osnm in os_list: + osln = len(osnm) + if nm[:osln].casefold() == osnm.casefold(): + return osnm + nm[osln:] + return nm[0].upper() + nm[1:] + + +def is_url(stg): + """Is string stg a URL?""" + idx = stg.find("://") + return idx >= 0 and stg[:idx] in ("http", "https", "ftp", "ftps") + + +def parse_date(stg, formats=DATE_FORMATS): + """ + Return time in Unix seconds or 0 if stg is not a valid date. + We recognize: %Y-%m-%dT%H:%M:%SZ, %Y-%m-%d %H:%M[:%S], and %Y-%m-%d + """ + bdate = 0 + for fmt in formats: + try: + tmdate = time.strptime(stg, fmt) + bdate = calendar.timegm(tmdate) + break + except ValueError: # as exc: + # print(f'date {stg} does not match {fmt}\n{exc}', file=sys.stderr) + pass + return bdate + + +def freq2secs(stg): + """Convert frequency to seconds (round up a bit), return 0 if not applicable""" + secs = FREQ_TO_SEC.get(stg) + if secs is None: + print(f'ERROR: replace frequency {stg}?', file=sys.stderr) + secs = 0 + return secs + + +def is_outdated(img, now=time.time()): + """return 1 if img (with build/regdate bdate) is outdated, + 2 if it's not hidden or marked, 3 if error""" + bdate = parse_date(img.properties.get('image_build_date', '')) + if not bdate: + bdate = parse_date(img.created_at, formats=STRICT_FORMATS) + replace_frequency = img.properties.get('replace_frequency', 'never') + if replace_frequency in ('never', 'critical_bug'): + return 0 + secs = FREQ_TO_SEC.get(replace_frequency) + if secs is None: + # NOTE this error is already handled by testcase scs-0102-prop-replace_frequency + logger.warning(f'invalid replace frequency {replace_frequency}') + return 0 + if now <= bdate + 1.1 * secs: + return 0 + # So we found an outdated image that should have been updated + # (5a1) Check whether we are past the provided_until date + until_str = img.properties["provided_until"] + if until_str in ("none", "notice"): + return 0 + until = parse_date(until_str) + if not until: + return 3 + if now > until: + return 0 + if img.is_hidden: + return 1 + parts = img.name.rsplit(" ", 1) + if len(parts) < 2: + return 2 + marker = parts[1] + if marker in OUTDATED_MARKERS or parse_date(marker, formats=MARKER_DATE_FORMATS): + return 1 + return 2 + + +def _log_error(cause, offenders, channel=logging.ERROR): + if not offenders: + return + names = [img.name for img in offenders] + logger.log(channel, f"{cause} for image(s): {', '.join(names)}") + + +def compute_scs_0102_prop_architecture(images, architectures=ARCHITECTURES): + offenders = [img for img in images if img.architecture not in architectures] + _log_error('property architecture not correct', offenders) + return not offenders + + +# NOTE I think this is a recommendation +def compute_scs_0102_prop_hash_algo(images): + offenders = [img for img in images if img.hash_algo not in ('sha256', 'sha512')] + _log_error('property hash_algo invalid', offenders) + return not offenders + + +def compute_scs_0102_prop_min_disk(images): + offenders1 = [img for img in images if not img.min_disk] + _log_error('property min_disk not set', offenders1) + offenders2 = [img for img in images if img.min_disk and img.min_disk * GIB < img.size] + _log_error('property min_disk smaller than size', offenders2) + return not offenders1 and not offenders2 + + +def compute_scs_0102_prop_min_ram(images): + offenders1 = [img for img in images if not img.min_ram] + _log_error('property min_ram not set', offenders1) + # emit a warning im RAM really low + # NOTE this will probably only get noticed if an error occurs as well + offenders2 = [img for img in images if img.min_ram and img.min_ram < 64] + _log_error('property min_ram < 64 MiB', offenders2, channel=logging.WARNING) + return not offenders1 + + +def compute_scs_0102_prop_os_version(images): + offenders = [img for img in images if not img.os_version] + _log_error('property os_version not set', offenders) + return not offenders + + +def compute_scs_0102_prop_os_distro(images): + offenders = [img for img in images if not img.os_distro] + _log_error('property os_distro not set', offenders) + return not offenders + + +def compute_scs_0102_prop_hw_disk_bus(images, hw_disk_buses=HW_DISK_BUSES): + offenders = [img for img in images if img.hw_disk_bus not in hw_disk_buses] + _log_error('property hw_disk_bus not correct', offenders) + return not offenders + + +def compute_scs_0102_prop_hypervisor_type(images, hypervisor_types=HYPERVISOR_TYPES): + offenders = [img for img in images if img.hypervisor_type not in hypervisor_types] + _log_error('property hypervisor_type not correct', offenders) + return not offenders + + +def compute_scs_0102_prop_hw_rng_model(images, hw_rng_models=HW_RNG_MODELS): + offenders = [img for img in images if img.hw_rng_model not in hw_rng_models] + _log_error('property hw_rng_model not correct', offenders) + return not offenders + + +def compute_scs_0102_prop_image_build_date(images, now=time.time()): + errors = 0 + for img in images: + rdate = parse_date(img.created_at, formats=STRICT_FORMATS) + bdate_str = img.properties.get('image_build_date', '') + bdate = parse_date(bdate_str) + if not bdate: + logger.error(f'Image "{img.name}": image_build_date "{bdate_str}" INVALID') + errors += 1 + elif bdate > rdate: + logger.error(f'Image "{img.name}": image_build_date {bdate_str} AFTER registration date {img.created_at}') + errors += 1 + if (bdate or rdate) > now: + logger.error(f'Image "{img.name}" has build time in the future: {bdate}') + errors += 1 + return not errors + + +# FIXME this is completely optional +# def compute_scs_0102_prop_patchlevel(image): +# return True + + +def compute_scs_0102_prop_image_original_user(images): + offenders = [img for img in images if not img.properties.get('image_original_user')] + _log_error('property image_original_user not set', offenders) + return not offenders + + +def compute_scs_0102_prop_image_source(images): + offenders = [ + img + for img in images + if img.properties.get('image_source') != 'private' + if not is_url(img.properties.get('image_source')) + ] + _log_error('property image_source INVALID (url or "private")', offenders) + return not offenders + + +def compute_scs_0102_prop_image_description(images): + offenders = [img for img in images if not img.properties.get('image_description')] + _log_error('property image_description not set', offenders) + return not offenders + + +def compute_scs_0102_prop_replace_frequency(images, replace_frequencies=FREQ_TO_SEC): + offenders = [img for img in images if img.properties.get('replace_frequency') not in replace_frequencies] + _log_error('property replace_frequency not correct', offenders) + return not offenders + + +def compute_scs_0102_prop_provided_until(images): + offenders = [img for img in images if not img.properties.get('provided_until')] + _log_error('property provided_until not set', offenders) + return not offenders + + +def compute_scs_0102_prop_uuid_validity(images): + errors = 0 + for img in images: + img_uuid_val = img.properties.get("uuid_validity") + if img_uuid_val in (None, "none", "notice", "forever"): + pass + elif img_uuid_val[:5] == "last-" and img_uuid_val[5:].isdecimal(): + pass + elif parse_date(img_uuid_val): + pass + else: + logger.error(f'Image "{img.name}": property uuid_validity INVALID: {img_uuid_val}') + errors += 1 + return not errors + + +def compute_scs_0102_prop_hotfix_hours(images): + errors = 0 + for img in images: + hotfix_hours = img.properties.get("hotfix_hours", '') + if not hotfix_hours or hotfix_hours.isdecimal(): + continue + logger.error(f'Image "{img.name}": property hotfix_hours INVALID: {hotfix_hours}') + errors += 1 + return not errors + + +def _find_replacement_image(by_name, img_name): + shortnm = img_name.rsplit(" ", 1)[0].rstrip() + if shortnm == img_name: + return None + replacement = by_name.get(shortnm) + if replacement is None or is_outdated(replacement): + return None + return replacement + + +def compute_scs_0102_image_recency(images): + by_name = {img.name: img for img in images} + if len(by_name) != len(images): + counter = Counter([img.name for img in images]) + duplicates = [name for name, count in counter.items() if count > 1] + logger.warning(f'duplicate names detected: {", ".join(duplicates)}') + errors = 0 + for img in images: + # This is a bit tricky: We need to disregard images that have been rotated out + # - os_hidden = True is a safe sign for this + # - A name with a date stamp or old or prev (and a newer exists) + outd = is_outdated(img) + if not outd: + continue # fine + if outd == 3: + logger.error(f'Image "{img.name}" does not provide a valid provided until date') + errors += 1 + continue # hopeless + # in case that outd in (1, 2) try to find a non-outdated version + if outd == 2: + logger.warning(f'Image "{img.name}" seems outdated (acc. to its repl freq) but is not hidden or otherwise marked') + # warnings += 1 + replacement = _find_replacement_image(by_name, img.name) + if replacement is None: + logger.error(f'Image "{img.name}" outdated without replacement') + errors += 1 + else: + logger.info(f'Image "{replacement.name}" is a valid replacement for outdated "{img.name}"') + return not errors diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 9e488bf69..1a9833301 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -71,8 +71,8 @@ modules: name: Image metadata v1 url: https://docs.scs.community/standards/scs-0102-v1-image-metadata run: - - executable: ./iaas/image-metadata/image-md-check.py - args: -c {os_cloud} -v -s + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} image-metadata-check # skip check of mand/recc/sugg images, for these were never authoritative, and they have been # superseded by scs-0104-v1 testcases: From d6619d0bb94263b78ef2451081daf284a50d57f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 26 Aug 2025 22:39:46 +0200 Subject: [PATCH 15/31] adapt check pertaining to scs-0103-standard-flavors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 21 ++++- .../flavor_names_check.py | 5 ++ .../flavor-manager-input.py | 0 .../flavors-openstack.py | 0 .../standard_flavors.py | 80 +++++++++++++++++++ Tests/scs-compatible-iaas.yaml | 4 +- 6 files changed, 107 insertions(+), 3 deletions(-) rename Tests/iaas/{standard-flavors => scs_0103_standard_flavors}/flavor-manager-input.py (100%) rename Tests/iaas/{standard-flavors => scs_0103_standard_flavors}/flavors-openstack.py (100%) create mode 100644 Tests/iaas/scs_0103_standard_flavors/standard_flavors.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 0f146ddd9..de6422f30 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -15,7 +15,8 @@ import openstack from scs_0100_flavor_naming.flavor_names_check import \ - compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check + compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, \ + compute_flavor_spec from scs_0101_entropy.entropy_check import \ compute_scs_0101_image_property, compute_scs_0101_flavor_property, compute_canonical_image, \ compute_collected_vm_output, compute_scs_0101_entropy_avail, compute_scs_0101_rngd, \ @@ -29,6 +30,8 @@ compute_scs_0102_prop_replace_frequency, compute_scs_0102_prop_provided_until, \ compute_scs_0102_prop_uuid_validity, compute_scs_0102_prop_hotfix_hours, \ compute_scs_0102_image_recency +from scs_0103_standard_flavors.standard_flavors import \ + SCS_0103_CANONICAL_NAMES, compute_flavor_lookup, compute_scs_0103_flavor logger = logging.getLogger(__name__) @@ -93,6 +96,22 @@ def make_container(cloud): c.scs_0102_prop_replace_frequency, c.scs_0102_prop_provided_until, c.scs_0102_prop_uuid_validity, c.scs_0102_image_recency, ))) + # scs_0103_standard_flavors + c.add_function('flavor_lookup', lambda c: compute_flavor_lookup(c.flavors)) + for canonical_name in SCS_0103_CANONICAL_NAMES: + nm = canonical_name.removeprefix('SCS-').lower().replace('-', '_') + # NOTE we need cn=canonical_name below because anon function only catches a variable's CELL, not its value + # i.e., if we use canonical_name inside it, we will only get its final value after the loop is done + c.add_function( + f'scs_0103_flavor_{nm}', + lambda c, cn=canonical_name: compute_scs_0103_flavor(c.flavor_lookup, compute_flavor_spec(cn)) + ) + c.add_function('standard_flavors_check', lambda c: all(( + c.scs_0103_flavor_1v_4, c.scs_0103_flavor_2v_8, c.scs_0103_flavor_4v_16, c.scs_0103_flavor_8v_32, + c.scs_0103_flavor_1v_2, c.scs_0103_flavor_2v_4, c.scs_0103_flavor_4v_8, c.scs_0103_flavor_8v_16, + c.scs_0103_flavor_16v_32, c.scs_0103_flavor_1v_8, c.scs_0103_flavor_2v_16, c.scs_0103_flavor_4v_32, + c.scs_0103_flavor_1l_1, c.scs_0103_flavor_2v_4_20s, c.scs_0103_flavor_4v_16_100s, + ))) return c diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py index ae3f661ae..0e48d0813 100644 --- a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py @@ -35,6 +35,11 @@ def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor] return result +def compute_flavor_spec(canonical_name: str) -> dict: + # this is a helper for tying together scs_0100 and scs_0103 + return flavor_names.flavorname_to_dict(flavor_names.parser_v3(canonical_name)) + + def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: problems = [flv.name for flv, flavorname in scs_flavors if not flavorname] if problems: diff --git a/Tests/iaas/standard-flavors/flavor-manager-input.py b/Tests/iaas/scs_0103_standard_flavors/flavor-manager-input.py similarity index 100% rename from Tests/iaas/standard-flavors/flavor-manager-input.py rename to Tests/iaas/scs_0103_standard_flavors/flavor-manager-input.py diff --git a/Tests/iaas/standard-flavors/flavors-openstack.py b/Tests/iaas/scs_0103_standard_flavors/flavors-openstack.py similarity index 100% rename from Tests/iaas/standard-flavors/flavors-openstack.py rename to Tests/iaas/scs_0103_standard_flavors/flavors-openstack.py diff --git a/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py b/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py new file mode 100644 index 000000000..aae58909b --- /dev/null +++ b/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py @@ -0,0 +1,80 @@ +import logging + + +logger = logging.getLogger(__name__) + + +NAME_KEY = "scs:name-v2" +SCS_0103_CANONICAL_NAMES = ( + "SCS-1V-4", + "SCS-2V-8", + "SCS-4V-16", + "SCS-8V-32", + "SCS-1V-2", + "SCS-2V-4", + "SCS-4V-8", + "SCS-8V-16", + "SCS-16V-32", + "SCS-1V-8", + "SCS-2V-16", + "SCS-4V-32", + "SCS-1L-1", + "SCS-2V-4-20s", + "SCS-4V-16-100s", + "SCS-1V-4-10", + "SCS-2V-8-20", + "SCS-4V-16-50", + "SCS-8V-32-100", + "SCS-1V-2-5", + "SCS-2V-4-10", + "SCS-4V-8-20", + "SCS-8V-16-50", + "SCS-16V-32-100", + "SCS-1V-8-20", + "SCS-2V-16-50", + "SCS-4V-32-100", + "SCS-1L-1-5", +) + + +def compute_flavor_lookup(flavors, name_key=NAME_KEY): + # look up via extra_spec given by name_key + by_name = { + flavor.extra_specs[name_key]: flavor + for flavor in flavors + if name_key in flavor.extra_specs + } + # as a fallback, also allow flavor to be found by its actual name + for flavor in flavors: + if not flavor.name.startswith('SCS-'): + continue + by_name.setdefault(flavor.name, flavor) + return by_name + + +def compute_scs_0103_flavor(flavor_lookup, flavor_spec): + canonical_name = flavor_spec[NAME_KEY] + flavor = flavor_lookup.get(canonical_name) + if flavor is None: + logger.error(f'standard flavor missing: {canonical_name}') + return False + # check that flavor matches flavor_spec + # cpu, ram, and disk should match, and they should match precisely for discoverability + # also check extra_specs (ours are prefixed with 'scs:') + comparison = [ + ('vcpus', flavor.vcpus, flavor_spec['cpus']), + ('ram', flavor.ram, 1024 * flavor_spec['ram']), + ('disk', flavor.disk, flavor_spec.get('disk', 0)), + ] + [ + (key, value, flavor.extra_specs.get(key)) + for key, value in flavor_spec.items() + if key.startswith("scs:") + ] + report = [ + f'{key}: {a_val!r} should be {b_val!r}' + for key, a_val, b_val in comparison + if a_val != b_val + ] + if report: + logger.error(f"Flavor '{flavor.name}' violating constraints: {'; '.join(report)}") + return not report diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 1a9833301..a557ce552 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -84,8 +84,8 @@ modules: name: Standard flavors url: https://docs.scs.community/standards/scs-0103-v1-standard-flavors run: - - executable: ./iaas/standard-flavors/flavors-openstack.py - args: -c {os_cloud} -d ./iaas/scs-0103-v1-flavors.yaml + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} standard-flavors-check testcases: - id: standard-flavors-check tags: [mandatory] From 0cb500b16603a0fc7577a7724dd2e4c5c195103f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 16:49:03 +0200 Subject: [PATCH 16/31] Adapt check pertaining to scs-0104-standard-images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 50 +++++++++++- .../images-openstack.py | 0 .../standard_images.py | 81 +++++++++++++++++++ Tests/scs-compatible-iaas.yaml | 23 ++++-- 4 files changed, 146 insertions(+), 8 deletions(-) rename Tests/iaas/{standard-images => scs_0104_standard_images}/images-openstack.py (100%) create mode 100644 Tests/iaas/scs_0104_standard_images/standard_images.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index de6422f30..0e8753ea4 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -32,6 +32,8 @@ compute_scs_0102_image_recency from scs_0103_standard_flavors.standard_flavors import \ SCS_0103_CANONICAL_NAMES, compute_flavor_lookup, compute_scs_0103_flavor +from scs_0104_standard_images.standard_images import \ + SCS_0104_IMAGE_SPECS, compute_scs_0104_source, compute_scs_0104_image logger = logging.getLogger(__name__) @@ -51,7 +53,8 @@ def make_container(cloud): # basic support attributes shared by multiple testcases c.add_function('conn', lambda _: openstack.connect(cloud=cloud, timeout=32)) c.add_function('flavors', lambda c: list(c.conn.list_flavors(get_extra=True))) - c.add_function('images', lambda c: [img for img in c.conn.list_images() if img.visibility in ('public', 'community')]) + c.add_function('images', lambda c: [img for img in c.conn.list_images(show_all=True) if img.visibility in ('public', 'community')]) + c.add_function('image_lookup', lambda c: {img.name: img for img in c.images}) # scs_0100_flavor_naming c.add_function('scs_flavors', lambda c: compute_scs_flavors(c.flavors)) c.add_function('scs_0100_syntax_check', lambda c: compute_scs_0100_syntax_check(c.scs_flavors)) @@ -112,6 +115,39 @@ def make_container(cloud): c.scs_0103_flavor_16v_32, c.scs_0103_flavor_1v_8, c.scs_0103_flavor_2v_16, c.scs_0103_flavor_4v_32, c.scs_0103_flavor_1l_1, c.scs_0103_flavor_2v_4_20s, c.scs_0103_flavor_4v_16_100s, ))) + # scs_0104_standard_images + c.add_function('scs_0104_source_capi_1', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['ubuntu-capi-image-1'])) + c.add_function('scs_0104_source_capi_2', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['ubuntu-capi-image-2'])) + c.add_function('scs_0104_source_ubuntu_2404', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Ubuntu 24.04'])) + c.add_function('scs_0104_source_ubuntu_2204', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Ubuntu 22.04'])) + c.add_function('scs_0104_source_ubuntu_2004', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Ubuntu 20.04'])) + c.add_function('scs_0104_source_debian_13', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 13'])) + c.add_function('scs_0104_source_debian_12', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 12'])) + c.add_function('scs_0104_source_debian_11', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 11'])) + c.add_function('scs_0104_source_debian_10', lambda c: compute_scs_0104_source(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 10'])) + c.add_function('scs_0104_image_capi_1', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['ubuntu-capi-image-1'])) + c.add_function('scs_0104_image_capi_2', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['ubuntu-capi-image-2'])) + c.add_function('scs_0104_image_ubuntu_2404', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Ubuntu 24.04'])) + c.add_function('scs_0104_image_ubuntu_2204', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Ubuntu 22.04'])) + c.add_function('scs_0104_image_ubuntu_2004', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Ubuntu 20.04'])) + c.add_function('scs_0104_image_debian_13', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 13'])) + c.add_function('scs_0104_image_debian_12', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 12'])) + c.add_function('scs_0104_image_debian_11', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 11'])) + c.add_function('scs_0104_image_debian_10', lambda c: compute_scs_0104_image(c.image_lookup, SCS_0104_IMAGE_SPECS['Debian 10'])) + # NOTE the following variant is correct for SCS-compatible IaaS v4 only + c.add_function('standard_images_check_1', lambda c: all(( + c.scs_0104_image_ubuntu_2204, + c.scs_0104_source_capi_1, c.scs_0104_source_capi_2, + c.scs_0104_source_ubuntu_2404, c.scs_0104_source_ubuntu_2204, c.scs_0104_source_ubuntu_2004, + c.scs_0104_source_debian_13, c.scs_0104_source_debian_12, c.scs_0104_source_debian_11, c.scs_0104_source_debian_10, + ))) + # NOTE the following variant is correct for SCS-compatible IaaS v5.1 only + c.add_function('standard_images_check_2', lambda c: all(( + c.scs_0104_image_ubuntu_2404, + c.scs_0104_source_capi_1, c.scs_0104_source_capi_2, + c.scs_0104_source_ubuntu_2404, c.scs_0104_source_ubuntu_2204, c.scs_0104_source_ubuntu_2004, + c.scs_0104_source_debian_13, c.scs_0104_source_debian_12, c.scs_0104_source_debian_11, c.scs_0104_source_debian_10, + ))) return c @@ -213,7 +249,14 @@ def main(argv): else: usage(2) - testcases = [t for t in args if t.endswith('-check') or t.startswith('scs-')] + # NOTE For historic reasons, there is precisely one testcase id, namely standard-images-check, + # whose meaning depends on the version of the certificate scope in question. We are in the process + # of transitioning away from this anti-feature. For the time being, however, we use the following + # hack to support it: One testcase may be implemented in multiple variants, denoted by suffixing + # a number, as in standard-images-check/1, standard-images-check/2. + # NOTE Also, the historic testcases have terrible naming. We will transition towards names + # that encode the corresponding standard, such as scs-0104-image-ubuntu-2404. + testcases = [t for t in args if t.rsplit('/', 1)[0].endswith('-check') or t.startswith('scs-')] if len(testcases) != len(args): unknown = [a for a in args if a not in testcases] logger.warning(f"ignoring unknown testcases: {','.join(unknown)}") @@ -224,7 +267,8 @@ def main(argv): c = make_container(cloud) for testcase in testcases: - harness(testcase, lambda: getattr(c, testcase.replace('-', '_'))) + testcase_name = testcase.rsplit('/', 1)[0] # see the note above + harness(testcase_name, lambda: getattr(c, testcase.replace('-', '_').replace('/', '_'))) return 0 diff --git a/Tests/iaas/standard-images/images-openstack.py b/Tests/iaas/scs_0104_standard_images/images-openstack.py similarity index 100% rename from Tests/iaas/standard-images/images-openstack.py rename to Tests/iaas/scs_0104_standard_images/images-openstack.py diff --git a/Tests/iaas/scs_0104_standard_images/standard_images.py b/Tests/iaas/scs_0104_standard_images/standard_images.py new file mode 100644 index 000000000..72007744f --- /dev/null +++ b/Tests/iaas/scs_0104_standard_images/standard_images.py @@ -0,0 +1,81 @@ +import logging +import re + +import yaml + + +logger = logging.getLogger(__name__) + + +_specs = yaml.safe_load(r""" +"ubuntu-capi-image-1": + name_scheme: "ubuntu-capi-image v[0-9]\\.[0-9]+(\\.[0-9]+)?" + source: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube +"ubuntu-capi-image-2": + # this name_scheme uses `-` to separate base name "ubuntu-capi-image" from version + # latest openstack-image-manager can be told to use `-` by setting `separator: "-"` on the image + name_scheme: "ubuntu-capi-image-v[0-9]\\.[0-9]+(\\.[0-9]+)?" + source: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube +"Ubuntu 24.04": + source: + - https://cloud-images.ubuntu.com/releases/noble/ + - https://cloud-images.ubuntu.com/noble/ +"Ubuntu 22.04": + source: + - https://cloud-images.ubuntu.com/releases/jammy/ + - https://cloud-images.ubuntu.com/jammy/ +"Ubuntu 20.04": + source: + - https://cloud-images.ubuntu.com/releases/focal/ + - https://cloud-images.ubuntu.com/focal/ +"Debian 13": + source: + - https://cloud.debian.org/images/cloud/trixie/ + - https://cdimage.debian.org/cdimage/cloud/trixie/ +"Debian 12": + source: + - https://cloud.debian.org/images/cloud/bookworm/ + - https://cdimage.debian.org/cdimage/cloud/bookworm/ +"Debian 11": + source: + - https://cloud.debian.org/images/cloud/bullseye/ + - https://cdimage.debian.org/cdimage/cloud/bullseye/ +"Debian 10": + source: + - https://cloud.debian.org/images/cloud/buster/ + - https://cdimage.debian.org/cdimage/cloud/buster/ +""") +SCS_0104_IMAGE_SPECS = {key: {'name': key, **val} for key, val in _specs.items()} + + +def _lookup_images(image_lookup, image_spec): + name_scheme = image_spec.get('name_scheme') + if name_scheme: + rex = re.compile(name_scheme) + return [img for name, img in image_lookup.items() if rex.match(name)] + img = image_lookup.get(image_spec['name']) + if img is None: + return [] + return [img] + + +def compute_scs_0104_source(image_lookup, image_spec): + matches = _lookup_images(image_lookup, image_spec) + errors = 0 + for image in matches: + img_source = image.properties['image_source'] + sources = image_spec['source'] + if not isinstance(sources, (tuple, list)): + sources = [sources] + if not any(img_source.startswith(src) for src in sources): + errors += 1 + logger.error(f"Image '{image.name}' source mismatch: {img_source} matches none of these prefixes: {', '.join(sources)}") + return not errors + + +def compute_scs_0104_image(image_lookup, image_spec): + matches = _lookup_images(image_lookup, image_spec) + if not matches: + logger.error(f"Missing image '{image_spec['name']}'") + return False + return True diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index a557ce552..2fe063c71 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -91,14 +91,27 @@ modules: tags: [mandatory] description: > Must fulfill all requirements of - - id: scs-0104-v1 + - id: scs-0104-v1-1 name: Standard images url: https://docs.scs.community/standards/scs-0104-v1-standard-images parameters: image_spec: address (URL) of an image-spec (YAML) file run: - - executable: ./iaas/standard-images/images-openstack.py - args: -c {os_cloud} -d {image_spec} + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} standard-images-check/1 + testcases: + - id: standard-images-check + tags: [mandatory] + description: > + Must fulfill all requirements of + - id: scs-0104-v1-2 + name: Standard images + url: https://docs.scs.community/standards/scs-0104-v1-standard-images + parameters: + image_spec: address (URL) of an image-spec (YAML) file + run: + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} standard-images-check/2 testcases: - id: standard-images-check tags: [mandatory] @@ -207,7 +220,7 @@ versions: - scs-0101-v1 - scs-0102-v1 - scs-0103-v1 - - ref: scs-0104-v1 + - ref: scs-0104-v1-2 parameters: image_spec: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/iaas/scs-0104-v1-images-v5.yaml - scs-0114-v1 @@ -228,7 +241,7 @@ versions: - scs-0101-v1 - scs-0102-v1 - scs-0103-v1 - - ref: scs-0104-v1 + - ref: scs-0104-v1-1 parameters: image_spec: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/iaas/scs-0104-v1-images.yaml targets: From 81e6b6cec3335f4dde98dda9707740fe58305850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 20:36:24 +0200 Subject: [PATCH 17/31] adapt checks pertaining to scs-0114-volume-types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 12 +++ .../volume-types-check.py | 0 .../scs_0114_volume_types/volume_types.py | 80 +++++++++++++++++++ Tests/scs-compatible-iaas.yaml | 4 +- 4 files changed, 94 insertions(+), 2 deletions(-) rename Tests/iaas/{volume-types => scs_0114_volume_types}/volume-types-check.py (100%) create mode 100644 Tests/iaas/scs_0114_volume_types/volume_types.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 0e8753ea4..e5225eb76 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -34,6 +34,8 @@ SCS_0103_CANONICAL_NAMES, compute_flavor_lookup, compute_scs_0103_flavor from scs_0104_standard_images.standard_images import \ SCS_0104_IMAGE_SPECS, compute_scs_0104_source, compute_scs_0104_image +from scs_0114_volume_types.volume_types import \ + compute_volume_type_lookup, compute_scs_0114_syntax_check, compute_scs_0114_aspect_type logger = logging.getLogger(__name__) @@ -148,6 +150,16 @@ def make_container(cloud): c.scs_0104_source_ubuntu_2404, c.scs_0104_source_ubuntu_2204, c.scs_0104_source_ubuntu_2004, c.scs_0104_source_debian_13, c.scs_0104_source_debian_12, c.scs_0104_source_debian_11, c.scs_0104_source_debian_10, ))) + # scs_0114_volume_types + c.add_function('volume_types', lambda c: c.conn.list_volume_types()) + c.add_function('volume_type_lookup', lambda c: compute_volume_type_lookup(c.volume_types)) + c.add_function('scs_0114_syntax_check', lambda c: compute_scs_0114_syntax_check(c.volume_type_lookup)) + c.add_function('scs_0114_encrypted_type', lambda c: compute_scs_0114_aspect_type(c.volume_type_lookup, 'encrypted')) + c.add_function('scs_0114_replicated_type', lambda c: compute_scs_0114_aspect_type(c.volume_type_lookup, 'replicated')) + c.add_function('volume_types_check', lambda c: all(( + c.scs_0114_syntax_check, + c.scs_0114_encrypted_type, c.scs_0114_replicated_type, + ))) return c diff --git a/Tests/iaas/volume-types/volume-types-check.py b/Tests/iaas/scs_0114_volume_types/volume-types-check.py similarity index 100% rename from Tests/iaas/volume-types/volume-types-check.py rename to Tests/iaas/scs_0114_volume_types/volume-types-check.py diff --git a/Tests/iaas/scs_0114_volume_types/volume_types.py b/Tests/iaas/scs_0114_volume_types/volume_types.py new file mode 100644 index 000000000..5a74c097b --- /dev/null +++ b/Tests/iaas/scs_0114_volume_types/volume_types.py @@ -0,0 +1,80 @@ +from collections import Counter, defaultdict +import logging +import re +import typing + + +logger = logging.getLogger(__name__) +RECOGNIZED_FEATURES = ('encrypted', 'replicated') +ERRONEOUS = object() # use this as key in volume_type_lookup for erroneous volume types + + +def _extract_feature_list(description, pattern=re.compile(r"\[scs:([^\[\]]*)\]")): + """Extract feature-list-like prefix + + If given `description` starts with a feature-list-like prefix, return list of features, + otherwise None. To be more precise, we look for a string of this form: + + `[scs:`feat1`, `...`, `...featN`]` + + where N >= 1 and featJ is a string that doesn't contain any comma or brackets. We return + the list [feat1, ..., featN] of substrings. + """ + if not description: + # The description can be None or empty - we need to catch this here + return + match = pattern.match(description) + if not match: + return + fs = match.group(1) + if not fs: + return [] + return [f.strip() for f in fs.split(",")] + + +def _test_feature_list(type_name: str, fl: typing.List[str], recognized=RECOGNIZED_FEATURES) -> typing.List[str]: + """Test given list of features and return list of errors""" + if not fl: + # either None (no feature list) or empty feature list: nothing to check + return + errors = [] + if fl != sorted(fl): + errors.append(f"{type_name}: feature list not sorted") + ctr = Counter(fl) + duplicates = [key for key, c in ctr.items() if c > 1] + if duplicates: + errors.append(f"{type_name}: duplicate features: {', '.join(duplicates)}") + unrecognized = [f for f in ctr if f not in recognized] + if unrecognized: + errors.append(f"{type_name}: unrecognized features: {', '.join(unrecognized)}") + return errors + + +def compute_volume_type_lookup(volume_types): + # collect volume types according to features + by_feature = defaultdict(list) + for typ in volume_types: + fl = _extract_feature_list(typ.description) + if fl is None: + continue + logger.debug(f"{typ.name}: feature list {fl!r}") + errors = _test_feature_list(typ.name, fl) + if errors: + by_feature[ERRONEOUS].extend(errors) + for feat in fl: + by_feature[feat].append(typ.name) + return by_feature + + +def compute_scs_0114_syntax_check(volume_type_lookup): + errors = volume_type_lookup.get(ERRONEOUS, ()) + for line in errors: + logger.error(line) + return not errors + + +def compute_scs_0114_aspect_type(volume_type_lookup, aspect): + applicable = volume_type_lookup[aspect] + if not applicable: + logger.error(f"no volume type having aspect {aspect}") + return bool(applicable) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 2fe063c71..ba11606b2 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -121,8 +121,8 @@ modules: name: Volume Types url: https://docs.scs.community/standards/scs-0114-v1-volume-type-standard run: - - executable: ./iaas/volume-types/volume-types-check.py - args: -c {os_cloud} -d + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} volume-types-check testcases: - id: volume-types-check tags: [volume-types-check] From e1810416205e634430710022ca2f1fcb23118b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 20:50:46 +0200 Subject: [PATCH 18/31] Adapt check pertaining to scs-0115-security-groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 5 + .../default-security-group-rules.py | 0 .../security_groups.py | 117 ++++++++++++++++++ Tests/scs-compatible-iaas.yaml | 4 +- 4 files changed, 124 insertions(+), 2 deletions(-) rename Tests/iaas/{security-groups => scs_0115_security_groups}/default-security-group-rules.py (100%) create mode 100644 Tests/iaas/scs_0115_security_groups/security_groups.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index e5225eb76..4f2ea7e56 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -36,6 +36,8 @@ SCS_0104_IMAGE_SPECS, compute_scs_0104_source, compute_scs_0104_image from scs_0114_volume_types.volume_types import \ compute_volume_type_lookup, compute_scs_0114_syntax_check, compute_scs_0114_aspect_type +from scs_0115_security_groups.security_groups import \ + compute_scs_0115_default_rules logger = logging.getLogger(__name__) @@ -160,6 +162,9 @@ def make_container(cloud): c.scs_0114_syntax_check, c.scs_0114_encrypted_type, c.scs_0114_replicated_type, ))) + # scs_0115_security_groups + c.add_function('scs_0115_default_rules', lambda c: compute_scs_0115_default_rules(c.conn)) + c.add_function('security_groups_default_rules_check', lambda c: c.scs_0115_default_rules) return c diff --git a/Tests/iaas/security-groups/default-security-group-rules.py b/Tests/iaas/scs_0115_security_groups/default-security-group-rules.py similarity index 100% rename from Tests/iaas/security-groups/default-security-group-rules.py rename to Tests/iaas/scs_0115_security_groups/default-security-group-rules.py diff --git a/Tests/iaas/scs_0115_security_groups/security_groups.py b/Tests/iaas/scs_0115_security_groups/security_groups.py new file mode 100644 index 000000000..8e5415684 --- /dev/null +++ b/Tests/iaas/scs_0115_security_groups/security_groups.py @@ -0,0 +1,117 @@ +from collections import Counter +import logging +import os +import sys + +import openstack +from openstack.exceptions import ResourceNotFound + +logger = logging.getLogger(__name__) + +SG_NAME = "scs-test-default-sg" +DESCRIPTION = "scs-test-default-sg" + + +def check_default_rules(rules, short=False): + """ + counts all verall ingress rules and egress rules, depending on the requested testing mode + + :param bool short + if short is True, the testing mode is set on short for older OpenStack versions + """ + ingress_rules = egress_rules = 0 + egress_vars = {'IPv4': {}, 'IPv6': {}} + for key, value in egress_vars.items(): + value['default'] = 0 + if not short: + value['custom'] = 0 + if not rules: + logger.info("No default security group rules defined.") + for rule in rules: + direction = rule["direction"] + ethertype = rule["ethertype"] + if direction == "ingress": + if not short: + # we allow ingress from the same security group + # but only for the default security group + if rule.remote_group_id == "PARENT" and not rule["used_in_non_default_sg"]: + continue + ingress_rules += 1 + elif direction == "egress" and ethertype in egress_vars: + egress_rules += 1 + if short: + egress_vars[ethertype]['default'] += 1 + continue + if rule.remote_ip_prefix: + # this rule does not allow traffic to all external ips + continue + # note: these two are not mutually exclusive + if rule["used_in_default_sg"]: + egress_vars[ethertype]['default'] += 1 + if rule["used_in_non_default_sg"]: + egress_vars[ethertype]['custom'] += 1 + # test whether there are no unallowed ingress rules + if ingress_rules: + logger.error(f"Expected no default ingress rules, found {ingress_rules}.") + # test whether all expected egress rules are present + missing = [(key, key2) for key, val in egress_vars.items() for key2, val2 in val.items() if not val2] + if missing: + logger.error( + "Expected rules for egress for IPv4 and IPv6 both for default and custom security groups. " + f"Missing rule types: {', '.join(str(x) for x in missing)}" + ) + logger.info(str({ + "Unallowed Ingress Rules": ingress_rules, + "Egress Rules": egress_rules, + })) + return not ingress_rules and not missing + + +def create_security_group(conn, sg_name: str = SG_NAME, description: str = DESCRIPTION): + """Create security group in openstack + + :returns: + ~openstack.network.v2.security_group.SecurityGroup: The new security group or None + """ + sg = conn.network.create_security_group(name=sg_name, description=description) + return sg.id + + +def delete_security_group(conn, sg_id): + conn.network.delete_security_group(sg_id) + # in case of a successful delete finding the sg will throw an exception + try: + conn.network.find_security_group(name_or_id=sg_id) + except ResourceNotFound: + logger.debug(f"Security group {sg_id} was deleted successfully.") + except Exception: + logger.critical(f"Security group {sg_id} was not deleted successfully") + raise + + +def altern_test_rules(connection: openstack.connection.Connection): + sg_id = create_security_group(connection) + try: + sg = connection.network.find_security_group(name_or_id=sg_id) + return check_default_rules(sg.security_group_rules, short=True) + finally: + delete_security_group(connection, sg_id) + + +def compute_scs_0115_default_rules(conn: openstack.connection.Connection): + try: + rules = list(conn.network.default_security_group_rules()) + except (ResourceNotFound, AttributeError) as exc: + # older versions of OpenStack don't have the endpoint and give ResourceNotFound + if isinstance(exc, ResourceNotFound) and 'default-security-group-rules' not in str(exc): + raise + # why we see the AttributeError in some environments is a mystery + if isinstance(exc, AttributeError) and 'default_security_group_rules' not in str(exc): + raise + logger.info( + "API call failed. OpenStack components might not be up to date. " + "Falling back to old-style test method. " + ) + return altern_test_rules(conn) + else: + return check_default_rules(rules) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index ba11606b2..abd89489c 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -132,8 +132,8 @@ modules: name: Default rules for security groups url: https://docs.scs.community/standards/scs-0115-v1-default-rules-for-security-groups run: - - executable: ./iaas/security-groups/default-security-group-rules.py - args: --os-cloud {os_cloud} --debug + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} security-groups-default-rules-check testcases: - id: security-groups-default-rules-check tags: [mandatory] From f1a9170c386543c4c07f2b5ac327d96ef15aadf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 20:51:53 +0200 Subject: [PATCH 19/31] Pacify flake8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/scs_0115_security_groups/security_groups.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Tests/iaas/scs_0115_security_groups/security_groups.py b/Tests/iaas/scs_0115_security_groups/security_groups.py index 8e5415684..52c409635 100644 --- a/Tests/iaas/scs_0115_security_groups/security_groups.py +++ b/Tests/iaas/scs_0115_security_groups/security_groups.py @@ -1,11 +1,9 @@ -from collections import Counter import logging -import os -import sys import openstack from openstack.exceptions import ResourceNotFound + logger = logging.getLogger(__name__) SG_NAME = "scs-test-default-sg" From 64dc965f1a22361759f0a52c0877dedeb609f359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 21:12:34 +0200 Subject: [PATCH 20/31] Adapt check pertaining to scs-0116-key-manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 11 ++ .../check-for-key-manager.py | 0 .../iaas/scs_0116_key_manager/key_manager.py | 119 ++++++++++++++++++ Tests/scs-compatible-iaas.yaml | 4 +- 4 files changed, 132 insertions(+), 2 deletions(-) rename Tests/iaas/{key-manager => scs_0116_key_manager}/check-for-key-manager.py (100%) create mode 100644 Tests/iaas/scs_0116_key_manager/key_manager.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 4f2ea7e56..84e4c157f 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -38,6 +38,8 @@ compute_volume_type_lookup, compute_scs_0114_syntax_check, compute_scs_0114_aspect_type from scs_0115_security_groups.security_groups import \ compute_scs_0115_default_rules +from scs_0116_key_manager.key_manager import \ + compute_services_lookup, compute_scs_0116_presence, compute_scs_0116_permissions logger = logging.getLogger(__name__) @@ -160,11 +162,20 @@ def make_container(cloud): c.add_function('scs_0114_replicated_type', lambda c: compute_scs_0114_aspect_type(c.volume_type_lookup, 'replicated')) c.add_function('volume_types_check', lambda c: all(( c.scs_0114_syntax_check, + # the following is recommended only, but we treat this whole monolithic testcase as recommended c.scs_0114_encrypted_type, c.scs_0114_replicated_type, ))) # scs_0115_security_groups c.add_function('scs_0115_default_rules', lambda c: compute_scs_0115_default_rules(c.conn)) c.add_function('security_groups_default_rules_check', lambda c: c.scs_0115_default_rules) + # scs_0115_key_manager + c.add_function('services_lookup', lambda c: compute_services_lookup(c.conn)) + c.add_function('scs_0116_presence', lambda c: compute_scs_0116_presence(c.services_lookup)) + c.add_function('scs_0116_permissions', lambda c: compute_scs_0116_permissions(c.conn, c.services_lookup)) + c.add_function('key_manager_check', lambda c: all(( + # recommended only: c.scs_0116_presence, + c.scs_0116_permissions, + ))) return c diff --git a/Tests/iaas/key-manager/check-for-key-manager.py b/Tests/iaas/scs_0116_key_manager/check-for-key-manager.py similarity index 100% rename from Tests/iaas/key-manager/check-for-key-manager.py rename to Tests/iaas/scs_0116_key_manager/check-for-key-manager.py diff --git a/Tests/iaas/scs_0116_key_manager/key_manager.py b/Tests/iaas/scs_0116_key_manager/key_manager.py new file mode 100644 index 000000000..5b01ea783 --- /dev/null +++ b/Tests/iaas/scs_0116_key_manager/key_manager.py @@ -0,0 +1,119 @@ +from collections import defaultdict +import logging + +import openstack + + +logger = logging.getLogger(__name__) + + +def fetch_roles(conn: openstack.connection.Connection) -> None: + """Checks whether the current user has at maximum privileges of the member role. + + :param conn: connection to an OpenStack cloud. + :returns: boolean, when role with most privileges is member + """ + role_names = set(conn.session.auth.get_access(conn.session).role_names) + if role_names & {"admin", "manager"}: + return False + if "reader" in role_names: + logger.info("User has reader role.") + custom_roles = sorted(role_names - {"reader", "member"}) + if custom_roles: + logger.info(f"User has custom roles {', '.join(custom_roles)}.") + return role_names + + +def compute_services_lookup(conn: openstack.connection.Connection) -> dict: + try: + services = conn.service_catalog + except Exception: + logger.critical("Could not access Catalog endpoint.") + raise + + result = defaultdict(list) + for svc in services: + svc_type = svc["type"] + result[svc_type].append(svc) + return result + + +def compute_scs_0116_presence(services_lookup): + services = services_lookup.get("key-manager", ()) + if not services: + logger.error("key-manager service not found") + return bool(services) + + +def _find_secrets(conn: openstack.connection.Connection, secret_name_or_id: str) -> list: + """Replacement method for finding secrets. + + Mimicks the behavior of Connection.key_manager.find_secret() + but fixes an issue with the internal implementation raising an + exception due to an unexpected microversion parameter. + Unlike find_secret(), we return a list with all secrets that match. + """ + secrets = conn.key_manager.secrets() + return [s for s in secrets if s.name == secret_name_or_id or s.id == secret_name_or_id] + + +def _delete_secret(conn: openstack.connection.Connection, secret: openstack.key_manager.v1.secret.Secret): + """Replacement method for deleting secrets + _delete_secret(connection, secret object) + + Workaround for SDK bugs: + - The id field in reality is a href (containg the UUID at the end) + - The delete_secret() function contrary to the documentation does + not accept openstack.key_manager.v1.secret.Secret objects nor the + hrefs, just plain UUIDs. + - It does not return an error when I try to delete a secret passing + an object or href, just silently does nothing. + The code here assumes that the SDK (when fixed) will continue to + accept UUIDs as argument for delete_secret() in the future. + Code is robust against those being passed directly in the .id attr + of the objects. (It would be even more robust to try to pass the + object first, then the href, then the UUID extracted from the href, + each time checking whether it was effective. But that's three delete + plus list calls and very ugly.) + """ + uuid = secret.id.rsplit('/', 1)[-1] + conn.key_manager.delete_secret(uuid) + + +def compute_scs_0116_permissions(conn: openstack.connection.Connection, services_lookup) -> None: + """ + After checking that the current user only has the member and maybe the + reader role, this method verifies that the user with a member role + has sufficient access to the Key Manager API functionality. + """ + if "member" not in fetch_roles(conn): + raise RuntimeError("Cannot test key-manager permissions. User has wrong roles") + if not services_lookup.get("key-manager", ()): + # this testcase only applies when a key manager is present + return True + secret_name = "scs-member-role-test-secret" + try: + existing_secrets = _find_secrets(conn, secret_name) + for secret in existing_secrets: + _delete_secret(conn, secret) + + if existing_secrets: + logger.debug(f'Deleted {len(existing_secrets)} secrets') + + conn.key_manager.create_secret( + name=secret_name, + payload_content_type="text/plain", + secret_type="opaque", + payload="foo", + ) + try: + new_secret = _find_secrets(conn, secret_name) + if not new_secret: + raise RuntimeError(f"Secret '{secret_name}' was not discoverable by the user") + finally: + _delete_secret(conn, new_secret[0]) + except (RuntimeError, openstack.exceptions.ForbiddenException): + logger.debug('exception details', exc_info=True) + logger.error("Unsuccessful at using Key Manager API") + return False + return True diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index abd89489c..3e2fda26a 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -143,8 +143,8 @@ modules: name: Key manager url: https://docs.scs.community/standards/scs-0116-v1-key-manager-standard run: - - executable: ./iaas/key-manager/check-for-key-manager.py - args: --os-cloud {os_cloud} --debug + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} key-manager-check testcases: - id: key-manager-check tags: [mandatory] From fbcb7a9ded734debe304e0ae84822fe557a51edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 21:28:49 +0200 Subject: [PATCH 21/31] Adapt check pertaining to scs-0117-volume-backup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 9 +- .../README.md | 0 .../volume-backup-tester.py | 90 +++++++++++++++ .../volume_backup.py} | 103 +++--------------- 4 files changed, 111 insertions(+), 91 deletions(-) rename Tests/iaas/{volume-backup => scs_0117_volume_backup}/README.md (100%) create mode 100755 Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py rename Tests/iaas/{volume-backup/volume-backup-tester.py => scs_0117_volume_backup/volume_backup.py} (69%) mode change 100755 => 100644 diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 84e4c157f..00b4f791a 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -40,6 +40,8 @@ compute_scs_0115_default_rules from scs_0116_key_manager.key_manager import \ compute_services_lookup, compute_scs_0116_presence, compute_scs_0116_permissions +from scs_0117_volume_backup.volume_backup import \ + compute_scs_0117_test_backup logger = logging.getLogger(__name__) @@ -168,7 +170,7 @@ def make_container(cloud): # scs_0115_security_groups c.add_function('scs_0115_default_rules', lambda c: compute_scs_0115_default_rules(c.conn)) c.add_function('security_groups_default_rules_check', lambda c: c.scs_0115_default_rules) - # scs_0115_key_manager + # scs_0116_key_manager c.add_function('services_lookup', lambda c: compute_services_lookup(c.conn)) c.add_function('scs_0116_presence', lambda c: compute_scs_0116_presence(c.services_lookup)) c.add_function('scs_0116_permissions', lambda c: compute_scs_0116_permissions(c.conn, c.services_lookup)) @@ -176,6 +178,11 @@ def make_container(cloud): # recommended only: c.scs_0116_presence, c.scs_0116_permissions, ))) + # scs_0117_volume_backup + c.add_function('scs_0117_test_backup', lambda c: compute_scs_0117_test_backup(c.conn)) + c.add_function('volume_backup_check', lambda c: all(( + c.scs_0117_test_backup, + ))) return c diff --git a/Tests/iaas/volume-backup/README.md b/Tests/iaas/scs_0117_volume_backup/README.md similarity index 100% rename from Tests/iaas/volume-backup/README.md rename to Tests/iaas/scs_0117_volume_backup/README.md diff --git a/Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py b/Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py new file mode 100755 index 000000000..6f57004e4 --- /dev/null +++ b/Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Volume Backup API tester for Block Storage API + +This test script executes basic operations on the Block Storage API centered +around volume backups. Its purpose is to verify that the Volume Backup API is +available and working as expected using simple operations such as creating and +restoring volume backups. + +It verifies that a properly configured backup driver is present to the extent +that aforementioned operations succeed on the API level. It does not by any +means verify that the backup and restore procedures actual handle the data +correctly (it only uses empty volumes and does not look at data for the sake +of simplicity). +""" + +import argparse +import getpass +import logging +import os +import sys + +import openstack + + +from volume_backup import DEFAULT_PREFIX, cleanup, compute_scs_0117_test_backup + + +def main(): + parser = argparse.ArgumentParser( + description="SCS Volume Backup API Conformance Checker") + parser.add_argument( + "--os-cloud", type=str, + help="Name of the cloud from clouds.yaml, alternative " + "to the OS_CLOUD environment variable" + ) + parser.add_argument( + "--ask", + help="Ask for password interactively instead of reading it from the " + "clouds.yaml", + action="store_true" + ) + parser.add_argument( + "--debug", action="store_true", + help="Enable OpenStack SDK debug logging" + ) + parser.add_argument( + "--prefix", type=str, + default=DEFAULT_PREFIX, + help=f"OpenStack resource name prefix for all resources to be created " + f"and/or cleaned up by this script within the configured domains " + f"(default: '{DEFAULT_PREFIX}')" + ) + parser.add_argument( + "--cleanup-only", action="store_true", + help="Instead of executing tests, cleanup all resources " + "with the prefix specified via '--prefix' (or its default)" + ) + args = parser.parse_args() + openstack.enable_logging(debug=False) + logging.basicConfig( + format="%(levelname)s: %(message)s", + level=logging.DEBUG if args.debug else logging.INFO, + ) + + # parse cloud name for lookup in clouds.yaml + cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) + if not cloud: + raise Exception( + "You need to have the OS_CLOUD environment variable set to your " + "cloud name or pass it via --os-cloud" + ) + password = getpass.getpass("Enter password: ") if args.ask else None + + with openstack.connect(cloud, password=password) as conn: + if args.cleanup_only: + if not cleanup(conn, prefix=args.prefix): + raise RuntimeError("cleanup failed") + else: + compute_scs_0117_test_backup(conn, prefix=args.prefix) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except SystemExit: + raise + except BaseException as exc: + logging.debug("traceback", exc_info=True) + logging.critical(str(exc)) + sys.exit(1) diff --git a/Tests/iaas/volume-backup/volume-backup-tester.py b/Tests/iaas/scs_0117_volume_backup/volume_backup.py old mode 100755 new mode 100644 similarity index 69% rename from Tests/iaas/volume-backup/volume-backup-tester.py rename to Tests/iaas/scs_0117_volume_backup/volume_backup.py index 0465bf244..ff5494997 --- a/Tests/iaas/volume-backup/volume-backup-tester.py +++ b/Tests/iaas/scs_0117_volume_backup/volume_backup.py @@ -1,29 +1,11 @@ -#!/usr/bin/env python3 -"""Volume Backup API tester for Block Storage API - -This test script executes basic operations on the Block Storage API centered -around volume backups. Its purpose is to verify that the Volume Backup API is -available and working as expected using simple operations such as creating and -restoring volume backups. - -It verifies that a properly configured backup driver is present to the extent -that aforementioned operations succeed on the API level. It does not by any -means verify that the backup and restore procedures actual handle the data -correctly (it only uses empty volumes and does not look at data for the sake -of simplicity). -""" - -import argparse from functools import partial -import getpass import logging -import os -import sys import time import typing import openstack + # prefix to be included in the names of any Keystone resources created # used by the cleanup routine to identify resources that can be safely deleted DEFAULT_PREFIX = "scs-test-" @@ -221,75 +203,16 @@ def cleanup(conn: openstack.connection.Connection, prefix=DEFAULT_PREFIX) -> boo return not cleanup_issues -def main(): - parser = argparse.ArgumentParser( - description="SCS Volume Backup API Conformance Checker") - parser.add_argument( - "--os-cloud", type=str, - help="Name of the cloud from clouds.yaml, alternative " - "to the OS_CLOUD environment variable" - ) - parser.add_argument( - "--ask", - help="Ask for password interactively instead of reading it from the " - "clouds.yaml", - action="store_true" - ) - parser.add_argument( - "--debug", action="store_true", - help="Enable OpenStack SDK debug logging" - ) - parser.add_argument( - "--prefix", type=str, - default=DEFAULT_PREFIX, - help=f"OpenStack resource name prefix for all resources to be created " - f"and/or cleaned up by this script within the configured domains " - f"(default: '{DEFAULT_PREFIX}')" - ) - parser.add_argument( - "--cleanup-only", action="store_true", - help="Instead of executing tests, cleanup all resources " - "with the prefix specified via '--prefix' (or its default)" - ) - args = parser.parse_args() - openstack.enable_logging(debug=False) - logging.basicConfig( - format="%(levelname)s: %(message)s", - level=logging.DEBUG if args.debug else logging.INFO, - ) - - # parse cloud name for lookup in clouds.yaml - cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) - if not cloud: - raise Exception( - "You need to have the OS_CLOUD environment variable set to your " - "cloud name or pass it via --os-cloud" - ) - password = getpass.getpass("Enter password: ") if args.ask else None - - with openstack.connect(cloud, password=password) as conn: - if not cleanup(conn, prefix=args.prefix): - raise RuntimeError("Initial cleanup failed") - if args.cleanup_only: - logging.info("Cleanup-only run finished.") - return - try: - test_backup(conn, prefix=args.prefix) - except BaseException: - print('volume-backup-check: FAIL') - raise - else: - print('volume-backup-check: PASS') - finally: - cleanup(conn, prefix=args.prefix) - - -if __name__ == "__main__": +def compute_scs_0117_test_backup(conn, prefix=DEFAULT_PREFIX): + if not cleanup(conn, prefix=prefix): + raise RuntimeError("Initial cleanup failed") try: - sys.exit(main()) - except SystemExit: - raise - except BaseException as exc: - logging.debug("traceback", exc_info=True) - logging.critical(str(exc)) - sys.exit(1) + test_backup(conn, prefix=prefix) + except BaseException: + logging.error('Backup test failed.') + logging.debug('exception details', exc_info=True) + return False + else: + return True + finally: + cleanup(conn, prefix=prefix) From 7185584150627cd02637a54d36e0a37cd211efe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 27 Aug 2025 21:31:09 +0200 Subject: [PATCH 22/31] fixup: forgot to update scs-compatible-iaas.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/scs-compatible-iaas.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 3e2fda26a..3d43a5eb2 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -154,8 +154,8 @@ modules: name: Volume backup url: https://docs.scs.community/standards/scs-0117-v1-volume-backup-service run: - - executable: ./iaas/volume-backup/volume-backup-tester.py - args: --os-cloud {os_cloud} --debug + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} volume-backup-check testcases: - id: volume-backup-check tags: [mandatory] From 637fb56d7a48e5bc5113bdc59d390f6d3e6448e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Thu, 28 Aug 2025 12:06:06 +0200 Subject: [PATCH 23/31] Adapt check pertaining to scs-0123-mandatory-services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/openstack_test.py | 17 ++ .../README.md | 0 .../mandatory-iaas-services.py | 0 .../mandatory_services.py | 153 ++++++++++++++++++ Tests/scs-compatible-iaas.yaml | 4 +- 5 files changed, 172 insertions(+), 2 deletions(-) rename Tests/iaas/{mandatory-services => scs_0123_mandatory_services}/README.md (100%) rename Tests/iaas/{mandatory-services => scs_0123_mandatory_services}/mandatory-iaas-services.py (100%) create mode 100644 Tests/iaas/scs_0123_mandatory_services/mandatory_services.py diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 00b4f791a..d934ad627 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -42,6 +42,8 @@ compute_services_lookup, compute_scs_0116_presence, compute_scs_0116_permissions from scs_0117_volume_backup.volume_backup import \ compute_scs_0117_test_backup +from scs_0123_mandatory_services.mandatory_services import \ + compute_scs_0123_service_presence, compute_scs_0123_swift_s3 logger = logging.getLogger(__name__) @@ -183,6 +185,21 @@ def make_container(cloud): c.add_function('volume_backup_check', lambda c: all(( c.scs_0117_test_backup, ))) + # scs_0123_mandatory_services + c.add_function('scs_0123_service_compute', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'compute')) + c.add_function('scs_0123_service_identity', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'identity')) + c.add_function('scs_0123_service_image', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'image')) + c.add_function('scs_0123_service_network', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'network')) + c.add_function('scs_0123_service_load_balancer', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'load-balancer')) + c.add_function('scs_0123_service_placement', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'placement')) + c.add_function('scs_0123_service_object_store', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'object-store')) + c.add_function('scs_0123_storage_apis', lambda c: compute_scs_0123_service_presence(c.services_lookup, 'volume', 'volumev3', 'block-storage')) + c.add_function('scs_0123_swift_s3', lambda c: compute_scs_0123_swift_s3(c.conn)) + c.add_function('service_apis_check', lambda c: all(( + c.scs_0123_service_compute, c.scs_0123_service_identity, c.scs_0123_service_image, + c.scs_0123_service_network, c.scs_0123_service_load_balancer, c.scs_0123_service_placement, + c.scs_0123_service_object_store, c.scs_0123_storage_apis, c.scs_0123_swift_s3, + ))) return c diff --git a/Tests/iaas/mandatory-services/README.md b/Tests/iaas/scs_0123_mandatory_services/README.md similarity index 100% rename from Tests/iaas/mandatory-services/README.md rename to Tests/iaas/scs_0123_mandatory_services/README.md diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/scs_0123_mandatory_services/mandatory-iaas-services.py similarity index 100% rename from Tests/iaas/mandatory-services/mandatory-iaas-services.py rename to Tests/iaas/scs_0123_mandatory_services/mandatory-iaas-services.py diff --git a/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py b/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py new file mode 100644 index 000000000..7991aeff2 --- /dev/null +++ b/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py @@ -0,0 +1,153 @@ +import json +import logging +import re +import uuid + +import boto3 +import openstack + + +TESTCONTNAME = "scs-test-container" +EC2MARKER = "TmpMandSvcTest" + +logger = logging.getLogger(__name__) +# NOTE suppress excessive logging (who knows what sensitive data might be in there) +# For the time being, I don't know where else to do it, but I guess it's fine. +logging.getLogger('botocore').setLevel(logging.WARNING) +logging.getLogger('boto3.resources').setLevel(logging.WARNING) + + +def compute_scs_0123_service_presence(services_lookup, *names): + services = [] + for name in names: + services.extend(services_lookup.get(name, ())) + if not services: + logger.error(f"No service of type(s) {', '.join(names)} found.") + return bool(services) + + +def s3_conn(creds, conn): + """Return an s3 client conn""" + cacert = conn.config.config.get("cacert") + # TODO: Handle self-signed certs (from ca_cert in openstack config) + if cacert: + logger.warning(f"Trust all Certificates in S3, OpenStack uses {cacert}") + return boto3.resource( + 's3', endpoint_url=creds["HOST"], verify=not cacert, + aws_access_key_id=creds["AK"], aws_secret_access_key=creds["SK"], + ) + + +def _parse_blob(cred): + try: + return json.loads(cred.blob) + except Exception as exc: + logger.debug(f"unable to parse credential {cred!r}: {exc!r}") + return None + + +def get_usable_credentials(conn): + """ + get all ec2 credentials for this project that carry meaningful data + + returns list of pairs (credential, parsed ec2 data) + """ + project_id = conn.identity.get_project_id() + candidates = [ + (cred, _parse_blob(cred)) + for cred in conn.identity.credentials() + if cred.type == "ec2" and cred.project_id == project_id + ] + return [ + (cred, parsed) + for cred, parsed in candidates + if parsed and parsed.get('access') and parsed.get('secret') + ] + + +def remove_leftovers(usable_credentials, conn): + """ + makes sure to delete any leftover set of ec2 credentials + """ + result = [] + for item in usable_credentials: + cred, parsed = item + if parsed.get("owner") == EC2MARKER: + logger.debug(f"Removing leftover credential {parsed['access']}") + conn.identity.delete_credential(cred) + else: + result.append(item) + return result + + +def ensure_ec2_credentials(usable_credentials, conn): + if usable_credentials: + return usable_credentials + parsed = { + "access": uuid.uuid4().hex, + "secret": uuid.uuid4().hex, + "owner": EC2MARKER, + } + blob = json.dumps(parsed) + try: + crd = conn.identity.create_credential( + type="ec2", blob=blob, user_id=conn.current_user_id, project_id=conn.current_project_id, + ) + except BaseException: + logger.warning("ec2 creds creation failed", exc_info=True) + raise + usable_credentials.append((crd, parsed)) + return usable_credentials # also return for chaining + + +def s3_from_ostack(usable_credentials, conn, rgx=re.compile(r"^(https*://[^/]*)/")): + """Set creds from openstack swift/keystone""" + # just use the first usable set of ec2 credentials + _, parsed = usable_credentials[0] + s3_creds = { + "AK": parsed["access"], + "SK": parsed["secret"], + } + m = rgx.match(conn.object_store.get_endpoint()) + if m: + s3_creds["HOST"] = m.group(1) + return s3_creds + + +def compute_scs_0123_swift_s3(conn: openstack.connection.Connection): + # we assume s3 is accessable via the service catalog, and Swift might exist too + usable_credentials = [] + s3_buckets = [] + # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) + try: + usable_credentials = remove_leftovers(get_usable_credentials(conn), conn) + s3_creds = s3_from_ostack(ensure_ec2_credentials(usable_credentials, conn), conn) + + # This is to be used for local debugging purposes ONLY + # logger.debug(f"using credentials {s3_creds}") + + s3 = s3_conn(s3_creds, conn) + buckets = list(s3.buckets.all()) + if not buckets: + s3.create_bucket(Bucket=TESTCONTNAME) + buckets = list(s3.buckets.all()) + if not buckets: + raise RuntimeError("failed to create S3 bucket") + + # actual test: buckets must equal containers (sort in case the order is different) + s3_buckets = sorted([b.name for b in buckets]) + sw_containers = sorted([c.name for c in conn.object_store.containers()]) + if s3_buckets == sw_containers: + return True + logger.error( + "S3 buckets and Swift cntainers differ:\n" + f"S3: {s3_buckets}\n" + f"SW: {sw_containers}" + ) + return False + finally: + # Cleanup created S3 bucket + if TESTCONTNAME in s3_buckets: + s3.delete_bucket(Bucket=TESTCONTNAME) + # Clean up ec2 cred IF we created one + remove_leftovers(usable_credentials, conn) diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 3d43a5eb2..bfcf61355 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -173,8 +173,8 @@ modules: name: Mandatory and Supported IaaS Services url: https://docs.scs.community/standards/scs-0123-v1-mandatory-and-supported-IaaS-services run: - - executable: ./iaas/mandatory-services/mandatory-iaas-services.py - args: --os-cloud {os_cloud} --debug + - executable: ./iaas/openstack_test.py + args: -c {os_cloud} service-apis-check testcases: - id: service-apis-check tags: [mandatory] From 6ddcfec8da825b76ddace26c021798c593f5ec1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Thu, 28 Aug 2025 21:46:52 +0200 Subject: [PATCH 24/31] Remove redundant stand-alone check scripts, improve documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- .../flavor-names-openstack.py | 108 ------- .../flavor_names_check.py | 17 +- Tests/iaas/scs_0101_entropy/entropy-check.py | 124 -------- Tests/iaas/scs_0101_entropy/entropy_check.py | 8 +- .../scs_0102_image_metadata/image-md-check.py | 138 -------- .../scs_0102_image_metadata/image_metadata.py | 31 ++ .../flavors-openstack.py | 193 ----------- .../standard_flavors.py | 8 + .../images-openstack.py | 166 ---------- .../standard_images.py | 10 + .../volume-types-check.py | 150 --------- .../scs_0114_volume_types/volume_types.py | 7 + .../default-security-group-rules.py | 186 ----------- .../security_groups.py | 7 +- .../check-for-key-manager.py | 181 ----------- .../iaas/scs_0116_key_manager/key_manager.py | 23 +- .../volume-backup-tester.py | 90 ------ .../scs_0117_volume_backup/volume_backup.py | 7 +- .../mandatory-iaas-services.py | 301 ------------------ 19 files changed, 102 insertions(+), 1653 deletions(-) delete mode 100755 Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py delete mode 100755 Tests/iaas/scs_0101_entropy/entropy-check.py delete mode 100755 Tests/iaas/scs_0102_image_metadata/image-md-check.py delete mode 100755 Tests/iaas/scs_0103_standard_flavors/flavors-openstack.py delete mode 100755 Tests/iaas/scs_0104_standard_images/images-openstack.py delete mode 100755 Tests/iaas/scs_0114_volume_types/volume-types-check.py delete mode 100755 Tests/iaas/scs_0115_security_groups/default-security-group-rules.py delete mode 100755 Tests/iaas/scs_0116_key_manager/check-for-key-manager.py delete mode 100755 Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py delete mode 100755 Tests/iaas/scs_0123_mandatory_services/mandatory-iaas-services.py diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py b/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py deleted file mode 100755 index 373829473..000000000 --- a/Tests/iaas/scs_0100_flavor_naming/flavor-names-openstack.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: - -"""Flavor naming checker - -Uses the flavor_names.py module. -Assumes a connection to an OpenStack tenant, -retrieves the list of flavors from there and validates them. -Something similar could be achieved by: -flavor-name-check.py -c $(openstack flavor list -f value -c Name) -In addition we check consistency by looking at the information -provided by openstack, such as the number of vCPUs and memory. - -(c) Kurt Garloff , 12/2022 -(c) Matthias Büchse , 1/2024 -SPDX-License-Identifier: CC-BY-SA 4.0 -""" - -import getopt -import logging -import os -import sys - -import openstack - -from flavor_names_check import \ - compute_scs_flavors, compute_scs_0100_syntax_check, compute_scs_0100_semantics_check, compute_flavor_name_check - - -logger = logging.getLogger(__name__) - - -def usage(rcode=1): - """help output""" - print("Usage: flavor-names-openstack.py [options]", file=sys.stderr) - print("Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env)", file=sys.stderr) - print("This tool retrieves the list of flavors from the OpenStack cloud OS_CLOUD", file=sys.stderr) - print(" and reports inconsistencies, errors etc. It returns 0 on success.", file=sys.stderr) - sys.exit(rcode) - - -def main(argv): - """Entry point -- main loop going over flavors""" - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) - openstack.enable_logging(debug=False) - cloud = None - - try: - cloud = os.environ["OS_CLOUD"] - except KeyError: - pass - try: - opts, args = getopt.gnu_getopt(argv, "c:C:vhq321o", - ("os-cloud=", "mand=", "verbose", "help", "quiet", "v2plus", - "v3", "v1prefer", "accept-old-mandatory")) - except getopt.GetoptError as exc: - print(f"CRITICAL: {exc!r}", file=sys.stderr) - usage(1) - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - usage(0) - elif opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - elif opt[0] == "-C" or opt[0] == "--mand": - if opt[1].split('/')[-1] != 'scs-0100-v3-flavors.yaml': - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - elif opt[0] == "-3" or opt[0] == "--v3": - # fnmck.disallow_old = True - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - elif opt[0] == "-2" or opt[0] == "--v2plus": - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - elif opt[0] == "-1" or opt[0] == "--v1prefer": - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - elif opt[0] == "-o" or opt[0] == "--accept-old-mandatory": - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - elif opt[0] == "-v" or opt[0] == "--verbose": - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - elif opt[0] == "-q" or opt[0] == "--quiet": - print(f'ignoring obsolete argument: {opt[0]}', file=sys.stderr) - else: - usage(2) - if len(args) > 0: - print(f"CRITICAL: Extra arguments {str(args)}", file=sys.stderr) - usage(1) - - if not cloud: - print("CRITICAL: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr) - sys.exit(1) - - with openstack.connect(cloud=cloud, timeout=32) as conn: - scs_flavors = compute_scs_flavors(conn.compute.flavors()) - result = compute_flavor_name_check( - compute_scs_0100_syntax_check(scs_flavors), - compute_scs_0100_semantics_check(scs_flavors), - ) - print("flavor-name-check: " + ('FAIL', 'PASS')[min(1, result)]) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main(sys.argv[1:])) - except SystemExit: - raise - except BaseException as exc: - print(f"CRITICAL: {exc!r}", file=sys.stderr) - sys.exit(1) diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py index 0e48d0813..c44a6259c 100644 --- a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: - import logging import typing @@ -22,6 +19,12 @@ def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor], parser=STRATEGY) -> list: + """ + Precompute the Flavorname instance for each openstack flavor where applicable. + + Returns a list of pairs of the form (flavor, flavorname_instance_or_none). + This information is (re)used for multiple testcases. + """ result = [] for flv in flavors: if not flv.name or flv.name[:4] != 'SCS-': @@ -41,6 +44,7 @@ def compute_flavor_spec(canonical_name: str) -> dict: def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: + """This test ensures that each SCS flavor is indeed named correctly.""" problems = [flv.name for flv, flavorname in scs_flavors if not flavorname] if problems: logger.error(f"scs-100-syntax-check: flavor(s) failed: {', '.join(sorted(problems))}") @@ -48,6 +52,13 @@ def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: + """ + This test ensures that each SCS flavor 'does what it says on the tin'. + + In particular, no flavor name may overpromise anything. + NOTE that this test is incomplete; it only checks the most obvious properties. + See also . + """ problems = set() for flv, flavorname in scs_flavors: if not flavorname: diff --git a/Tests/iaas/scs_0101_entropy/entropy-check.py b/Tests/iaas/scs_0101_entropy/entropy-check.py deleted file mode 100755 index 22c107193..000000000 --- a/Tests/iaas/scs_0101_entropy/entropy-check.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -"""Entropy checker - -Check given cloud for conformance with SCS standard regarding -entropy, to be found under /Standards/scs-0101-v1-entropy.md - -Return code is 0 precisely when it could be verified that the standard is satisfied. -Otherwise the return code is the number of errors that occurred (up to 127 due to OS -restrictions); for further information, see the log messages on various channels: - CRITICAL for problems preventing the test to complete, - ERROR for violations of requirements, - WARNING for violations of recommendations, - DEBUG for background information and problems that don't hinder the test. -""" -import getopt -import logging -import os -import sys -import warnings - -import openstack -import openstack.cloud - - -import entropy_check - - -logger = logging.getLogger(__name__) - - -def print_usage(file=sys.stderr): - """Help output""" - print("""Usage: entropy-check.py [options] -This tool checks the requested images and flavors according to the SCS Standard 0101 "Entropy". -Options: - [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) - [-d/--debug] enables DEBUG logging channel - [-i/--images IMAGE_LIST] sets images to be tested, separated by comma. - [-V/--image-visibility VIS_LIST] filters images by visibility - (default: 'public,community'; use '*' to disable) -""", end='', file=file) - - -def print_result(check_id, passed): - print(check_id + ": " + ('FAIL', 'PASS')[bool(passed)]) - - -def main(argv): - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - warnings.filterwarnings("ignore", "search_floating_ips") - - try: - opts, args = getopt.gnu_getopt(argv, "c:i:hdV:", ["os-cloud=", "images=", "help", "debug", "image-visibility="]) - except getopt.GetoptError as exc: - logger.critical(f"{exc}") - print_usage() - return 1 - - cloud = os.environ.get("OS_CLOUD") - image_visibility = set() - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - print_usage() - return 0 - if opt[0] == "-i" or opt[0] == "--images": - logger.info("ignoring obsolete option -i") - if opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-d" or opt[0] == "--debug": - logging.getLogger().setLevel(logging.DEBUG) - if opt[0] == "-V" or opt[0] == "--image-visibility": - image_visibility.update([v.strip() for v in opt[1].split(',')]) - - if not cloud: - logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") - return 1 - - if not image_visibility: - image_visibility.update(("public", "community")) - - try: - logger.debug(f"Connecting to cloud '{cloud}'") - with openstack.connect(cloud=cloud, timeout=32) as conn: - all_images = conn.list_images() - all_flavors = conn.list_flavors(get_extra=True) - - if '*' not in image_visibility: - logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") - all_images = [img for img in all_images if img.visibility in image_visibility] - all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] - logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") - - if not all_images: - logger.critical("Can't run this test without image") - return 1 - - logger.debug("Checking images and flavors for recommended attributes") - print_result('entropy-check-image-properties', entropy_check.compute_scs_0101_image_property(all_images)) - print_result('entropy-check-flavor-properties', entropy_check.compute_scs_0101_flavor_property(all_flavors)) - - logger.debug("Checking dynamic instance properties") - canonical_image = entropy_check.compute_canonical_image(all_images) - collected_vm_output = entropy_check.compute_collected_vm_output(conn, all_flavors, canonical_image) - print_result('entropy-check-rngd', entropy_check.compute_scs_0101_rngd(collected_vm_output, canonical_image.name)) - scs_0101_entropy_avail_result = entropy_check.compute_scs_0101_entropy_avail(collected_vm_output, canonical_image.name) - scs_0101_fips_test_result = entropy_check.compute_scs_0101_fips_test(collected_vm_output, canonical_image.name) - entropy_check_result = entropy_check.compute_scs_0101_entropy_check( - scs_0101_entropy_avail_result, - scs_0101_fips_test_result, - ) - print_result('entropy-check-entropy-avail', scs_0101_entropy_avail_result) - print_result('entropy-check-fips-test', scs_0101_fips_test_result) - print_result('entropy-check', entropy_check_result) - return not entropy_check_result - except BaseException as e: - logger.critical(f"{e!r}") - logger.debug("Exception info", exc_info=True) - return 1 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tests/iaas/scs_0101_entropy/entropy_check.py b/Tests/iaas/scs_0101_entropy/entropy_check.py index 593127d02..a0b180ad3 100644 --- a/Tests/iaas/scs_0101_entropy/entropy_check.py +++ b/Tests/iaas/scs_0101_entropy/entropy_check.py @@ -59,6 +59,7 @@ def compute_scs_0101_image_property(images, attributes=IMAGE_ATTRIBUTES): + """This test ensures that each image has the relevant properties.""" candidates = [ (image.name, [f"{key}={value}" for key, value in attributes.items() if image.get(key) != value]) for image in images @@ -71,6 +72,7 @@ def compute_scs_0101_image_property(images, attributes=IMAGE_ATTRIBUTES): def compute_scs_0101_flavor_property(flavors, attributes=FLAVOR_ATTRIBUTES, optional=FLAVOR_OPTIONAL): + """This test ensures that each flavor has the relevant extra_spec.""" offenses = 0 for flavor in flavors: extra_specs = flavor['extra_specs'] @@ -89,6 +91,7 @@ def compute_scs_0101_flavor_property(flavors, attributes=FLAVOR_ATTRIBUTES, opti def compute_scs_0101_entropy_avail(collected_vm_output, image_name): + """This test ensures that the `entropy_avail` value is correct for a test VM.""" lines = collected_vm_output['entropy-avail'] entropy_avail = lines[0].strip() if entropy_avail != "256": @@ -101,6 +104,7 @@ def compute_scs_0101_entropy_avail(collected_vm_output, image_name): def compute_scs_0101_rngd(collected_vm_output, image_name): + """This test ensures that the `rngd` service is running on a test VM.""" lines = collected_vm_output['rngd'] if "could not be found" in '\n'.join(lines): logger.error(f"VM '{image_name}' doesn't provide the recommended service rngd") @@ -109,6 +113,7 @@ def compute_scs_0101_rngd(collected_vm_output, image_name): def compute_scs_0101_fips_test(collected_vm_output, image_name): + """This test ensures that the 'fips test' via `rngtest` is passed on a test VM.""" lines = collected_vm_output['fips-test'] try: fips_data = '\n'.join(lines) @@ -137,6 +142,7 @@ def compute_scs_0101_fips_test(collected_vm_output, image_name): # FIXME this is not actually being used AFAICT -- mbuechse def compute_scs_0101_virtio_rng(collected_vm_output, image_name): + """This test ensures that the virtualized rng device is value inside a test VM.""" lines = collected_vm_output['virtio-rng'] try: # `cat` can fail with return code 1 if special file does not exist @@ -345,7 +351,7 @@ def _convert_to_collected(lines, marker=MARKER): def compute_collected_vm_output(conn, all_flavors, image): - # Check a VM for services and requirements + """Creates a test VM, collects and returns its output for later evaluation.""" logger.debug(f"Selected image: {image.name} ({image.id})") flavor = select_flavor_for_image(all_flavors, image) userdata = SERVER_USERDATA.get(image.os_distro, SERVER_USERDATA_GENERIC) diff --git a/Tests/iaas/scs_0102_image_metadata/image-md-check.py b/Tests/iaas/scs_0102_image_metadata/image-md-check.py deleted file mode 100755 index 54d8bd6af..000000000 --- a/Tests/iaas/scs_0102_image_metadata/image-md-check.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: -# -# SCS/Docs/tools/image-md-check.py - -""" -Retrieve metadata from (public) images and check for compliance -with SCS specifications. - -(c) Kurt Garloff , 09/2022 -SPDX-License-Identifier: CC-BY-SA-4.0 -""" - -from collections import Counter -import getopt -import logging -import os -import sys - -import openstack - - -from image_metadata import \ - compute_scs_0102_prop_architecture, compute_scs_0102_prop_hash_algo, compute_scs_0102_prop_min_disk, \ - compute_scs_0102_prop_min_ram, compute_scs_0102_prop_os_version, compute_scs_0102_prop_os_distro, \ - compute_scs_0102_prop_hw_disk_bus, compute_scs_0102_prop_hypervisor_type, compute_scs_0102_prop_hw_rng_model, \ - compute_scs_0102_prop_image_build_date, compute_scs_0102_prop_image_original_user, \ - compute_scs_0102_prop_image_source, compute_scs_0102_prop_image_description, \ - compute_scs_0102_prop_replace_frequency, compute_scs_0102_prop_provided_until, \ - compute_scs_0102_prop_uuid_validity, compute_scs_0102_prop_hotfix_hours, \ - compute_scs_0102_image_recency - - -logger = logging.getLogger(__name__) - - -def usage(ret): - "Usage information" - print("Usage: image-md-check.py [options] [images]") - print("image-md-check.py will create a report on public images by retrieving") - print(" the image metadata (properties) and comparing this against the image") - print(" metadata spec from SCS.") - print("Options: --os-cloud CLOUDNAME: Use this cloud config, default is $OS_CLOUD") - print(" -p/--private : Also consider private images") - print(" -v/--verbose : Be more verbose") - print(" -h/--help : Print this usage information") - print(" [-V/--image-visibility VIS_LIST] : filters images by visibility") - print(" (default: 'public,community'; use '*' to disable)") - print("If you pass images, only these will be validated, otherwise all images") - print("(filtered according to -p, -V) from the catalog will be processed.") - sys.exit(ret) - - -def main(argv): - "Main entry point" - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - image_visibility = set() - private = False - cloud = os.environ.get("OS_CLOUD") - err = 0 - try: - opts, args = getopt.gnu_getopt(argv[1:], "phvc:sV:", - ("private", "help", "os-cloud=", "verbose", "skip-completeness", "image-visibility=")) - except getopt.GetoptError: # as exc: - print("CRITICAL: Command-line syntax error", file=sys.stderr) - usage(1) - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - usage(0) - elif opt[0] == "-p" or opt[0] == "--private": - private = True # only keep this for backwards compatibility (we have -V now) - elif opt[0] == "-v" or opt[0] == "--verbose": - logging.getLogger().setLevel(logging.DEBUG) - elif opt[0] == "-s" or opt[0] == "--skip-completeness": - logger.info("ignoring obsolete command-line option -s") - elif opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-V" or opt[0] == "--image-visibility": - image_visibility.update([v.strip() for v in opt[1].split(',')]) - image_names = args - if not cloud: - print("CRITICAL: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr) - usage(1) - if not image_visibility: - image_visibility.update(("public", "community")) - if private: - image_visibility.add("private") - try: - conn = openstack.connect(cloud=cloud, timeout=24) - all_images = list(conn.image.images()) - if '*' not in image_visibility: - logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") - all_images = [img for img in all_images if img.visibility in image_visibility] - all_image_names = [f"{img.name} ({img.visibility})" for img in all_images] - logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") - by_name = {img.name: img for img in all_images} - if len(by_name) != len(all_images): - counter = Counter([img.name for img in all_images]) - duplicates = [name for name, count in counter.items() if count > 1] - print(f'WARNING: duplicate names detected: {", ".join(duplicates)}', file=sys.stderr) - if image_names: - images = [by_name[nm] for nm in image_names] - else: - images = all_images - result = all(( - compute_scs_0102_prop_architecture(images), - compute_scs_0102_prop_min_disk(images), - compute_scs_0102_prop_min_ram(images), - compute_scs_0102_prop_os_version(images), - compute_scs_0102_prop_os_distro(images), - compute_scs_0102_prop_hw_disk_bus(images), - compute_scs_0102_prop_image_build_date(images), - compute_scs_0102_prop_image_original_user(images), - compute_scs_0102_prop_image_source(images), - compute_scs_0102_prop_image_description(images), - compute_scs_0102_prop_replace_frequency(images), - compute_scs_0102_prop_provided_until(images), - compute_scs_0102_prop_uuid_validity(images), - compute_scs_0102_prop_hotfix_hours(images), - compute_scs_0102_image_recency(images), - )) - print("image-metadata-check: " + ('FAIL', 'PASS')[min(1, result)]) - # recommended stuff - _ = all(( - compute_scs_0102_prop_hash_algo(images), - compute_scs_0102_prop_hypervisor_type(images), - compute_scs_0102_prop_hw_rng_model(images), - )) - except BaseException as exc: - print(f"CRITICAL: {exc!r}", file=sys.stderr) - return 1 + err - return err - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/Tests/iaas/scs_0102_image_metadata/image_metadata.py b/Tests/iaas/scs_0102_image_metadata/image_metadata.py index ac1e92c1a..233b046b0 100644 --- a/Tests/iaas/scs_0102_image_metadata/image_metadata.py +++ b/Tests/iaas/scs_0102_image_metadata/image_metadata.py @@ -117,6 +117,7 @@ def _log_error(cause, offenders, channel=logging.ERROR): def compute_scs_0102_prop_architecture(images, architectures=ARCHITECTURES): + """This test ensures that each image has a proper value for the property `architecture`.""" offenders = [img for img in images if img.architecture not in architectures] _log_error('property architecture not correct', offenders) return not offenders @@ -124,12 +125,14 @@ def compute_scs_0102_prop_architecture(images, architectures=ARCHITECTURES): # NOTE I think this is a recommendation def compute_scs_0102_prop_hash_algo(images): + """This test ensures that each image has a proper value for the property `hash_algo`.""" offenders = [img for img in images if img.hash_algo not in ('sha256', 'sha512')] _log_error('property hash_algo invalid', offenders) return not offenders def compute_scs_0102_prop_min_disk(images): + """This test ensures that each image has a proper value for the property `min_disk`.""" offenders1 = [img for img in images if not img.min_disk] _log_error('property min_disk not set', offenders1) offenders2 = [img for img in images if img.min_disk and img.min_disk * GIB < img.size] @@ -138,6 +141,7 @@ def compute_scs_0102_prop_min_disk(images): def compute_scs_0102_prop_min_ram(images): + """This test ensures that each image has a proper value for the property `min_ram`.""" offenders1 = [img for img in images if not img.min_ram] _log_error('property min_ram not set', offenders1) # emit a warning im RAM really low @@ -148,36 +152,46 @@ def compute_scs_0102_prop_min_ram(images): def compute_scs_0102_prop_os_version(images): + """This test ensures that each image has a proper value for the property `os_version`.""" + # NOTE currently we are content when the property is not empty, but we could be more strict, + # because the standard was recently edited to refer to the OpenStack docs, which prescribe + # certain values for common operating systems. offenders = [img for img in images if not img.os_version] _log_error('property os_version not set', offenders) return not offenders def compute_scs_0102_prop_os_distro(images): + """This test ensures that each image has a proper value for the property `os_distro`.""" + # NOTE see note in `compute_scs_0102_prop_os_version` offenders = [img for img in images if not img.os_distro] _log_error('property os_distro not set', offenders) return not offenders def compute_scs_0102_prop_hw_disk_bus(images, hw_disk_buses=HW_DISK_BUSES): + """This test ensures that each image has a proper value for the property `hw_disk_bus`.""" offenders = [img for img in images if img.hw_disk_bus not in hw_disk_buses] _log_error('property hw_disk_bus not correct', offenders) return not offenders def compute_scs_0102_prop_hypervisor_type(images, hypervisor_types=HYPERVISOR_TYPES): + """This test ensures that each image has a proper value for the property `hypervisor_type`.""" offenders = [img for img in images if img.hypervisor_type not in hypervisor_types] _log_error('property hypervisor_type not correct', offenders) return not offenders def compute_scs_0102_prop_hw_rng_model(images, hw_rng_models=HW_RNG_MODELS): + """This test ensures that each image has a proper value for the property `hw_rng_model`.""" offenders = [img for img in images if img.hw_rng_model not in hw_rng_models] _log_error('property hw_rng_model not correct', offenders) return not offenders def compute_scs_0102_prop_image_build_date(images, now=time.time()): + """This test ensures that each image has a proper value for the property `image_build_date`.""" errors = 0 for img in images: rdate = parse_date(img.created_at, formats=STRICT_FORMATS) @@ -201,12 +215,14 @@ def compute_scs_0102_prop_image_build_date(images, now=time.time()): def compute_scs_0102_prop_image_original_user(images): + """This test ensures that each image has a proper value for the property `image_original_user`.""" offenders = [img for img in images if not img.properties.get('image_original_user')] _log_error('property image_original_user not set', offenders) return not offenders def compute_scs_0102_prop_image_source(images): + """This test ensures that each image has a proper value for the property `image_source`.""" offenders = [ img for img in images @@ -218,24 +234,28 @@ def compute_scs_0102_prop_image_source(images): def compute_scs_0102_prop_image_description(images): + """This test ensures that each image has a proper value for the property `image_description`.""" offenders = [img for img in images if not img.properties.get('image_description')] _log_error('property image_description not set', offenders) return not offenders def compute_scs_0102_prop_replace_frequency(images, replace_frequencies=FREQ_TO_SEC): + """This test ensures that each image has a proper value for the property `replace_frequency`.""" offenders = [img for img in images if img.properties.get('replace_frequency') not in replace_frequencies] _log_error('property replace_frequency not correct', offenders) return not offenders def compute_scs_0102_prop_provided_until(images): + """This test ensures that each image has a proper value for the property `provided_until`.""" offenders = [img for img in images if not img.properties.get('provided_until')] _log_error('property provided_until not set', offenders) return not offenders def compute_scs_0102_prop_uuid_validity(images): + """This test ensures that each image has a proper value for the property `uuid_validity`.""" errors = 0 for img in images: img_uuid_val = img.properties.get("uuid_validity") @@ -252,6 +272,7 @@ def compute_scs_0102_prop_uuid_validity(images): def compute_scs_0102_prop_hotfix_hours(images): + """This test ensures that each image has a proper value for the property `hotfix_hours`.""" errors = 0 for img in images: hotfix_hours = img.properties.get("hotfix_hours", '') @@ -273,6 +294,16 @@ def _find_replacement_image(by_name, img_name): def compute_scs_0102_image_recency(images): + """ + This test ensures that each image is as recent as claimed by `replace_frequency`. + + If an image is outdated, it is not deemed critical in the following two cases: + + 1. If the image is past its `provided_until` date, since no guarantees are given for these. + 2. If a recent variant of the image is found. + + However, a warning is issued if an outdated image is not hidden. + """ by_name = {img.name: img for img in images} if len(by_name) != len(images): counter = Counter([img.name for img in images]) diff --git a/Tests/iaas/scs_0103_standard_flavors/flavors-openstack.py b/Tests/iaas/scs_0103_standard_flavors/flavors-openstack.py deleted file mode 100755 index 2680fa822..000000000 --- a/Tests/iaas/scs_0103_standard_flavors/flavors-openstack.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -"""Standard flavors checker for OpenStack - -Check given cloud for conformance with SCS standard regarding standard flavors, -to be found under /Standards/scs-0103-v1-standard-flavors.md - -The respective list of flavors is defined in a corresponding yaml file; this script -expects the path of such a file as its only positional argument. - -Return code is 0 precisely when it could be verified that the standard is satisfied. -Otherwise the return code is the number of errors that occurred (up to 127 due to OS -restrictions); for further information, see the log messages on various channels: - CRITICAL for problems preventing the test to complete, - ERROR for violations of requirements, - INFO for violations of recommendations, - DEBUG for background information and problems that don't hinder the test. -""" -from collections import Counter -import getopt -import logging -import os -import sys - -import openstack -import openstack.cloud -import yaml - - -logger = logging.getLogger(__name__) -# do not enforce this part of the standard, because it doesn't work for the customers -# RESERVED_KEYS = ('scs:name-v1', 'scs:name-v2') -RESERVED_KEYS = () - - -def print_usage(file=sys.stderr): - """Help output""" - print("""Usage: flavors-openstack.py [options] YAML -This tool checks the flavors according to the SCS Standard 0103 "Standard Flavors". -Arguments: - YAML path to the file containing the flavor definitions corresponding to the version - of the standard to be tested -Options: - [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) - [-d/--debug] enables DEBUG logging channel -""", end='', file=file) - - -class CountingHandler(logging.Handler): - def __init__(self, level=logging.NOTSET): - super().__init__(level=level) - self.bylevel = Counter() - - def handle(self, record): - self.bylevel[record.levelno] += 1 - - -def main(argv): - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - # count the number of log records per level (used for summary and return code) - counting_handler = CountingHandler(level=logging.INFO) - logger.addHandler(counting_handler) - - try: - opts, args = getopt.gnu_getopt(argv, "c:hd", ["os-cloud=", "help", "debug"]) - except getopt.GetoptError as exc: - logger.critical(f"{exc}") - print_usage() - return 1 - - if len(args) != 1: - logger.critical("Missing YAML argument, or too many arguments") - print_usage() - return 1 - - yaml_path = args[0] - cloud = os.environ.get("OS_CLOUD") - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - print_usage() - return 0 - if opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-d" or opt[0] == "--debug": - logging.getLogger().setLevel(logging.DEBUG) - - if not cloud: - logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") - return 1 - - try: - with open(yaml_path, "rb") as fileobj: - flavor_spec_data = yaml.safe_load(fileobj) - except Exception as e: - logger.critical(f"Unable to load '{yaml_path}': {e!r}") - logger.debug("Exception info", exc_info=True) - return 1 - - if 'meta' not in flavor_spec_data or 'name_key' not in flavor_spec_data['meta']: - logger.critical("Flavor definition missing 'meta' field or field incomplete") - return 1 - - if 'flavor_groups' not in flavor_spec_data: - logger.critical("Flavor definition missing 'flavor_groups' field") - - name_key = flavor_spec_data['meta']['name_key'] - # compute union of all flavor groups, copying group info (mainly "status") to each flavor - # check if the spec is complete while we are at it - flavor_specs = [] - for flavor_group in flavor_spec_data['flavor_groups']: - group_info = dict(flavor_group) - group_info.pop('list') - missing = {'status'} - set(group_info) - if missing: - logging.critical(f"Flavor group missing attributes: {', '.join(missing)}") - return 1 - for flavor_spec in flavor_group['list']: - missing = {'name', 'cpus', 'ram'} - set(flavor_spec) - if missing: - logging.critical(f"Flavor spec missing attributes: {', '.join(missing)}") - return 1 - flavor_specs.append({"_group": group_info, **flavor_spec}) - - try: - logger.debug(f"Fetching flavors from cloud '{cloud}'") - with openstack.connect(cloud=cloud, timeout=32) as conn: - present_flavors = conn.list_flavors(get_extra=True) - by_name = { - flavor.extra_specs[name_key]: flavor - for flavor in present_flavors - if name_key in flavor.extra_specs - } - by_legacy_name = {flavor.name: flavor for flavor in present_flavors} - # for reserved keys, keep track of all flavors that don't have a matching spec - unmatched = {flavor.id: (flavor, keys) for flavor, keys in [ - (flavor, tuple(key for key in RESERVED_KEYS if key in flavor.extra_specs)) - for flavor in present_flavors - ] if keys} - - logger.debug(f"Checking {len(flavor_specs)} flavor specs against {len(present_flavors)} flavors") - for flavor_spec in flavor_specs: - flavor = by_name.get(flavor_spec[name_key]) - if not flavor: - flavor = by_legacy_name.get(flavor_spec[name_key]) - if flavor: - logger.warning(f"Flavor '{flavor_spec['name']}' found via name only, missing property {name_key!r}") - else: - status = flavor_spec['_group']['status'] - level = {"mandatory": logging.ERROR}.get(status, logging.WARNING) - logger.log(level, f"Missing {status} flavor '{flavor_spec['name']}'") - continue - # this flavor has a matching spec - unmatched.pop(flavor.id, None) - # check that flavor matches flavor_spec - # cpu, ram, and disk should match, and they should match precisely for discoverability - if flavor.vcpus != flavor_spec['cpus']: - logger.error(f"Flavor '{flavor.name}' violating CPU constraint: {flavor.vcpus} != {flavor_spec['cpus']}") - if flavor.ram != 1024 * flavor_spec['ram']: - logger.error(f"Flavor '{flavor.name}' violating RAM constraint: {flavor.ram} != {1024 * flavor_spec['ram']}") - if flavor.disk != flavor_spec.get('disk', 0): - logger.error(f"Flavor '{flavor.name}' violating disk constraint: {flavor.disk} != {flavor_spec.get('disk', 0)}") - # other fields besides name, cpu, ram, and disk should also match - report = [ - f"{key}: {es_value!r} should be {value!r}" - for key, value, es_value in [ - (key, value, flavor.extra_specs.get(key)) - for key, value in flavor_spec.items() - if key.startswith("scs:") - ] - if value != es_value - ] - if report: - logger.error(f"Flavor '{flavor.name}' violating property constraints: {'; '.join(report)}") - if unmatched: - logger.error( - "The following flavors are not standard, yet use a reserved property: " + ", ".join( - f"{flavor.name}: {', '.join(keys)}" for flavor, keys in unmatched.values() if keys - ) - ) - except BaseException as e: - logger.critical(f"{e!r}") - logger.debug("Exception info", exc_info=True) - - c = counting_handler.bylevel - logger.debug(f"Total critical / error / info: {c[logging.CRITICAL]} / {c[logging.ERROR]} / {c[logging.INFO]}") - if not c[logging.CRITICAL]: - print("standard-flavors-check: " + ('PASS', 'FAIL')[min(1, c[logging.ERROR])]) - return min(127, c[logging.CRITICAL] + c[logging.ERROR]) # cap at 127 due to OS restrictions - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py b/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py index aae58909b..83136e8a2 100644 --- a/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py +++ b/Tests/iaas/scs_0103_standard_flavors/standard_flavors.py @@ -53,6 +53,14 @@ def compute_flavor_lookup(flavors, name_key=NAME_KEY): def compute_scs_0103_flavor(flavor_lookup, flavor_spec): + """ + This test ensures that the flavor given via `flavor_spec` is present. + + The primary way to arrive at a flavor spec is to parse an SCS flavor name and + convert it to a dictionary. This can be done using the module `flavor_names.py` + in the `iaas/scs_0100_flavor_naming` directory. See `iaas/openstack_test.py` + for details. + """ canonical_name = flavor_spec[NAME_KEY] flavor = flavor_lookup.get(canonical_name) if flavor is None: diff --git a/Tests/iaas/scs_0104_standard_images/images-openstack.py b/Tests/iaas/scs_0104_standard_images/images-openstack.py deleted file mode 100755 index 6f192e5d0..000000000 --- a/Tests/iaas/scs_0104_standard_images/images-openstack.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""Standard images checker for OpenStack - -Check given cloud for conformance with SCS standard regarding standard images, -to be found under /Standards/scs-0104-v1-standard-images.md - -The respective list of images is defined in a corresponding yaml file; this script -expects the path of such a file as its only positional argument. - -Return code is 0 precisely when it could be verified that the standard is satisfied. -Otherwise the return code is the number of errors that occurred (up to 127 due to OS -restrictions); for further information, see the log messages on various channels: - CRITICAL for problems preventing the test to complete, - ERROR for violations of requirements, - WARNING for violations of recommendations, - DEBUG for background information and problems that don't hinder the test. -""" -from collections import Counter -import getopt -import logging -import os -import re -import sys - -import openstack -import openstack.cloud -import yaml - - -logger = logging.getLogger(__name__) - - -def print_usage(file=sys.stderr): - """Help output""" - print("""Usage: images-openstack.py [options] YAML -This tool checks the flavors according to the SCS Standard 0104 "Standard Images". -Arguments: - YAML path to the file containing the image definitions corresponding to the version - of the standard to be tested -Options: - [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) - [-d/--debug] enables DEBUG logging channel - [-V/--image-visibility VIS_LIST] filters images by visibility - (default: 'public,community'; use '*' to disable) -""", end='', file=file) - - -class CountingHandler(logging.Handler): - def __init__(self, level=logging.NOTSET): - super().__init__(level=level) - self.bylevel = Counter() - - def handle(self, record): - self.bylevel[record.levelno] += 1 - - -def main(argv): - # configure logging, disable verbose library logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - # count the number of log records per level (used for summary and return code) - counting_handler = CountingHandler(level=logging.INFO) - logger.addHandler(counting_handler) - - try: - opts, args = getopt.gnu_getopt(argv, "c:hdV:", ["os-cloud=", "help", "debug", "image-visibility="]) - except getopt.GetoptError as exc: - logger.critical(f"{exc}") - print_usage() - return 1 - - if len(args) != 1: - logger.critical("Missing YAML argument, or too many arguments") - print_usage() - return 1 - - yaml_path = args[0] - cloud = os.environ.get("OS_CLOUD") - image_visibility = set() - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - print_usage() - return 0 - if opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-d" or opt[0] == "--debug": - logging.getLogger().setLevel(logging.DEBUG) - if opt[0] == "-V" or opt[0] == "--image-visibility": - image_visibility.update([v.strip() for v in opt[1].split(',')]) - - if not cloud: - logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") - return 1 - - if not image_visibility: - image_visibility.update(("public", "community")) - - # we only support local files; but we allow specifying the following URLs for the sake of - # better documentation - prefix = next(p for p in ( - 'https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/', - 'https://github.com/SovereignCloudStack/standards/blob/main/Tests/', - '', # sentinel (do not remove!) - ) if yaml_path.startswith(p)) - if prefix: - yaml_path = yaml_path[len(prefix):] - try: - with open(yaml_path, "rb") as fileobj: - image_data = yaml.safe_load(fileobj) - except Exception as e: - logger.critical(f"Unable to load '{yaml_path}': {e!r}") - logger.debug("Exception info", exc_info=True) - return 1 - - if 'images' not in image_data: - logger.critical("Image definition missing 'images' field") - return 1 - - image_specs = image_data['images'] - try: - logger.debug(f"Fetching image list from cloud '{cloud}'") - with openstack.connect(cloud=cloud, timeout=32) as conn: - present_images = conn.list_images(show_all=True) - if '*' not in image_visibility: - logger.debug(f"Images: filter for visibility {', '.join(sorted(image_visibility))}") - present_images = [img for img in present_images if img.visibility in image_visibility] - all_image_names = [f"{img.name} ({img.visibility})" for img in present_images] - logger.debug(f"Images: {', '.join(all_image_names) or '(NONE)'}") - by_name = { - image.name: image - for image in present_images - } - - logger.debug(f"Checking {len(image_specs)} image specs against {len(present_images)} images") - for image_spec in image_specs: - name_scheme = image_spec.get('name_scheme') - if name_scheme: - rex = re.compile(name_scheme) - matches = [img for name, img in by_name.items() if rex.match(name)] - else: - matches = [img for img in (by_name.get(image_spec['name']), ) if img is not None] - if not matches: - status = image_spec.get('status', 'optional') - level = {"mandatory": logging.ERROR, "recommended": logging.WARNING}.get(status, logging.DEBUG) - logger.log(level, f"Missing {status} image '{image_spec['name']}'") - continue - for image in matches: - img_source = image.properties['image_source'] - sources = image_spec['source'] - if not isinstance(sources, (tuple, list)): - sources = [sources] - if not any(img_source.startswith(src) for src in sources): - logger.error(f"Image '{image.name}' source mismatch: {img_source} matches none of these prefixes: {', '.join(sources)}") - except BaseException as e: - logger.critical(f"{e!r}") - logger.debug("Exception info", exc_info=True) - - c = counting_handler.bylevel - logger.debug(f"Total critical / error / warning: {c[logging.CRITICAL]} / {c[logging.ERROR]} / {c[logging.WARNING]}") - if not c[logging.CRITICAL]: - print("standard-images-check: " + ('PASS', 'FAIL')[min(1, c[logging.ERROR])]) - return min(127, c[logging.CRITICAL] + c[logging.ERROR]) # cap at 127 due to OS restrictions - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tests/iaas/scs_0104_standard_images/standard_images.py b/Tests/iaas/scs_0104_standard_images/standard_images.py index 72007744f..1950ee341 100644 --- a/Tests/iaas/scs_0104_standard_images/standard_images.py +++ b/Tests/iaas/scs_0104_standard_images/standard_images.py @@ -60,6 +60,11 @@ def _lookup_images(image_lookup, image_spec): def compute_scs_0104_source(image_lookup, image_spec): + """ + This test ensures that every image matching `image_spec` has the correct `image_source`. + + For an impression of what these specs look like, refer to `SCS_0104_IMAGE_SPECS`. + """ matches = _lookup_images(image_lookup, image_spec) errors = 0 for image in matches: @@ -74,6 +79,11 @@ def compute_scs_0104_source(image_lookup, image_spec): def compute_scs_0104_image(image_lookup, image_spec): + """ + This test ensures that a certain image is present, as specified by `image_spec`. + + For an impression of what these specs look like, refer to `SCS_0104_IMAGE_SPECS`. + """ matches = _lookup_images(image_lookup, image_spec) if not matches: logger.error(f"Missing image '{image_spec['name']}'") diff --git a/Tests/iaas/scs_0114_volume_types/volume-types-check.py b/Tests/iaas/scs_0114_volume_types/volume-types-check.py deleted file mode 100755 index 199c1c5dd..000000000 --- a/Tests/iaas/scs_0114_volume_types/volume-types-check.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -"""Volume types checker - -Check given cloud for conformance with SCS standard regarding -volume types, to be found under /Standards/scs-0112-v1-volume-types.md - -Return code is 0 precisely when it could be verified that the standard is satisfied. -Otherwise the return code is the number of errors that occurred (up to 127 due to OS -restrictions); for further information, see the log messages on various channels: - CRITICAL for problems preventing the test to complete, - ERROR for violations of requirements, - INFO for violations of recommendations, - DEBUG for background information and problems that don't hinder the test. -""" -from collections import Counter, defaultdict -import getopt -import logging -import os -import re -import sys - -import openstack -import openstack.cloud - - -logger = logging.getLogger(__name__) -RECOGNIZED_FEATURES = ('encrypted', 'replicated') - - -def extract_feature_list(description, pattern=re.compile(r"\[scs:([^\[\]]*)\]")): - """Extract feature-list-like prefix - - If given `description` starts with a feature-list-like prefix, return list of features, - otherwise None. To be more precise, we look for a string of this form: - - `[scs:`feat1`, `...`, `...featN`]` - - where N >= 1 and featJ is a string that doesn't contain any comma or brackets. We return - the list [feat1, ..., featN] of substrings. - """ - if not description: - # The description can be None or empty - we need to catch this here - return - match = pattern.match(description) - if not match: - return - fs = match.group(1) - if not fs: - return [] - return [f.strip() for f in fs.split(",")] - - -def test_feature_list(type_name: str, fl: list[str], recognized=RECOGNIZED_FEATURES): - """Test given list of features and report errors to error channel""" - if not fl: - # either None (no feature list) or empty feature list: nothing to check - return - if fl != sorted(fl): - logger.error(f"{type_name}: feature list not sorted") - ctr = Counter(fl) - duplicates = [key for key, c in ctr.items() if c > 1] - if duplicates: - logger.error(f"{type_name}: duplicate features: {', '.join(duplicates)}") - unrecognized = [f for f in ctr if f not in recognized] - if unrecognized: - logger.error(f"{type_name}: unrecognized features: {', '.join(unrecognized)}") - - -def print_usage(file=sys.stderr): - """Help output""" - print("""Usage: volume-types-check.py [options] -This tool checks volume types according to the SCS Standard 0112 "Volume Types". -Options: - [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) - [-d/--debug] enables DEBUG logging channel -""", end='', file=file) - - -class CountingHandler(logging.Handler): - def __init__(self, level=logging.NOTSET): - super().__init__(level=level) - self.bylevel = Counter() - - def handle(self, record): - self.bylevel[record.levelno] += 1 - - -def main(argv): - # configure logging - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - openstack.enable_logging(debug=False) - # count the number of log records per level (used for summary and return code) - counting_handler = CountingHandler(level=logging.INFO) - logger.addHandler(counting_handler) - - try: - opts, args = getopt.gnu_getopt(argv, "c:i:hd", ["os-cloud=", "help", "debug"]) - except getopt.GetoptError as exc: - logger.critical(f"{exc}") - print_usage() - return 1 - - cloud = os.environ.get("OS_CLOUD") - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - print_usage() - return 0 - if opt[0] == "-c" or opt[0] == "--os-cloud": - cloud = opt[1] - if opt[0] == "-d" or opt[0] == "--debug": - logging.getLogger().setLevel(logging.DEBUG) - - if not cloud: - logger.critical("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.") - return 1 - - try: - logger.debug(f"Connecting to cloud '{cloud}'") - with openstack.connect(cloud=cloud, timeout=32) as conn: - volume_types = conn.list_volume_types() - # collect volume types according to features - by_feature = defaultdict(list) - for typ in volume_types: - fl = extract_feature_list(typ.description) - if fl is None: - continue - logger.debug(f"{typ.name}: feature list {fl!r}") - test_feature_list(typ.name, fl) - for feat in fl: - by_feature[feat].append(typ.name) - logger.debug(f"volume types by feature: {dict(by_feature)}") - for feat in ('encrypted', 'replicated'): - if not by_feature[feat]: - logger.warning(f"Recommendation violated: missing {feat} volume type") - except BaseException as e: - logger.critical(f"{e!r}") - logger.debug("Exception info", exc_info=True) - - c = counting_handler.bylevel - logger.debug( - "Total critical / error / warning: " - f"{c[logging.CRITICAL]} / {c[logging.ERROR]} / {c[logging.WARNING]}" - ) - if not c[logging.CRITICAL]: - print("volume-types-check: " + ('PASS', 'FAIL')[min(1, c[logging.ERROR] + c[logging.WARNING])]) - return min(127, c[logging.CRITICAL] + c[logging.ERROR]) # cap at 127 due to OS restrictions - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tests/iaas/scs_0114_volume_types/volume_types.py b/Tests/iaas/scs_0114_volume_types/volume_types.py index 5a74c097b..2bfd41b3e 100644 --- a/Tests/iaas/scs_0114_volume_types/volume_types.py +++ b/Tests/iaas/scs_0114_volume_types/volume_types.py @@ -67,6 +67,10 @@ def compute_volume_type_lookup(volume_types): def compute_scs_0114_syntax_check(volume_type_lookup): + """ + This test ensures that, for every volume type description, + the aspect list, if present, is formatted according to the standard. + """ errors = volume_type_lookup.get(ERRONEOUS, ()) for line in errors: logger.error(line) @@ -74,6 +78,9 @@ def compute_scs_0114_syntax_check(volume_type_lookup): def compute_scs_0114_aspect_type(volume_type_lookup, aspect): + """ + This test ensures that a volume type with a certain `aspect` is present. + """ applicable = volume_type_lookup[aspect] if not applicable: logger.error(f"no volume type having aspect {aspect}") diff --git a/Tests/iaas/scs_0115_security_groups/default-security-group-rules.py b/Tests/iaas/scs_0115_security_groups/default-security-group-rules.py deleted file mode 100755 index 303540e46..000000000 --- a/Tests/iaas/scs_0115_security_groups/default-security-group-rules.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 -"""Default Security Group Rules Checker - -This script tests the absence of any ingress default security group rule -except for ingress rules from the same Security Group. Furthermore the -presence of default rules for egress traffic is checked. -""" -import argparse -from collections import Counter -import logging -import os -import sys - -import openstack -from openstack.exceptions import ResourceNotFound - -logger = logging.getLogger(__name__) - -SG_NAME = "scs-test-default-sg" -DESCRIPTION = "scs-test-default-sg" - - -def check_default_rules(rules, short=False): - """ - counts all verall ingress rules and egress rules, depending on the requested testing mode - - :param bool short - if short is True, the testing mode is set on short for older OpenStack versions - """ - ingress_rules = egress_rules = 0 - egress_vars = {'IPv4': {}, 'IPv6': {}} - for key, value in egress_vars.items(): - value['default'] = 0 - if not short: - value['custom'] = 0 - if not rules: - logger.info("No default security group rules defined.") - for rule in rules: - direction = rule["direction"] - ethertype = rule["ethertype"] - if direction == "ingress": - if not short: - # we allow ingress from the same security group - # but only for the default security group - if rule.remote_group_id == "PARENT" and not rule["used_in_non_default_sg"]: - continue - ingress_rules += 1 - elif direction == "egress" and ethertype in egress_vars: - egress_rules += 1 - if short: - egress_vars[ethertype]['default'] += 1 - continue - if rule.remote_ip_prefix: - # this rule does not allow traffic to all external ips - continue - # note: these two are not mutually exclusive - if rule["used_in_default_sg"]: - egress_vars[ethertype]['default'] += 1 - if rule["used_in_non_default_sg"]: - egress_vars[ethertype]['custom'] += 1 - # test whether there are no unallowed ingress rules - if ingress_rules: - logger.error(f"Expected no default ingress rules, found {ingress_rules}.") - # test whether all expected egress rules are present - missing = [(key, key2) for key, val in egress_vars.items() for key2, val2 in val.items() if not val2] - if missing: - logger.error( - "Expected rules for egress for IPv4 and IPv6 both for default and custom security groups. " - f"Missing rule types: {', '.join(str(x) for x in missing)}" - ) - logger.info(str({ - "Unallowed Ingress Rules": ingress_rules, - "Egress Rules": egress_rules, - })) - - -def create_security_group(conn, sg_name: str = SG_NAME, description: str = DESCRIPTION): - """Create security group in openstack - - :returns: - ~openstack.network.v2.security_group.SecurityGroup: The new security group or None - """ - sg = conn.network.create_security_group(name=sg_name, description=description) - return sg.id - - -def delete_security_group(conn, sg_id): - conn.network.delete_security_group(sg_id) - # in case of a successful delete finding the sg will throw an exception - try: - conn.network.find_security_group(name_or_id=sg_id) - except ResourceNotFound: - logger.debug(f"Security group {sg_id} was deleted successfully.") - except Exception: - logger.critical(f"Security group {sg_id} was not deleted successfully") - raise - - -def altern_test_rules(connection: openstack.connection.Connection): - sg_id = create_security_group(connection) - try: - sg = connection.network.find_security_group(name_or_id=sg_id) - check_default_rules(sg.security_group_rules, short=True) - finally: - delete_security_group(connection, sg_id) - - -def test_rules(connection: openstack.connection.Connection): - try: - rules = list(connection.network.default_security_group_rules()) - except (ResourceNotFound, AttributeError) as exc: - # older versions of OpenStack don't have the endpoint and give ResourceNotFound - if isinstance(exc, ResourceNotFound) and 'default-security-group-rules' not in str(exc): - raise - # why we see the AttributeError in some environments is a mystery - if isinstance(exc, AttributeError) and 'default_security_group_rules' not in str(exc): - raise - logger.info( - "API call failed. OpenStack components might not be up to date. " - "Falling back to old-style test method. " - ) - altern_test_rules(connection) - else: - check_default_rules(rules) - - -class CountingHandler(logging.Handler): - def __init__(self, level=logging.NOTSET): - super().__init__(level=level) - self.bylevel = Counter() - - def handle(self, record): - self.bylevel[record.levelno] += 1 - - -def main(): - parser = argparse.ArgumentParser( - description="SCS Default Security Group Rules Checker", - ) - parser.add_argument( - "--os-cloud", - type=str, - help="Name of the cloud from clouds.yaml, alternative " - "to the OS_CLOUD environment variable", - ) - parser.add_argument( - "--debug", action="store_true", help="Enable debug logging", - ) - args = parser.parse_args() - openstack.enable_logging(debug=False) # never leak sensitive data (enable this locally) - logging.basicConfig( - format="%(levelname)s: %(message)s", - level=logging.DEBUG if args.debug else logging.INFO, - ) - - # count the number of log records per level (used for summary and return code) - counting_handler = CountingHandler(level=logging.INFO) - logger.addHandler(counting_handler) - - # parse cloud name for lookup in clouds.yaml - cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) - if not cloud: - raise ValueError( - "You need to have the OS_CLOUD environment variable set to your cloud " - "name or pass it via --os-cloud" - ) - - with openstack.connect(cloud) as conn: - test_rules(conn) - - c = counting_handler.bylevel - logger.debug(f"Total critical / error / warning: {c[logging.CRITICAL]} / {c[logging.ERROR]} / {c[logging.WARNING]}") - if not c[logging.CRITICAL]: - print("security-groups-default-rules-check: " + ('PASS', 'FAIL')[min(1, c[logging.ERROR])]) - return min(127, c[logging.CRITICAL] + c[logging.ERROR]) # cap at 127 due to OS restrictions - - -if __name__ == "__main__": - try: - sys.exit(main()) - except SystemExit: - raise - except BaseException as exc: - logging.debug("traceback", exc_info=True) - logging.critical(str(exc)) - sys.exit(1) diff --git a/Tests/iaas/scs_0115_security_groups/security_groups.py b/Tests/iaas/scs_0115_security_groups/security_groups.py index 52c409635..3c5fcb254 100644 --- a/Tests/iaas/scs_0115_security_groups/security_groups.py +++ b/Tests/iaas/scs_0115_security_groups/security_groups.py @@ -96,7 +96,12 @@ def altern_test_rules(connection: openstack.connection.Connection): delete_security_group(connection, sg_id) -def compute_scs_0115_default_rules(conn: openstack.connection.Connection): +def compute_scs_0115_default_rules(conn: openstack.connection.Connection) -> bool: + """ + This test checks the absence of any ingress default security group rule + except for ingress rules from the same security group. Furthermore the + presence of default rules for egress traffic is checked. + """ try: rules = list(conn.network.default_security_group_rules()) except (ResourceNotFound, AttributeError) as exc: diff --git a/Tests/iaas/scs_0116_key_manager/check-for-key-manager.py b/Tests/iaas/scs_0116_key_manager/check-for-key-manager.py deleted file mode 100755 index 9ea85e7e5..000000000 --- a/Tests/iaas/scs_0116_key_manager/check-for-key-manager.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -"""Key Manager service checker for scs-0116-v1-key-manager-standard.md - -This script retrieves the endpoint catalog from Keystone using the OpenStack -SDK and checks whether a key manager API endpoint is present. -It then checks whether a user with the maximum of a member role can create secrets. -This will only work after policy adjustments or with the new secure RBAC roles and policies. -The script relies on an OpenStack SDK compatible clouds.yaml file for -authentication with Keystone. -""" - -import argparse -import logging -import os -import sys - -import openstack - -logger = logging.getLogger(__name__) - - -def initialize_logging(): - logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) - - -def check_for_member_role(conn: openstack.connection.Connection) -> None: - """Checks whether the current user has at maximum privileges of the member role. - - :param conn: connection to an OpenStack cloud. - :returns: boolean, when role with most privileges is member - """ - role_names = set(conn.session.auth.get_access(conn.session).role_names) - if role_names & {"admin", "manager"}: - return False - if "reader" in role_names: - logger.info("User has reader role.") - custom_roles = sorted(role_names - {"reader", "member"}) - if custom_roles: - logger.info(f"User has custom roles {', '.join(custom_roles)}.") - return "member" in role_names - - -def check_presence_of_key_manager(conn: openstack.connection.Connection) -> None: - try: - services = conn.service_catalog - except Exception: - logger.critical("Could not access Catalog endpoint.") - raise - - for svc in services: - svc_type = svc["type"] - if svc_type == "key-manager": - # key-manager is present - # now we want to check whether a user with member role - # can create and access secrets - logger.info("Key Manager is present") - return True - - -def _find_secrets(conn: openstack.connection.Connection, secret_name_or_id: str) -> list: - """Replacement method for finding secrets. - - Mimicks the behavior of Connection.key_manager.find_secret() - but fixes an issue with the internal implementation raising an - exception due to an unexpected microversion parameter. - Unlike find_secret(), we return a list with all secrets that match. - """ - secrets = conn.key_manager.secrets() - return [s for s in secrets if s.name == secret_name_or_id or s.id == secret_name_or_id] - - -def _delete_secret(conn: openstack.connection.Connection, secret: openstack.key_manager.v1.secret.Secret): - """Replacement method for deleting secrets - _delete_secret(connection, secret object) - - Workaround for SDK bugs: - - The id field in reality is a href (containg the UUID at the end) - - The delete_secret() function contrary to the documentation does - not accept openstack.key_manager.v1.secret.Secret objects nor the - hrefs, just plain UUIDs. - - It does not return an error when I try to delete a secret passing - an object or href, just silently does nothing. - The code here assumes that the SDK (when fixed) will continue to - accept UUIDs as argument for delete_secret() in the future. - Code is robust against those being passed directly in the .id attr - of the objects. (It would be even more robust to try to pass the - object first, then the href, then the UUID extracted from the href, - each time checking whether it was effective. But that's three delete - plus list calls and very ugly.) - """ - uuid = secret.id.rsplit('/', 1)[-1] - conn.key_manager.delete_secret(uuid) - - -def check_key_manager_permissions(conn: openstack.connection.Connection) -> None: - """ - After checking that the current user only has the member and maybe the - reader role, this method verifies that the user with a member role - has sufficient access to the Key Manager API functionality. - """ - secret_name = "scs-member-role-test-secret" - try: - existing_secrets = _find_secrets(conn, secret_name) - for secret in existing_secrets: - _delete_secret(conn, secret) - - if existing_secrets: - logger.debug(f'Deleted {len(existing_secrets)} secrets') - - conn.key_manager.create_secret( - name=secret_name, - payload_content_type="text/plain", - secret_type="opaque", - payload="foo", - ) - try: - new_secret = _find_secrets(conn, secret_name) - if not new_secret: - raise ValueError(f"Secret '{secret_name}' was not discoverable by the user") - finally: - _delete_secret(conn, new_secret[0]) - except openstack.exceptions.ForbiddenException: - logger.debug('exception details', exc_info=True) - logger.error( - "Users with the 'member' role can use Key Manager API: FAIL" - ) - return 1 - logger.info( - "Users with the 'member' role can use Key Manager API: PASS" - ) - - -def main(): - initialize_logging() - parser = argparse.ArgumentParser(description="SCS Mandatory IaaS Service Checker") - parser.add_argument( - "--os-cloud", - type=str, - help="Name of the cloud from clouds.yaml, alternative " - "to the OS_CLOUD environment variable", - ) - parser.add_argument( - "--debug", action="store_true", help="Enable OpenStack SDK debug logging" - ) - args = parser.parse_args() - # @mbuechse: I think this is so much as to be unusable! - # (If necessary, a developer can always uncomment) - # openstack.enable_logging(debug=args.debug) - if args.debug: - logger.setLevel(logging.DEBUG) - - # parse cloud name for lookup in clouds.yaml - cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) - if not cloud: - logger.critical( - "You need to have the OS_CLOUD environment variable set to your cloud " - "name or pass it via --os-cloud" - ) - return 2 - - with openstack.connect(cloud=cloud) as conn: - if not check_for_member_role(conn): - logger.critical("Cannot test key-manager permissions. User has wrong roles") - return 2 - if check_presence_of_key_manager(conn): - return check_key_manager_permissions(conn) - else: - # not an error, because key manager is merely recommended - logger.warning("There is no key-manager endpoint in the cloud.") - - -if __name__ == "__main__": - try: - sys.exit(main() or 0) - except SystemExit as e: - if e.code < 2: - print("key-manager-check: " + ('PASS', 'FAIL')[min(1, e.code)]) - raise - except BaseException: - logger.critical("exception", exc_info=True) - sys.exit(2) diff --git a/Tests/iaas/scs_0116_key_manager/key_manager.py b/Tests/iaas/scs_0116_key_manager/key_manager.py index 5b01ea783..a8399f7bb 100644 --- a/Tests/iaas/scs_0116_key_manager/key_manager.py +++ b/Tests/iaas/scs_0116_key_manager/key_manager.py @@ -7,15 +7,18 @@ logger = logging.getLogger(__name__) -def fetch_roles(conn: openstack.connection.Connection) -> None: - """Checks whether the current user has at maximum privileges of the member role. +def ensure_unprivileged(conn: openstack.connection.Connection) -> list: + """ + Retrieves role names. + Raises exception if elevated privileges (admin, manager) are present. + Otherwise returns list of role names. :param conn: connection to an OpenStack cloud. - :returns: boolean, when role with most privileges is member + :returns: list of role names """ role_names = set(conn.session.auth.get_access(conn.session).role_names) if role_names & {"admin", "manager"}: - return False + raise RuntimeError("user privileges too high: admin/manager roles detected") if "reader" in role_names: logger.info("User has reader role.") custom_roles = sorted(role_names - {"reader", "member"}) @@ -38,7 +41,8 @@ def compute_services_lookup(conn: openstack.connection.Connection) -> dict: return result -def compute_scs_0116_presence(services_lookup): +def compute_scs_0116_presence(services_lookup: dict) -> bool: + """This test checks that a service of type key-manager is present.""" services = services_lookup.get("key-manager", ()) if not services: logger.error("key-manager service not found") @@ -80,13 +84,12 @@ def _delete_secret(conn: openstack.connection.Connection, secret: openstack.key_ conn.key_manager.delete_secret(uuid) -def compute_scs_0116_permissions(conn: openstack.connection.Connection, services_lookup) -> None: +def compute_scs_0116_permissions(conn: openstack.connection.Connection, services_lookup) -> bool: """ - After checking that the current user only has the member and maybe the - reader role, this method verifies that the user with a member role - has sufficient access to the Key Manager API functionality. + After checking that the current user is not privileged, this test verifies that the user has + sufficient access to the Key Manager API functionality by creating and deleting a secret. """ - if "member" not in fetch_roles(conn): + if "member" not in ensure_unprivileged(conn): raise RuntimeError("Cannot test key-manager permissions. User has wrong roles") if not services_lookup.get("key-manager", ()): # this testcase only applies when a key manager is present diff --git a/Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py b/Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py deleted file mode 100755 index 6f57004e4..000000000 --- a/Tests/iaas/scs_0117_volume_backup/volume-backup-tester.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -"""Volume Backup API tester for Block Storage API - -This test script executes basic operations on the Block Storage API centered -around volume backups. Its purpose is to verify that the Volume Backup API is -available and working as expected using simple operations such as creating and -restoring volume backups. - -It verifies that a properly configured backup driver is present to the extent -that aforementioned operations succeed on the API level. It does not by any -means verify that the backup and restore procedures actual handle the data -correctly (it only uses empty volumes and does not look at data for the sake -of simplicity). -""" - -import argparse -import getpass -import logging -import os -import sys - -import openstack - - -from volume_backup import DEFAULT_PREFIX, cleanup, compute_scs_0117_test_backup - - -def main(): - parser = argparse.ArgumentParser( - description="SCS Volume Backup API Conformance Checker") - parser.add_argument( - "--os-cloud", type=str, - help="Name of the cloud from clouds.yaml, alternative " - "to the OS_CLOUD environment variable" - ) - parser.add_argument( - "--ask", - help="Ask for password interactively instead of reading it from the " - "clouds.yaml", - action="store_true" - ) - parser.add_argument( - "--debug", action="store_true", - help="Enable OpenStack SDK debug logging" - ) - parser.add_argument( - "--prefix", type=str, - default=DEFAULT_PREFIX, - help=f"OpenStack resource name prefix for all resources to be created " - f"and/or cleaned up by this script within the configured domains " - f"(default: '{DEFAULT_PREFIX}')" - ) - parser.add_argument( - "--cleanup-only", action="store_true", - help="Instead of executing tests, cleanup all resources " - "with the prefix specified via '--prefix' (or its default)" - ) - args = parser.parse_args() - openstack.enable_logging(debug=False) - logging.basicConfig( - format="%(levelname)s: %(message)s", - level=logging.DEBUG if args.debug else logging.INFO, - ) - - # parse cloud name for lookup in clouds.yaml - cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) - if not cloud: - raise Exception( - "You need to have the OS_CLOUD environment variable set to your " - "cloud name or pass it via --os-cloud" - ) - password = getpass.getpass("Enter password: ") if args.ask else None - - with openstack.connect(cloud, password=password) as conn: - if args.cleanup_only: - if not cleanup(conn, prefix=args.prefix): - raise RuntimeError("cleanup failed") - else: - compute_scs_0117_test_backup(conn, prefix=args.prefix) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except SystemExit: - raise - except BaseException as exc: - logging.debug("traceback", exc_info=True) - logging.critical(str(exc)) - sys.exit(1) diff --git a/Tests/iaas/scs_0117_volume_backup/volume_backup.py b/Tests/iaas/scs_0117_volume_backup/volume_backup.py index ff5494997..9bd1b9d3c 100644 --- a/Tests/iaas/scs_0117_volume_backup/volume_backup.py +++ b/Tests/iaas/scs_0117_volume_backup/volume_backup.py @@ -139,7 +139,6 @@ def cleanup(conn: openstack.connection.Connection, prefix=DEFAULT_PREFIX) -> boo Returns False if there were any errors during cleanup which might leave resources behind. Otherwise returns True to indicate cleanup success. """ - logging.info(f"Performing cleanup for resources with the '{prefix}' prefix ...") cleanup_issues = 0 # count failed cleanup operations @@ -204,6 +203,12 @@ def cleanup(conn: openstack.connection.Connection, prefix=DEFAULT_PREFIX) -> boo def compute_scs_0117_test_backup(conn, prefix=DEFAULT_PREFIX): + """ + This test verifies that a properly configured backup driver is present to the extent + that backup and restore operations succeed on the API level. It does not verify that + the restored volume is correct (for the sake of simplicity, it only uses empty volumes + and does not look at data). + """ if not cleanup(conn, prefix=prefix): raise RuntimeError("Initial cleanup failed") try: diff --git a/Tests/iaas/scs_0123_mandatory_services/mandatory-iaas-services.py b/Tests/iaas/scs_0123_mandatory_services/mandatory-iaas-services.py deleted file mode 100755 index 25f6c8213..000000000 --- a/Tests/iaas/scs_0123_mandatory_services/mandatory-iaas-services.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -"""Mandatory APIs checker -This script retrieves the endpoint catalog from Keystone using the OpenStack -SDK and checks whether all mandatory APi endpoints, are present. -The script relies on an OpenStack SDK compatible clouds.yaml file for -authentication with Keystone. -As the s3 endpoint might differ, a missing one will only result in a warning. -""" - -import argparse -from collections import Counter -import json -import logging -import os -import re -import sys -import uuid - -import boto3 -import openstack - - -TESTCONTNAME = "scs-test-container" -EC2MARKER = "TmpMandSvcTest" - -logger = logging.getLogger(__name__) -mandatory_services = ["compute", "identity", "image", "network", - "load-balancer", "placement", "object-store"] -block_storage_service = ["volume", "volumev3", "block-storage"] - - -def check_presence_of_mandatory_services(conn: openstack.connection.Connection, s3_credentials=None): - services = conn.service_catalog - - if s3_credentials: - mandatory_services.remove("object-store") - for svc in services: - svc_type = svc['type'] - if svc_type in mandatory_services: - mandatory_services.remove(svc_type) - elif svc_type in block_storage_service: - block_storage_service.remove(svc_type) - - bs_service_not_present = 0 - if len(block_storage_service) == 3: - # neither block-storage nor volume nor volumev3 is present - # we must assume, that there is no volume service - logger.error("No block-storage (volume) endpoint found.") - mandatory_services.append(block_storage_service[0]) - bs_service_not_present = 1 - if mandatory_services: - # some mandatory APIs were not found - logger.error(f"The following endpoints are missing: " - f"{', '.join(mandatory_services)}.") - return len(mandatory_services) + bs_service_not_present - - -def list_containers(conn): - "Gets a list of buckets" - return [cont.name for cont in conn.object_store.containers()] - - -def create_container(conn, name): - "Creates a test container" - conn.object_store.create_container(name) - return list_containers(conn) - - -def del_container(conn, name): - "Deletes a test container" - conn.object_store.delete(name) - # return list_containers(conn) - - -def s3_conn(creds, conn=None): - "Return an s3 client conn" - vrfy = True - if conn: - cacert = conn.config.config.get("cacert") - # TODO: Handle self-signed certs (from ca_cert in openstack config) - if cacert: - logger.warning(f"Trust all Certificates in S3, OpenStack uses {cacert}") - vrfy = False - return boto3.resource('s3', aws_access_key_id=creds["AK"], - aws_secret_access_key=creds["SK"], - endpoint_url=creds["HOST"], - verify=vrfy) - - -def list_s3_buckets(s3): - "Get a list of s3 buckets" - return [buck.name for buck in s3.buckets.all()] - - -def create_bucket(s3, name): - "Create an s3 bucket" - # bucket = s3.Bucket(name) - # bucket.create() - s3.create_bucket(Bucket=name) - return list_s3_buckets(s3) - - -def del_bucket(s3, name): - "Delete an s3 bucket" - buck = s3.Bucket(name=name) - buck.delete() - # s3.delete_bucket(Bucket=name) - - -def s3_from_env(creds, fieldnm, env, prefix=""): - "Set creds[fieldnm] to os.environ[env] if set" - if env in os.environ: - creds[fieldnm] = prefix + os.environ[env] - if fieldnm not in creds: - logger.warning(f"s3_creds[{fieldnm}] not set") - - -def s3_from_ostack(creds, conn, endpoint): - """Set creds from openstack swift/keystone - Returns credential ID *if* an ec2 credential was created, - None otherwise.""" - rgx = re.compile(r"^(https*://[^/]*)/") - match = rgx.match(endpoint) - if match: - creds["HOST"] = match.group(1) - # Use first ec2 cred that matches the project (if one exists) - project_id = conn.identity.get_project_id() - ec2_creds = [cred for cred in conn.identity.credentials() - if cred.type == "ec2" and cred.project_id == project_id] - found_ec2 = None - for cred in ec2_creds: - try: - ec2_dict = json.loads(cred.blob) - except Exception: - logger.warning(f"unable to parse credential {cred!r}", exc_info=True) - continue - # Clean up old EC2 creds and jump over - if ec2_dict.get("owner") == EC2MARKER: - logger.debug(f"Removing leftover credential {ec2_dict['access']}") - conn.identity.delete_credential(cred) - continue - found_ec2 = ec2_dict - if found_ec2: - creds["AK"] = found_ec2["access"] - creds["SK"] = found_ec2["secret"] - return - # Generate keyid and secret - ak = uuid.uuid4().hex - sk = uuid.uuid4().hex - blob = f'{{"access": "{ak}", "secret": "{sk}", "owner": "{EC2MARKER}"}}' - try: - crd = conn.identity.create_credential(type="ec2", blob=blob, - user_id=conn.current_user_id, - project_id=conn.current_project_id) - except BaseException: - logger.warning("ec2 creds creation failed", exc_info=True) - return - creds["AK"] = ak - creds["SK"] = sk - return crd.id - - -def check_for_s3_and_swift(conn: openstack.connection.Connection, s3_credentials=None): - # If we get credentials, we assume that there is no Swift and only test s3 - if s3_credentials: - try: - s3 = s3_conn(s3_credentials) - except Exception: - logger.debug("details", exc_info=True) - logger.error("Connection to S3 failed.") - return 1 - s3_buckets = list_s3_buckets(s3) or create_bucket(s3, TESTCONTNAME) - if not s3_buckets: - raise RuntimeError("failed to create S3 bucket") - if s3_buckets == [TESTCONTNAME]: - del_bucket(s3, TESTCONTNAME) - # everything worked, and we don't need to test for Swift: - logger.info("SUCCESS: S3 exists") - return 0 - # there were no credentials given, so we assume s3 is accessable via - # the service catalog and Swift might exist too - s3_creds = {} - try: - endpoint = conn.object_store.get_endpoint() - except Exception: - logger.exception( - "No object store endpoint found. No testing for the s3 service possible." - ) - return 1 - # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) - try: - ec2_cred = s3_from_ostack(s3_creds, conn, endpoint) - # Overrides (var names are from libs3, in case you wonder) - s3_from_env(s3_creds, "HOST", "S3_HOSTNAME", "https://") - s3_from_env(s3_creds, "AK", "S3_ACCESS_KEY_ID") - s3_from_env(s3_creds, "SK", "S3_SECRET_ACCESS_KEY") - - # This is to be used for local debugging purposes ONLY - # logger.info(f"using credentials {s3_creds}") - - s3 = s3_conn(s3_creds, conn) - s3_buckets = list_s3_buckets(s3) or create_bucket(s3, TESTCONTNAME) - if not s3_buckets: - raise RuntimeError("failed to create S3 bucket") - - # If we got till here, s3 is working, now swift - swift_containers = list_containers(conn) - # if not swift_containers: - # swift_containers = create_container(conn, TESTCONTNAME) - result = 0 - # Compare number of buckets/containers - # FIXME: Could compare list of sorted names - if Counter(s3_buckets) != Counter(swift_containers): - logger.error("S3 buckets and Swift Containers differ:\n" - f"S3: {sorted(s3_buckets)}\nSW: {sorted(swift_containers)}") - result = 1 - else: - logger.info("SUCCESS: S3 and Swift exist and agree") - # No need to clean up swift container, as we did not create one - # (If swift and S3 agree, there will be a S3 bucket that we clean up with S3.) - # if swift_containers == [TESTCONTNAME]: - # del_container(conn, TESTCONTNAME) - # Cleanup created S3 bucket - if s3_buckets == [TESTCONTNAME]: - del_bucket(s3, TESTCONTNAME) - # Clean up ec2 cred IF we created one - finally: - if ec2_cred: - conn.identity.delete_credential(ec2_cred) - return result - - -def main(): - parser = argparse.ArgumentParser( - description="SCS Mandatory IaaS Service Checker") - parser.add_argument( - "--os-cloud", type=str, - help="Name of the cloud from clouds.yaml, alternative " - "to the OS_CLOUD environment variable" - ) - parser.add_argument( - "--s3-endpoint", type=str, - help="URL to the s3 service." - ) - parser.add_argument( - "--s3-access", type=str, - help="Access Key to connect to the s3 service." - ) - parser.add_argument( - "--s3-access-secret", type=str, - help="Access secret to connect to the s3 service." - ) - parser.add_argument( - "--debug", action="store_true", - help="Enable OpenStack SDK debug logging" - ) - args = parser.parse_args() - logging.basicConfig( - format="%(levelname)s: %(message)s", - level=logging.DEBUG if args.debug else logging.INFO, - ) - openstack.enable_logging(debug=False) - - # parse cloud name for lookup in clouds.yaml - cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) - if not cloud: - raise RuntimeError( - "You need to have the OS_CLOUD environment variable set to your " - "cloud name or pass it via --os-cloud" - ) - - s3_credentials = None - if args.s3_endpoint: - if (not args.s3_access) or (not args.s3_access_secret): - logger.warning("test for external s3 needs access key and access secret.") - s3_credentials = { - "AK": args.s3_access, - "SK": args.s3_access_secret, - "HOST": args.s3_endpoint - } - elif args.s3_access or args.s3_access_secret: - logger.warning("access to s3 was given, but no endpoint provided.") - - with openstack.connect(cloud) as conn: - result = check_presence_of_mandatory_services(conn, s3_credentials) - result += check_for_s3_and_swift(conn, s3_credentials) - - print('service-apis-check: ' + ('PASS', 'FAIL')[min(1, result)]) - - return result - - -if __name__ == "__main__": - try: - sys.exit(main()) - except SystemExit: - raise - except BaseException as exc: - logging.debug("traceback", exc_info=True) - logging.critical(str(exc)) - sys.exit(1) From 6b30de7e8870ef042ed7dcb13aa93ac7ce549304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Thu, 28 Aug 2025 22:05:53 +0200 Subject: [PATCH 25/31] started adapting testing notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- ...w1-flavor-naming-implementation-testing.md | 9 +++-- ...-0101-w1-entropy-implementation-testing.md | 12 +++++- Standards/scs-0102-v1-image-metadata.md | 6 --- ...1-image-metadata-implementation-testing.md | 39 ++++++++++++++----- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/Standards/scs-0100-w1-flavor-naming-implementation-testing.md b/Standards/scs-0100-w1-flavor-naming-implementation-testing.md index 868215476..1bf8c5561 100644 --- a/Standards/scs-0100-w1-flavor-naming-implementation-testing.md +++ b/Standards/scs-0100-w1-flavor-naming-implementation-testing.md @@ -167,9 +167,12 @@ None so far. ### Implementation -The script [`flavor-names-openstack.py`](https://github.com/SovereignCloudStack/standards/tree/main/Tests/iaas/flavor-naming/flavor-names-openstack.py) -talks to the OpenStack API of the cloud specified by the `OS_CLOUD` environment and queries properties and -checks the names for standards compliance. +We implemented two testcases, paralleling the two items in the "Errors" section above: + +- `scs-0100-syntax-check` ensures that any name starting with `SCS-` adheres to the standard; +- `scs-0100-semantics-check` ensures that any such name is telling the truth as specified in the standard. + +These testcases can be checked using [`openstack_test.py`](https://github.com/SovereignCloudStack/standards/tree/main/Tests/iaas/openstack_test.py). ## Manual tests diff --git a/Standards/scs-0101-w1-entropy-implementation-testing.md b/Standards/scs-0101-w1-entropy-implementation-testing.md index a18ad61ce..213300cb2 100644 --- a/Standards/scs-0101-w1-entropy-implementation-testing.md +++ b/Standards/scs-0101-w1-entropy-implementation-testing.md @@ -60,8 +60,16 @@ as ensured by the image metadata standard. ### Implementation -The script [`entropy-check.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/entropy/entropy-check.py) -connects to OpenStack and performs the checks described in this section. +We implemented the following testcases that reflect the items in the above section +on automated tests: + +- `scs_0101_image_property`, +- `scs_0101_flavor_property`, +- `scs_0101_entropy_avail`, +- `scs_0101_rngd`, +- `scs_0101_fips_test` (covers both the error and warning case). + +These testcases can be checked using [`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). ## Manual tests diff --git a/Standards/scs-0102-v1-image-metadata.md b/Standards/scs-0102-v1-image-metadata.md index 8b78ed81c..578f84363 100644 --- a/Standards/scs-0102-v1-image-metadata.md +++ b/Standards/scs-0102-v1-image-metadata.md @@ -239,9 +239,3 @@ A boolean property that is not present is considered to be `false`. contact for issues with this image. Note that this field must only be set if the service provider does provide support for this image included in the image/flavor pricing (but it might be provided by a contracted 3rd party, e.g. the OS vendor). - -### Conformance Tests - -The script `image-md-check.py` retrieves the -image list from a configured cloud and checks each image for the -completeness and consistency of mandatory properties. diff --git a/Standards/scs-0102-w1-image-metadata-implementation-testing.md b/Standards/scs-0102-w1-image-metadata-implementation-testing.md index b2d9f5b75..5aad2c6f7 100644 --- a/Standards/scs-0102-w1-image-metadata-implementation-testing.md +++ b/Standards/scs-0102-w1-image-metadata-implementation-testing.md @@ -16,16 +16,35 @@ for these images. ## Automated tests -### Images sample - -Some checks need to be performed on a live instance. All publicly available images on this instance -will be checked for either only the mandatory properties or possibly also the recommended ones. -Additionally, a user can also decide to test their private images, although this isn't a necessity. - -### Implementation - -The script [`image-md-check.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/image-metadata/image-md-check.py) -connects to OpenStack and performs the checks described in this section. +We implemented a host of testcases to reflect the requirements and recommendations of the standard. The following +testcases ensure that fields have proper values: + +- `scs_0102_prop_architecture`, +- `scs_0102_prop_hash_algo`, +- `scs_0102_prop_min_disk`, +- `scs_0102_prop_min_ram`, +- `scs_0102_prop_os_version`, +- `scs_0102_prop_os_distro`, +- `scs_0102_prop_hw_disk_bus`, +- `scs_0102_prop_hypervisor_type`, +- `scs_0102_prop_hw_rng_model`, +- `scs_0102_prop_image_build_date`, +- `scs_0102_prop_image_original_user`, +- `scs_0102_prop_image_source`, +- `scs_0102_prop_image_description`, +- `scs_0102_prop_replace_frequency`, +- `scs_0102_prop_provided_until`, +- `scs_0102_prop_uuid_validity`, +- `scs_0102_prop_hotfix_hours`. + +The property `patchlevel` is not tested because it is entirely optional. + +The following testcase ensures that each image is as recent as claimed by its `replace_frequency`: + +- `scs_0102_image_recency` + +The script [`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py) +can be used to check these testcases. ## Manual tests From 6e6ea9dc249519bdb0fb269e77ef668309186405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 29 Aug 2025 13:15:09 +0200 Subject: [PATCH 26/31] adapt testing notes, contd. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Standards/scs-0103-v1-standard-flavors.md | 21 --------------- ...0103-w1-standard-flavors-implementation.md | 27 +++++++++++++++++++ Standards/scs-0104-v1-standard-images.md | 15 ----------- ...-0104-w1-standard-images-implementation.md | 17 ++++++++++++ 4 files changed, 44 insertions(+), 36 deletions(-) create mode 100644 Standards/scs-0103-w1-standard-flavors-implementation.md diff --git a/Standards/scs-0103-v1-standard-flavors.md b/Standards/scs-0103-v1-standard-flavors.md index cee9282f2..b85d66288 100644 --- a/Standards/scs-0103-v1-standard-flavors.md +++ b/Standards/scs-0103-v1-standard-flavors.md @@ -156,27 +156,6 @@ with `block_device_mapping_v2`, e.g. to create a bootable 12G cinder volume from image `IMGUUID` that gets tied to the VM instance life cycle.) -## Conformance Tests - -The script [`flavors-openstack.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/standard-flavors/flavors-openstack.py) -will read the lists of mandatory and recommended flavors -from a yaml file provided as command-line argument, connect to an OpenStack installation, -and check whether the flavors are present and their extra_specs are correct. - -Missing flavors will be reported on various logging channels: error for mandatory, warning for -recommended flavors. Incorrect extra_specs will be reported as error in any case. -The return code will be non-zero if the test could not be performed or if any error was -reported. - -The script does not check whether a name given via the extra_spec `scs:name-vN` is indeed valid according -to any major version of the SCS standard on flavor naming. - -## Operational tooling - -The [openstack-flavor-manager](https://github.com/osism/openstack-flavor-manager) is able to -create all standard, mandatory SCS flavors for you. It takes input that can be generated by -`flavor-manager-input.py`. - ## Previous standard versions The list of standard flavors used to be part of the flavor naming standard up until diff --git a/Standards/scs-0103-w1-standard-flavors-implementation.md b/Standards/scs-0103-w1-standard-flavors-implementation.md new file mode 100644 index 000000000..ce332d3cc --- /dev/null +++ b/Standards/scs-0103-w1-standard-flavors-implementation.md @@ -0,0 +1,27 @@ +--- +title: "SCS Standard Flavors: Implementation Notes" +type: Supplement +track: IaaS +status: Draft +supplements: + - scs-0103-v1-standard-flavors.md +--- + +## Operational tooling + +The [openstack-flavor-manager](https://github.com/osism/openstack-flavor-manager) is able to +create all standard, mandatory SCS flavors for you. It takes input that can be generated by +[`flavor-manager-input.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/scs_0100_flavor_naming/flavor-manager-input.py). + +## Automated tests + +We implemented a set of testcases corresponding 1:1 to the standard flavors: + +- `scs_0103_flavor_X` with varying `X`: ensures that flavor `SCS-X` is present and has the + required `extra_specs`. + +_NOTE_: We still need to add testcases to ensure that the `extra_specs` of non-standard +flavors are correct as well. + +The testcases can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). diff --git a/Standards/scs-0104-v1-standard-images.md b/Standards/scs-0104-v1-standard-images.md index 06e87a8f6..dab829a93 100644 --- a/Standards/scs-0104-v1-standard-images.md +++ b/Standards/scs-0104-v1-standard-images.md @@ -128,18 +128,3 @@ next version. This standard makes no statement as to what is supposed to happen corresponding images in a live cloud environment. It is recommended to keep the once-mandatory images in the live environment. As for new environments, it is up to the operator whether to provide any or all of these images, as stated above. - -## Conformance Tests - -The script `images-openstack.py` will read the lists of mandatory and recommended images -from a yaml file provided as command-line argument, connect to an OpenStack installation, -and check whether the images are present. Missing images will be reported on various -logging channels: error for mandatory, info for recommended images. Additionally, images -whose `image_source` does not conform with the specifications will be reported on the -error channel. The return code will be non-zero if the test could not be performed or -if any errors have been reported. - -## Operational tooling - -The [openstack-image-manager](https://github.com/osism/openstack-image-manager) is able to -create all standard, mandatory SCS images for you given image definitions from a YAML file. diff --git a/Standards/scs-0104-w1-standard-images-implementation.md b/Standards/scs-0104-w1-standard-images-implementation.md index 9a18a9056..029d6998e 100644 --- a/Standards/scs-0104-w1-standard-images-implementation.md +++ b/Standards/scs-0104-w1-standard-images-implementation.md @@ -59,3 +59,20 @@ Run the test script on your environment and check the error messages :) The [openstack-image-manager](https://github.com/osism/openstack-image-manager) is able to create all standard, mandatory SCS images for you given image definitions from a YAML file. Please see [its documentation](https://docs.scs.community/docs/iaas/components/image-manager/) for details. + +## Automated tests + +We implemented testcases reflecting the information given in the YAML file(s). Be advised that +the approach with the YAML file has considerable drawbacks, and it will be abandoned in +future versions of the standard in favor of a more classical approach. The testcases already +anticipate this future development. + +There are two classes of testcases: + +- `scs_0104_source_X` with varying `X`: + these ensure that certain images have the correct `image_source`; +- `scs_0104_image_X` with varying `X`: + these ensure that certain images can be found in the list of public images. + +The testcases can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). From c14e7a7410320071acbbf667a768c057520c05ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 29 Aug 2025 15:42:49 +0200 Subject: [PATCH 27/31] adapt testing notes, contd. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- ...-0101-w1-entropy-implementation-testing.md | 10 +++--- ...1-image-metadata-implementation-testing.md | 36 +++++++++---------- ...0103-w1-standard-flavors-implementation.md | 4 +-- ...-0104-w1-standard-images-implementation.md | 6 ++-- Standards/scs-0114-v1-volume-type-standard.md | 12 ------- .../scs-0114-w1-volume-type-implementation.md | 20 +++++++++++ ...15-v1-default-rules-for-security-groups.md | 5 --- ...-0115-w1-security-groups-implementation.md | 22 ++++++++++++ ...6-w1-key-manager-implementation-testing.md | 10 +++--- .../scs-0117-v1-volume-backup-service.md | 12 ------- ...w1-volume-backup-service-implementation.md | 26 ++++++++++++++ ...-supported-IaaS-services-implementation.md | 19 ++++++++++ 12 files changed, 120 insertions(+), 62 deletions(-) create mode 100644 Standards/scs-0114-w1-volume-type-implementation.md create mode 100644 Standards/scs-0115-w1-security-groups-implementation.md create mode 100644 Standards/scs-0117-w1-volume-backup-service-implementation.md create mode 100644 Standards/scs-0123-w1-mandatory-and-supported-IaaS-services-implementation.md diff --git a/Standards/scs-0101-w1-entropy-implementation-testing.md b/Standards/scs-0101-w1-entropy-implementation-testing.md index 213300cb2..c163b3b6b 100644 --- a/Standards/scs-0101-w1-entropy-implementation-testing.md +++ b/Standards/scs-0101-w1-entropy-implementation-testing.md @@ -63,11 +63,11 @@ as ensured by the image metadata standard. We implemented the following testcases that reflect the items in the above section on automated tests: -- `scs_0101_image_property`, -- `scs_0101_flavor_property`, -- `scs_0101_entropy_avail`, -- `scs_0101_rngd`, -- `scs_0101_fips_test` (covers both the error and warning case). +- `scs-0101-image-property`, +- `scs-0101-flavor-property`, +- `scs-0101-entropy-avail`, +- `scs-0101-rngd`, +- `scs-0101-fips-test` (covers both the error and warning case). These testcases can be checked using [`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). diff --git a/Standards/scs-0102-w1-image-metadata-implementation-testing.md b/Standards/scs-0102-w1-image-metadata-implementation-testing.md index 5aad2c6f7..3a1d8b11f 100644 --- a/Standards/scs-0102-w1-image-metadata-implementation-testing.md +++ b/Standards/scs-0102-w1-image-metadata-implementation-testing.md @@ -19,29 +19,29 @@ for these images. We implemented a host of testcases to reflect the requirements and recommendations of the standard. The following testcases ensure that fields have proper values: -- `scs_0102_prop_architecture`, -- `scs_0102_prop_hash_algo`, -- `scs_0102_prop_min_disk`, -- `scs_0102_prop_min_ram`, -- `scs_0102_prop_os_version`, -- `scs_0102_prop_os_distro`, -- `scs_0102_prop_hw_disk_bus`, -- `scs_0102_prop_hypervisor_type`, -- `scs_0102_prop_hw_rng_model`, -- `scs_0102_prop_image_build_date`, -- `scs_0102_prop_image_original_user`, -- `scs_0102_prop_image_source`, -- `scs_0102_prop_image_description`, -- `scs_0102_prop_replace_frequency`, -- `scs_0102_prop_provided_until`, -- `scs_0102_prop_uuid_validity`, -- `scs_0102_prop_hotfix_hours`. +- `scs-0102-prop-architecture`, +- `scs-0102-prop-hash_algo`, +- `scs-0102-prop-min_disk`, +- `scs-0102-prop-min_ram`, +- `scs-0102-prop-os_version`, +- `scs-0102-prop-os_distro`, +- `scs-0102-prop-hw_disk_bus`, +- `scs-0102-prop-hypervisor_type`, +- `scs-0102-prop-hw_rng_model`, +- `scs-0102-prop-image_build_date`, +- `scs-0102-prop-image_original_user`, +- `scs-0102-prop-image_source`, +- `scs-0102-prop-image_description`, +- `scs-0102-prop-replace_frequency`, +- `scs-0102-prop-provided_until`, +- `scs-0102-prop-uuid_validity`, +- `scs-0102-prop-hotfix_hours`. The property `patchlevel` is not tested because it is entirely optional. The following testcase ensures that each image is as recent as claimed by its `replace_frequency`: -- `scs_0102_image_recency` +- `scs-0102-image-recency` The script [`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py) can be used to check these testcases. diff --git a/Standards/scs-0103-w1-standard-flavors-implementation.md b/Standards/scs-0103-w1-standard-flavors-implementation.md index ce332d3cc..b47318cce 100644 --- a/Standards/scs-0103-w1-standard-flavors-implementation.md +++ b/Standards/scs-0103-w1-standard-flavors-implementation.md @@ -1,5 +1,5 @@ --- -title: "SCS Standard Flavors: Implementation Notes" +title: "SCS Standard Flavors: Implementation and Testing Notes" type: Supplement track: IaaS status: Draft @@ -17,7 +17,7 @@ create all standard, mandatory SCS flavors for you. It takes input that can be g We implemented a set of testcases corresponding 1:1 to the standard flavors: -- `scs_0103_flavor_X` with varying `X`: ensures that flavor `SCS-X` is present and has the +- `scs-0103-flavor-X` with varying `X`: ensures that flavor `SCS-X` is present and has the required `extra_specs`. _NOTE_: We still need to add testcases to ensure that the `extra_specs` of non-standard diff --git a/Standards/scs-0104-w1-standard-images-implementation.md b/Standards/scs-0104-w1-standard-images-implementation.md index 029d6998e..e21617222 100644 --- a/Standards/scs-0104-w1-standard-images-implementation.md +++ b/Standards/scs-0104-w1-standard-images-implementation.md @@ -1,5 +1,5 @@ --- -title: "SCS Standard Images: Implementation Notes" +title: "SCS Standard Images: Implementation and Testing Notes" type: Supplement track: IaaS status: Draft @@ -69,9 +69,9 @@ anticipate this future development. There are two classes of testcases: -- `scs_0104_source_X` with varying `X`: +- `scs-0104-source-X` with varying `X`: these ensure that certain images have the correct `image_source`; -- `scs_0104_image_X` with varying `X`: +- `scs-0104-image-X` with varying `X`: these ensure that certain images can be found in the list of public images. The testcases can be run using the script diff --git a/Standards/scs-0114-v1-volume-type-standard.md b/Standards/scs-0114-v1-volume-type-standard.md index 003db9a24..16b02224a 100644 --- a/Standards/scs-0114-v1-volume-type-standard.md +++ b/Standards/scs-0114-v1-volume-type-standard.md @@ -124,15 +124,3 @@ It should look like the following part: ## Related Documents - corresponding [decision record document](https://github.com/SovereignCloudStack/standards/blob/main/Standards/scs-0111-v1-volume-type-decisions.md) - -## Conformance Tests - -The script `/Tests/iaas/volume-types/volume-types-check.py` connects to an OpenStack environment and tests -the following: - -- for each volume type: if its description starts with `[scs:....]`, then this prefix is a feature list - (sorted, each entry at most once), and each entry is one of the possible features described here, -- the recommended volume types are present (otherwise, a WARNING is produced). - -The return code is zero precisely when the test could be performed and the conditions are satisfied. -Otherwise, detailed errors and warnings are output to stderr. diff --git a/Standards/scs-0114-w1-volume-type-implementation.md b/Standards/scs-0114-w1-volume-type-implementation.md new file mode 100644 index 000000000..13997f4fb --- /dev/null +++ b/Standards/scs-0114-w1-volume-type-implementation.md @@ -0,0 +1,20 @@ +--- +title: "SCS Volume Types: Testing Notes" +type: Supplement +track: IaaS +status: Draft +supplements: + - scs-0114-v1-volume-type-standard.md +--- + +## Automated tests + +We implemented the following testcases: + +- `scs-0114-syntax-check` ensures that, for every volume type description, + the list of aspects, if present, is formatted according to the standard. +- `scs-0114-encrypted-type` ensures that a volume type featuring encryption is present. +- `scs-0114-replicated-type` ensures that a volume type featuring replication is present. + +The testcases can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). diff --git a/Standards/scs-0115-v1-default-rules-for-security-groups.md b/Standards/scs-0115-v1-default-rules-for-security-groups.md index 8809a2857..7dbdb1ea5 100644 --- a/Standards/scs-0115-v1-default-rules-for-security-groups.md +++ b/Standards/scs-0115-v1-default-rules-for-security-groups.md @@ -134,8 +134,3 @@ $ openstack default security group rule list The spec for introducing configurability for the default Security Groups Rules can be found [here](https://specs.openstack.org/openstack/neutron-specs/specs/2023.2/configurable-default-sg-rules.html). More about Security Groups as a resource in OpenStack can be found [here](https://docs.openstack.org/nova/latest/user/security-groups.html). - -## Conformance Tests - -The conformance tests should check for the absence of any ingress traffic rules except traffic from the same Security Group in the `openstack default security group rule list`. -As having egress rules is allowed by this standard, but not forced and can be set in various ways, the tests should check for presence of any egress rules. diff --git a/Standards/scs-0115-w1-security-groups-implementation.md b/Standards/scs-0115-w1-security-groups-implementation.md new file mode 100644 index 000000000..09a1d66d6 --- /dev/null +++ b/Standards/scs-0115-w1-security-groups-implementation.md @@ -0,0 +1,22 @@ +--- +title: "Default Rules for Security Groups: Implementation and Testing Notes" +type: Supplement +track: IaaS +status: Draft +supplements: + - scs-0115-v1-default-rules-for-security-groups.md +--- + +## Automated tests + +We implemented a single test case, + +- `scs-0115-default-rules`, + +which ensures + +1. the absence of any ingress traffic rules except traffic from the same Security Group in the `openstack default security group rule list`, +2. the presence of any egress traffic rules. + +The testcase can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). diff --git a/Standards/scs-0116-w1-key-manager-implementation-testing.md b/Standards/scs-0116-w1-key-manager-implementation-testing.md index d3acc6b4c..23d9f1f76 100644 --- a/Standards/scs-0116-w1-key-manager-implementation-testing.md +++ b/Standards/scs-0116-w1-key-manager-implementation-testing.md @@ -41,13 +41,13 @@ This can be done with a small change in the policy.yaml file. The `creator` has ## Automated Tests -The check for the presence of a Key Manager is done with a test script, that checks the presence of a Key Manager service in the catalog endpoint of Openstack. -This check can eventually be moved to the checks for the mandatory an supported service/API list, in case of a promotion of the Key Manager to the mandatory list. +We implemented the following testcases, in accordance with the standard: -### Implementation +- `scs-0116-presence` ensures that a service of type "key-manager" occurs in the service catalog; +- `scs-0116-permissions` ensures that a regular user has suitable access to the key-manager API. -The script [`check-for-key-manager.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/key-manager/check-for-key-manager.py) -connects to OpenStack and performs the checks described in this section. +The testcases can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). ## Manual Tests diff --git a/Standards/scs-0117-v1-volume-backup-service.md b/Standards/scs-0117-v1-volume-backup-service.md index 9838536fa..f08bef96b 100644 --- a/Standards/scs-0117-v1-volume-backup-service.md +++ b/Standards/scs-0117-v1-volume-backup-service.md @@ -84,15 +84,3 @@ The volume backup target storage SHOULD be a separate storage system from the on - [OpenStack Block Storage v3 Backup API reference](https://docs.openstack.org/api-ref/block-storage/v3/index.html#backups-backups) - [OpenStack Volume Backup Drivers](https://docs.openstack.org/cinder/latest/configuration/block-storage/backup-drivers.html) - -## Conformance Tests - -Conformance tests include using the `/v3/{project_id}/backups` Block Storage API endpoint to create a volume and a backup of it as a non-admin user and subsequently restore the backup on a new volume while verifying the success of each operation. -These tests verify the mandatory part of the standard: providing the Volume Backup API. - -There is a test suite in [`volume-backup-tester.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/volume-backup/volume-backup-tester.py). -The test suite connects to the OpenStack API and executes basic operations using the volume backup API to verify that the functionality requested by the standard is available. -Please consult the associated [README.md](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/volume-backup/README.md) for detailed setup and testing instructions. - -Note that these tests don't verify the optional part of the standard: providing a separate storage backend for Cinder volume backups. -This cannot be checked from outside of the infrastructure as it is an architectural property of the infrastructure itself and transparent to customers. diff --git a/Standards/scs-0117-w1-volume-backup-service-implementation.md b/Standards/scs-0117-w1-volume-backup-service-implementation.md new file mode 100644 index 000000000..f9f9d6028 --- /dev/null +++ b/Standards/scs-0117-w1-volume-backup-service-implementation.md @@ -0,0 +1,26 @@ +--- +title: "Volume Backup Functionality: Implementation and Testing Notes" +type: Supplement +track: IaaS +status: Draft +supplements: + - scs-0117-v1-volume-backup-service.md +--- + +## Automated tests + +We implemented a single testcase, + +- `scs-0117-test-backup`, + +which verifies that a non-admin user can backup and restore volumes. + +To this end, the testcase uses the `/v3/{project_id}/backups` Block Storage API endpoint to create a volume and a backup of it and subsequently restores the backup on a new volume while verifying the success of each operation. + +The testcase can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). + +## Manual tests + +Note that the automated tests don't verify the optional part of the standard: providing a separate storage backend for Cinder volume backups. +This cannot be checked from outside of the infrastructure as it is an architectural property of the infrastructure itself and transparent to customers. diff --git a/Standards/scs-0123-w1-mandatory-and-supported-IaaS-services-implementation.md b/Standards/scs-0123-w1-mandatory-and-supported-IaaS-services-implementation.md new file mode 100644 index 000000000..defa9903a --- /dev/null +++ b/Standards/scs-0123-w1-mandatory-and-supported-IaaS-services-implementation.md @@ -0,0 +1,19 @@ +--- +title: "Mandatory and Supported IaaS Services: Implementation and Testing Notes" +type: Supplement +track: IaaS +status: Draft +supplements: + - scs-0123-v1-mandatory-and-supported-IaaS-services.md +--- + +## Automated tests + +We implemented the following testcases in accordance with the standard: + +- `scs-0123-service-X` (with varying `X`) ensures that a service of type X can be found in the service catalog, +- `scs-0123-storage-apis` ensures that a service of one of the following types can be found: "volume", "volumev3", "block-storage", +- `scs-0123-swift-s3` ensures that S3 can be used to access object storage using EC2 credentials from the identity API. + +The testcases can be run using the script +[`openstack_test.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/openstack_test.py). From 95515ab84c6efd58c1abbb71962f4f6e63584615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 29 Aug 2025 16:07:49 +0200 Subject: [PATCH 28/31] added missing doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/scs_0123_mandatory_services/mandatory_services.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py b/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py index 7991aeff2..4812e2ab0 100644 --- a/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py +++ b/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py @@ -115,6 +115,11 @@ def s3_from_ostack(usable_credentials, conn, rgx=re.compile(r"^(https*://[^/]*)/ def compute_scs_0123_swift_s3(conn: openstack.connection.Connection): + """ + This test ensures that S3 can be used to access object storage using EC2 credentials from the identity API. + It will abort with an exception if no service of type object-storage is present. As of now, we deem + this behavior adequate. + """ # we assume s3 is accessable via the service catalog, and Swift might exist too usable_credentials = [] s3_buckets = [] From 49964f3302c14249e59f90649648476693753574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 29 Aug 2025 16:33:50 +0200 Subject: [PATCH 29/31] adapt testcase evaluation and reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/scs-compliance-check.py | 9 ++++++--- Tests/scs_cert_lib.py | 28 +++++++++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Tests/scs-compliance-check.py b/Tests/scs-compliance-check.py index c7c615fd6..315794259 100755 --- a/Tests/scs-compliance-check.py +++ b/Tests/scs-compliance-check.py @@ -282,16 +282,19 @@ def run_suite(suite: TestSuite, runner: CheckRunner): def print_report(subject: str, suite: TestSuite, targets: dict, results: dict, verbose=False): print(f"{subject} {suite.name}:") for tname, target_spec in targets.items(): - failed, missing, passed = suite.select(tname, target_spec).eval_buckets(results) - verdict = 'FAIL' if failed else 'TENTATIVE pass' if missing else 'PASS' + by_value = suite.select(tname, target_spec).eval_buckets(results) + missing, failed, aborted, passed = by_value[None], by_value[-1], by_value[0], by_value[1] + verdict = 'FAIL' if failed or aborted else 'TENTATIVE pass' if missing else 'PASS' summary_parts = [f"{len(passed)} passed"] if failed: summary_parts.append(f"{len(failed)} failed") + if aborted: + summary_parts.append(f"{len(aborted)} aborted") if missing: summary_parts.append(f"{len(missing)} missing") verdict += f" ({', '.join(summary_parts)})" print(f"- {tname}: {verdict}") - reportcateg = [(failed, 'FAILED'), (missing, 'MISSING')] + reportcateg = [(failed, 'FAILED'), (aborted, 'ABORTED'), (missing, 'MISSING')] if verbose: reportcateg.append((passed, 'PASSED')) for offenders, category in reportcateg: diff --git a/Tests/scs_cert_lib.py b/Tests/scs_cert_lib.py index 968ede809..25957bc20 100644 --- a/Tests/scs_cert_lib.py +++ b/Tests/scs_cert_lib.py @@ -23,12 +23,10 @@ 'testcases': ('lifetime', 'id', 'description', 'tags'), 'include': ('ref', 'parameters'), } -# The canonical result values are -1, 0, and 1, for FAIL, MISS (or DNF), and PASS, respectively; -# these concrete numbers are important because we do rely on their ordering. Note that MISS/DNF should -# not be reported because it is tantamount to a result being absent. (Therefore the NIL_RESULT default -# value below.) -TESTCASE_VERDICTS = {'PASS': 1, 'FAIL': -1} -NIL_RESULT = {'result': 0} +# The canonical result values are -1, 0, and 1, for FAIL, ABORT, and PASS, respectively; +# -- in addition, None is used to encode a missing value, but must not be included in a formal report! -- +# these concrete numbers are important because we do rely on their ordering. +TESTCASE_VERDICTS = {'PASS': 1, 'ABORT': 0, 'FAIL': -1} def _check_keywords(ctx, d, keywords=KEYWORDS): @@ -208,18 +206,26 @@ def select(self, name, selectors): suite.include_testcases([tc for tc in self.testcases if test_selectors(selectors, tc['tags'])]) return suite - def eval_buckets(self, results) -> tuple[list[dict], list[dict], list[dict]]: - """returns lists of (failed, missing, passed) test cases""" + def eval_buckets(self, results) -> dict: + """ + returns buckets of test cases by means of a mapping + + None: list of missing testcases + -1: list of failed testcases + 0: list of aborted testcases + 1: list of passed testcases + """ by_value = defaultdict(list) for testcase in self.testcases: - value = results.get(testcase['id'], NIL_RESULT).get('result', 0) + value = results.get(testcase['id'], {}).get('result') by_value[value].append(testcase) - return by_value[-1], by_value[0], by_value[1] + return by_value def evaluate(self, results) -> int: """returns overall result""" return min([ - results.get(testcase['id'], NIL_RESULT).get('result', 0) + # here, we treat None (MISSING) as 0 (ABORT) + results.get(testcase['id'], {}).get('result') or 0 for testcase in self.testcases ], default=0) From 30f1f923218f5245a1e43c5c8124a69f9906f762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 2 Sep 2025 22:01:26 +0200 Subject: [PATCH 30/31] remove obsolete documentation and scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/iaas/scs_0100_flavor_naming/README.md | 44 +--------- .../flavor-manager-input.py | 84 ------------------- Tests/iaas/scs_0117_volume_backup/README.md | 70 ---------------- .../scs_0123_mandatory_services/README.md | 66 --------------- 4 files changed, 2 insertions(+), 262 deletions(-) delete mode 100755 Tests/iaas/scs_0103_standard_flavors/flavor-manager-input.py delete mode 100644 Tests/iaas/scs_0117_volume_backup/README.md delete mode 100644 Tests/iaas/scs_0123_mandatory_services/README.md diff --git a/Tests/iaas/scs_0100_flavor_naming/README.md b/Tests/iaas/scs_0100_flavor_naming/README.md index ccd2953af..f43eec9a0 100644 --- a/Tests/iaas/scs_0100_flavor_naming/README.md +++ b/Tests/iaas/scs_0100_flavor_naming/README.md @@ -1,12 +1,10 @@ # SCS flavor name tooling -This directory contains three basic tools: +This directory contains two basic tools: - `cli.py`, a command-line interface for syntax-checking flavor names, turning them into prose descriptions or yaml documents, and interactively constructing flavor names; -- `flavor-form.py`, a CGI script for a web form that offers roughly the same features, -- `flavor-names-openstack.py`, a script that connects to an OpenStack environment and checks the syntax - of all publicly available flavors. +- `flavor-form.py`, a CGI script for a web form that offers roughly the same features. Besides, it contains the scripts `flavor-name-check.py` and `flavor-name-describe.py`, which have been superseded by `cli.py`, and it contains `flavor-form-server.py`, which is a most simple webserver that can @@ -188,41 +186,3 @@ The form is accessible at http://0.0.0.0:8000/cgi-bin/flavor-form.py ^C Keyboard interrupt received, exiting. ``` - -## flavor-names-openstack.py - -As with `cli.py`, we have a look at the built-in help: - -```console -$ ./flavor-names-openstack.py -h -Usage: flavor-names-openstack.py [options] -Options: [-c/--os-cloud OS_CLOUD] sets cloud environment (default from OS_CLOUD env) - [-C/--mand mand.yaml] overrides the list of mandatory flavor names - [-1/--v1prefer] prefer v1 flavor names (but still tolerates v2 - [-o/--accept-old-mandatory] prefer v2 flavor names, but v1 ones can fulfill mand list - [-2/--v2plus] only accepts v2 flavor names, old ones result in errors - [-3/--v3] differentiate b/w mand and recommended flavors - [-v/--verbose] [-q/--quiet] control verbosity of output -This tool retrieves the list of flavors from the OpenStack cloud OS_CLOUD - and checks for the presence of the mandatory SCS flavors (read from mand.yaml) - and reports inconsistencies, errors etc. It returns 0 on success. -``` - -Example invocation: - -```console -$ OS_CLOUD=wave1-compliance ./flavor-names-openstack.py -wave1-compliance: - OtherFlavorSummary: - TotalAmount: 0 - SCSFlavorSummary: - FlavorsWithWarnings: 0 - MandatoryFlavorsMissing: 0 - MandatoryFlavorsPresent: 26 - OptionalFlavorsValid: 37 - OptionalFlavorsWrong: 0 - TotalAmount: 63 - TotalSummary: - Errors: 0 - Warnings: 0 -``` diff --git a/Tests/iaas/scs_0103_standard_flavors/flavor-manager-input.py b/Tests/iaas/scs_0103_standard_flavors/flavor-manager-input.py deleted file mode 100755 index b420e8f47..000000000 --- a/Tests/iaas/scs_0103_standard_flavors/flavor-manager-input.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -""" -Convert yaml format from the one expected by flavors-openstack.py to the one expected -by osism's flavor manager, cf. https://github.com/osism/openstack-flavor-manager . - -The conversion is performed from stdin to stdout. -""" -import logging -import sys - -import yaml - - -logger = logging.getLogger(__name__) - - -def main(argv): - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - - # load the yaml, checking for sanity - try: - flavor_spec_data = yaml.safe_load(sys.stdin) - except Exception as e: - logger.critical(f"Unable to load: {e!r}") - logger.debug("Exception info", exc_info=True) - return 1 - - if 'meta' not in flavor_spec_data or 'name_key' not in flavor_spec_data['meta']: - logger.critical("Flavor definition missing 'meta' field or field incomplete") - return 1 - - if 'flavor_groups' not in flavor_spec_data: - logger.critical("Flavor definition missing 'flavor_groups' field") - - # boilerplate / scaffolding - result = yaml.safe_load(""" -reference: - - field: name - mandatory_prefix: SCS- - - field: public - default: true - - field: disabled - default: false - - field: cpus - - field: ram - - field: disk -mandatory: [] -recommended: [] -""") - - # transfer the info into the result yaml, again checking for sanity - for flavor_group in flavor_spec_data['flavor_groups']: - group_info = dict(flavor_group) - group_info.pop('list') - missing = {'status'} - set(group_info) - if missing: - logging.critical(f"Flavor group missing attributes: {', '.join(missing)}") - return 1 - group_info.pop('status') - result_list = result[flavor_group['status']] - for flavor_spec in flavor_group['list']: - missing = {'name', 'cpus', 'ram'} - set(flavor_spec) - if missing: - logging.critical(f"Flavor spec missing attributes: {', '.join(missing)}") - return 1 - flavor_spec = {**group_info, **flavor_spec} - flavor_spec['ram'] = flavor_spec['ram'] * 1024 - extra_specs = { - f"scs:{key}": value - for key, value in flavor_spec.items() - if key not in ('name', 'cpus', 'ram', 'disk') - } - flavor_spec = { - key: value - for key, value in flavor_spec.items() - if key in ('name', 'cpus', 'ram', 'disk') - } - result_list.append({**flavor_spec, **extra_specs}) - - print(yaml.dump(result, sort_keys=False)) - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tests/iaas/scs_0117_volume_backup/README.md b/Tests/iaas/scs_0117_volume_backup/README.md deleted file mode 100644 index 2b6cd4716..000000000 --- a/Tests/iaas/scs_0117_volume_backup/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Volume Backup API Test Suite - -## Test Environment Setup - -### Test Execution Environment - -> **NOTE:** The test execution procedure does not require cloud admin rights. - -To execute the test suite a valid cloud configuration for the OpenStack SDK in the shape of "`clouds.yaml`" is mandatory[^1]. -**The file is expected to be located in the current working directory where the test script is executed unless configured otherwise.** - -[^1]: [OpenStack Documentation: Configuring OpenStack SDK Applications](https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html) - -The test execution environment can be located on any system outside of the cloud infrastructure that has OpenStack API access. -Make sure that the API access is configured properly in "`clouds.yaml`". - -It is recommended to use a Python virtual environment[^2]. -Next, install the OpenStack SDK required by the test suite: - -```bash -pip3 install openstacksdk -``` - -Within this environment execute the test suite. - -[^2]: [Python 3 Documentation: Virtual Environments and Packages](https://docs.python.org/3/tutorial/venv.html) - -## Test Execution - -The test suite is executed as follows: - -```bash -python3 volume-backup-tester.py --os-cloud mycloud -``` - -As an alternative to "`--os-cloud`", the "`OS_CLOUD`" environment variable may be specified instead. -The parameter is used to look up the correct cloud configuration in "`clouds.yaml`". -For the example command above, this file should contain a `clouds.mycloud` section like this: - -```yaml ---- -clouds: - mycloud: - auth: - auth_url: ... - ... - ... -``` - -If the test suite fails and leaves test resources behind, the "`--cleanup-only`" flag may be used to delete those resources from the domains: - -```bash -python3 volume-backup-tester.py --os-cloud mycloud --cleanup-only -``` - -For any further options consult the output of "`python3 volume-backup-tester.py --help`". - -### Script Behavior & Test Results - -> **NOTE:** Before any execution of test batches, the script will automatically perform a cleanup of volumes and volume backups matching a special prefix (see the "`--prefix`" flag). -> This cleanup behavior is identical to "`--cleanup-only`". - -The script will print all cleanup actions and passed tests to `stdout`. - -If all tests pass, the script will return with an exit code of `0`. - -If any test fails, the script will halt, print the exact error to `stderr` and return with a non-zero exit code. - -In case of a failed test, cleanup is not performed automatically, allowing for manual inspection of the cloud state for debugging purposes. -Although unnecessary due to automatic cleanup upon next execution, you can manually trigger a cleanup using the "`--cleanup-only`" flag of this script. diff --git a/Tests/iaas/scs_0123_mandatory_services/README.md b/Tests/iaas/scs_0123_mandatory_services/README.md deleted file mode 100644 index 33a66d7f4..000000000 --- a/Tests/iaas/scs_0123_mandatory_services/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Mandatory IaaS Service APIs Test Suite - -## Test Environment Setup - -### Test Execution Environment - -> **NOTE:** The test execution procedure does not require cloud admin rights. - -To execute the test suite a valid cloud configuration for the OpenStack SDK in the shape of "`clouds.yaml`" is mandatory[^1]. -**The file is expected to be located in the current working directory where the test script is executed unless configured otherwise.** - -[^1]: [OpenStack Documentation: Configuring OpenStack SDK Applications](https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html) - -The test execution environment can be located on any system outside of the cloud infrastructure that has OpenStack API access. -Make sure that the API access is configured properly in "`clouds.yaml`". - -It is recommended to use a Python virtual environment[^2]. -Next, install the OpenStack SDK and boto3 required by the test suite: - -```bash -pip3 install openstacksdk -pip3 install boto3 -``` - -Within this environment execute the test suite. - -[^2]: [Python 3 Documentation: Virtual Environments and Packages](https://docs.python.org/3/tutorial/venv.html) - -## Test Execution - -The test suite is executed as follows: - -```bash -python3 mandatory-iaas-services.py --os-cloud mycloud -``` - -As an alternative to "`--os-cloud`", the "`OS_CLOUD`" environment variable may be specified instead. -The parameter is used to look up the correct cloud configuration in "`clouds.yaml`". -For the example command above, this file should contain a `clouds.mycloud` section like this: - -```yaml ---- -clouds: - mycloud: - auth: - auth_url: ... - ... - ... -``` - -If the deployment uses s3 only and does not have the object store endpoint specified in the service catalog, the "`--s3-endpoint`" flag may be used to specify the s3 endpoint. -In that case the "`--s3-access`" and "`--s3-access-secret`" flags must also be set, to give all necessery credentials to the test suite: - -```bash -python3 mandatory-iaas-services3.py --os-cloud mycloud2 --s3-endpoint "http://s3-endpoint:9000" --s3-access test-user --s3-access-secret test-user-secret -``` - -For any further options consult the output of "`python3 volume-backup-tester.py --help`". - -### Script Behavior & Test Results - -If all tests pass, the script will return with an exit code of `0`. - -If any test fails, the script will halt, print the exact error to `stderr` and return with a non-zero exit code. - -There is no cleanup done by this test as it mainly only inspect the service catalog and only for the object store creates a bucket, which is then promptly deleted. From dae1f8cf55aa378843e158b0ffc274e634c8d7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Fri, 5 Sep 2025 22:04:41 +0200 Subject: [PATCH 31/31] Fix typos; thanks @depressiveRobot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marvin Frommhold Signed-off-by: Matthias Büchse --- Tests/iaas/scs_0123_mandatory_services/mandatory_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py b/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py index 4812e2ab0..33888b686 100644 --- a/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py +++ b/Tests/iaas/scs_0123_mandatory_services/mandatory_services.py @@ -120,7 +120,7 @@ def compute_scs_0123_swift_s3(conn: openstack.connection.Connection): It will abort with an exception if no service of type object-storage is present. As of now, we deem this behavior adequate. """ - # we assume s3 is accessable via the service catalog, and Swift might exist too + # we assume s3 is accessible via the service catalog, and Swift might exist too usable_credentials = [] s3_buckets = [] # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) @@ -145,7 +145,7 @@ def compute_scs_0123_swift_s3(conn: openstack.connection.Connection): if s3_buckets == sw_containers: return True logger.error( - "S3 buckets and Swift cntainers differ:\n" + "S3 buckets and Swift containers differ:\n" f"S3: {s3_buckets}\n" f"SW: {sw_containers}" )