diff --git a/Tests/iaas/openstack_test.py b/Tests/iaas/openstack_test.py index 2cccff674..518e405a9 100755 --- a/Tests/iaas/openstack_test.py +++ b/Tests/iaas/openstack_test.py @@ -205,6 +205,35 @@ def make_container(cloud): return c +class _Filter: + _instance = None + + def __init__(self): + self.components = [] + + def __call__(self, record): + if len(self.components) == 1 or len(self.components) == 2 and self.components[0].replace('-', '_') == self.components[1]: + record.msg = f'{self.components[0]}: {record.msg}' + elif self.components: + record.msg = f'[{self.components[-1]}] {record.msg}' + return True + + @classmethod + def install(cls, logger): + if cls._instance: + return + cls._instance = cls() + logger.handlers[0].filters.append(cls._instance) + + @classmethod + def push(cls, name): + cls._instance.components.append(name) + + @classmethod + def pop(cls): + del cls._instance.components[-1] + + class Container: """ This class does lazy evaluation and memoization. You register any potential value either @@ -232,13 +261,17 @@ def __init__(self): def __getattr__(self, key): val = self._values.get(key) if val is None: - logger.debug(f'... {key}') + # I thought this was too verbose, but it massively helps classifying log messages + # logger.debug(f'... {key}') + _Filter.push(key) try: ret = self._functions[key](self) except BaseException as e: val = (True, e) else: val = (False, ret) + _Filter.pop() + # logger.debug(f'... /{key}') self._values[key] = val error, ret = val if error: @@ -256,6 +289,27 @@ def add_value(self, name, value): self._values[name] = value +def _eval_result(result, messages): + """evaluates `result` from calling a check function + + returns 0 if check function succeeded, otherwise 1; + appends to list `messages` if the result contains messages + """ + # either a pair (success, messages) + if isinstance(result, tuple): + success, msgs = result + messages.extend(messages) + return not success + # or just a list of messages + if isinstance(result, list): + if result: + messages.extend(result) + return 1 + return 0 + # or just a scalar (no messages) + return not result + + def harness(name, *check_fns): """Harness for evaluating testcase `name`. @@ -268,15 +322,25 @@ def harness(name, *check_fns): - 'PASS' otherwise """ logger.debug(f'** {name}') + _Filter.push(name) + + messages = [] try: - result = all(check_fn() for check_fn in check_fns) + results = [check_fn() for check_fn in check_fns] except BaseException: logger.debug('exception during check', exc_info=True) result = 'ABORT' else: - result = ['FAIL', 'PASS'][min(1, result)] + fails = 0 + for r in results: + fails += _eval_result(r, messages) + result = ['FAIL', 'PASS'][fails == 0] + finally: + _Filter.pop() # this is quite redundant # logger.debug(f'** computation end for {name}') + for msg in messages: + print(f"{name}: {msg}") print(f"{name}: {result}") @@ -297,6 +361,7 @@ def run_sanity_checks(container): def main(argv): # configure logging, disable verbose library logging logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) + _Filter.install(logging.getLogger()) openstack.enable_logging(debug=False) cloud = None diff --git a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py index 9bbab493b..bfce69d6d 100644 --- a/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py +++ b/Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py @@ -9,7 +9,6 @@ logger = logging.getLogger(__name__) -TESTCASES = ('scs-0100-syntax-check', 'scs-0100-semantics-check', 'flavor-name-check') STRATEGY = flavor_names.ParsingStrategy( vstr='v3', parsers=(flavor_names.parser_v3, ), @@ -32,18 +31,14 @@ def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor] try: flavorname = parser(flv.name) except ValueError as exc: - logger.info(f"error parsing {flv.name}: {exc}") - flavorname = None + flavorname = f"error parsing {flv.name}: {exc}" result.append((flv, flavorname)) return result def compute_scs_0100_syntax_check(scs_flavors: list) -> bool: """This test ensures that each SCS flavor is indeed named correctly.""" - problems = [flv.name for flv, flavorname in scs_flavors if not flavorname] - if problems: - logger.error(f"scs-100-syntax-check: flavor(s) failed: {', '.join(sorted(problems))}") - return not problems + return [flavorname for _, flavorname in scs_flavors if isinstance(flavorname, str)] def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: @@ -54,36 +49,31 @@ def compute_scs_0100_semantics_check(scs_flavors: list) -> bool: NOTE that this test is incomplete; it only checks the most obvious properties. See also . """ - problems = set() + problems = [] for flv, flavorname in scs_flavors: - if not flavorname: + if isinstance(flavorname, str): continue # this case is handled by syntax check cpuram = flavorname.cpuram if flv.vcpus < cpuram.cpus: - logger.error(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}") - problems.add(flv.name) + problems.append(f"CPU overpromise for {flv.name!r}: {flv.vcpus} < {cpuram.cpus}") elif flv.vcpus > cpuram.cpus: - logger.info(f"Flavor {flv.name} CPU underpromise: {flv.vcpus} > {cpuram.cpus}") + logger.info(f"CPU underpromise for {flv.name!r}: {flv.vcpus} > {cpuram.cpus}") # RAM flvram = int((flv.ram + 51) / 102.4) / 10 # Warn for strange sizes (want integer numbers, half allowed for < 10GiB) if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2): - logger.info(f"Flavor {flv.name} uses discouraged uneven size of memory {flvram:.1f} GiB") + logger.info(f"Discouraged uneven size of memory for {flv.name!r}: {flvram:.1f} GiB") if flvram < cpuram.ram: - logger.error(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}") - problems.add(flv.name) + problems.append(f"RAM overpromise for {flv.name!r}: {flvram:.1f} < {cpuram.ram:.1f}") elif flvram > cpuram.ram: - logger.info(f"Flavor {flv.name} RAM underpromise {flvram:.1f} > {cpuram.ram:.1f}") + logger.info(f"RAM underpromise for {flv.name!r}: {flvram:.1f} > {cpuram.ram:.1f}") # Disk could have been omitted disksize = flavorname.disk.disksize if flavorname.disk else 0 # We have a recommendation for disk size steps if disksize not in ACC_DISK: - logger.info(f"Flavor {flv.name} non-standard disk size {disksize}, should have (5, 10, 20, 50, 100, 200, ...)") + logger.info(f"Non-standard disk size for {flv.name!r}: {disksize} not in (5, 10, 20, 50, 100, 200, ...)") if flv.disk < disksize: - logger.error(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}") - problems.add(flv.name) + problems.append(f"Disk overpromise for {flv.name!r}: {flv.disk} < {disksize}") elif flv.disk > disksize: - logger.info(f"Flavor {flv.name} disk underpromise {flv.disk} > {disksize}") - if problems: - logger.error(f"scs-100-semantics-check: flavor(s) failed: {', '.join(sorted(problems))}") - return not problems + logger.info(f"Disk underpromise for {flv.name!r}: {flv.disk} > {disksize}") + return problems diff --git a/Tests/iaas/scs_0102_image_metadata/image_metadata.py b/Tests/iaas/scs_0102_image_metadata/image_metadata.py index a22501241..87f8efb04 100644 --- a/Tests/iaas/scs_0102_image_metadata/image_metadata.py +++ b/Tests/iaas/scs_0102_image_metadata/image_metadata.py @@ -100,45 +100,49 @@ def is_outdated(img, now=time.time()): def _log_error(cause, offenders, channel=logging.ERROR): + """helper function to construct report messages""" if not offenders: - return + return [] names = [img.name for img in offenders] - logger.log(channel, f"{cause} for image(s): {', '.join(names)}") + message = f"{cause} for image(s): {', '.join(names)}" + logger.log(channel, message) + return [message] def compute_scs_0102_prop_architecture(images, architectures=ARCHITECTURES): """This test ensures that each image has a proper value for the property `architecture`.""" offenders = [img for img in images if img.architecture not in architectures] - _log_error('property architecture not correct', offenders) - return not offenders + return _log_error('property architecture not correct', offenders) # NOTE I think this is a recommendation def compute_scs_0102_prop_hash_algo(images): """This test ensures that each image has a proper value for the property `hash_algo`.""" offenders = [img for img in images if img.hash_algo not in ('sha256', 'sha512')] - _log_error('property hash_algo invalid', offenders) - return not offenders + return _log_error('property hash_algo invalid', offenders) def compute_scs_0102_prop_min_disk(images): """This test ensures that each image has a proper value for the property `min_disk`.""" - offenders1 = [img for img in images if not img.min_disk] - _log_error('property min_disk not set', offenders1) - offenders2 = [img for img in images if img.min_disk and img.min_disk * GIB < img.size] - _log_error('property min_disk smaller than size', offenders2) - return not offenders1 and not offenders2 + msgs1 = _log_error( + 'property min_disk not set', + [img for img in images if not img.min_disk], + ) + msgs2 = _log_error( + 'property min_disk smaller than size', + [img for img in images if img.min_disk and img.min_disk * GIB < img.size], + ) + return msgs1 + msgs2 def compute_scs_0102_prop_min_ram(images): """This test ensures that each image has a proper value for the property `min_ram`.""" - offenders1 = [img for img in images if not img.min_ram] - _log_error('property min_ram not set', offenders1) # emit a warning im RAM really low # NOTE this will probably only get noticed if an error occurs as well offenders2 = [img for img in images if img.min_ram and img.min_ram < 64] _log_error('property min_ram < 64 MiB', offenders2, channel=logging.WARNING) - return not offenders1 + offenders1 = [img for img in images if not img.min_ram] + return _log_error('property min_ram not set', offenders1) def compute_scs_0102_prop_os_version(images): @@ -148,8 +152,7 @@ def compute_scs_0102_prop_os_version(images): # certain values for common operating systems. # - os_version not matching regexp r'[0-9\.]*' (should be a numeric version no) offenders = [img for img in images if not img.os_version] - _log_error('property os_version not set', offenders) - return not offenders + return _log_error('property os_version not set', offenders) def compute_scs_0102_prop_os_distro(images): @@ -158,55 +161,49 @@ def compute_scs_0102_prop_os_distro(images): # - os_distro not being all-lowercase (they all should be acc. to # https://docs.openstack.org/glance/2025.1/admin/useful-image-properties.html offenders = [img for img in images if not img.os_distro] - _log_error('property os_distro not set', offenders) - return not offenders + return _log_error('property os_distro not set', offenders) def compute_scs_0102_prop_os_purpose(images, os_purposes=OS_PURPOSES): """This test ensures that each image has a proper value for the property `os_distro`.""" offenders = [img for img in images if img.properties.get('os_purpose') not in os_purposes] - _log_error('property os_purpose not set or not correct', offenders) - return not offenders + return _log_error('property os_purpose not set or not correct', offenders) def compute_scs_0102_prop_hw_disk_bus(images, hw_disk_buses=HW_DISK_BUSES): """This test ensures that each image has a proper value for the property `hw_disk_bus`.""" offenders = [img for img in images if img.hw_disk_bus not in hw_disk_buses] - _log_error('property hw_disk_bus not correct', offenders) - return not offenders + return _log_error('property hw_disk_bus not correct', offenders) def compute_scs_0102_prop_hypervisor_type(images, hypervisor_types=HYPERVISOR_TYPES): """This test ensures that each image has a proper value for the property `hypervisor_type`.""" offenders = [img for img in images if img.hypervisor_type not in hypervisor_types] - _log_error('property hypervisor_type not correct', offenders) - return not offenders + return _log_error('property hypervisor_type not correct', offenders) def compute_scs_0102_prop_hw_rng_model(images, hw_rng_models=HW_RNG_MODELS): """This test ensures that each image has a proper value for the property `hw_rng_model`.""" offenders = [img for img in images if img.hw_rng_model not in hw_rng_models] - _log_error('property hw_rng_model not correct', offenders) - return not offenders + return _log_error('property hw_rng_model not correct', offenders) def compute_scs_0102_prop_image_build_date(images, now=time.time()): """This test ensures that each image has a proper value for the property `image_build_date`.""" - errors = 0 + problems = [] for img in images: rdate = parse_date(img.created_at, formats=STRICT_FORMATS) bdate_str = img.properties.get('image_build_date', '') bdate = parse_date(bdate_str) - if not bdate: - logger.error(f'Image "{img.name}": image_build_date "{bdate_str}" INVALID') - errors += 1 + if not bdate_str: + problems.append(f'image_build_date NOT SET for {img.name!r}') + elif not bdate: + problems.append(f'image_build_date INVALID for {img.name!r}: {bdate_str!r}') elif bdate > rdate: - logger.error(f'Image "{img.name}": image_build_date {bdate_str} AFTER registration date {img.created_at}') - errors += 1 + problems.append(f'image_build_date AFTER registration for {img.name!r}: {bdate_str} > {img.created_at}') if (bdate or rdate) > now: - logger.error(f'Image "{img.name}" has build time in the future: {bdate}') - errors += 1 - return not errors + problems.append(f'image_build_date in the future for {img.name!r}: {bdate}') + return problems # FIXME this is completely optional @@ -217,46 +214,45 @@ def compute_scs_0102_prop_image_build_date(images, now=time.time()): def compute_scs_0102_prop_image_original_user(images): """This test ensures that each image has a proper value for the property `image_original_user`.""" offenders = [img for img in images if not img.properties.get('image_original_user')] - _log_error('property image_original_user not set', offenders) - return not offenders + return _log_error('property image_original_user not set', offenders) def compute_scs_0102_prop_image_source(images): """This test ensures that each image has a proper value for the property `image_source`.""" - offenders = [ - img - for img in images - if img.properties.get('image_source') != 'private' - if not is_url(img.properties.get('image_source', '')) - ] - _log_error('property image_source INVALID (url or "private")', offenders) - return not offenders + problems = [] + for img in images: + src = img.properties.get('image_source') + if not src: + problems.append(f"image_source MISSING for {img.name}") + continue + if src == 'private': + continue + if not is_url(src): + problems.append(f'image_source INVALID (url or "private") for {img.name}') + return problems def compute_scs_0102_prop_image_description(images): """This test ensures that each image has a proper value for the property `image_description`.""" offenders = [img for img in images if not img.properties.get('image_description')] - _log_error('property image_description not set', offenders) - return not offenders + return _log_error('property image_description not set', offenders) def compute_scs_0102_prop_replace_frequency(images, replace_frequencies=FREQ_TO_SEC): """This test ensures that each image has a proper value for the property `replace_frequency`.""" offenders = [img for img in images if img.properties.get('replace_frequency') not in replace_frequencies] - _log_error('property replace_frequency not correct', offenders) - return not offenders + return _log_error('property replace_frequency not correct', offenders) def compute_scs_0102_prop_provided_until(images): """This test ensures that each image has a proper value for the property `provided_until`.""" offenders = [img for img in images if not img.properties.get('provided_until')] - _log_error('property provided_until not set', offenders) - return not offenders + return _log_error('property provided_until not set', offenders) def compute_scs_0102_prop_uuid_validity(images): """This test ensures that each image has a proper value for the property `uuid_validity`.""" - errors = 0 + offenders = [] for img in images: img_uuid_val = img.properties.get("uuid_validity") if img_uuid_val in (None, "none", "notice", "forever"): @@ -267,20 +263,20 @@ def compute_scs_0102_prop_uuid_validity(images): pass else: logger.error(f'Image "{img.name}": property uuid_validity INVALID: {img_uuid_val}') - errors += 1 - return not errors + offenders.append(img) + return _log_error('uuid_validity INVALID', offenders, channel=logging.DEBUG) def compute_scs_0102_prop_hotfix_hours(images): """This test ensures that each image has a proper value for the property `hotfix_hours`.""" - errors = 0 + offenders = [] for img in images: hotfix_hours = img.properties.get("hotfix_hours", '') if not hotfix_hours or hotfix_hours.isdecimal(): continue logger.error(f'Image "{img.name}": property hotfix_hours INVALID: {hotfix_hours}') - errors += 1 - return not errors + offenders.append(img) + return _log_error('hotfix_hours INVALID', offenders, channel=logging.DEBUG) def _find_replacement_image(by_name, img_name): @@ -309,7 +305,7 @@ def compute_scs_0102_image_recency(images): counter = Counter([img.name for img in images]) duplicates = [name for name, count in counter.items() if count > 1] logger.warning(f'duplicate names detected: {", ".join(duplicates)}') - errors = 0 + problems = [] for img in images: # This is a bit tricky: We need to disregard images that have been rotated out # - os_hidden = True is a safe sign for this @@ -318,17 +314,15 @@ def compute_scs_0102_image_recency(images): if not outd: continue # fine if outd == 3: - logger.error(f'Image "{img.name}" does not provide a valid provided until date') - errors += 1 + problems.append(f'property provided_until INVALID for {img.name!r}') continue # hopeless # in case that outd in (1, 2) try to find a non-outdated version if outd == 2: - logger.warning(f'Image "{img.name}" seems outdated (acc. to its repl freq) but is not hidden or otherwise marked') + logger.warning(f'Image {img.name!r} seems outdated (acc. to its repl freq) but is not hidden or otherwise marked') # warnings += 1 replacement = _find_replacement_image(by_name, img.name) if replacement is None: - logger.error(f'Image "{img.name}" outdated without replacement') - errors += 1 + problems.append(f'outdated image w/o replacement: {img.name!r}') else: - logger.info(f'Image "{replacement.name}" is a valid replacement for outdated "{img.name}"') - return not errors + logger.info(f'Image {replacement.name!r} is a valid replacement for outdated {img.name!r}') + return problems diff --git a/Tests/iaas/scs_0104_standard_images/standard_images.py b/Tests/iaas/scs_0104_standard_images/standard_images.py index 6f8c827e8..749453a6f 100644 --- a/Tests/iaas/scs_0104_standard_images/standard_images.py +++ b/Tests/iaas/scs_0104_standard_images/standard_images.py @@ -70,16 +70,15 @@ def compute_scs_0104_source(image_lookup, image_spec): For an impression of what these specs look like, refer to `SCS_0104_IMAGE_SPECS`. """ matches = _lookup_images(image_lookup, image_spec) - errors = 0 + errors = [] for image in matches: img_source = image.properties.get('image_source', '') sources = image_spec['source'] if not isinstance(sources, (tuple, list)): sources = [sources] if not any(img_source.startswith(src) for src in sources): - errors += 1 - logger.error(f"Image '{image.name}' source mismatch: '{img_source}' matches none of these prefixes: {', '.join(sources)}") - return not errors + errors.append(f"Image '{image.name}' source mismatch: '{img_source}' matches none of these prefixes: {', '.join(sources)}") + return errors def compute_scs_0104_image(image_lookup, image_spec): @@ -89,7 +88,7 @@ def compute_scs_0104_image(image_lookup, image_spec): For an impression of what these specs look like, refer to `SCS_0104_IMAGE_SPECS`. """ matches = _lookup_images(image_lookup, image_spec) + errors = [] if not matches: - logger.error(f"Missing image '{image_spec['name']}'") - return False - return True + errors.append(f"Missing image '{image_spec['name']}'") + return errors