Implement output of testcase-specific report messages

Tests/iaas/openstack_test.py

@@ -232,6 +232,7 @@ def __init__(self):
     def __getattr__(self, key):
         val = self._values.get(key)
         if val is None:
+            # I thought this was too verbose, but it massively helps classifying log messages
             logger.debug(f'... {key}')
             try:
                 ret = self._functions[key](self)
@@ -256,6 +257,27 @@ def add_value(self, name, value):
         self._values[name] = value
 
 
+def _eval_result(result, messages):
+    """evaluates `result` from calling a check function
+
+    returns 0 if check function succeeded, otherwise 1;
+    appends to list `messages` if the result contains messages
+    """
+    # either a pair (success, messages)
+    if isinstance(result, tuple):
+        success, msgs = result
+        messages.extend(messages)
+        return not success
+    # or just a list of messages
+    if isinstance(result, list):
+        if result:
+            messages.extend(result)
+            return 1
+        return 0
+    # or just a scalar (no messages)
+    return not result
+
+
 def harness(name, *check_fns):
     """Harness for evaluating testcase `name`.
 
@@ -268,15 +290,21 @@ def harness(name, *check_fns):
     - 'PASS' otherwise
     """
     logger.debug(f'** {name}')
+    messages = []
     try:
-        result = all(check_fn() for check_fn in check_fns)
+        results = [check_fn() for check_fn in check_fns]
     except BaseException:
         logger.debug('exception during check', exc_info=True)
         result = 'ABORT'
     else:
-        result = ['FAIL', 'PASS'][min(1, result)]
+        fails = 0
+        for r in results:
+            fails += _eval_result(r, messages)
+        result = ['FAIL', 'PASS'][fails == 0]
     # this is quite redundant
     # logger.debug(f'** computation end for {name}')
+    for msg in messages:
+        print(f"{name}: {msg}")
     print(f"{name}: {result}")
 
 

Tests/iaas/scs_0100_flavor_naming/flavor_names_check.py

@@ -9,7 +9,6 @@
 logger = logging.getLogger(__name__)
 
 
-TESTCASES = ('scs-0100-syntax-check', 'scs-0100-semantics-check', 'flavor-name-check')
 STRATEGY = flavor_names.ParsingStrategy(
     vstr='v3',
     parsers=(flavor_names.parser_v3, ),
@@ -32,18 +31,14 @@ def compute_scs_flavors(flavors: typing.List[openstack.compute.v2.flavor.Flavor]
         try:
             flavorname = parser(flv.name)
         except ValueError as exc:
-            logger.info(f"error parsing {flv.name}: {exc}")
-            flavorname = None
+            flavorname = f"error parsing {flv.name}: {exc}"
         result.append((flv, flavorname))
     return result
 
 
 def compute_scs_0100_syntax_check(scs_flavors: list) -> bool:
     """This test ensures that each SCS flavor is indeed named correctly."""
-    problems = [flv.name for flv, flavorname in scs_flavors if not flavorname]
-    if problems:
-        logger.error(f"scs-100-syntax-check: flavor(s) failed: {', '.join(sorted(problems))}")
-    return not problems
+    return [flavorname for _, flavorname in scs_flavors if isinstance(flavorname, str)]
 
 
 def compute_scs_0100_semantics_check(scs_flavors: list) -> bool:
@@ -54,36 +49,31 @@ def compute_scs_0100_semantics_check(scs_flavors: list) -> bool:
     NOTE that this test is incomplete; it only checks the most obvious properties.
     See also <https://github.com/SovereignCloudStack/standards/issues/554>.
     """
-    problems = set()
+    problems = []
     for flv, flavorname in scs_flavors:
-        if not flavorname:
+        if isinstance(flavorname, str):
             continue  # this case is handled by syntax check
         cpuram = flavorname.cpuram
         if flv.vcpus < cpuram.cpus:
-            logger.error(f"Flavor {flv.name} CPU overpromise: {flv.vcpus} < {cpuram.cpus}")
-            problems.add(flv.name)
+            problems.append(f"CPU overpromise for {flv.name!r}: {flv.vcpus} < {cpuram.cpus}")
         elif flv.vcpus > cpuram.cpus:
-            logger.info(f"Flavor {flv.name} CPU underpromise: {flv.vcpus} > {cpuram.cpus}")
+            logger.info(f"CPU underpromise for {flv.name!r}: {flv.vcpus} > {cpuram.cpus}")
         # RAM
         flvram = int((flv.ram + 51) / 102.4) / 10
         # Warn for strange sizes (want integer numbers, half allowed for < 10GiB)
         if flvram >= 10 and flvram != int(flvram) or flvram * 2 != int(flvram * 2):
-            logger.info(f"Flavor {flv.name} uses discouraged uneven size of memory {flvram:.1f} GiB")
+            logger.info(f"Discouraged uneven size of memory for {flv.name!r}: {flvram:.1f} GiB")
         if flvram < cpuram.ram:
-            logger.error(f"Flavor {flv.name} RAM overpromise {flvram:.1f} < {cpuram.ram:.1f}")
-            problems.add(flv.name)
+            problems.append(f"RAM overpromise for {flv.name!r}: {flvram:.1f} < {cpuram.ram:.1f}")
         elif flvram > cpuram.ram:
-            logger.info(f"Flavor {flv.name} RAM underpromise {flvram:.1f} > {cpuram.ram:.1f}")
+            logger.info(f"RAM underpromise for {flv.name!r}: {flvram:.1f} > {cpuram.ram:.1f}")
         # Disk could have been omitted
         disksize = flavorname.disk.disksize if flavorname.disk else 0
         # We have a recommendation for disk size steps
         if disksize not in ACC_DISK:
-            logger.info(f"Flavor {flv.name} non-standard disk size {disksize}, should have (5, 10, 20, 50, 100, 200, ...)")
+            logger.info(f"Non-standard disk size for {flv.name!r}: {disksize} not in (5, 10, 20, 50, 100, 200, ...)")
         if flv.disk < disksize:
-            logger.error(f"Flavor {flv.name} disk overpromise {flv.disk} < {disksize}")
-            problems.add(flv.name)
+            problems.append(f"Disk overpromise for {flv.name!r}: {flv.disk} < {disksize}")
         elif flv.disk > disksize:
-            logger.info(f"Flavor {flv.name} disk underpromise {flv.disk} > {disksize}")
-    if problems:
-        logger.error(f"scs-100-semantics-check: flavor(s) failed: {', '.join(sorted(problems))}")
-    return not problems
+            logger.info(f"Disk underpromise for {flv.name!r}: {flv.disk} > {disksize}")
+    return problems