Merge branch '251-ci-fail-if-there-are-unconfigured-tests' into 'master'

tests/testfw: fail if there are unconfigured tests

Closes #251

See merge request !212

test/Makefile

@@ -2,6 +2,7 @@ PYTHON := python3
 XUNIT ?= 0
 
 TEST_FLAGS = -p`nproc`
+TEST_OPTS_test_debug := --no-strict
 
 all: test coverage
 
@@ -9,7 +10,7 @@ elaborate:
 	$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
 
 test_%: tests_%.yml FORCE
-	$(PYTHON) run.py test $< -- $(TEST_FLAGS)
+	$(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
 
 coverage:
 	lcov --capture --directory build --output-file code_coverage.info

test/testfw/__init__.py

@@ -73,8 +73,10 @@ def create():
 @cli.command()
 @click.argument('config', type=click.Path())
 @click.argument('vunit_args', nargs=-1)
+@click.option('--strict/--no-strict', default=True,
+              help='Return non-zero if an unconfigured test was found.')
 @click.pass_obj
-def test(obj, config, vunit_args):
+def test(obj, config, strict, vunit_args):
     """Run the tests. Configuration is passed in YAML config file.
 
     You mas pass arguments directly to VUnit by appending them at the command end.
@@ -105,40 +107,46 @@ def test(obj, config, vunit_args):
     build.mkdir(exist_ok=True)
     os.chdir(str(build))
 
-    run_unit = 'unit' in config
-    run_feature = 'feature' in config
-    run_sanity = 'sanity' in config
-    run_reference = 'reference' in config
-
     ui = create_vunit(obj, vunit_args, out_basename)
 
     lib = ui.add_library("lib")
     add_common_sources(lib, ui)
 
-    tests = []
-    if run_unit:
-        tests.append(test_unit.UnitTests(ui, lib, config['unit'], build, base))
-    if run_sanity:
-        tests.append(test_sanity.SanityTests(ui, lib, config['sanity'], build, base))
-    if run_feature:
-        tests.append(test_feature.FeatureTests(ui, lib, config['feature'], build, base))
-    if run_reference:
-        tests.append(test_reference.ReferenceTests(ui, lib, config['reference'], build, base))
+    tests_classes = [
+        # key in config, factory
+        ('unit', test_unit.UnitTests),
+        ('sanity', test_sanity.SanityTests),
+        ('feature', test_feature.FeatureTests),
+        ('reference', test_reference.ReferenceTests),
+    ]
 
+    tests = []
+    for cfg_key, factory in tests_classes:
+        if cfg_key in config:
+            tests.append(factory(ui, lib, config[cfg_key], build, base))
 
     for t in tests:
         t.add_sources()
+
     add_flags(ui, lib, build)
-    for t in tests:
-        t.configure()
+
+    conf_ok = [t.configure() for t in tests]
 
     # check for unknown tests
     all_benches = lib.get_test_benches('*')
-    unknown_tests = [tb for tb in all_benches if not re.match('tb_.*?_unit_test|tb_sanity|tb_feature|tb_reference_wrapper', tb.name)]
+    pattern = 'tb_.*?_unit_test|tb_sanity|tb_feature|tb_reference_wrapper'
+    unknown_tests = [tb for tb in all_benches
+                     if not re.match(pattern, tb.name)]
     if len(unknown_tests):
-        log.warn('Unknown tests (defaults will be used): {}'.format(', '.join(tb.name for tb in unknown_tests)))
-
-    vunit_run(ui, build, out_basename)
+        log.warn('Unknown tests (defaults will be used): {}'
+                 .format(', '.join(tb.name for tb in unknown_tests)))
+
+    res = vunit_run(ui, build, out_basename)
+    if not all(conf_ok) and strict:
+        log.error('Some test cases were discovered but not configured (see above).')
+        if res == 0:
+            res = 1
+    sys.exit(res)
 
 
 def create_vunit(obj, vunit_args, out_basename):
@@ -152,7 +160,7 @@ def create_vunit(obj, vunit_args, out_basename):
     return ui
 
 
-def vunit_run(ui, build, out_basename):
+def vunit_run(ui, build, out_basename) -> int:
     try:
         vunit_ifc.run(ui)
         res = None
@@ -168,7 +176,7 @@ def vunit_run(ui, build, out_basename):
             print('<?xml-stylesheet href="xunit.xsl" type="text/xsl"?>', file=f)
             f.write(c)
         out.unlink()
-    sys.exit(res)
+    return res
 
 
 """

test/testfw/test_common.py

@@ -7,6 +7,7 @@ from pathlib import Path
 from jinja2 import Environment, PackageLoader
 from pprint import pprint
 import random
+from typing import List
 
 __all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files',
            'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options',
@@ -30,13 +31,17 @@ class TestsBase:
     def jinja_env(self):
         return jinja_env
 
-    def add_sources(self):
+    def add_sources(self) -> None:
         raise NotImplementedError()
 
-    def configure(self):
+    def configure(self) -> bool:
+        """Configure the tests.
+
+        Return False if there were unconfigured tests found."""
+
         raise NotImplementedError()
 
-    def add_modelsim_gui_file(self, tb, cfg, name):
+    def add_modelsim_gui_file(self, tb, cfg, name) -> None:
         if 'wave' in cfg:
             tcl = self.base / cfg['wave']
             if not tcl.exists():
@@ -58,7 +63,7 @@ class TestsBase:
         tb.set_sim_option("modelsim.init_file.gui", str(tcl))
 
 
-def add_sources(lib, patterns):
+def add_sources(lib, patterns) -> None:
     for pattern in patterns:
         p = join(str(d.parent), pattern)
         log.debug('Adding sources matching {}'.format(p))
@@ -67,19 +72,19 @@ def add_sources(lib, patterns):
                 lib.add_source_file(str(f))
 
 
-def add_common_sources(lib, ui):
+def add_common_sources(lib, ui) -> None:
     add_sources(lib, ['../src/**/*.vhd'])
     ui.enable_check_preprocessing()
     ui.enable_location_preprocessing() #(additional_subprograms=['log'])
     add_sources(lib, ['*.vhd', 'lib/*.vhd'])
 
 
-def get_common_modelsim_init_files():
+def get_common_modelsim_init_files() -> List[str]:
     modelsim_init_files = '../lib/test_lib.tcl,modelsim_init.tcl'
     modelsim_init_files = [str(d/x) for x in modelsim_init_files.split(',')]
     return modelsim_init_files
 
-def add_flags(ui, lib, build):
+def add_flags(ui, lib, build) -> None:
     unit_tests = lib.get_test_benches('*_unit_test', allow_empty=True)
     for ut in unit_tests:
         ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
@@ -96,11 +101,11 @@ def add_flags(ui, lib, build):
     ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
 
 
-def get_seed(cfg):
+def get_seed(cfg) -> int:
     if 'seed' in cfg and 'randomize' in cfg:
         log.warning('Both "seed" and "randomize" are set - seed takes precedence')
     if 'seed' in cfg:
-        seed = cfg['seed']
+        seed = int(cfg['seed'], 0)
     elif cfg.get('randomize', False):
         # only 31 bits
         seed = int(random.random() * 2**31) & 0x7FFFFFFF
@@ -109,14 +114,14 @@ def get_seed(cfg):
     return seed
 
 
-def dict_merge(up, *lowers):
+def dict_merge(up, *lowers) -> None:
     for lower in lowers:
         for k, v in lower.items():
             if k not in up:
                 up[k] = v
 
 
-def vhdl_serialize(o):
+def vhdl_serialize(o) -> str:
     if isinstance(o, Iterable):
         ss = []
         for x in o:
@@ -126,7 +131,7 @@ def vhdl_serialize(o):
         return str(o)
 
 
-def dump_sim_options(lib):
+def dump_sim_options(lib) -> None:
     for tb in lib.get_test_benches('*'):
         for cfgs in tb._test_bench.get_configuration_dicts():
             for name, cfg in cfgs.items():

test/testfw/test_feature.py

@@ -26,7 +26,7 @@ class FeatureTests(TestsBase):
         tb = self.lib.get_test_benches('*tb_feature')[0]
         tb.scan_tests_from_file(str(wrname))
 
-    def configure(self) -> None:
+    def configure(self) -> bool:
         tb = self.lib.get_test_benches('*tb_feature')[0]
         default = self.config['default']
         self.add_modelsim_gui_file(tb, default, 'feature')
@@ -60,9 +60,9 @@ class FeatureTests(TestsBase):
                 'seed'         : get_seed(cfg)
             }
             tb.add_config(name, generics=generics)
-        self._check_for_unconfigured()
+        return self._check_for_unconfigured()
 
-    def _check_for_unconfigured(self):
+    def _check_for_unconfigured(self) -> bool:
         config = self.config
         # check for unconfigured unit tests
         test_files = self.base.glob('feature/*_feature_tb.vhd')
@@ -72,6 +72,7 @@ class FeatureTests(TestsBase):
         unconfigured = [tb for tb in all_tests if tb not in configured]
         if len(unconfigured):
             log.warn("Feature tests with no configuration found (will not be run): {}".format(', '.join(unconfigured)))
+        return len(unconfigured) == 0
 
     def _create_wrapper(self, ofile: Path) -> None:
         template = self.jinja_env.get_template('pkg_feature_exec_dispath-body.vhd')

test/testfw/test_reference.py

@@ -18,7 +18,7 @@ class ReferenceTests(TestsBase):
         sources.append('reference/vunit_reference_wrapper.vhd')
         add_sources(self.lib, sources)
 
-    def configure(self) -> None:
+    def configure(self) -> bool:
         tb = self.lib.get_test_benches('*reference*')[0]
         default = self.config['default']
         self.add_modelsim_gui_file(tb, default, 'sanity')
@@ -47,3 +47,4 @@ class ReferenceTests(TestsBase):
             init_files += [str(tcl)]
             tb.set_sim_option("modelsim.init_files.after_load", init_files)
             self.add_modelsim_gui_file(tb, cfg, data_set)
+        return True

test/testfw/test_sanity.py

@@ -73,3 +73,4 @@ class SanityTests(TestsBase):
                 'gauss_iter'   : vhdl_serialize(cfg['gauss_iter']),
             }
             tb.add_config(name, generics=generics)
+        return True

test/testfw/test_unit.py

@@ -9,11 +9,11 @@ log = logging.getLogger(__name__)
 
 
 class UnitTests(TestsBase):
-    def add_sources(self):
+    def add_sources(self) -> None:
         add_sources(self.lib, ['unit/**/*.vhd'])
         self._create_wrapper(self.build / "tb_wrappers.vhd")
 
-    def configure(self):
+    def configure(self) -> bool:
         lib, config, build = self.lib, self.config, self.build
         default = config['default']
         unit_tests = lib.get_test_benches('*_unit_test')
@@ -44,9 +44,9 @@ class UnitTests(TestsBase):
             init_files += [str(tcl)]
             tb.set_sim_option("modelsim.init_files.after_load", init_files)
             self.add_modelsim_gui_file(tb, cfg, name)
-        self._check_for_unconfigured()
+        return self._check_for_unconfigured()
 
-    def _check_for_unconfigured(self):
+    def _check_for_unconfigured(self) -> bool:
         lib, config = self.lib, self.config
         # check for unconfigured unit tests
         unit_tests = lib.get_test_benches('*tb_*_unit_test')
@@ -55,8 +55,9 @@ class UnitTests(TestsBase):
         unconfigured = [tb for tb in unit_tests if tb.name not in configured]
         if len(unconfigured):
             log.warn("Unit tests with no configuration found (defaults will be used): {}".format(', '.join(tb.name for tb in unconfigured)))
+        return len(unconfigured) == 0
 
-    def _create_wrapper(self, fname):
+    def _create_wrapper(self, fname) -> None:
         fname = str(fname)
         files = self.lib.get_source_files()
         tests = []
@@ -67,8 +68,6 @@ class UnitTests(TestsBase):
                     m = r.match(l)
                     if m:
                         tests.append(m.group(1))
-        configs = []
-        tbs = []
         c = self.jinja_env.get_template('unit_wrappers.vhd').render(tests=tests)
         with open(fname, "wt", encoding='utf-8') as f:
             f.write(c)