Commit 60acecf8 authored by Martin Jeřábek's avatar Martin Jeřábek

Merge branch '251-ci-fail-if-there-are-unconfigured-tests' into 'master'

tests/testfw: fail if there are unconfigured tests

Closes #251

See merge request !212
parents 6b920cd3 006315e7
Pipeline #6199 canceled with stage
...@@ -2,6 +2,7 @@ PYTHON := python3 ...@@ -2,6 +2,7 @@ PYTHON := python3
XUNIT ?= 0 XUNIT ?= 0
TEST_FLAGS = -p`nproc` TEST_FLAGS = -p`nproc`
TEST_OPTS_test_debug := --no-strict
all: test coverage all: test coverage
...@@ -9,7 +10,7 @@ elaborate: ...@@ -9,7 +10,7 @@ elaborate:
$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS) $(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
test_%: tests_%.yml FORCE test_%: tests_%.yml FORCE
$(PYTHON) run.py test $< -- $(TEST_FLAGS) $(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
coverage: coverage:
lcov --capture --directory build --output-file code_coverage.info lcov --capture --directory build --output-file code_coverage.info
......
...@@ -73,8 +73,10 @@ def create(): ...@@ -73,8 +73,10 @@ def create():
@cli.command() @cli.command()
@click.argument('config', type=click.Path()) @click.argument('config', type=click.Path())
@click.argument('vunit_args', nargs=-1) @click.argument('vunit_args', nargs=-1)
@click.option('--strict/--no-strict', default=True,
help='Return non-zero if an unconfigured test was found.')
@click.pass_obj @click.pass_obj
def test(obj, config, vunit_args): def test(obj, config, strict, vunit_args):
"""Run the tests. Configuration is passed in YAML config file. """Run the tests. Configuration is passed in YAML config file.
You mas pass arguments directly to VUnit by appending them at the command end. You mas pass arguments directly to VUnit by appending them at the command end.
...@@ -105,40 +107,46 @@ def test(obj, config, vunit_args): ...@@ -105,40 +107,46 @@ def test(obj, config, vunit_args):
build.mkdir(exist_ok=True) build.mkdir(exist_ok=True)
os.chdir(str(build)) os.chdir(str(build))
run_unit = 'unit' in config
run_feature = 'feature' in config
run_sanity = 'sanity' in config
run_reference = 'reference' in config
ui = create_vunit(obj, vunit_args, out_basename) ui = create_vunit(obj, vunit_args, out_basename)
lib = ui.add_library("lib") lib = ui.add_library("lib")
add_common_sources(lib, ui) add_common_sources(lib, ui)
tests = [] tests_classes = [
if run_unit: # key in config, factory
tests.append(test_unit.UnitTests(ui, lib, config['unit'], build, base)) ('unit', test_unit.UnitTests),
if run_sanity: ('sanity', test_sanity.SanityTests),
tests.append(test_sanity.SanityTests(ui, lib, config['sanity'], build, base)) ('feature', test_feature.FeatureTests),
if run_feature: ('reference', test_reference.ReferenceTests),
tests.append(test_feature.FeatureTests(ui, lib, config['feature'], build, base)) ]
if run_reference:
tests.append(test_reference.ReferenceTests(ui, lib, config['reference'], build, base))
tests = []
for cfg_key, factory in tests_classes:
if cfg_key in config:
tests.append(factory(ui, lib, config[cfg_key], build, base))
for t in tests: for t in tests:
t.add_sources() t.add_sources()
add_flags(ui, lib, build) add_flags(ui, lib, build)
for t in tests:
t.configure() conf_ok = [t.configure() for t in tests]
# check for unknown tests # check for unknown tests
all_benches = lib.get_test_benches('*') all_benches = lib.get_test_benches('*')
unknown_tests = [tb for tb in all_benches if not re.match('tb_.*?_unit_test|tb_sanity|tb_feature|tb_reference_wrapper', tb.name)] pattern = 'tb_.*?_unit_test|tb_sanity|tb_feature|tb_reference_wrapper'
unknown_tests = [tb for tb in all_benches
if not re.match(pattern, tb.name)]
if len(unknown_tests): if len(unknown_tests):
log.warn('Unknown tests (defaults will be used): {}'.format(', '.join(tb.name for tb in unknown_tests))) log.warn('Unknown tests (defaults will be used): {}'
.format(', '.join(tb.name for tb in unknown_tests)))
vunit_run(ui, build, out_basename)
res = vunit_run(ui, build, out_basename)
if not all(conf_ok) and strict:
log.error('Some test cases were discovered but not configured (see above).')
if res == 0:
res = 1
sys.exit(res)
def create_vunit(obj, vunit_args, out_basename): def create_vunit(obj, vunit_args, out_basename):
...@@ -152,7 +160,7 @@ def create_vunit(obj, vunit_args, out_basename): ...@@ -152,7 +160,7 @@ def create_vunit(obj, vunit_args, out_basename):
return ui return ui
def vunit_run(ui, build, out_basename): def vunit_run(ui, build, out_basename) -> int:
try: try:
vunit_ifc.run(ui) vunit_ifc.run(ui)
res = None res = None
...@@ -168,7 +176,7 @@ def vunit_run(ui, build, out_basename): ...@@ -168,7 +176,7 @@ def vunit_run(ui, build, out_basename):
print('<?xml-stylesheet href="xunit.xsl" type="text/xsl"?>', file=f) print('<?xml-stylesheet href="xunit.xsl" type="text/xsl"?>', file=f)
f.write(c) f.write(c)
out.unlink() out.unlink()
sys.exit(res) return res
""" """
......
...@@ -7,6 +7,7 @@ from pathlib import Path ...@@ -7,6 +7,7 @@ from pathlib import Path
from jinja2 import Environment, PackageLoader from jinja2 import Environment, PackageLoader
from pprint import pprint from pprint import pprint
import random import random
from typing import List
__all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files', __all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files',
'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options', 'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options',
...@@ -30,13 +31,17 @@ class TestsBase: ...@@ -30,13 +31,17 @@ class TestsBase:
def jinja_env(self): def jinja_env(self):
return jinja_env return jinja_env
def add_sources(self): def add_sources(self) -> None:
raise NotImplementedError() raise NotImplementedError()
def configure(self): def configure(self) -> bool:
"""Configure the tests.
Return False if there were unconfigured tests found."""
raise NotImplementedError() raise NotImplementedError()
def add_modelsim_gui_file(self, tb, cfg, name): def add_modelsim_gui_file(self, tb, cfg, name) -> None:
if 'wave' in cfg: if 'wave' in cfg:
tcl = self.base / cfg['wave'] tcl = self.base / cfg['wave']
if not tcl.exists(): if not tcl.exists():
...@@ -58,7 +63,7 @@ class TestsBase: ...@@ -58,7 +63,7 @@ class TestsBase:
tb.set_sim_option("modelsim.init_file.gui", str(tcl)) tb.set_sim_option("modelsim.init_file.gui", str(tcl))
def add_sources(lib, patterns): def add_sources(lib, patterns) -> None:
for pattern in patterns: for pattern in patterns:
p = join(str(d.parent), pattern) p = join(str(d.parent), pattern)
log.debug('Adding sources matching {}'.format(p)) log.debug('Adding sources matching {}'.format(p))
...@@ -67,19 +72,19 @@ def add_sources(lib, patterns): ...@@ -67,19 +72,19 @@ def add_sources(lib, patterns):
lib.add_source_file(str(f)) lib.add_source_file(str(f))
def add_common_sources(lib, ui): def add_common_sources(lib, ui) -> None:
add_sources(lib, ['../src/**/*.vhd']) add_sources(lib, ['../src/**/*.vhd'])
ui.enable_check_preprocessing() ui.enable_check_preprocessing()
ui.enable_location_preprocessing() #(additional_subprograms=['log']) ui.enable_location_preprocessing() #(additional_subprograms=['log'])
add_sources(lib, ['*.vhd', 'lib/*.vhd']) add_sources(lib, ['*.vhd', 'lib/*.vhd'])
def get_common_modelsim_init_files(): def get_common_modelsim_init_files() -> List[str]:
modelsim_init_files = '../lib/test_lib.tcl,modelsim_init.tcl' modelsim_init_files = '../lib/test_lib.tcl,modelsim_init.tcl'
modelsim_init_files = [str(d/x) for x in modelsim_init_files.split(',')] modelsim_init_files = [str(d/x) for x in modelsim_init_files.split(',')]
return modelsim_init_files return modelsim_init_files
def add_flags(ui, lib, build): def add_flags(ui, lib, build) -> None:
unit_tests = lib.get_test_benches('*_unit_test', allow_empty=True) unit_tests = lib.get_test_benches('*_unit_test', allow_empty=True)
for ut in unit_tests: for ut in unit_tests:
ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd")) ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
...@@ -96,11 +101,11 @@ def add_flags(ui, lib, build): ...@@ -96,11 +101,11 @@ def add_flags(ui, lib, build):
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files) ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
def get_seed(cfg): def get_seed(cfg) -> int:
if 'seed' in cfg and 'randomize' in cfg: if 'seed' in cfg and 'randomize' in cfg:
log.warning('Both "seed" and "randomize" are set - seed takes precedence') log.warning('Both "seed" and "randomize" are set - seed takes precedence')
if 'seed' in cfg: if 'seed' in cfg:
seed = cfg['seed'] seed = int(cfg['seed'], 0)
elif cfg.get('randomize', False): elif cfg.get('randomize', False):
# only 31 bits # only 31 bits
seed = int(random.random() * 2**31) & 0x7FFFFFFF seed = int(random.random() * 2**31) & 0x7FFFFFFF
...@@ -109,14 +114,14 @@ def get_seed(cfg): ...@@ -109,14 +114,14 @@ def get_seed(cfg):
return seed return seed
def dict_merge(up, *lowers): def dict_merge(up, *lowers) -> None:
for lower in lowers: for lower in lowers:
for k, v in lower.items(): for k, v in lower.items():
if k not in up: if k not in up:
up[k] = v up[k] = v
def vhdl_serialize(o): def vhdl_serialize(o) -> str:
if isinstance(o, Iterable): if isinstance(o, Iterable):
ss = [] ss = []
for x in o: for x in o:
...@@ -126,7 +131,7 @@ def vhdl_serialize(o): ...@@ -126,7 +131,7 @@ def vhdl_serialize(o):
return str(o) return str(o)
def dump_sim_options(lib): def dump_sim_options(lib) -> None:
for tb in lib.get_test_benches('*'): for tb in lib.get_test_benches('*'):
for cfgs in tb._test_bench.get_configuration_dicts(): for cfgs in tb._test_bench.get_configuration_dicts():
for name, cfg in cfgs.items(): for name, cfg in cfgs.items():
......
...@@ -26,7 +26,7 @@ class FeatureTests(TestsBase): ...@@ -26,7 +26,7 @@ class FeatureTests(TestsBase):
tb = self.lib.get_test_benches('*tb_feature')[0] tb = self.lib.get_test_benches('*tb_feature')[0]
tb.scan_tests_from_file(str(wrname)) tb.scan_tests_from_file(str(wrname))
def configure(self) -> None: def configure(self) -> bool:
tb = self.lib.get_test_benches('*tb_feature')[0] tb = self.lib.get_test_benches('*tb_feature')[0]
default = self.config['default'] default = self.config['default']
self.add_modelsim_gui_file(tb, default, 'feature') self.add_modelsim_gui_file(tb, default, 'feature')
...@@ -60,9 +60,9 @@ class FeatureTests(TestsBase): ...@@ -60,9 +60,9 @@ class FeatureTests(TestsBase):
'seed' : get_seed(cfg) 'seed' : get_seed(cfg)
} }
tb.add_config(name, generics=generics) tb.add_config(name, generics=generics)
self._check_for_unconfigured() return self._check_for_unconfigured()
def _check_for_unconfigured(self): def _check_for_unconfigured(self) -> bool:
config = self.config config = self.config
# check for unconfigured unit tests # check for unconfigured unit tests
test_files = self.base.glob('feature/*_feature_tb.vhd') test_files = self.base.glob('feature/*_feature_tb.vhd')
...@@ -72,6 +72,7 @@ class FeatureTests(TestsBase): ...@@ -72,6 +72,7 @@ class FeatureTests(TestsBase):
unconfigured = [tb for tb in all_tests if tb not in configured] unconfigured = [tb for tb in all_tests if tb not in configured]
if len(unconfigured): if len(unconfigured):
log.warn("Feature tests with no configuration found (will not be run): {}".format(', '.join(unconfigured))) log.warn("Feature tests with no configuration found (will not be run): {}".format(', '.join(unconfigured)))
return len(unconfigured) == 0
def _create_wrapper(self, ofile: Path) -> None: def _create_wrapper(self, ofile: Path) -> None:
template = self.jinja_env.get_template('pkg_feature_exec_dispath-body.vhd') template = self.jinja_env.get_template('pkg_feature_exec_dispath-body.vhd')
......
...@@ -18,7 +18,7 @@ class ReferenceTests(TestsBase): ...@@ -18,7 +18,7 @@ class ReferenceTests(TestsBase):
sources.append('reference/vunit_reference_wrapper.vhd') sources.append('reference/vunit_reference_wrapper.vhd')
add_sources(self.lib, sources) add_sources(self.lib, sources)
def configure(self) -> None: def configure(self) -> bool:
tb = self.lib.get_test_benches('*reference*')[0] tb = self.lib.get_test_benches('*reference*')[0]
default = self.config['default'] default = self.config['default']
self.add_modelsim_gui_file(tb, default, 'sanity') self.add_modelsim_gui_file(tb, default, 'sanity')
...@@ -47,3 +47,4 @@ class ReferenceTests(TestsBase): ...@@ -47,3 +47,4 @@ class ReferenceTests(TestsBase):
init_files += [str(tcl)] init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files) tb.set_sim_option("modelsim.init_files.after_load", init_files)
self.add_modelsim_gui_file(tb, cfg, data_set) self.add_modelsim_gui_file(tb, cfg, data_set)
return True
...@@ -73,3 +73,4 @@ class SanityTests(TestsBase): ...@@ -73,3 +73,4 @@ class SanityTests(TestsBase):
'gauss_iter' : vhdl_serialize(cfg['gauss_iter']), 'gauss_iter' : vhdl_serialize(cfg['gauss_iter']),
} }
tb.add_config(name, generics=generics) tb.add_config(name, generics=generics)
return True
...@@ -9,11 +9,11 @@ log = logging.getLogger(__name__) ...@@ -9,11 +9,11 @@ log = logging.getLogger(__name__)
class UnitTests(TestsBase): class UnitTests(TestsBase):
def add_sources(self): def add_sources(self) -> None:
add_sources(self.lib, ['unit/**/*.vhd']) add_sources(self.lib, ['unit/**/*.vhd'])
self._create_wrapper(self.build / "tb_wrappers.vhd") self._create_wrapper(self.build / "tb_wrappers.vhd")
def configure(self): def configure(self) -> bool:
lib, config, build = self.lib, self.config, self.build lib, config, build = self.lib, self.config, self.build
default = config['default'] default = config['default']
unit_tests = lib.get_test_benches('*_unit_test') unit_tests = lib.get_test_benches('*_unit_test')
...@@ -44,9 +44,9 @@ class UnitTests(TestsBase): ...@@ -44,9 +44,9 @@ class UnitTests(TestsBase):
init_files += [str(tcl)] init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files) tb.set_sim_option("modelsim.init_files.after_load", init_files)
self.add_modelsim_gui_file(tb, cfg, name) self.add_modelsim_gui_file(tb, cfg, name)
self._check_for_unconfigured() return self._check_for_unconfigured()
def _check_for_unconfigured(self): def _check_for_unconfigured(self) -> bool:
lib, config = self.lib, self.config lib, config = self.lib, self.config
# check for unconfigured unit tests # check for unconfigured unit tests
unit_tests = lib.get_test_benches('*tb_*_unit_test') unit_tests = lib.get_test_benches('*tb_*_unit_test')
...@@ -55,8 +55,9 @@ class UnitTests(TestsBase): ...@@ -55,8 +55,9 @@ class UnitTests(TestsBase):
unconfigured = [tb for tb in unit_tests if tb.name not in configured] unconfigured = [tb for tb in unit_tests if tb.name not in configured]
if len(unconfigured): if len(unconfigured):
log.warn("Unit tests with no configuration found (defaults will be used): {}".format(', '.join(tb.name for tb in unconfigured))) log.warn("Unit tests with no configuration found (defaults will be used): {}".format(', '.join(tb.name for tb in unconfigured)))
return len(unconfigured) == 0
def _create_wrapper(self, fname): def _create_wrapper(self, fname) -> None:
fname = str(fname) fname = str(fname)
files = self.lib.get_source_files() files = self.lib.get_source_files()
tests = [] tests = []
...@@ -67,8 +68,6 @@ class UnitTests(TestsBase): ...@@ -67,8 +68,6 @@ class UnitTests(TestsBase):
m = r.match(l) m = r.match(l)
if m: if m:
tests.append(m.group(1)) tests.append(m.group(1))
configs = []
tbs = []
c = self.jinja_env.get_template('unit_wrappers.vhd').render(tests=tests) c = self.jinja_env.get_template('unit_wrappers.vhd').render(tests=tests)
with open(fname, "wt", encoding='utf-8') as f: with open(fname, "wt", encoding='utf-8') as f:
f.write(c) f.write(c)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment