Commit 60acecf8 authored by Martin Jeřábek's avatar Martin Jeřábek

Merge branch '251-ci-fail-if-there-are-unconfigured-tests' into 'master'

tests/testfw: fail if there are unconfigured tests

Closes #251

See merge request !212
parents 6b920cd3 006315e7
Pipeline #6199 canceled with stage
......@@ -2,6 +2,7 @@ PYTHON := python3
XUNIT ?= 0
TEST_FLAGS = -p`nproc`
TEST_OPTS_test_debug := --no-strict
all: test coverage
......@@ -9,7 +10,7 @@ elaborate:
$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
test_%: tests_%.yml FORCE
$(PYTHON) run.py test $< -- $(TEST_FLAGS)
$(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
coverage:
lcov --capture --directory build --output-file code_coverage.info
......
......@@ -73,8 +73,10 @@ def create():
@cli.command()
@click.argument('config', type=click.Path())
@click.argument('vunit_args', nargs=-1)
@click.option('--strict/--no-strict', default=True,
help='Return non-zero if an unconfigured test was found.')
@click.pass_obj
def test(obj, config, vunit_args):
def test(obj, config, strict, vunit_args):
"""Run the tests. Configuration is passed in YAML config file.
You mas pass arguments directly to VUnit by appending them at the command end.
......@@ -105,40 +107,46 @@ def test(obj, config, vunit_args):
build.mkdir(exist_ok=True)
os.chdir(str(build))
run_unit = 'unit' in config
run_feature = 'feature' in config
run_sanity = 'sanity' in config
run_reference = 'reference' in config
ui = create_vunit(obj, vunit_args, out_basename)
lib = ui.add_library("lib")
add_common_sources(lib, ui)
tests = []
if run_unit:
tests.append(test_unit.UnitTests(ui, lib, config['unit'], build, base))
if run_sanity:
tests.append(test_sanity.SanityTests(ui, lib, config['sanity'], build, base))
if run_feature:
tests.append(test_feature.FeatureTests(ui, lib, config['feature'], build, base))
if run_reference:
tests.append(test_reference.ReferenceTests(ui, lib, config['reference'], build, base))
tests_classes = [
# key in config, factory
('unit', test_unit.UnitTests),
('sanity', test_sanity.SanityTests),
('feature', test_feature.FeatureTests),
('reference', test_reference.ReferenceTests),
]
tests = []
for cfg_key, factory in tests_classes:
if cfg_key in config:
tests.append(factory(ui, lib, config[cfg_key], build, base))
for t in tests:
t.add_sources()
add_flags(ui, lib, build)
for t in tests:
t.configure()
conf_ok = [t.configure() for t in tests]
# check for unknown tests
all_benches = lib.get_test_benches('*')
unknown_tests = [tb for tb in all_benches if not re.match('tb_.*?_unit_test|tb_sanity|tb_feature|tb_reference_wrapper', tb.name)]
pattern = 'tb_.*?_unit_test|tb_sanity|tb_feature|tb_reference_wrapper'
unknown_tests = [tb for tb in all_benches
if not re.match(pattern, tb.name)]
if len(unknown_tests):
log.warn('Unknown tests (defaults will be used): {}'.format(', '.join(tb.name for tb in unknown_tests)))
vunit_run(ui, build, out_basename)
log.warn('Unknown tests (defaults will be used): {}'
.format(', '.join(tb.name for tb in unknown_tests)))
res = vunit_run(ui, build, out_basename)
if not all(conf_ok) and strict:
log.error('Some test cases were discovered but not configured (see above).')
if res == 0:
res = 1
sys.exit(res)
def create_vunit(obj, vunit_args, out_basename):
......@@ -152,7 +160,7 @@ def create_vunit(obj, vunit_args, out_basename):
return ui
def vunit_run(ui, build, out_basename):
def vunit_run(ui, build, out_basename) -> int:
try:
vunit_ifc.run(ui)
res = None
......@@ -168,7 +176,7 @@ def vunit_run(ui, build, out_basename):
print('<?xml-stylesheet href="xunit.xsl" type="text/xsl"?>', file=f)
f.write(c)
out.unlink()
sys.exit(res)
return res
"""
......
......@@ -7,6 +7,7 @@ from pathlib import Path
from jinja2 import Environment, PackageLoader
from pprint import pprint
import random
from typing import List
__all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files',
'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options',
......@@ -30,13 +31,17 @@ class TestsBase:
def jinja_env(self):
return jinja_env
def add_sources(self):
def add_sources(self) -> None:
raise NotImplementedError()
def configure(self):
def configure(self) -> bool:
"""Configure the tests.
Return False if there were unconfigured tests found."""
raise NotImplementedError()
def add_modelsim_gui_file(self, tb, cfg, name):
def add_modelsim_gui_file(self, tb, cfg, name) -> None:
if 'wave' in cfg:
tcl = self.base / cfg['wave']
if not tcl.exists():
......@@ -58,7 +63,7 @@ class TestsBase:
tb.set_sim_option("modelsim.init_file.gui", str(tcl))
def add_sources(lib, patterns):
def add_sources(lib, patterns) -> None:
for pattern in patterns:
p = join(str(d.parent), pattern)
log.debug('Adding sources matching {}'.format(p))
......@@ -67,19 +72,19 @@ def add_sources(lib, patterns):
lib.add_source_file(str(f))
def add_common_sources(lib, ui):
def add_common_sources(lib, ui) -> None:
add_sources(lib, ['../src/**/*.vhd'])
ui.enable_check_preprocessing()
ui.enable_location_preprocessing() #(additional_subprograms=['log'])
add_sources(lib, ['*.vhd', 'lib/*.vhd'])
def get_common_modelsim_init_files():
def get_common_modelsim_init_files() -> List[str]:
modelsim_init_files = '../lib/test_lib.tcl,modelsim_init.tcl'
modelsim_init_files = [str(d/x) for x in modelsim_init_files.split(',')]
return modelsim_init_files
def add_flags(ui, lib, build):
def add_flags(ui, lib, build) -> None:
unit_tests = lib.get_test_benches('*_unit_test', allow_empty=True)
for ut in unit_tests:
ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
......@@ -96,11 +101,11 @@ def add_flags(ui, lib, build):
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
def get_seed(cfg):
def get_seed(cfg) -> int:
if 'seed' in cfg and 'randomize' in cfg:
log.warning('Both "seed" and "randomize" are set - seed takes precedence')
if 'seed' in cfg:
seed = cfg['seed']
seed = int(cfg['seed'], 0)
elif cfg.get('randomize', False):
# only 31 bits
seed = int(random.random() * 2**31) & 0x7FFFFFFF
......@@ -109,14 +114,14 @@ def get_seed(cfg):
return seed
def dict_merge(up, *lowers):
def dict_merge(up, *lowers) -> None:
for lower in lowers:
for k, v in lower.items():
if k not in up:
up[k] = v
def vhdl_serialize(o):
def vhdl_serialize(o) -> str:
if isinstance(o, Iterable):
ss = []
for x in o:
......@@ -126,7 +131,7 @@ def vhdl_serialize(o):
return str(o)
def dump_sim_options(lib):
def dump_sim_options(lib) -> None:
for tb in lib.get_test_benches('*'):
for cfgs in tb._test_bench.get_configuration_dicts():
for name, cfg in cfgs.items():
......
......@@ -26,7 +26,7 @@ class FeatureTests(TestsBase):
tb = self.lib.get_test_benches('*tb_feature')[0]
tb.scan_tests_from_file(str(wrname))
def configure(self) -> None:
def configure(self) -> bool:
tb = self.lib.get_test_benches('*tb_feature')[0]
default = self.config['default']
self.add_modelsim_gui_file(tb, default, 'feature')
......@@ -60,9 +60,9 @@ class FeatureTests(TestsBase):
'seed' : get_seed(cfg)
}
tb.add_config(name, generics=generics)
self._check_for_unconfigured()
return self._check_for_unconfigured()
def _check_for_unconfigured(self):
def _check_for_unconfigured(self) -> bool:
config = self.config
# check for unconfigured unit tests
test_files = self.base.glob('feature/*_feature_tb.vhd')
......@@ -72,6 +72,7 @@ class FeatureTests(TestsBase):
unconfigured = [tb for tb in all_tests if tb not in configured]
if len(unconfigured):
log.warn("Feature tests with no configuration found (will not be run): {}".format(', '.join(unconfigured)))
return len(unconfigured) == 0
def _create_wrapper(self, ofile: Path) -> None:
template = self.jinja_env.get_template('pkg_feature_exec_dispath-body.vhd')
......
......@@ -18,7 +18,7 @@ class ReferenceTests(TestsBase):
sources.append('reference/vunit_reference_wrapper.vhd')
add_sources(self.lib, sources)
def configure(self) -> None:
def configure(self) -> bool:
tb = self.lib.get_test_benches('*reference*')[0]
default = self.config['default']
self.add_modelsim_gui_file(tb, default, 'sanity')
......@@ -47,3 +47,4 @@ class ReferenceTests(TestsBase):
init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files)
self.add_modelsim_gui_file(tb, cfg, data_set)
return True
......@@ -73,3 +73,4 @@ class SanityTests(TestsBase):
'gauss_iter' : vhdl_serialize(cfg['gauss_iter']),
}
tb.add_config(name, generics=generics)
return True
......@@ -9,11 +9,11 @@ log = logging.getLogger(__name__)
class UnitTests(TestsBase):
def add_sources(self):
def add_sources(self) -> None:
add_sources(self.lib, ['unit/**/*.vhd'])
self._create_wrapper(self.build / "tb_wrappers.vhd")
def configure(self):
def configure(self) -> bool:
lib, config, build = self.lib, self.config, self.build
default = config['default']
unit_tests = lib.get_test_benches('*_unit_test')
......@@ -44,9 +44,9 @@ class UnitTests(TestsBase):
init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files)
self.add_modelsim_gui_file(tb, cfg, name)
self._check_for_unconfigured()
return self._check_for_unconfigured()
def _check_for_unconfigured(self):
def _check_for_unconfigured(self) -> bool:
lib, config = self.lib, self.config
# check for unconfigured unit tests
unit_tests = lib.get_test_benches('*tb_*_unit_test')
......@@ -55,8 +55,9 @@ class UnitTests(TestsBase):
unconfigured = [tb for tb in unit_tests if tb.name not in configured]
if len(unconfigured):
log.warn("Unit tests with no configuration found (defaults will be used): {}".format(', '.join(tb.name for tb in unconfigured)))
return len(unconfigured) == 0
def _create_wrapper(self, fname):
def _create_wrapper(self, fname) -> None:
fname = str(fname)
files = self.lib.get_source_files()
tests = []
......@@ -67,8 +68,6 @@ class UnitTests(TestsBase):
m = r.match(l)
if m:
tests.append(m.group(1))
configs = []
tbs = []
c = self.jinja_env.get_template('unit_wrappers.vhd').render(tests=tests)
with open(fname, "wt", encoding='utf-8') as f:
f.write(c)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment