Skip to content

GitLab

C
CTU CAN FD IP Core
Commit e935c73e authored by Martin Jeřábek's avatar Martin Jeřábek
Browse files
Options

testfw: fix accumulating sim_options, extract common bits, cleanup

parent 27d09104
Hide whitespace changes
Inline Side-by-side
Showing with 166 additions and 192 deletions
test/testfw/__init__.py
View file @ e935c73e
......@@ -27,7 +27,7 @@ setup_logging()
from . import vunit_ifc
from . import test_unit, test_sanity, test_feature, test_reference
from vunit.ui import VUnit
from .test_common import add_common_sources, add_flags
from .test_common import add_common_sources, get_compile_options
#-------------------------------------------------------------------------------
......@@ -150,7 +150,11 @@ def test(obj, *, config, strict, create_ghws, vunit_args):
for t in tests:
t.add_sources()
add_flags(ui, lib, build)
c = get_compile_options()
for k, v in c.items():
lib.set_compile_option(k, v)
conf_ok = [t.configure() for t in tests]
# check for unknown tests
......@@ -198,27 +202,3 @@ def vunit_run(ui, build, out_basename) -> int:
f.write(c)
out.unlink()
return res
"""
+ vunit configurations
+ pass modelsim gui file via ui.set_sim_option("modelsim.init_file.gui", ...)
+ include the standard library files in ui.set_sim_option("modelsim.init_files.after_load", [...])
+ set TCOMP global variable
- allow preprocessed calls to log()
- use some log from vunit?
- use random from unit?
+ use per-test default configurations (with set tcl files etc.), different sanity configurations
x pass encoded composite generics (sanity test)
+ use watchdog - pass the time in config: test_runner_watchdog(runner, 10 ms);
- bash completion for files & tests:
- click._bashcompletion.get_choices -> extend the if to check if the given argument is an instance of XXX
and implement completion method for that instance. Complete test names.
- feature tests
- sanity - optimize bus delay shift registers
"""
test/testfw/test_common.py
View file @ e935c73e
......@@ -8,11 +8,13 @@ from jinja2 import Environment, PackageLoader
from pprint import pprint
import random
from .gtkwave import tcl2gtkw
from typing import List
from typing import List, Tuple
import copy
import re
__all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files',
'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options',
'TestsBase', 'get_seed']
__all__ = ['add_sources', 'add_common_sources',
'dict_merge', 'vhdl_serialize', 'dump_sim_options',
'TestsBase', 'get_seed', 'OptionsDict']
d = Path(abspath(__file__)).parent
log = logging.getLogger(__name__)
......@@ -20,6 +22,43 @@ log = logging.getLogger(__name__)
jinja_env = Environment(loader=PackageLoader(__package__, 'data'), autoescape=False)
class OptionsDict(dict):
# def __getattr__(self, key):
# return self[key]
def __iadd__(self, upper: dict):
self.__merge(self, upper)
return self
def __add__(self, upper: dict) -> 'OptionsDict':
res = copy.deepcopy(self)
res += upper
return res
def __radd__(self, lower: dict) -> 'OptionsDict':
res = copy.deepcopy(lower)
res += self
return res
@classmethod
def __merge(cls, lower, upper) -> None:
if isinstance(lower, OptionsDict):
if not isinstance(upper, OptionsDict):
raise TypeError('Cannot merge {} and {}'.format(type(lower), type('upper')))
for k, v in upper.items():
if k not in lower:
lower[k] = v
else:
cls.__merge(lower[k], v)
elif isinstance(lower, list):
if not isinstance(upper, list):
raise TypeError('Cannot merge {} and {}'.format(type(lower), type('upper')))
lower.extend(upper)
else:
raise TypeError('Cannot merge {} and {}'.format(type(lower), type('upper')))
class TestsBase:
def __init__(self, ui, lib, config, build, base, create_ghws: bool):
self.ui = ui
......@@ -43,9 +82,18 @@ class TestsBase:
raise NotImplementedError()
def add_modelsim_gui_file(self, tb, cfg, name, tcl_init_files: List[str] = None) -> None:
if tcl_init_files is None:
tcl_init_files = get_common_modelsim_init_files()
def generate_init_tcl(self, fname: str, tcomp: str) -> OptionsDict:
tcl = self.build / fname
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
global TCOMP
set TCOMP {}
'''.format(tcomp)), file=f)
return OptionsDict({"modelsim.init_files.after_load": [str(tcl)]})
def add_modelsim_gui_file(self, tb, cfg, name, tcl_init_files: List[str]) -> OptionsDict:
"""Return sim_options to add to the testcase."""
sim_options = OptionsDict({'ghdl.sim_flags': []})
if 'wave' in cfg:
tcl = self.base / cfg['wave']
if not tcl.exists():
......@@ -65,7 +113,7 @@ class TestsBase:
get_test_results
'''.format(name)), file=f)
tb.set_sim_option("modelsim.init_file.gui", str(tcl))
sim_options["modelsim.init_file.gui"] = str(tcl)
if 'gtkw' in cfg:
gtkw = self.base / cfg['gtkw']
if not gtkw.exists():
......@@ -79,9 +127,7 @@ class TestsBase:
# the conversion now.
if self.create_ghws:
log.info('Will generate {}'.format(ghw_file))
sim_flags = get_common_sim_flags()
sim_flags += ['--wave=' + str(ghw_file)]
tb.set_sim_option("ghdl.sim_flags", sim_flags)
sim_options["ghdl.sim_flags"] += ['--wave=' + str(ghw_file)]
else:
if not ghw_file.exists():
log.warning("Cannot convert wave file {} to gtkw, because"
......@@ -94,12 +140,32 @@ class TestsBase:
if gtkw:
try:
tb.set_sim_option("ghdl.gtkwave_flags", ['--save='+str(gtkw)])
tb.set_sim_option("ghdl.gtkwave_flags", [])
sim_options["ghdl.gtkwave_flags"] = ['--save='+str(gtkw)]
except ValueError:
try:
tb.set_sim_option("ghdl.gtkw_file", str(gtkw))
tb.set_sim_option("ghdl.gtkw_file", "")
sim_options["ghdl.gtkw_file"] = str(gtkw)
except ValueError:
log.warning('Setting GTKW file per test is not supported in this VUnit version.')
return OptionsDict(sim_options)
def get_default_sim_options(self) -> OptionsDict:
c, s = get_default_compile_and_sim_options()
return s
def add_psl_cov(self, name) -> OptionsDict:
name = re.sub(r'[^a-zA-Z0-9_-]', '_', name)
psl_path = self.build / "functional_coverage" / "coverage_data" \
/ "psl_cov_{}.json".format(name)
sim_flags = ["--psl-report={}".format(psl_path)]
return OptionsDict({"ghdl.sim_flags": sim_flags})
@staticmethod
def set_sim_options(tb, options: OptionsDict) -> None:
for k, v in options.items():
tb.set_sim_option(k, v)
def add_sources(lib, patterns) -> None:
for pattern in patterns:
......@@ -109,6 +175,7 @@ def add_sources(lib, patterns) -> None:
if f != "tb_wrappers.vhd":
lib.add_source_file(str(f))
def add_common_sources(lib, ui) -> None:
add_sources(lib, ['../src/**/*.vhd'])
ui.enable_check_preprocessing()
......@@ -116,38 +183,41 @@ def add_common_sources(lib, ui) -> None:
add_sources(lib, ['*.vhd', 'lib/*.vhd', 'models/*.vhd'])
def get_common_modelsim_init_files() -> List[str]:
modelsim_init_files = ['../lib/test_lib.tcl', 'modelsim_init.tcl']
modelsim_init_files = [str(d/x) for x in modelsim_init_files]
return modelsim_init_files
def get_common_sim_flags() -> List[str]:
return ["--ieee-asserts=disable-at-0"]
def add_flags(ui, lib, build) -> None:
unit_tests = lib.get_test_benches('*_unit_test', allow_empty=True)
for ut in unit_tests:
ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
reference_tests = lib.get_test_benches('*reference*', allow_empty=True)
for rt in reference_tests:
rt.scan_tests_from_file(str(build / "../reference/vunit_reference_wrapper.vhd"))
#lib.add_compile_option("ghdl.flags", ["-Wc,-g"])
lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage", "-fpsl", "-g"])
elab_flags = ["-Wl,-lgcov", "-g"]
elab_flags.append("-Wl,--coverage")
elab_flags.append("-Wl,-no-pie")
elab_flags.append("-fpsl")
ui.set_sim_option("ghdl.elab_flags", elab_flags)
# Global simulation flags
sim_flags = get_common_sim_flags()
ui.set_sim_option("ghdl.sim_flags", sim_flags)
modelsim_init_files = get_common_modelsim_init_files()
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
def get_default_compile_and_sim_options() -> Tuple[OptionsDict, OptionsDict]:
# TODO: move to config
debug = True
coverage = True
psl = True
compile_flags = [] # type: List[str]
elab_flags = ["-Wl,-no-pie"]
if debug:
compile_flags += ['-g']
elab_flags += ['-g']
if coverage:
compile_flags += ["-fprofile-arcs", "-ftest-coverage"]
elab_flags += ["-Wl,-lgcov", "-Wl,--coverage"]
if psl:
compile_flags += ['-fpsl']
elab_flags += ['-fpsl']
compile_options = OptionsDict()
compile_options["ghdl.flags"] = compile_flags
cmif = ['../lib/test_lib.tcl', 'modelsim_init.tcl']
common_modelsim_init_files = [str(d/x) for x in cmif]
sim_options = OptionsDict({
"ghdl.elab_flags": elab_flags,
"modelsim.init_files.after_load": common_modelsim_init_files,
"ghdl.sim_flags": ["--ieee-asserts=disable-at-0"],
})
return compile_options, sim_options
def get_compile_options() -> OptionsDict:
c, s = get_default_compile_and_sim_options()
return c
def get_seed(cfg) -> int:
......
test/testfw/test_feature.py
View file @ e935c73e
import logging
from pathlib import Path
from .test_common import add_sources, TestsBase, dict_merge, \
get_common_modelsim_init_files, get_seed
get_seed, OptionsDict
from textwrap import dedent
import re
......@@ -26,25 +26,14 @@ class FeatureTests(TestsBase):
tb = self.lib.get_test_benches('*tb_feature')[0]
tb.scan_tests_from_file(str(wrname))
def create_psl_cov_file_opt(self, name):
psl_path = "functional_coverage/coverage_data/psl_cov_feature_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self) -> bool:
tb = self.lib.get_test_benches('*tb_feature')[0]
default = self.config['default']
sim_options = self.get_default_sim_options()
# generate & set per-test modelsim tcl file
tcl = self.build / 'modelsim_init_feature.tcl'
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
global TCOMP
set TCOMP tb_feature/test_comp
'''), file=f)
init_files = get_common_modelsim_init_files()
init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files)
sim_options += self.generate_init_tcl('modelsim_init_feature.tcl', 'tb_feature/test_comp')
sim_options += self.add_modelsim_gui_file(tb, default, 'feature', sim_options['modelsim.init_files.after_load'])
for name, cfg in self.config['tests'].items():
if cfg is None:
......@@ -64,12 +53,13 @@ class FeatureTests(TestsBase):
'seed' : get_seed(cfg)
}
if (cfg['psl_coverage']):
psl_opts = self.create_psl_cov_file_opt(name)
tb.add_config(name, generics=generics, sim_options=psl_opts)
else:
tb.add_config(name, generics=generics)
self.add_modelsim_gui_file(tb, default, 'feature', init_files)
local_sim_options = OptionsDict()
if cfg['psl_coverage']:
local_sim_options += self.add_psl_cov('{}.{}'.format(tb.name, name))
local_sim_options = sim_options + local_sim_options
tb.add_config(name, generics=generics, sim_options=local_sim_options)
return self._check_for_unconfigured()
def _check_for_unconfigured(self) -> bool:
......
test/testfw/test_reference.py
View file @ e935c73e
import logging
from pathlib import Path
from .test_common import add_sources, TestsBase, dict_merge, \
get_common_modelsim_init_files, get_seed
from textwrap import dedent
get_seed, OptionsDict
log = logging.getLogger(__name__)
......@@ -18,28 +17,20 @@ class ReferenceTests(TestsBase):
sources.append('reference/vunit_reference_wrapper.vhd')
add_sources(self.lib, sources)
def create_psl_cov_file_opt(self, name):
psl_path = "functional_coverage/coverage_data/psl_cov_reference_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self) -> bool:
tb = self.lib.get_test_benches('*reference*')[0]
default = self.config['default']
tcl = self.build / 'modelsim_init_reference.tcl'
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
global TCOMP
set TCOMP tb_reference_wrapper/i_test
'''), file=f)
# TODO: is this necessary?
tb.scan_tests_from_file(str(self.base / "reference/vunit_reference_wrapper.vhd"))
init_files = get_common_modelsim_init_files()
init_files += [str(tcl)]
sim_options = self.get_default_sim_options()
# generate & set per-test modelsim tcl file
sim_options += self.generate_init_tcl('modelsim_init_reference.tcl', 'tb_reference_wrapper/i_test')
sim_options += self.add_modelsim_gui_file(tb, default, 'reference', sim_options['modelsim.init_files.after_load'])
for data_set, cfg in self.config['tests'].items():
dict_merge(cfg, default)
# bm = len_to_matrix(cfg['topology'], cfg['bus_len_v'])
generics = {
'timeout' : cfg['timeout'],
'iterations' : cfg['iterations'],
......@@ -48,13 +39,10 @@ class ReferenceTests(TestsBase):
'seed' : get_seed(cfg),
'data_path' : str(self.build) + '/../' + cfg['data_path'],
}
local_sim_options = OptionsDict()
if cfg['psl_coverage']:
local_sim_options += self.add_psl_cov('{}.{}'.format(tb.name, data_set))
local_sim_options = sim_options + local_sim_options
tb.add_config(data_set, generics=generics, sim_options=local_sim_options)
if (cfg['psl_coverage']):
psl_opts = self.create_psl_cov_file_opt(data_set)
tb.add_config(data_set, generics=generics, sim_options=psl_opts)
else:
tb.add_config(data_set, generics=generics)
tb.set_sim_option("modelsim.init_files.after_load", init_files)
self.add_modelsim_gui_file(tb, default, 'reference', init_files)
return True
test/testfw/test_sanity.py
View file @ e935c73e
import logging
from textwrap import dedent
from .test_common import TestsBase, add_sources, dict_merge, vhdl_serialize, \
get_seed, get_common_modelsim_init_files
get_seed, OptionsDict
log = logging.getLogger(__name__)
def len_to_matrix(topology, bus_len):
l = bus_len
if topology == 'bus':
bm = [[0.0, l[1], l[1]+l[2], l[1]+l[2]+l[3]],
[l[1], 0.0, l[2], l[2]+l[3]],
[l[1]+l[2], l[2], 0.0, l[3]],
[l[1]+l[2]+l[3], l[2]+l[3], l[3], 0.0]]
elif topology == 'star':
bm = [[0.0, l[1]+l[2], l[1]+l[3], l[1]+l[4]],
[l[1]+l[2], 0.0, l[2]+l[3], l[2]+l[4]],
[l[1]+l[3], l[2]+l[3], 0.0, l[3]+l[4]],
[l[1]+l[4], l[2]+l[4], l[3]+l[4], 0.0]]
elif topology == 'tree':
bm = [[0.0, l[1]+l[2], l[1]+l[3]+l[5], l[1]+l[4]+l[5]],
[l[1]+l[2], 0.0, l[2]+l[3]+l[5], l[2]+l[4]+l[5]],
[l[1]+l[3]+l[5], l[2]+l[3]+l[5], 0.0, l[3]+l[4]],
[l[1]+l[4]+l[5], l[2]+l[4]+l[5], l[3]+l[4], 0.0]]
elif topology == 'ring':
bm = [[0.0, min(l[1], l[2]+l[3]+l[4]), min(l[1]+l[2],l[3]+l[4]), min(l[4],l[1]+l[2]+l[3])],
[min(l[1], l[2]+l[3]+l[4]), 0.0, min(l[2], l[1]+l[3]+l[4]), min(l[2]+l[3],l[1]+l[4])],
[min(l[1]+l[2],l[3]+l[4]), min(l[2], l[1]+l[3]+l[4]), 0.0, min(l[3],l[1]+l[2]+l[4])],
[min(l[4],l[1]+l[2]+l[3]), min(l[2]+l[3],l[1]+l[4]), min(l[3],l[1]+l[2]+l[4]), 0.0]]
elif topology == 'custom':
bm = [[0.0, l[1], l[2], l[3]],
[l[1], 0.0, l[4], l[5]],
[l[2], l[4], 0.0, l[6]],
[l[3], l[6], l[6], 0.0]]
else:
raise ValueError("Invalid bus topology.")
return bm
class SanityTests(TestsBase):
def add_sources(self):
add_sources(self.lib, ['sanity/**/*.vhd'])
......@@ -48,25 +15,14 @@ class SanityTests(TestsBase):
valid_name = valid_name.replace("/", "_")
return valid_name
def create_psl_cov_file_opt(self, name):
test_name = "psl_cov_sanity_{}.json".format(self.format_valid_test_name(name))
psl_path = "functional_coverage/coverage_data/{}".format(test_name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self):
tb = self.lib.get_test_benches('*tb_sanity')[0]
default = self.config['default']
tcl = self.build / 'modelsim_init_sanity.tcl'
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
global TCOMP
set TCOMP tb_sanity
'''), file=f)
init_files = get_common_modelsim_init_files()
init_files += [str(tcl)]
sim_options = self.get_default_sim_options()
# generate & set per-test modelsim tcl file
sim_options += self.generate_init_tcl('modelsim_init_sanity.tcl', 'tb_sanity')
sim_options += self.add_modelsim_gui_file(tb, default, 'sanity', sim_options['modelsim.init_files.after_load'])
for name, cfg in self.config['tests'].items():
if 'wave' in cfg:
......@@ -74,7 +30,6 @@ class SanityTests(TestsBase):
' (set it in default instead)'.format(name))
dict_merge(cfg, default)
# bm = len_to_matrix(cfg['topology'], cfg['bus_len_v'])
generics = {
'timeout' : cfg['timeout'],
'iterations' : cfg['iterations'],
......@@ -94,13 +49,9 @@ class SanityTests(TestsBase):
'gauss_iter' : vhdl_serialize(cfg['gauss_iter']),
}
sanity_cfg_name = name.replace(" ", "_").replace("/", "_").strip('"')
local_sim_options = OptionsDict()
if cfg['psl_coverage']:
psl_opts = self.create_psl_cov_file_opt(name)
tb.add_config(name, generics=generics, sim_options=psl_opts)
else:
tb.add_config(name, generics=generics)
self.add_modelsim_gui_file(tb, default, 'sanity', init_files)
local_sim_options += self.add_psl_cov('{}.{}'.format(tb.name, name))
local_sim_options = sim_options + local_sim_options
tb.add_config(name, generics=generics, sim_options=local_sim_options)
return True
test/testfw/test_unit.py
View file @ e935c73e
......@@ -2,7 +2,7 @@ import re
import logging
from textwrap import dedent
from .test_common import add_sources, dict_merge, TestsBase, \
get_common_modelsim_init_files, get_seed
get_seed, OptionsDict
from pprint import pprint
log = logging.getLogger(__name__)
......@@ -13,15 +13,14 @@ class UnitTests(TestsBase):
add_sources(self.lib, ['unit/**/*.vhd'])
self._create_wrapper(self.build / "tb_wrappers.vhd")
def add_psl_cov_file(self, tb, name):
psl_path = "functional_coverage/coverage_data/psl_cov_unit_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
tb.set_sim_option("ghdl.sim_flags", [psl_flag])
def configure(self) -> bool:
lib, config, build = self.lib, self.config, self.build
default = config['default']
unit_tests = lib.get_test_benches('*_unit_test')
for ut in unit_tests:
ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
for name, cfg in config['tests'].items():
dict_merge(cfg, default)
tb = lib.get_test_benches('*tb_{}_unit_test'.format(name),
......@@ -39,19 +38,15 @@ class UnitTests(TestsBase):
tb.set_generic('error_tol', cfg['error_tolerance'])
tb.set_generic('seed', get_seed(cfg))
sim_options = self.get_default_sim_options()
# generate & set per-test modelsim tcl file
tcl = build / 'modelsim_init_{}.tcl'.format(name)
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
global TCOMP
set TCOMP tb_{}_unit_test/tb/i_test
'''.format(name)), file=f)
init_files = get_common_modelsim_init_files()
init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files)
if (cfg['psl_coverage']):
self.add_psl_cov_file(tb, name)
self.add_modelsim_gui_file(tb, cfg, name, tcl_init_files=init_files)
sim_options += self.generate_init_tcl('modelsim_init_{}.tcl'.format(name), 'tb_{}_unit_test/tb/i_test'.format(name))
sim_options += self.add_modelsim_gui_file(tb, cfg, name, sim_options['modelsim.init_files.after_load'])
if cfg['psl_coverage']:
sim_options += self.add_psl_cov(tb.name)
self.set_sim_options(tb, sim_options)
return self._check_for_unconfigured()
def _check_for_unconfigured(self) -> bool:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment