Commit b5146d9a authored by Ille, Ondrej, Ing.'s avatar Ille, Ondrej, Ing.

Merge branch '200-bring-up-ghdl-functional-coverage' into 'master'

Resolve "Bring-up GHDL functional coverage."

Closes #200

See merge request !213
parents 60acecf8 4feef64e
Pipeline #6201 passed with stages
in 14 minutes and 47 seconds
......@@ -82,6 +82,7 @@ test_ip_fast:
- cd test
- make test_fast
- make coverage
- make functional_coverage
coverage: "/lines......: ([^%]+%)/"
artifacts:
when: always
......@@ -89,6 +90,7 @@ test_ip_fast:
- test/code_html
- test/tests_fast.xml
- test/xunit.xsl
- test/build/functional_coverage
reports:
junit: [test/tests_fast.xml]
......@@ -118,6 +120,7 @@ pages:
- cp test/tests_*.xml public/
- mv test/code_html public/coverage || true
- cp doc/core/Progdokum.pdf public/ || true
- cp -R test/build/functional_coverage/* public/functional_coverage/ || true
only:
- master
artifacts:
......
......@@ -3,6 +3,8 @@
[![pipeline status](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/pipeline.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/tests_fast.xml)
[![coverage report](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/coverage.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/coverage/)
[![documentation](https://img.shields.io/badge/documentation--blue.svg)]( http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/Progdokum.pdf)
[![functional coverage](https://img.shields.io/badge/functional%20coverage--orange.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/functional_coverage/functional_coverage_report.html)
CAN FD IP Core written in VHDL, originally developed at Czech Technical University -- Faculty of Electrical Engineering -- Department of Measurement.
......
Subproject commit 24f6fe411be6ed81ce4ab27d2dfeadaca425d999
Subproject commit 3baadf8b367cb06e83ceaa58cc088d93949b5175
......@@ -253,4 +253,30 @@ begin
txt_buf_ready => txt_buf_ready
);
----------------------------------------------------------------------------
----------------------------------------------------------------------------
-- Functional coverage
----------------------------------------------------------------------------
----------------------------------------------------------------------------
func_cov_block : block
begin
-- psl default clock is rising_edge(clk_sys);
-- Each SW command active
-- psl txtb_set_ready_cov : cover (txt_sw_cmd.set_rdy = '1' and sw_cbs = '1');
-- psl txtb_set_empty_cov : cover (txt_sw_cmd.set_ety = '1' and sw_cbs = '1');
-- psl txtb_set_abort_cov : cover (txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
-- HW Commands
-- psl txtb_hw_lock : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1');
-- psl txtb_hw_unlock : cover (txt_hw_cmd.unlock = '1' and hw_cbs = '1');
-- psl txtb_hw_valid : cover (txt_hw_cmd.valid = '1' and hw_cbs = '1');
-- psl txtb_hw_err : cover (txt_hw_cmd.err = '1' and hw_cbs = '1');
-- psl txtb_hw_arbl : cover (txt_hw_cmd.arbl = '1' and hw_cbs = '1');
-- psl txtb_hw_failed : cover (txt_hw_cmd.failed = '1' and hw_cbs = '1');
end block;
end architecture;
......@@ -357,4 +357,31 @@ begin
end if;
end process;
----------------------------------------------------------------------------
----------------------------------------------------------------------------
-- Functional coverage
----------------------------------------------------------------------------
----------------------------------------------------------------------------
func_cov_block : block
begin
-- psl default clock is rising_edge(clk_sys);
-- Each FSM state
-- psl txtb_fsm_empty : cover (buf_fsm = txt_empty);
-- psl txtb_fsm_ready : cover (buf_fsm = txt_ready);
-- psl txtb_fsm_tx_prog : cover (buf_fsm = txt_tx_prog);
-- psl txtb_fsm_ab_prog : cover (buf_fsm = txt_ab_prog);
-- psl txtb_fsm_error : cover (buf_fsm = txt_error);
-- psl txtb_fsm_aborted : cover (buf_fsm = txt_aborted);
-- psl txtb_fsm_tx_ok : cover (buf_fsm = txt_ok);
-- Simultaneous HW and SW Commands
-- psl txtb_rdy_hazard : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1' and
-- txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
end block;
end architecture;
PYTHON := python3
XUNIT ?= 0
BUILD_DIR = build
TESTFW_DIR = testfw
TEST_FLAGS = -p`nproc`
TEST_OPTS_test_debug := --no-strict
all: test coverage
all: test coverage functional_coverage
elaborate:
$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
......@@ -13,9 +15,12 @@ test_%: tests_%.yml FORCE
$(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
coverage:
lcov --capture --directory build --output-file code_coverage.info
lcov --capture --directory $(BUILD_DIR) --output-file code_coverage.info
genhtml code_coverage.info --output-directory code_html
functional_coverage:
$(PYTHON) $(TESTFW_DIR)/test_parse_psl_coverage.py
clean:
-rm -Rf build
......
......@@ -10,6 +10,7 @@ from os.path import abspath
from .log import MyLogRecord
d = Path(abspath(__file__)).parent
func_cov_dir = os.path.join(str(d.parent), "build/functional_coverage")
def setup_logging() -> None:
......@@ -125,11 +126,14 @@ def test(obj, config, strict, vunit_args):
if cfg_key in config:
tests.append(factory(ui, lib, config[cfg_key], build, base))
if not os.path.exists(func_cov_dir):
os.makedirs(func_cov_dir);
os.makedirs(os.path.join(func_cov_dir, "html"))
os.makedirs(os.path.join(func_cov_dir, "coverage_data"))
for t in tests:
t.add_sources()
add_flags(ui, lib, build)
conf_ok = [t.configure() for t in tests]
# check for unknown tests
......
......@@ -62,7 +62,6 @@ class TestsBase:
'''.format(name)), file=f)
tb.set_sim_option("modelsim.init_file.gui", str(tcl))
def add_sources(lib, patterns) -> None:
for pattern in patterns:
p = join(str(d.parent), pattern)
......@@ -71,7 +70,6 @@ def add_sources(lib, patterns) -> None:
if f != "tb_wrappers.vhd":
lib.add_source_file(str(f))
def add_common_sources(lib, ui) -> None:
add_sources(lib, ['../src/**/*.vhd'])
ui.enable_check_preprocessing()
......@@ -95,8 +93,17 @@ def add_flags(ui, lib, build) -> None:
#lib.add_compile_option("ghdl.flags", ["-Wc,-g"])
lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage", "-fpsl"])
ui.set_sim_option("ghdl.elab_flags", ["-Wl,-lgcov", "-Wl,--coverage", "-Wl,-no-pie", "-fpsl"])
ui.set_sim_option("ghdl.sim_flags", ["--ieee-asserts=disable-at-0"])
elab_flags = ["-Wl,-lgcov"]
elab_flags.append("-Wl,--coverage");
elab_flags.append("-Wl,-no-pie");
elab_flags.append("-fpsl");
ui.set_sim_option("ghdl.elab_flags",elab_flags)
# Global simulation flags
sim_flags = ["--ieee-asserts=disable-at-0"]
ui.set_sim_option("ghdl.sim_flags", sim_flags)
modelsim_init_files = get_common_modelsim_init_files()
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
......
......@@ -26,6 +26,11 @@ class FeatureTests(TestsBase):
tb = self.lib.get_test_benches('*tb_feature')[0]
tb.scan_tests_from_file(str(wrname))
def create_psl_cov_file_opt(self, name):
psl_path = "functional_coverage/coverage_data/psl_cov_feature_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self) -> bool:
tb = self.lib.get_test_benches('*tb_feature')[0]
default = self.config['default']
......@@ -59,7 +64,13 @@ class FeatureTests(TestsBase):
'test_name' : name,
'seed' : get_seed(cfg)
}
tb.add_config(name, generics=generics)
if (cfg['psl_coverage']):
psl_opts = self.create_psl_cov_file_opt(name)
tb.add_config(name, generics=generics, sim_options=psl_opts)
else:
tb.add_config(name, generics=generics)
return self._check_for_unconfigured()
def _check_for_unconfigured(self) -> bool:
......
This diff is collapsed.
......@@ -18,6 +18,11 @@ class ReferenceTests(TestsBase):
sources.append('reference/vunit_reference_wrapper.vhd')
add_sources(self.lib, sources)
def create_psl_cov_file_opt(self, name):
psl_path = "functional_coverage/coverage_data/psl_cov_reference_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self) -> bool:
tb = self.lib.get_test_benches('*reference*')[0]
default = self.config['default']
......@@ -34,8 +39,13 @@ class ReferenceTests(TestsBase):
'seed' : get_seed(cfg),
'data_path' : str(self.build) + '/../' + cfg['data_path'],
}
tb.add_config(data_set, generics=generics)
if (cfg['psl_coverage']):
psl_opts = self.create_psl_cov_file_opt(data_set)
tb.add_config(data_set, generics=generics, sim_options=psl_opts)
else:
tb.add_config(data_set, generics=generics)
tcl = self.build / 'modelsim_init_{}.tcl'.format(data_set)
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
......
......@@ -42,6 +42,18 @@ class SanityTests(TestsBase):
def add_sources(self):
add_sources(self.lib, ['sanity/**/*.vhd'])
def format_valid_test_name(self, name):
valid_name = name.replace(" ", '_')
valid_name = valid_name.replace("/", "_")
valid_name = valid_name.replace("", "_")
return valid_name
def create_psl_cov_file_opt(self, name):
test_name = self.format_valid_test_name("psl_cov_sanity_{}.json".format(name))
psl_path = "functional_coverage/coverage_data/{}".format(test_name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self):
# TODO: wave
tb = self.lib.get_test_benches('*tb_sanity')[0]
......@@ -72,5 +84,13 @@ class SanityTests(TestsBase):
'timing_config': vhdl_serialize(cfg['timing_config']),
'gauss_iter' : vhdl_serialize(cfg['gauss_iter']),
}
tb.add_config(name, generics=generics)
return True
sanity_cfg_name = name.replace(" ", "_").replace("/", "_").strip('"')
if (cfg['psl_coverage']):
psl_opts = self.create_psl_cov_file_opt(name)
tb.add_config(name, generics=generics, sim_options=psl_opts)
else:
tb.add_config(name, generics=generics)
return True
......@@ -13,6 +13,11 @@ class UnitTests(TestsBase):
add_sources(self.lib, ['unit/**/*.vhd'])
self._create_wrapper(self.build / "tb_wrappers.vhd")
def add_psl_cov_file(self, tb, name):
psl_path = "functional_coverage/coverage_data/psl_cov_unit_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
tb.set_sim_option("ghdl.sim_flags", [psl_flag])
def configure(self) -> bool:
lib, config, build = self.lib, self.config, self.build
default = config['default']
......@@ -43,6 +48,8 @@ class UnitTests(TestsBase):
init_files = get_common_modelsim_init_files()
init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files)
if (cfg['psl_coverage']):
self.add_psl_cov_file(tb, name)
self.add_modelsim_gui_file(tb, cfg, name)
return self._check_for_unconfigured()
......
......@@ -11,3 +11,4 @@ feature:
wave: feature/feature_env_setup.tcl
tests:
txt_buffer_hazard:
iterations: 1
_default: &default
log_level: info
psl_coverage: true
error_tolerance: 0
# seed: 0 # optional; use to reconstruct results from randomized runs
# randomize: false
......@@ -72,7 +73,7 @@ feature:
iterations: 1
timestamp_registers:
txt_buffer_hazard:
iterations: 5
iterations: 20
reference:
default:
<<: *default
......
_default: &default
log_level: info
psl_coverage: true
error_tolerance: 0
randomize: true
unit:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment