Merge branch '200-bring-up-ghdl-functional-coverage' into 'master'

Resolve "Bring-up GHDL functional coverage."

Closes #200

See merge request !213

.gitlab-ci.yml

@@ -82,6 +82,7 @@ test_ip_fast:
         - cd test
         - make test_fast
         - make coverage
+        - make functional_coverage
     coverage: "/lines......: ([^%]+%)/"
     artifacts:
         when: always
@@ -89,6 +90,7 @@ test_ip_fast:
             - test/code_html
             - test/tests_fast.xml
             - test/xunit.xsl
+            - test/build/functional_coverage
         reports:
             junit: [test/tests_fast.xml]
 
@@ -118,6 +120,7 @@ pages:
         - cp test/tests_*.xml public/
         - mv test/code_html public/coverage || true
         - cp doc/core/Progdokum.pdf public/ || true
+        - cp -R test/build/functional_coverage/* public/functional_coverage/ || true
     only:
         - master
     artifacts:

README.md

@@ -3,6 +3,8 @@
 [![pipeline status](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/pipeline.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/tests_fast.xml)
 [![coverage report](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/coverage.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/coverage/)
 [![documentation](https://img.shields.io/badge/documentation--blue.svg)]( http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/Progdokum.pdf)
+[![functional coverage](https://img.shields.io/badge/functional%20coverage--orange.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/functional_coverage/functional_coverage_report.html)
+
 
 CAN FD IP Core written in VHDL, originally developed at Czech Technical University -- Faculty of Electrical Engineering -- Department of Measurement.
 

pyXact_generator @ 3baadf8b

-Subproject commit 24f6fe411be6ed81ce4ab27d2dfeadaca425d999
+Subproject commit 3baadf8b367cb06e83ceaa58cc088d93949b5175

src/txt_buffer/txt_buffer.vhd

@@ -253,4 +253,30 @@ begin
         txt_buf_ready          => txt_buf_ready
     );
 
+    
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    -- Functional coverage
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    func_cov_block : block
+    begin 
+    
+    -- psl default clock is rising_edge(clk_sys);
+
+    -- Each SW command active
+    -- psl txtb_set_ready_cov : cover (txt_sw_cmd.set_rdy = '1' and sw_cbs = '1');
+    -- psl txtb_set_empty_cov : cover (txt_sw_cmd.set_ety = '1' and sw_cbs = '1');
+    -- psl txtb_set_abort_cov : cover (txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
+      
+    -- HW Commands
+    -- psl txtb_hw_lock : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1');
+    -- psl txtb_hw_unlock : cover (txt_hw_cmd.unlock = '1' and hw_cbs = '1');
+    -- psl txtb_hw_valid : cover (txt_hw_cmd.valid = '1' and hw_cbs = '1');
+    -- psl txtb_hw_err : cover (txt_hw_cmd.err = '1' and hw_cbs = '1');
+    -- psl txtb_hw_arbl : cover (txt_hw_cmd.arbl = '1' and hw_cbs = '1');
+    -- psl txtb_hw_failed : cover (txt_hw_cmd.failed = '1' and hw_cbs = '1');
+    
+    end block;
+
 end architecture;

src/txt_buffer/txt_buffer_fsm.vhd

@@ -357,4 +357,31 @@ begin
         end if;
     end process;
 
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    -- Functional coverage
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    func_cov_block : block
+    begin 
+    
+    -- psl default clock is rising_edge(clk_sys);
+
+    -- Each FSM state
+    -- psl txtb_fsm_empty : cover (buf_fsm = txt_empty);
+    -- psl txtb_fsm_ready : cover (buf_fsm = txt_ready);
+    -- psl txtb_fsm_tx_prog : cover (buf_fsm = txt_tx_prog);
+    -- psl txtb_fsm_ab_prog : cover (buf_fsm = txt_ab_prog);
+    -- psl txtb_fsm_error : cover (buf_fsm = txt_error);
+    -- psl txtb_fsm_aborted : cover (buf_fsm = txt_aborted);
+    -- psl txtb_fsm_tx_ok : cover (buf_fsm = txt_ok);
+    
+    -- Simultaneous HW and SW Commands
+    -- psl txtb_rdy_hazard : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1' and
+    --                              txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
+
+    
+    
+    end block;
+
 end architecture;

test/Makefile

 PYTHON := python3
 XUNIT ?= 0
+BUILD_DIR = build
+TESTFW_DIR = testfw
 
 TEST_FLAGS = -p`nproc`
 TEST_OPTS_test_debug := --no-strict
 
-all: test coverage
+all: test coverage functional_coverage
 
 elaborate:
 	$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
@@ -13,9 +15,12 @@ test_%: tests_%.yml FORCE
 	$(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
 
 coverage:
-	lcov --capture --directory build --output-file code_coverage.info
+	lcov --capture --directory $(BUILD_DIR) --output-file code_coverage.info
 	genhtml code_coverage.info --output-directory code_html
 
+functional_coverage:
+	$(PYTHON) $(TESTFW_DIR)/test_parse_psl_coverage.py
+
 clean:
 	-rm -Rf build
 

test/testfw/__init__.py

@@ -10,6 +10,7 @@ from os.path import abspath
 from .log import MyLogRecord
 
 d = Path(abspath(__file__)).parent
+func_cov_dir = os.path.join(str(d.parent), "build/functional_coverage")
 
 
 def setup_logging() -> None:
@@ -125,11 +126,14 @@ def test(obj, config, strict, vunit_args):
         if cfg_key in config:
             tests.append(factory(ui, lib, config[cfg_key], build, base))
 
+    if not os.path.exists(func_cov_dir):
+        os.makedirs(func_cov_dir);
+        os.makedirs(os.path.join(func_cov_dir, "html"))
+        os.makedirs(os.path.join(func_cov_dir, "coverage_data"))
+
     for t in tests:
         t.add_sources()
-
     add_flags(ui, lib, build)
-
     conf_ok = [t.configure() for t in tests]
 
     # check for unknown tests

test/testfw/test_common.py

@@ -62,7 +62,6 @@ class TestsBase:
                     '''.format(name)), file=f)
         tb.set_sim_option("modelsim.init_file.gui", str(tcl))
 
-
 def add_sources(lib, patterns) -> None:
     for pattern in patterns:
         p = join(str(d.parent), pattern)
@@ -71,7 +70,6 @@ def add_sources(lib, patterns) -> None:
             if f != "tb_wrappers.vhd":
                 lib.add_source_file(str(f))
 
-
 def add_common_sources(lib, ui) -> None:
     add_sources(lib, ['../src/**/*.vhd'])
     ui.enable_check_preprocessing()
@@ -95,8 +93,17 @@ def add_flags(ui, lib, build) -> None:
 
     #lib.add_compile_option("ghdl.flags", ["-Wc,-g"])
     lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage", "-fpsl"])
-    ui.set_sim_option("ghdl.elab_flags", ["-Wl,-lgcov", "-Wl,--coverage", "-Wl,-no-pie", "-fpsl"])
-    ui.set_sim_option("ghdl.sim_flags", ["--ieee-asserts=disable-at-0"])
+
+    elab_flags = ["-Wl,-lgcov"]
+    elab_flags.append("-Wl,--coverage");
+    elab_flags.append("-Wl,-no-pie");
+    elab_flags.append("-fpsl");
+    ui.set_sim_option("ghdl.elab_flags",elab_flags)
+
+    # Global simulation flags
+    sim_flags = ["--ieee-asserts=disable-at-0"]
+    ui.set_sim_option("ghdl.sim_flags", sim_flags)
+
     modelsim_init_files = get_common_modelsim_init_files()
     ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
 

test/testfw/test_feature.py

@@ -26,6 +26,11 @@ class FeatureTests(TestsBase):
         tb = self.lib.get_test_benches('*tb_feature')[0]
         tb.scan_tests_from_file(str(wrname))
 
+    def create_psl_cov_file_opt(self, name):
+        psl_path = "functional_coverage/coverage_data/psl_cov_feature_{}.json".format(name)
+        psl_flag = "--psl-report={}".format(psl_path)
+        return {"ghdl.sim_flags" : [psl_flag]}
+
     def configure(self) -> bool:
         tb = self.lib.get_test_benches('*tb_feature')[0]
         default = self.config['default']
@@ -59,7 +64,13 @@ class FeatureTests(TestsBase):
                 'test_name'    : name,
                 'seed'         : get_seed(cfg)
             }
-            tb.add_config(name, generics=generics)
+
+            if (cfg['psl_coverage']):
+                psl_opts = self.create_psl_cov_file_opt(name)
+                tb.add_config(name, generics=generics, sim_options=psl_opts)
+            else:
+                tb.add_config(name, generics=generics)	
+
         return self._check_for_unconfigured()
 
     def _check_for_unconfigured(self) -> bool:

test/testfw/test_reference.py

@@ -18,6 +18,11 @@ class ReferenceTests(TestsBase):
         sources.append('reference/vunit_reference_wrapper.vhd')
         add_sources(self.lib, sources)
 
+    def create_psl_cov_file_opt(self, name):
+        psl_path = "functional_coverage/coverage_data/psl_cov_reference_{}.json".format(name)
+        psl_flag = "--psl-report={}".format(psl_path)
+        return {"ghdl.sim_flags" : [psl_flag]}
+
     def configure(self) -> bool:
         tb = self.lib.get_test_benches('*reference*')[0]
         default = self.config['default']
@@ -34,8 +39,13 @@ class ReferenceTests(TestsBase):
                 'seed'         : get_seed(cfg),
                 'data_path'    : str(self.build) + '/../' + cfg['data_path'],
             }
-            tb.add_config(data_set, generics=generics)
 
+            if (cfg['psl_coverage']):
+                psl_opts = self.create_psl_cov_file_opt(data_set)
+                tb.add_config(data_set, generics=generics, sim_options=psl_opts)
+            else:
+                tb.add_config(data_set, generics=generics)	
+            
             tcl = self.build / 'modelsim_init_{}.tcl'.format(data_set)
             with tcl.open('wt', encoding='utf-8') as f:
                 print(dedent('''\

test/testfw/test_sanity.py

@@ -42,6 +42,18 @@ class SanityTests(TestsBase):
     def add_sources(self):
         add_sources(self.lib, ['sanity/**/*.vhd'])
 
+    def format_valid_test_name(self, name):
+        valid_name = name.replace(" ", '_')
+        valid_name = valid_name.replace("/", "_")
+        valid_name = valid_name.replace("", "_")
+        return valid_name
+
+    def create_psl_cov_file_opt(self, name):
+        test_name = self.format_valid_test_name("psl_cov_sanity_{}.json".format(name))
+        psl_path = "functional_coverage/coverage_data/{}".format(test_name)
+        psl_flag = "--psl-report={}".format(psl_path)
+        return {"ghdl.sim_flags" : [psl_flag]}
+
     def configure(self):
         # TODO: wave
         tb = self.lib.get_test_benches('*tb_sanity')[0]
@@ -72,5 +84,13 @@ class SanityTests(TestsBase):
                 'timing_config': vhdl_serialize(cfg['timing_config']),
                 'gauss_iter'   : vhdl_serialize(cfg['gauss_iter']),
             }
-            tb.add_config(name, generics=generics)
-        return True
+
+            sanity_cfg_name = name.replace(" ", "_").replace("/", "_").strip('"')
+
+            if (cfg['psl_coverage']):
+                psl_opts = self.create_psl_cov_file_opt(name)
+                tb.add_config(name, generics=generics, sim_options=psl_opts)
+            else:
+                tb.add_config(name, generics=generics)	
+
+            return True

test/testfw/test_unit.py

@@ -13,6 +13,11 @@ class UnitTests(TestsBase):
         add_sources(self.lib, ['unit/**/*.vhd'])
         self._create_wrapper(self.build / "tb_wrappers.vhd")
 
+    def add_psl_cov_file(self, tb, name):
+        psl_path = "functional_coverage/coverage_data/psl_cov_unit_{}.json".format(name)
+        psl_flag = "--psl-report={}".format(psl_path)
+        tb.set_sim_option("ghdl.sim_flags", [psl_flag])
+
     def configure(self) -> bool:
         lib, config, build = self.lib, self.config, self.build
         default = config['default']
@@ -43,6 +48,8 @@ class UnitTests(TestsBase):
             init_files = get_common_modelsim_init_files()
             init_files += [str(tcl)]
             tb.set_sim_option("modelsim.init_files.after_load", init_files)
+            if (cfg['psl_coverage']):
+                self.add_psl_cov_file(tb, name)
             self.add_modelsim_gui_file(tb, cfg, name)
         return self._check_for_unconfigured()
 

test/tests_debug.yml

@@ -11,3 +11,4 @@ feature:
         wave: feature/feature_env_setup.tcl
     tests:
         txt_buffer_hazard:
+            iterations: 1

test/tests_fast.yml

 _default: &default
     log_level: info
+    psl_coverage: true
     error_tolerance: 0
     # seed: 0 # optional; use to reconstruct results from randomized runs
     # randomize: false
@@ -72,7 +73,7 @@ feature:
             iterations: 1
         timestamp_registers:
         txt_buffer_hazard:
-            iterations: 5
+            iterations: 20
 reference:
     default:
         <<: *default

test/tests_nightly.yml

 _default: &default
     log_level: info
+    psl_coverage: true
     error_tolerance: 0
     randomize: true
 unit: