Merge branch '200-bring-up-ghdl-functional-coverage' into 'master'

Resolve "Bring-up GHDL functional coverage."

Closes #200

See merge request !213

.gitlab-ci.yml

@@ -82,6 +82,7 @@ test_ip_fast:
         - cd test
         - make test_fast
         - make coverage
+        - make functional_coverage
     coverage: "/lines......: ([^%]+%)/"
     artifacts:
         when: always
@@ -89,6 +90,7 @@ test_ip_fast:
             - test/code_html
             - test/tests_fast.xml
             - test/xunit.xsl
+            - test/build/functional_coverage
         reports:
             junit: [test/tests_fast.xml]
 
@@ -118,6 +120,7 @@ pages:
         - cp test/tests_*.xml public/
         - mv test/code_html public/coverage || true
         - cp doc/core/Progdokum.pdf public/ || true
+        - cp -R test/build/functional_coverage/* public/functional_coverage/ || true
     only:
         - master
     artifacts:

README.md

@@ -3,6 +3,8 @@
 [![pipeline status](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/pipeline.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/tests_fast.xml)
 [![coverage report](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/coverage.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/coverage/)
 [![documentation](https://img.shields.io/badge/documentation--blue.svg)]( http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/Progdokum.pdf)
+[![functional coverage](https://img.shields.io/badge/functional%20coverage--orange.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/functional_coverage/functional_coverage_report.html)
+
 
 CAN FD IP Core written in VHDL, originally developed at Czech Technical University -- Faculty of Electrical Engineering -- Department of Measurement.
 

pyXact_generator @ 3baadf8b

-Subproject commit 24f6fe411be6ed81ce4ab27d2dfeadaca425d999
+Subproject commit 3baadf8b367cb06e83ceaa58cc088d93949b5175

src/txt_buffer/txt_buffer.vhd

@@ -253,4 +253,30 @@ begin
         txt_buf_ready          => txt_buf_ready
     );
 
+    
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    -- Functional coverage
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    func_cov_block : block
+    begin 
+    
+    -- psl default clock is rising_edge(clk_sys);
+
+    -- Each SW command active
+    -- psl txtb_set_ready_cov : cover (txt_sw_cmd.set_rdy = '1' and sw_cbs = '1');
+    -- psl txtb_set_empty_cov : cover (txt_sw_cmd.set_ety = '1' and sw_cbs = '1');
+    -- psl txtb_set_abort_cov : cover (txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
+      
+    -- HW Commands
+    -- psl txtb_hw_lock : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1');
+    -- psl txtb_hw_unlock : cover (txt_hw_cmd.unlock = '1' and hw_cbs = '1');
+    -- psl txtb_hw_valid : cover (txt_hw_cmd.valid = '1' and hw_cbs = '1');
+    -- psl txtb_hw_err : cover (txt_hw_cmd.err = '1' and hw_cbs = '1');
+    -- psl txtb_hw_arbl : cover (txt_hw_cmd.arbl = '1' and hw_cbs = '1');
+    -- psl txtb_hw_failed : cover (txt_hw_cmd.failed = '1' and hw_cbs = '1');
+    
+    end block;
+
 end architecture;

src/txt_buffer/txt_buffer_fsm.vhd

@@ -357,4 +357,31 @@ begin
         end if;
     end process;
 
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    -- Functional coverage
+    ----------------------------------------------------------------------------
+    ----------------------------------------------------------------------------
+    func_cov_block : block
+    begin 
+    
+    -- psl default clock is rising_edge(clk_sys);
+
+    -- Each FSM state
+    -- psl txtb_fsm_empty : cover (buf_fsm = txt_empty);
+    -- psl txtb_fsm_ready : cover (buf_fsm = txt_ready);
+    -- psl txtb_fsm_tx_prog : cover (buf_fsm = txt_tx_prog);
+    -- psl txtb_fsm_ab_prog : cover (buf_fsm = txt_ab_prog);
+    -- psl txtb_fsm_error : cover (buf_fsm = txt_error);
+    -- psl txtb_fsm_aborted : cover (buf_fsm = txt_aborted);
+    -- psl txtb_fsm_tx_ok : cover (buf_fsm = txt_ok);
+    
+    -- Simultaneous HW and SW Commands
+    -- psl txtb_rdy_hazard : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1' and
+    --                              txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
+
+    
+    
+    end block;
+
 end architecture;

test/Makefile

 PYTHON := python3
 XUNIT ?= 0
+BUILD_DIR = build
+TESTFW_DIR = testfw
 
 TEST_FLAGS = -p`nproc`
 TEST_OPTS_test_debug := --no-strict
 
-all: test coverage
+all: test coverage functional_coverage
 
 elaborate:
 	$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
@@ -13,9 +15,12 @@ test_%: tests_%.yml FORCE
 	$(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
 
 coverage:
-	lcov --capture --directory build --output-file code_coverage.info
+	lcov --capture --directory $(BUILD_DIR) --output-file code_coverage.info
 	genhtml code_coverage.info --output-directory code_html
 
+functional_coverage:
+	$(PYTHON) $(TESTFW_DIR)/test_parse_psl_coverage.py
+
 clean:
 	-rm -Rf build
 

test/testfw/__init__.py

@@ -10,6 +10,7 @@ from os.path import abspath
 from .log import MyLogRecord
 
 d = Path(abspath(__file__)).parent
+func_cov_dir = os.path.join(str(d.parent), "build/functional_coverage")
 
 
 def setup_logging() -> None:
@@ -125,11 +126,14 @@ def test(obj, config, strict, vunit_args):
         if cfg_key in config:
             tests.append(factory(ui, lib, config[cfg_key], build, base))
 
+    if not os.path.exists(func_cov_dir):
+        os.makedirs(func_cov_dir);
+        os.makedirs(os.path.join(func_cov_dir, "html"))
+        os.makedirs(os.path.join(func_cov_dir, "coverage_data"))
+
     for t in tests:
         t.add_sources()
-
     add_flags(ui, lib, build)
-
     conf_ok = [t.configure() for t in tests]
 
     # check for unknown tests

test/testfw/test_common.py

@@ -62,7 +62,6 @@ class TestsBase:
                     '''.format(name)), file=f)
         tb.set_sim_option("modelsim.init_file.gui", str(tcl))
 
-
 def add_sources(lib, patterns) -> None:
     for pattern in patterns:
         p = join(str(d.parent), pattern)
@@ -71,7 +70,6 @@ def add_sources(lib, patterns) -> None:
             if f != "tb_wrappers.vhd":
                 lib.add_source_file(str(f))
 
-
 def add_common_sources(lib, ui) -> None:
     add_sources(lib, ['../src/**/*.vhd'])
     ui.enable_check_preprocessing()
@@ -95,8 +93,17 @@ def add_flags(ui, lib, build) -> None:
 
     #lib.add_compile_option("ghdl.flags", ["-Wc,-g"])
     lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage", "-fpsl"])
-    ui.set_sim_option("ghdl.elab_flags", ["-Wl,-lgcov", "-Wl,--coverage", "-Wl,-no-pie", "-fpsl"])
-    ui.set_sim_option("ghdl.sim_flags", ["--ieee-asserts=disable-at-0"])
+
+    elab_flags = ["-Wl,-lgcov"]
+    elab_flags.append("-Wl,--coverage");
+    elab_flags.append("-Wl,-no-pie");
+    elab_flags.append("-fpsl");
+    ui.set_sim_option("ghdl.elab_flags",elab_flags)
+
+    # Global simulation flags
+    sim_flags = ["--ieee-asserts=disable-at-0"]
+    ui.set_sim_option("ghdl.sim_flags", sim_flags)
+
     modelsim_init_files = get_common_modelsim_init_files()
     ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
 

test/testfw/test_feature.py

@@ -26,6 +26,11 @@ class FeatureTests(TestsBase):
         tb = self.lib.get_test_benches('*tb_feature')[0]
         tb.scan_tests_from_file(str(wrname))
 
+    def create_psl_cov_file_opt(self, name):
+        psl_path = "functional_coverage/coverage_data/psl_cov_feature_{}.json".format(name)
+        psl_flag = "--psl-report={}".format(psl_path)
+        return {"ghdl.sim_flags" : [psl_flag]}
+
     def configure(self) -> bool:
         tb = self.lib.get_test_benches('*tb_feature')[0]
         default = self.config['default']
@@ -59,7 +64,13 @@ class FeatureTests(TestsBase):
                 'test_name'    : name,
                 'seed'         : get_seed(cfg)
             }
-            tb.add_config(name, generics=generics)
+
+            if (cfg['psl_coverage']):
+                psl_opts = self.create_psl_cov_file_opt(name)
+                tb.add_config(name, generics=generics, sim_options=psl_opts)
+            else:
+                tb.add_config(name, generics=generics)	
+
         return self._check_for_unconfigured()
 
     def _check_for_unconfigured(self) -> bool:

test/testfw/test_parse_psl_coverage.py

+import os
+import sys
+from json2html import *
+import random
+import logging
+from os.path import join, abspath
+from pathlib import Path
+import json
+from yattag import Doc
+
+test_dir = Path(Path(abspath(__file__)).parent).parent
+build_dir = os.path.join(str(test_dir.absolute()), "build")
+func_cov_dir = os.path.join(str(build_dir), "functional_coverage")
+psl_dir = os.path.join(str(func_cov_dir), "coverage_data")
+html_dir = os.path.join(str(func_cov_dir), "html")
+
+dut_top = " "
+
+log = logging.getLogger(__name__)
+
+
+def merge_psl_coverage_files(out_file, in_file_prefix):
+	"""
+	Merge PSL coverage details from multiple files to single file
+	"""
+	if(out_file.startswith(in_file_prefix)):
+		log.error("File name for merging should not have the same prefix as merged files")
+		system.exit(0)
+
+	json_out_path = os.path.join(func_cov_dir, out_file)
+	json_out_file = open(json_out_path, 'w')
+	json_out_list = []
+	for filename in os.listdir(psl_dir):
+		if (not (filename.startswith(in_file_prefix) and \
+		         filename.endswith(".json"))):
+			continue
+
+		in_filename = os.path.join(psl_dir, filename)
+		print("Merging JSON PSL coverage from: {}\n".format(in_filename))
+		json_in_file = open(in_filename, 'r')
+		json_obj = json.load(json_in_file)
+
+		# Add test name to each PSL point
+		for psl_point in json_obj["details"]:
+			psl_point["test"] = filename.strip(in_file_prefix).replace(".json","")
+
+		json_out_list.extend(json_obj["details"])
+
+	json_str = json.dumps(json_out_list, indent=1)
+	json_out_file.write(json_str)
+	json_out_file.close()
+
+
+def collapse_psl_coverage_files(non_collapsed):
+	"""
+	Collapses PSL coverage which is output from multiple testcase/testbench
+	runs into single psl_coverage output.
+	If DUT is instantiated in multiple testbenches, above levels of
+	hierarchy from "dut_top" will be ignored and these files will be collapsed.
+	E.g. if "dut_top" = "can_top_level",
+	then multiple instances of CTU CAN FD will not generate multiple PSL outputs.
+	Collapsing policy is following:
+		- cover - If at least one of collapsed points is covered -> COVERED
+		- assert - If at least one of collapsed points is failed -> FAILED
+	Each cover point which is covered has also appended a testcase name where
+	it was covered.
+	"""
+	log.info("Collapsing PSL points with common hierarchy below: {}".format(dut_top))
+	collapsed = []
+
+	# We do stupid quadratic sort because we don't really care if it is gonna last 10
+	# or 40 seconds... If we ever get to the point that this takes too long we know
+	# that we have reeealy lot of PSL points and we turned into Semiconductor monster!
+	for psl_in in non_collapsed:
+
+		found = False
+		for psl_out in collapsed:
+
+			# Check if name in output list is equal to searched name from "dut_top"
+			# entity down. Skip if not
+			in_name = psl_in["name"].split(dut_top)[-1]
+			out_name = psl_out["name"].split(dut_top)[-1]
+			if (out_name != in_name):
+				continue
+
+			if (not ("colapsed_points" in psl_out)):
+				psl_out["colapsed_name"] = str(dut_top + in_name)
+				psl_out["colapsed_points"] = []
+
+			psl_out["colapsed_points"].append(psl_in)
+
+			# If any of colapsed points is covered -> whole point is covered
+			if (psl_in["status"] == "covered"):
+				psl_out["status"] = "covered"
+				psl_out["count"] += psl_in["count"]
+
+			# If any of colapsed points is failed -> whole point is failed
+			if (psl_in["status"] == "failed"):
+				psl_out["status"] = "failed"
+
+			# Assertion hits add up for both failed and passed
+			if (psl_out["directive"] == "assertion"):
+				psl_out["count"] += psl_in["count"]
+
+			found = True
+			break;
+
+		# Input point was not collapsed into any of output points -> Add directly
+		if (not found):
+			collapsed.append(psl_in)
+
+	return collapsed
+
+
+def get_collapsed_file_name(psl_point):
+	"""
+	Create unique file name for collapsed PSL points
+	"""
+	file_name = dut_top + psl_point["name"].split(dut_top)[-1]
+	file_name = file_name.replace(".","_")
+	file_name = file_name.replace(" ","_")
+	file_name = file_name.replace(")","_")
+	file_name = file_name.replace("(","_")
+	file_name = file_name.replace("@","_")
+	file_name = file_name + "_" + str(psl_point["line"])
+	return file_name
+
+
+def load_json_psl_coverage(filename):
+	"""
+	Load PSL Coverage JSON file to JSON object.
+	"""
+	psl_cov_path = os.path.join(func_cov_dir, filename)
+	
+	# Read JSON string from file
+	log.info("Loading JSON PSL output: {}".format(psl_cov_path))
+	json_file = open(psl_cov_path, 'r')
+	return json.load(json_file)
+
+
+def split_json_coverage_by_file(json):
+	"""
+	Parse input PSL Coverage JSON file. Group PSL endpoints by file.
+	Return dictionary in format:
+		{filename : psl_points} where psl_points is a list of PSL points in
+		filename.
+	"""
+	file_dict = {}
+	for psl_point in json:
+
+		# Create new list if first PSL of a file is parsed
+		if (not(psl_point["file"] in file_dict)):
+			file_dict[psl_point["file"]] = []
+		
+		file_dict[psl_point["file"]].append(psl_point)
+
+	return file_dict
+
+
+def add_html_table_header(doc, tag, text, headers, back_color="White"):
+	"""
+	Add header to HTML table.
+	"""
+	with tag('tr'):
+		for header in headers:
+			with tag('th', bgcolor=back_color):
+				text(header)
+
+
+def calc_coverage_results(psl_points, psl_type):
+	"""
+	Calculate coverage results from list of PSL points in JSON format.
+	"""
+	ok = 0
+	nok = 0
+	for psl_point in psl_points:
+		if (psl_point["directive"] != psl_type):
+			continue;
+		if (psl_point["status"] == "passed" or
+			psl_point["status"] == "covered"):
+			ok += 1
+		else:
+			nok +=1
+	return [ok, nok]
+
+
+def calc_coverage_color(coverage):
+	"""
+	Return color based on coverage result.
+	"""
+	if (coverage < 0 or coverage > 100):
+		log("Invalid coverage input should be between 0 - 100 %")
+		return
+
+	if (coverage > 90):
+		return "Lime"
+	elif (coverage > 80):
+		return "Orange"
+	elif (coverage > 70):
+		return "OrangeRed"
+	else:
+		return "Red"
+
+
+def print_cov_cell_percentage(doc, tag, text, psl_points, coverage_type, merge_abs_vals):
+	"""
+	"""
+	[ok, nok] = calc_coverage_results(psl_points, coverage_type)
+	summ = max(1, ok + nok)
+	percents = ok/summ * 100
+	color = calc_coverage_color(percents)
+
+	if (merge_abs_vals):
+		if (ok + nok > 0):
+			with tag('td', bgcolor=color):
+				text("({}/{}) {}%".format(ok, summ, percents))
+		else:
+			with tag('td', bgcolor="Silver"):
+				text("NA")
+	else:
+		with tag('td'):
+			text(ok)
+		with tag('td'):
+			text(nok)
+
+		if (ok + nok > 0):
+			with tag('td', bgcolor=color):
+				text("{}%".format(percents))
+		else:
+			with tag('td', bgcolor="Silver"):
+				text("NA")	
+				
+
+def add_psl_html_header(doc, tag, text, filename, psl_points):
+	"""
+	Create HTML page header with info about coverage data within list of
+	PSL points in JSON format.
+	"""
+	with tag('table', width='100%', border=0, cellspacing=0, cellpadding=0):
+		with tag('tr'):			
+			with tag('th', ('class','title')):
+				with tag('font', size=10):
+					text("GHDL PSL Functional coverage report")
+			with tag('table', width='100%', border="1px solid black"):
+				headers = ["Filename"]
+				headers.append("Covered")
+				headers.append("Not-Covered")
+				headers.append("Functional coverage")
+				headers.append("Passed")
+				headers.append("Failed")
+				headers.append("Assertions passed")
+				add_html_table_header(doc, tag, text, headers, back_color="Aquamarine")
+
+				with tag('td'):
+					text(filename)
+
+				# Calculate results for each type
+				coverage_types = ["cover", "assertion"]
+				for coverage_type in coverage_types:
+					print_cov_cell_percentage(doc, tag, text, psl_points, \
+						coverage_type, merge_abs_vals=False)
+
+
+def add_non_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White"):
+	"""
+	Add HTML table entry for non-collapsed PSL functional coverage point. 
+
+	"""
+	with tag('td'):
+		text(psl_point["name"].split(".")[-1])		
+	with tag('td'):
+		text(psl_point["test"])
+	with tag('td', width="50%", style="word-break:break-all;"):
+		text(dut_top + psl_point["name"])
+	with tag('td'):
+		text(psl_point["line"])
+	with tag('td'):
+		text(psl_point["count"])
+
+	if (psl_point["status"] == "covered" or \
+		psl_point["status"] == "passed"):
+		color = "Lime"
+	else:
+		color = "red"
+
+	with tag('td', ('bgcolor',color)):
+		text(psl_point["status"])
+
+
+def add_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White"):
+	"""
+	Add HTML table entry for collapsed PSL functional coverage point. Adds
+	llink reference to collapsed entries on separate site.
+	"""
+	with tag('td'):
+		text(psl_point["name"].split(".")[-1])
+	
+	with tag('td'):
+		file_name = get_collapsed_file_name(psl_point)
+		with tag('a', href=file_name+".html"):
+			text("Open collapsed tests")
+	with tag('td'):
+		text(dut_top + psl_point["name"].split(dut_top)[-1])
+	with tag('td'):
+		text(psl_point["line"])
+	with tag('td'):
+		text(psl_point["count"])
+
+	if (psl_point["status"] == "covered" or \
+		psl_point["status"] == "passed"):
+		color = "Lime"
+	else:
+		color = "red"
+
+	with tag('td', ('bgcolor',color)):
+		text(psl_point["status"])
+
+
+def add_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White"):
+	"""
+	Add PSL point in JSON format to HTML table. For collapsed entries,
+	overall result is shown and link to collapsed points is inserted.
+	"""
+	# Add default entry (single or collapsed)
+	with tag('tr', ('bgcolor',def_bg_color)):
+		if ("colapsed_points" in psl_point):
+			add_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White")
+		else:
+			add_non_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White")
+
+	# Create separate page with collapsed PSL points for this PSL statement
+	# Add unique filename
+	if ("colapsed_points" in psl_point):
+		file_name = os.path.join(html_dir, get_collapsed_file_name(psl_point))
+		create_psl_file_page(file_name, psl_point["colapsed_points"]);
+
+
+def create_psl_file_page(filename, psl_points):
+	"""
+	Create HTML file with list of PSL coverage statements.
+	"""
+	parsed_file_name = os.path.basename(filename)
+	html_cov_path = os.path.join(html_dir, 
+						"{}.html".format(parsed_file_name))
+	html_file = open(html_cov_path, 'w')
+
+	doc, tag, text = Doc().tagtext()
+
+	# Add Common header
+	add_psl_html_header(doc, tag, text, parsed_file_name, psl_points)
+
+	# Add "Cover" and "Assertion" points
+	psl_types = [{"name" : "Cover Points" , "type" : "cover"}, \
+				 {"name" : "Assertions" , "type" : "assertion"}]
+	for psl_type in psl_types:
+		with tag('p'):
+			with tag('table', width='100%', border="1px solid black"):
+				with tag('caption'):
+					with tag('font', size=5):
+						text(psl_type["name"])
+				titles = ["PSL Point Name", "Test name", "Full Path Name", "Line", "Count", "Status"]
+				add_html_table_header(doc, tag, text, titles, back_color="Peru")
+				for psl_point in psl_points:
+					if (psl_point["directive"] == psl_type["type"]):
+						add_psl_table_entry(doc, tag, text, psl_point)
+
+	html_file.write(doc.getvalue())
+	html_file.close()
+
+
+def create_psl_file_refs_table(doc, tag, text, psl_by_files):
+	"""
+	Create entries to HTML table for each file. Calculates
+	coverage summary for each file. Adds Reference to files.
+	"""
+	for file_name, psl_list in psl_by_files.items():
+		with tag('tr'):
+			with tag('td'):
+				name = os.path.basename(file_name)
+				with tag('a', href= os.path.join("html", name + ".html")):
+					text(name)
+			coverage_types = ["cover", "assertion"]
+			for coverage_type in coverage_types:
+				print_cov_cell_percentage(doc, tag, text, psl_list, \
+						coverage_type, merge_abs_vals=True)
+
+
+def create_psl_report(psl_by_files, psl_orig):
+	"""
+	Generates PSL report. Each list within psl_by_files has separate
+	HTML page. Summary page is created from psl_orig
+	"""
+	# Create HTML page for each source file
+	for file_name, psl_list in psl_by_files.items():
+		create_psl_file_page(file_name, psl_list)
+
+	html_rep_path = os.path.join(func_cov_dir, "functional_coverage_report.html")
+	html_file = open(html_rep_path, 'w')
+
+	doc, tag, text = Doc().tagtext()
+
+	# Add Common Header
+	add_psl_html_header(doc, tag, text, "TOP LEVEL", psl_orig)
+
+	with tag('p'):
+		with tag('table', width="100%", border="1px solid black"):
+			header = ["File name", "Coverage", "Asserts"]
+			add_html_table_header(doc, tag, text, header, back_color="Peru")
+			create_psl_file_refs_table(doc, tag, text, psl_by_files)
+