Commit b5146d9a authored by Ille, Ondrej, Ing.'s avatar Ille, Ondrej, Ing.

Merge branch '200-bring-up-ghdl-functional-coverage' into 'master'

Resolve "Bring-up GHDL functional coverage."

Closes #200

See merge request !213
parents 60acecf8 4feef64e
Pipeline #6201 passed with stages
in 14 minutes and 47 seconds
......@@ -82,6 +82,7 @@ test_ip_fast:
- cd test
- make test_fast
- make coverage
- make functional_coverage
coverage: "/lines......: ([^%]+%)/"
artifacts:
when: always
......@@ -89,6 +90,7 @@ test_ip_fast:
- test/code_html
- test/tests_fast.xml
- test/xunit.xsl
- test/build/functional_coverage
reports:
junit: [test/tests_fast.xml]
......@@ -118,6 +120,7 @@ pages:
- cp test/tests_*.xml public/
- mv test/code_html public/coverage || true
- cp doc/core/Progdokum.pdf public/ || true
- cp -R test/build/functional_coverage/* public/functional_coverage/ || true
only:
- master
artifacts:
......
......@@ -3,6 +3,8 @@
[![pipeline status](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/pipeline.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/tests_fast.xml)
[![coverage report](https://gitlab.fel.cvut.cz/canbus/ctucanfd_ip_core/badges/master/coverage.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/coverage/)
[![documentation](https://img.shields.io/badge/documentation--blue.svg)]( http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/Progdokum.pdf)
[![functional coverage](https://img.shields.io/badge/functional%20coverage--orange.svg)](http://canbus.pages.fel.cvut.cz/ctucanfd_ip_core/functional_coverage/functional_coverage_report.html)
CAN FD IP Core written in VHDL, originally developed at Czech Technical University -- Faculty of Electrical Engineering -- Department of Measurement.
......
Subproject commit 24f6fe411be6ed81ce4ab27d2dfeadaca425d999
Subproject commit 3baadf8b367cb06e83ceaa58cc088d93949b5175
......@@ -253,4 +253,30 @@ begin
txt_buf_ready => txt_buf_ready
);
----------------------------------------------------------------------------
----------------------------------------------------------------------------
-- Functional coverage
----------------------------------------------------------------------------
----------------------------------------------------------------------------
func_cov_block : block
begin
-- psl default clock is rising_edge(clk_sys);
-- Each SW command active
-- psl txtb_set_ready_cov : cover (txt_sw_cmd.set_rdy = '1' and sw_cbs = '1');
-- psl txtb_set_empty_cov : cover (txt_sw_cmd.set_ety = '1' and sw_cbs = '1');
-- psl txtb_set_abort_cov : cover (txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
-- HW Commands
-- psl txtb_hw_lock : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1');
-- psl txtb_hw_unlock : cover (txt_hw_cmd.unlock = '1' and hw_cbs = '1');
-- psl txtb_hw_valid : cover (txt_hw_cmd.valid = '1' and hw_cbs = '1');
-- psl txtb_hw_err : cover (txt_hw_cmd.err = '1' and hw_cbs = '1');
-- psl txtb_hw_arbl : cover (txt_hw_cmd.arbl = '1' and hw_cbs = '1');
-- psl txtb_hw_failed : cover (txt_hw_cmd.failed = '1' and hw_cbs = '1');
end block;
end architecture;
......@@ -357,4 +357,31 @@ begin
end if;
end process;
----------------------------------------------------------------------------
----------------------------------------------------------------------------
-- Functional coverage
----------------------------------------------------------------------------
----------------------------------------------------------------------------
func_cov_block : block
begin
-- psl default clock is rising_edge(clk_sys);
-- Each FSM state
-- psl txtb_fsm_empty : cover (buf_fsm = txt_empty);
-- psl txtb_fsm_ready : cover (buf_fsm = txt_ready);
-- psl txtb_fsm_tx_prog : cover (buf_fsm = txt_tx_prog);
-- psl txtb_fsm_ab_prog : cover (buf_fsm = txt_ab_prog);
-- psl txtb_fsm_error : cover (buf_fsm = txt_error);
-- psl txtb_fsm_aborted : cover (buf_fsm = txt_aborted);
-- psl txtb_fsm_tx_ok : cover (buf_fsm = txt_ok);
-- Simultaneous HW and SW Commands
-- psl txtb_rdy_hazard : cover (txt_hw_cmd.lock = '1' and hw_cbs = '1' and
-- txt_sw_cmd.set_abt = '1' and sw_cbs = '1');
end block;
end architecture;
PYTHON := python3
XUNIT ?= 0
BUILD_DIR = build
TESTFW_DIR = testfw
TEST_FLAGS = -p`nproc`
TEST_OPTS_test_debug := --no-strict
all: test coverage
all: test coverage functional_coverage
elaborate:
$(PYTHON) run.py test tests_fast.yml -- --elaborate $(TEST_FLAGS)
......@@ -13,9 +15,12 @@ test_%: tests_%.yml FORCE
$(PYTHON) run.py test $(TEST_OPTS_$@) $< -- $(TEST_FLAGS)
coverage:
lcov --capture --directory build --output-file code_coverage.info
lcov --capture --directory $(BUILD_DIR) --output-file code_coverage.info
genhtml code_coverage.info --output-directory code_html
functional_coverage:
$(PYTHON) $(TESTFW_DIR)/test_parse_psl_coverage.py
clean:
-rm -Rf build
......
......@@ -10,6 +10,7 @@ from os.path import abspath
from .log import MyLogRecord
d = Path(abspath(__file__)).parent
func_cov_dir = os.path.join(str(d.parent), "build/functional_coverage")
def setup_logging() -> None:
......@@ -125,11 +126,14 @@ def test(obj, config, strict, vunit_args):
if cfg_key in config:
tests.append(factory(ui, lib, config[cfg_key], build, base))
if not os.path.exists(func_cov_dir):
os.makedirs(func_cov_dir);
os.makedirs(os.path.join(func_cov_dir, "html"))
os.makedirs(os.path.join(func_cov_dir, "coverage_data"))
for t in tests:
t.add_sources()
add_flags(ui, lib, build)
conf_ok = [t.configure() for t in tests]
# check for unknown tests
......
......@@ -62,7 +62,6 @@ class TestsBase:
'''.format(name)), file=f)
tb.set_sim_option("modelsim.init_file.gui", str(tcl))
def add_sources(lib, patterns) -> None:
for pattern in patterns:
p = join(str(d.parent), pattern)
......@@ -71,7 +70,6 @@ def add_sources(lib, patterns) -> None:
if f != "tb_wrappers.vhd":
lib.add_source_file(str(f))
def add_common_sources(lib, ui) -> None:
add_sources(lib, ['../src/**/*.vhd'])
ui.enable_check_preprocessing()
......@@ -95,8 +93,17 @@ def add_flags(ui, lib, build) -> None:
#lib.add_compile_option("ghdl.flags", ["-Wc,-g"])
lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage", "-fpsl"])
ui.set_sim_option("ghdl.elab_flags", ["-Wl,-lgcov", "-Wl,--coverage", "-Wl,-no-pie", "-fpsl"])
ui.set_sim_option("ghdl.sim_flags", ["--ieee-asserts=disable-at-0"])
elab_flags = ["-Wl,-lgcov"]
elab_flags.append("-Wl,--coverage");
elab_flags.append("-Wl,-no-pie");
elab_flags.append("-fpsl");
ui.set_sim_option("ghdl.elab_flags",elab_flags)
# Global simulation flags
sim_flags = ["--ieee-asserts=disable-at-0"]
ui.set_sim_option("ghdl.sim_flags", sim_flags)
modelsim_init_files = get_common_modelsim_init_files()
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
......
......@@ -26,6 +26,11 @@ class FeatureTests(TestsBase):
tb = self.lib.get_test_benches('*tb_feature')[0]
tb.scan_tests_from_file(str(wrname))
def create_psl_cov_file_opt(self, name):
psl_path = "functional_coverage/coverage_data/psl_cov_feature_{}.json".format(name)
psl_flag = "--psl-report={}".format(psl_path)
return {"ghdl.sim_flags" : [psl_flag]}
def configure(self) -> bool:
tb = self.lib.get_test_benches('*tb_feature')[0]
default = self.config['default']
......@@ -59,7 +64,13 @@ class FeatureTests(TestsBase):
'test_name' : name,
'seed' : get_seed(cfg)
}
tb.add_config(name, generics=generics)
if (cfg['psl_coverage']):
psl_opts = self.create_psl_cov_file_opt(name)
tb.add_config(name, generics=generics, sim_options=psl_opts)
else:
tb.add_config(name, generics=generics)
return self._check_for_unconfigured()
def _check_for_unconfigured(self) -> bool:
......
import os
import sys
from json2html import *
import random
import logging
from os.path import join, abspath
from pathlib import Path
import json
from yattag import Doc
test_dir = Path(Path(abspath(__file__)).parent).parent
build_dir = os.path.join(str(test_dir.absolute()), "build")
func_cov_dir = os.path.join(str(build_dir), "functional_coverage")
psl_dir = os.path.join(str(func_cov_dir), "coverage_data")
html_dir = os.path.join(str(func_cov_dir), "html")
dut_top = " "
log = logging.getLogger(__name__)
def merge_psl_coverage_files(out_file, in_file_prefix):
"""
Merge PSL coverage details from multiple files to single file
"""
if(out_file.startswith(in_file_prefix)):
log.error("File name for merging should not have the same prefix as merged files")
system.exit(0)
json_out_path = os.path.join(func_cov_dir, out_file)
json_out_file = open(json_out_path, 'w')
json_out_list = []
for filename in os.listdir(psl_dir):
if (not (filename.startswith(in_file_prefix) and \
filename.endswith(".json"))):
continue
in_filename = os.path.join(psl_dir, filename)
print("Merging JSON PSL coverage from: {}\n".format(in_filename))
json_in_file = open(in_filename, 'r')
json_obj = json.load(json_in_file)
# Add test name to each PSL point
for psl_point in json_obj["details"]:
psl_point["test"] = filename.strip(in_file_prefix).replace(".json","")
json_out_list.extend(json_obj["details"])
json_str = json.dumps(json_out_list, indent=1)
json_out_file.write(json_str)
json_out_file.close()
def collapse_psl_coverage_files(non_collapsed):
"""
Collapses PSL coverage which is output from multiple testcase/testbench
runs into single psl_coverage output.
If DUT is instantiated in multiple testbenches, above levels of
hierarchy from "dut_top" will be ignored and these files will be collapsed.
E.g. if "dut_top" = "can_top_level",
then multiple instances of CTU CAN FD will not generate multiple PSL outputs.
Collapsing policy is following:
- cover - If at least one of collapsed points is covered -> COVERED
- assert - If at least one of collapsed points is failed -> FAILED
Each cover point which is covered has also appended a testcase name where
it was covered.
"""
log.info("Collapsing PSL points with common hierarchy below: {}".format(dut_top))
collapsed = []
# We do stupid quadratic sort because we don't really care if it is gonna last 10
# or 40 seconds... If we ever get to the point that this takes too long we know
# that we have reeealy lot of PSL points and we turned into Semiconductor monster!
for psl_in in non_collapsed:
found = False
for psl_out in collapsed:
# Check if name in output list is equal to searched name from "dut_top"
# entity down. Skip if not
in_name = psl_in["name"].split(dut_top)[-1]
out_name = psl_out["name"].split(dut_top)[-1]
if (out_name != in_name):
continue
if (not ("colapsed_points" in psl_out)):
psl_out["colapsed_name"] = str(dut_top + in_name)
psl_out["colapsed_points"] = []
psl_out["colapsed_points"].append(psl_in)
# If any of colapsed points is covered -> whole point is covered
if (psl_in["status"] == "covered"):
psl_out["status"] = "covered"
psl_out["count"] += psl_in["count"]
# If any of colapsed points is failed -> whole point is failed
if (psl_in["status"] == "failed"):
psl_out["status"] = "failed"
# Assertion hits add up for both failed and passed
if (psl_out["directive"] == "assertion"):
psl_out["count"] += psl_in["count"]
found = True
break;
# Input point was not collapsed into any of output points -> Add directly
if (not found):
collapsed.append(psl_in)
return collapsed
def get_collapsed_file_name(psl_point):
"""
Create unique file name for collapsed PSL points
"""
file_name = dut_top + psl_point["name"].split(dut_top)[-1]
file_name = file_name.replace(".","_")
file_name = file_name.replace(" ","_")
file_name = file_name.replace(")","_")
file_name = file_name.replace("(","_")
file_name = file_name.replace("@","_")
file_name = file_name + "_" + str(psl_point["line"])
return file_name
def load_json_psl_coverage(filename):
"""
Load PSL Coverage JSON file to JSON object.
"""
psl_cov_path = os.path.join(func_cov_dir, filename)
# Read JSON string from file
log.info("Loading JSON PSL output: {}".format(psl_cov_path))
json_file = open(psl_cov_path, 'r')
return json.load(json_file)
def split_json_coverage_by_file(json):
"""
Parse input PSL Coverage JSON file. Group PSL endpoints by file.
Return dictionary in format:
{filename : psl_points} where psl_points is a list of PSL points in
filename.
"""
file_dict = {}
for psl_point in json:
# Create new list if first PSL of a file is parsed
if (not(psl_point["file"] in file_dict)):
file_dict[psl_point["file"]] = []
file_dict[psl_point["file"]].append(psl_point)
return file_dict
def add_html_table_header(doc, tag, text, headers, back_color="White"):
"""
Add header to HTML table.
"""
with tag('tr'):
for header in headers:
with tag('th', bgcolor=back_color):
text(header)
def calc_coverage_results(psl_points, psl_type):
"""
Calculate coverage results from list of PSL points in JSON format.
"""
ok = 0
nok = 0
for psl_point in psl_points:
if (psl_point["directive"] != psl_type):
continue;
if (psl_point["status"] == "passed" or
psl_point["status"] == "covered"):
ok += 1
else:
nok +=1
return [ok, nok]
def calc_coverage_color(coverage):
"""
Return color based on coverage result.
"""
if (coverage < 0 or coverage > 100):
log("Invalid coverage input should be between 0 - 100 %")
return
if (coverage > 90):
return "Lime"
elif (coverage > 80):
return "Orange"
elif (coverage > 70):
return "OrangeRed"
else:
return "Red"
def print_cov_cell_percentage(doc, tag, text, psl_points, coverage_type, merge_abs_vals):
"""
"""
[ok, nok] = calc_coverage_results(psl_points, coverage_type)
summ = max(1, ok + nok)
percents = ok/summ * 100
color = calc_coverage_color(percents)
if (merge_abs_vals):
if (ok + nok > 0):
with tag('td', bgcolor=color):
text("({}/{}) {}%".format(ok, summ, percents))
else:
with tag('td', bgcolor="Silver"):
text("NA")
else:
with tag('td'):
text(ok)
with tag('td'):
text(nok)
if (ok + nok > 0):
with tag('td', bgcolor=color):
text("{}%".format(percents))
else:
with tag('td', bgcolor="Silver"):
text("NA")
def add_psl_html_header(doc, tag, text, filename, psl_points):
"""
Create HTML page header with info about coverage data within list of
PSL points in JSON format.
"""
with tag('table', width='100%', border=0, cellspacing=0, cellpadding=0):
with tag('tr'):
with tag('th', ('class','title')):
with tag('font', size=10):
text("GHDL PSL Functional coverage report")
with tag('table', width='100%', border="1px solid black"):
headers = ["Filename"]
headers.append("Covered")
headers.append("Not-Covered")
headers.append("Functional coverage")
headers.append("Passed")
headers.append("Failed")
headers.append("Assertions passed")
add_html_table_header(doc, tag, text, headers, back_color="Aquamarine")
with tag('td'):
text(filename)
# Calculate results for each type
coverage_types = ["cover", "assertion"]
for coverage_type in coverage_types:
print_cov_cell_percentage(doc, tag, text, psl_points, \
coverage_type, merge_abs_vals=False)
def add_non_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White"):
"""
Add HTML table entry for non-collapsed PSL functional coverage point.
"""
with tag('td'):
text(psl_point["name"].split(".")[-1])
with tag('td'):
text(psl_point["test"])
with tag('td', width="50%", style="word-break:break-all;"):
text(dut_top + psl_point["name"])
with tag('td'):
text(psl_point["line"])
with tag('td'):
text(psl_point["count"])
if (psl_point["status"] == "covered" or \
psl_point["status"] == "passed"):
color = "Lime"
else:
color = "red"
with tag('td', ('bgcolor',color)):
text(psl_point["status"])
def add_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White"):
"""
Add HTML table entry for collapsed PSL functional coverage point. Adds
llink reference to collapsed entries on separate site.
"""
with tag('td'):
text(psl_point["name"].split(".")[-1])
with tag('td'):
file_name = get_collapsed_file_name(psl_point)
with tag('a', href=file_name+".html"):
text("Open collapsed tests")
with tag('td'):
text(dut_top + psl_point["name"].split(dut_top)[-1])
with tag('td'):
text(psl_point["line"])
with tag('td'):
text(psl_point["count"])
if (psl_point["status"] == "covered" or \
psl_point["status"] == "passed"):
color = "Lime"
else:
color = "red"
with tag('td', ('bgcolor',color)):
text(psl_point["status"])
def add_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White"):
"""
Add PSL point in JSON format to HTML table. For collapsed entries,
overall result is shown and link to collapsed points is inserted.
"""
# Add default entry (single or collapsed)
with tag('tr', ('bgcolor',def_bg_color)):
if ("colapsed_points" in psl_point):
add_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White")
else:
add_non_colapsed_psl_table_entry(doc, tag, text, psl_point, def_bg_color="White")
# Create separate page with collapsed PSL points for this PSL statement
# Add unique filename
if ("colapsed_points" in psl_point):
file_name = os.path.join(html_dir, get_collapsed_file_name(psl_point))
create_psl_file_page(file_name, psl_point["colapsed_points"]);
def create_psl_file_page(filename, psl_points):
"""
Create HTML file with list of PSL coverage statements.
"""
parsed_file_name = os.path.basename(filename)
html_cov_path = os.path.join(html_dir,
"{}.html".format(parsed_file_name))
html_file = open(html_cov_path, 'w')
doc, tag, text = Doc().tagtext()
# Add Common header
add_psl_html_header(doc, tag, text, parsed_file_name, psl_points)
# Add "Cover" and "Assertion" points
psl_types = [{"name" : "Cover Points" , "type" : "cover"}, \
{"name" : "Assertions" , "type" : "assertion"}]
for psl_type in psl_types:
with tag('p'):
with tag('table', width='100%', border="1px solid black"):
with tag('caption'):
with tag('font', size=5):
text(psl_type["name"])
titles = ["PSL Point Name", "Test name", "Full Path Name", "Line", "Count", "Status"]
add_html_table_header(doc, tag, text, titles, back_color="Peru")
for psl_point in psl_points:
if (psl_point["directive"] == psl_type["type"]):
add_psl_table_entry(doc, tag, text, psl_point)
html_file.write(doc.getvalue())
html_file.close()
def create_psl_file_refs_table(doc, tag, text, psl_by_files):
"""
Create entries to HTML table for each file. Calculates
coverage summary for each file. Adds Reference to files.
"""
for file_name, psl_list in psl_by_files.items():
with tag('tr'):
with tag('td'):
name = os.path.basename(file_name)
with tag('a', href= os.path.join("html", name + ".html")):
text(name)
coverage_types = ["cover", "assertion"]
for coverage_type in coverage_types:
print_cov_cell_percentage(doc, tag, text, psl_list, \
coverage_type, merge_abs_vals=True)
def create_psl_report(psl_by_files, psl_orig):
"""
Generates PSL report. Each list within psl_by_files has separate
HTML page. Summary page is created from psl_orig
"""
# Create HTML page for each source file
for file_name, psl_list in psl_by_files.items():
create_psl_file_page(file_name, psl_list)
html_rep_path = os.path.join(func_cov_dir, "functional_coverage_report.html")
html_file = open(html_rep_path, 'w')
doc, tag, text = Doc().tagtext()
# Add Common Header
add_psl_html_header(doc, tag, text, "TOP LEVEL", psl_orig)
with tag('p'):
with tag('table', width="100%", border="1px solid black"):
header = ["File name", "Coverage", "Asserts"]
add_html_table_header(doc, tag, text, header, back_color="Peru")
create_psl_file_refs_table(doc, tag, text, psl_by_files)