Commit 35301574 authored by Martin Jeřábek's avatar Martin Jeřábek

testfw: logging, wave files, test termination

parent 5f7acf5f
......@@ -1527,15 +1527,17 @@ package body CANtestLib is
status <= passed;
wait for 200 ns;
report "Test result: SUCCESS" severity note;
--std.env.stop(0);
else
status <= failed;
wait for 200 ns;
report "Test result: FAILURE" severity error;
--std.env.stop(1);
end if;
-- Finish the test
wait for 200 ns;
report "Test END" severity failure;
--wait for 200 ns;
--report "Test END" severity failure;
end procedure;
......
......@@ -47,9 +47,9 @@ proc add_test_status_waves {} {
global TCOMP
add wave -noupdate -divider -height 20 "Test details"
add wave status
add wave run
add wave errors
#add wave status
#add wave run
#add wave errors
add wave $TCOMP/iterations
add wave $TCOMP/log_level
add wave $TCOMP/error_beh
......
################################################################################
##
## CAN with Flexible Data-Rate IP Core
##
## Copyright (C) 2017 Ondrej Ille <ondrej.ille@gmail.com>
##
## Project advisor: Jiri Novak <jnovak@fel.cvut.cz>
## Department of Measurement (http://meas.fel.cvut.cz/)
## Faculty of Electrical Engineering (http://www.fel.cvut.cz)
## Czech Technical University (http://www.cvut.cz/)
##
## Permission is hereby granted, free of charge, to any person obtaining a copy
## of this VHDL component and associated documentation files (the "Component"),
## to deal in the Component without restriction, including without limitation
## the rights to use, copy, modify, merge, publish, distribute, sublicense,
## and/or sell copies of the Component, and to permit persons to whom the
## Component is furnished to do so, subject to the following conditions:
##
## The above copyright notice and this permission notice shall be included in
## all copies or substantial portions of the Component.
##
## THE COMPONENT IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
## AUTHORS OR COPYRIGHTHOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
## FROM, OUT OF OR IN CONNECTION WITH THE COMPONENT OR THE USE OR OTHER DEALINGS
## IN THE COMPONENT.
##
## The CAN protocol is developed by Robert Bosch GmbH and protected by patents.
## Anybody who wants to implement this IP core on silicon has to obtain a CAN
## protocol license from Bosch.
##
################################################################################
################################################################################
## Description:
################################################################################
##########################################################
#Adds the basic waveforms which are common for all tests
##########################################################
proc add_common_waves {} {
add wave -noupdate -divider -height 20 "Test details"
add wave status
add wave run
add wave errors
add wave test_comp/iterations
add wave test_comp/log_level
add wave test_comp/error_beh
}
import click
import yaml
from glob import glob
from os.path import join, dirname, abspath
from vunit.ui import VUnit
import logging
import logging.config
from pathlib import Path
import click
from glob import glob
import os
import re
import sys
from textwrap import dedent
from pathlib import Path
from pprint import pprint
from .log import MyLogRecord
from . import vunit_ifc
from vunit.ui import VUnit
d = Path(abspath(__file__)).parent
def setup_logging() -> None:
with Path(d / 'logging.yaml').open('rt', encoding='utf-8') as f:
cfg = yaml.load(f)
logging.setLogRecordFactory(MyLogRecord)
logging.config.dictConfig(cfg)
global log
log = logging.getLogger('fw')
#-------------------------------------------------------------------------------
def add_sources(ui):
lib = ui.add_library("lib")
for pattern in ['../src/**/*.vhd', '*.vhd', 'unit/**/*.vhd', 'sanity/*.vhd', 'lib/*.vhd']:
class AttrDict:
def __init__(self, data):
self.data = data
def __getitem__(self, key):
return getattr(self.data, key)
class AliasedGroup(click.Group):
def get_command(self, ctx, cmd_name):
rv = super().get_command(ctx, cmd_name)
if rv is not None:
return rv
matches = [x for x in self.list_commands(ctx)
if x.startswith(cmd_name)]
if not matches:
return None
elif len(matches) == 1:
return click.Group.get_command(self, ctx, matches[0])
ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
#-------------------------------------------------------------------------------
def add_sources(lib, patterns):
for pattern in patterns:
p = join(str(d.parent), pattern)
print(p)
log.debug('Adding sources matching {}'.format(p))
for f in glob(p, recursive=True):
if f != "tb_wrappers.vhd":
lib.add_source_file(str(f))
return lib
def add_common_sources(lib):
return add_sources(lib, ['../src/**/*.vhd', '*.vhd', 'lib/*.vhd'])
def get_common_modelsim_init_files():
modelsim_init_files = '../lib/test_lib.tcl,modelsim_init.tcl'
modelsim_init_files = [str(d/x) for x in modelsim_init_files.split(',')]
return modelsim_init_files
def add_flags(ui, lib, build):
unit_tests = lib.get_test_benches('*_unit_test')
......@@ -32,7 +72,8 @@ def add_flags(ui, lib, build):
lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage"])
ui.set_sim_option("ghdl.elab_flags", ["-Wl,-lgcov", "-Wl,--coverage", "-Wl,-no-pie"])
ui.set_sim_option("ghdl.sim_flags", ["--ieee-asserts=disable-at-0"])
ui.set_sim_option("modelsim.init_files.after_load", [str(d / 'modelsim_init.tcl')])
modelsim_init_files = get_common_modelsim_init_files()
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
def create_wrapper(lib, fname):
fname = str(fname)
......@@ -94,31 +135,45 @@ def create_wrapper(lib, fname):
lib.add_source_file(fname)
@click.group()
def cli():
@click.group(cls=AliasedGroup)
@click.option('--compile', is_flag=True) #, hidden=True
@click.pass_context
def cli(ctx, compile):
setup_logging()
ctx.obj = {'compile': compile}
pass
@cli.command()
def create():
pass
@cli.group()
@cli.group(cls=AliasedGroup)
def test():
sys.argv[0] = abspath(sys.argv[0])
pass
@test.command()
@click.argument('config', type=click.Path(exists=True))
@click.argument('config', type=click.Path())
@click.argument('vunit_args', nargs=-1)
def unit(config, vunit_args):
config_file = config
with open(config_file, 'rt', encoding='utf-8') as f:
@click.pass_obj
def unit(obj, config, vunit_args):
config_file = d.parent / config
with config_file.open('rt', encoding='utf-8') as f:
config = yaml.load(f)
base = Path(config_file).resolve().parent
build = d.parent / 'build'
build.mkdir(exist_ok=True)
os.chdir(str(build))
args = []
if obj['compile']:
args += ['--compile']
args += ['--xunit-xml', '../test_unit.xml1'] + list(vunit_args)
ui = VUnit.from_argv(args)
ui = VUnit.from_argv(['lib.tb_*_unit_test.*', '--xunit-xml', '../test_unit.xml1'] + list(vunit_args))
lib = add_sources(ui)
lib = ui.add_library("lib")
add_common_sources(lib)
add_sources(lib, ['unit/**/*.vhd'])
create_wrapper(lib, build / "tb_wrappers.vhd")
add_flags(ui, lib, build)
......@@ -131,16 +186,52 @@ def unit(config, vunit_args):
tb = lib.get_test_benches('*tb_{}_unit_test'.format(name), allow_empty=True)
if not len(tb):
pprint([x.name for x in unit_tests])
raise RuntimeError('Testbench {}_unit_test does not exist (buf specified in config).'.format(name))
raise RuntimeError('Testbench {}_unit_test does not exist (but specified in config).'.format(name))
assert len(tb) == 1
tb = tb[0]
tb.set_generic('iterations', cfg['iterations'])
tb.set_generic('log_level', cfg['log_level'] + '_l')
tb.set_generic('error_tol', cfg['error_tolerance'])
# generate & set per-test modelsim tcl file
tcl = build / 'modelsim_init_{}.tcl'.format(name)
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
global TCOMP
set TCOMP tb_{}_unit_test/tb/i_test
'''.format(name)), file=f)
init_files = get_common_modelsim_init_files()
init_files += [str(tcl)]
tb.set_sim_option("modelsim.init_files.after_load", init_files)
if 'wave' in cfg:
tb.set_sim_option("modelsim.init_file.gui", str(base / cfg['wave']))
path = base / cfg['wave']
if not path.exists():
log.warn('Wave file {} not found'.format(cfg['wave']))
tb.set_sim_option("modelsim.init_file.gui", str(path))
else:
tcl = build / 'modelsim_gui_{}.tcl'.format(name)
with tcl.open('wt', encoding='utf-8') as f:
print(dedent('''\
start_CAN_simulation "dummy"
global IgnoreAddWaveErrors
puts "Automatically adding common waves. Failures are handled gracefully."
set IgnoreAddWaveErrors 1
add_test_status_waves
add_system_waves
set IgnoreAddWaveErrors 0
run_simulation
get_test_results
'''.format(name)), file=f)
tb.set_sim_option("modelsim.init_file.gui", str(tcl))
os.chdir(str(build))
# check for unconfigured testbenches
all_benches = lib.get_test_benches('*')
configured = ['tb_{}_unit_test'.format(name) for name in config['tests'].keys()]
log.debug('Configured tests: {}'.format(', '.join(configured)))
unconfigured = [tb for tb in all_benches if tb.name not in configured]
if len(unconfigured):
log.warn("Testbenches with no configuration found: {}".format(', '.join(x.name for x in unconfigured)))
try:
vunit_ifc.run(ui)
......@@ -156,7 +247,6 @@ def unit(config, vunit_args):
f.write(c)
out.unlink()
"""
- vunit configurations
- pass modelsim gui file via ui.set_sim_option("modelsim.init_file.gui", ...)
......@@ -172,3 +262,11 @@ def unit(config, vunit_args):
- use watchdog - pass the time in config: test_runner_watchdog(runner, 10 ms);
"""
def dump_sim_options(lib):
for tb in lib.get_test_benches('*'):
for cfgs in tb._test_bench.get_configuration_dicts():
for name, cfg in cfgs.items():
print('{}#{}:'.format(tb.name, name))
#pprint(cfg.__dict__)
pprint(cfg.sim_options)
import logging
class MyLogRecord(logging.LogRecord):
_slvls = {
logging.DEBUG: '--',
logging.INFO: 'II',
logging.WARNING: 'WW',
logging.ERROR: 'EE',
}
_colors = {
logging.DEBUG: '',
logging.INFO: '\033[1;34m',
logging.WARNING: '\033[1;33m',
logging.ERROR: '\033[1;31m',
}
_msgcolors = {
logging.DEBUG: '',
logging.INFO: '',
logging.WARNING: '\033[1;33m',
logging.ERROR: '\033[1;31m',
}
def __init__(self, name, level, fn, line, msg, args, exc_info, func=None, sinfo=None, **kwargs):
super().__init__(name, level, fn, line, msg, args, exc_info, func=func, sinfo=sinfo, **kwargs)
self.slvl = self._slvls.get(level, level)
self.color = self._colors.get(level, '')
self.msgcolor = self._msgcolors.get(level, '')
self.crst = '\033[1;m'
version: 1
formatters:
colored:
format: '[%(asctime)s] (%(color)s%(slvl)s%(crst)s) %(name)-6s %(msgcolor)s%(message)s%(crst)s'
nocolor:
format: '[%(asctime)s] (%(slvl)s) %(name)-6s %(message)s'
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: colored
stream: ext://sys.stderr
loggers:
vunit:
level: INFO
handlers: [console]
propagate: yes
root:
level: DEBUG
handlers: [console]
global TCOMP
set TCOMP tb_bit_stuffing_unit_test/tb/i_test
proc start_CAN_simulation {test_wrapper} {
}
proc run_simulation {} {
vunit_run
}
proc get_test_results_feature {} {
}
proc get_test_results {} {
}
rename add _add
proc safe_add {args} {
if {[catch {_add {*}$args} fid]} {
puts stderr $fid
}
}
global IgnoreAddWaveErrors
set IgnoreAddWaveErrors 0
proc add {args} {
global IgnoreAddWaveErrors
if {$IgnoreAddWaveErrors} {
safe_add {*}$args
} else {
_add {*}$args
}
}
proc sim_restart {} {
vunit_restart
vunit_user_init
}
rename vunit_help _vunit_help
proc vunit_help {} {
_vunit_help
puts "sim_restart:"
puts " - recompile, add waves, restart (a.k.a vunit_restart; vunit_user_init)"
}
default:
log_level: info
log_level: warning
error_tolerance: 0
iterations: 50
tests:
bit_stuffing:
iterations: 10
wave: Bit_Stuffing/Bit_Stuffing.tcl
wave: unit/Bit_Stuffing/bsdt_unit.tcl
apb:
iterations: 1
bus_sync:
wave: unit/Bus_Sampling/bsnc_unit.tcl
crc:
wave: unit/CRC/crct_unit.tcl
fault_confinement:
wave: unit/Fault_confinement/fault_conf_unit.tcl
int_man:
wave: unit/Int_Manager/intm_unit.tcl
mess_filt:
wave: unit/Message_filter/msft_unit.tcl
presc:
wave: unit/Prescaler/prsc_unit.tcl
protocol_control:
wave: unit/Protocol_Control/pctl_unit.tcl
rx_buf:
wave: unit/RX_Buffer/rxbf_unit.tcl
iterations: 10
tx_arb:
wave: unit/TX_Arbitrator/txar_unit.tcl
tx_buf:
wave: unit/TX_Buffer/txbf_unit.tcl
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment