Commit 26142377 authored by Martin Jeřábek's avatar Martin Jeřábek
Browse files

Merge branch '114-bring-the-whole-test-framework-to-ghdl' into 'master'

Allow for test randomization in testfw

See merge request illeondr/CAN_FD_IP_Core!110
parents f944cf98 6ead60ee
......@@ -315,6 +315,8 @@ entity tb_feature is
-- Timeout in simulation time. 0 means no limit
timeout : string := "0 ms";
seed : natural := 0;
hw_reset_on_new_test : boolean := true;
test_name : string
......@@ -360,7 +362,7 @@ architecture tb of tb_feature is
signal bl_force : boolean := false;
-- test internal signals
signal iteration_done : boolean;
signal iteration_done : boolean := false;
signal mem_bus : mem_bus_arr_t := (OTHERS => mem_bus_init);
......@@ -409,10 +411,14 @@ begin
variable o : feature_outputs_t;
begin
test_runner_setup(runner, runner_cfg);
--Set the process to run and wait until it comes out of reset
iteration_done <= false;
run <= true;
error_ctr <= 0;
apply_rand_seed(seed, 0, rand_ctr);
report "Restarting mem_bus(1)";
restart_mem_bus(mem_bus(1));
report "Restarting mem_bus(1)";
......
......@@ -583,7 +583,7 @@ package CANtestLib is
procedure log(
constant message : in String;
constant log_severity : in log_lvl_type;
signal log_level : in log_lvl_type
constant log_level : in log_lvl_type
);
......@@ -634,10 +634,10 @@ package CANtestLib is
-- error_tol Error tolerance of test.
----------------------------------------------------------------------------
procedure print_test_info(
signal iterations : in natural;
signal log_level : in log_lvl_type;
signal error_beh : in err_beh_type;
signal error_tol : in natural
constant iterations : in natural;
constant log_level : in log_lvl_type;
constant error_beh : in err_beh_type;
constant error_tol : in natural
);
......@@ -657,6 +657,12 @@ package CANtestLib is
);
-- variation of above, only returns the seed value
function apply_rand_seed(
constant seed : in natural;
constant offset : in natural
) return natural;
----------------------------------------------------------------------------
-- Decode data length code from value as defined in CAN FD Standard to
-- length of frame in bytes.
......@@ -1689,7 +1695,7 @@ package body CANtestLib is
procedure log(
constant Message : in String;
constant log_severity : in log_lvl_type;
signal log_level : in log_lvl_type
constant log_level : in log_lvl_type
)is
begin
......@@ -1769,10 +1775,10 @@ package body CANtestLib is
procedure print_test_info(
signal iterations :in natural;
signal log_level :in log_lvl_type;
signal error_beh :in err_beh_type;
signal error_tol :in natural
constant iterations : in natural;
constant log_level : in log_lvl_type;
constant error_beh : in err_beh_type;
constant error_tol : in natural
)is
begin
report "Test info:";
......@@ -1796,15 +1802,25 @@ package body CANtestLib is
end;
function apply_rand_seed(
constant seed : in natural;
constant offset : in natural
) return natural is
variable tmp : natural;
begin
tmp := seed + offset;
report "Random initialized with seed " & natural'image(seed);
return tmp mod RAND_POOL_SIZE;
end function;
procedure apply_rand_seed(
constant seed : in natural;
constant offset : in natural;
signal rand_ctr : out natural range 0 to RAND_POOL_SIZE
)is
variable tmp : natural;
begin
tmp := seed + offset;
rand_ctr <= tmp mod RAND_POOL_SIZE;
rand_ctr <= apply_rand_seed(seed, offset);
wait for 0 ns;
end procedure;
......
......@@ -73,62 +73,62 @@ use work.CAN_FD_register_map.all;
use work.CAN_FD_frame_format.all;
entity sanity_test is
port (
-- Input trigger, test starts running when true
signal run :in boolean;
generic (
seed : natural := 0;
-- Number of iterations that test should do
signal iterations :in natural;
iterations : natural;
-- Logging level, severity which should be shown
signal log_level :in log_lvl_type;
log_level : log_lvl_type;
-- Test behaviour when error occurs: Quit, or Go on
signal error_beh :in err_beh_type;
error_beh : err_beh_type;
-- Error tolerance, error counter should not exceed this value in order
-- for the test to pass
signal error_tol :in natural;
-- Status of the test
signal status :out test_status_type;
-- Amount of errors which appeared in the test
signal errors :out natural ;
--TODO: Error log results
error_tol : natural;
----------------------------------------------
-- Test configuration
----------------------------------------------
-- Uncertainties of each clock (in ppm)
signal epsilon_v : epsilon_type;
epsilon_v : epsilon_type;
-- Transceiver delay parameters
signal trv_del_v : trv_del_type;
trv_del_v : trv_del_type;
-- Configuration of bus topology
signal bus_matrix : bus_matrix_type;
bus_matrix : bus_matrix_type;
-- Number of iterations in Gauss distribution calculation
signal iter_am : natural := 40;
iter_am : natural := 40;
-- Noise pulse width mean in nanaoseconds
signal nw_mean : real;
nw_mean : real;
-- Noise pulse width variance
signal nw_var : real;
nw_var : real;
-- Gap between two noise pulses mean in nanoseconds
signal ng_mean : real;
ng_mean : real;
-- Gap variance
signal ng_var : real;
ng_var : real;
topology : string (1 to 50);
signal topology : string (1 to 50);
timing_config : bit_time_config_type
);
port (
-- Input trigger, test starts running when true
signal run :in boolean;
-- Status of the test
signal status :out test_status_type;
signal timing_config : bit_time_config_type
-- Amount of errors which appeared in the test
signal errors :out natural
--TODO: Error log results
);
end entity;
......@@ -278,7 +278,8 @@ architecture behavioral of sanity_test is
array (1 to NODE_COUNT) of natural range 0 to RAND_POOL_SIZE;
signal rand_ctr_gen : rand_ctr_array_type;
signal rand_ident_ctr : natural range 0 to RAND_POOL_SIZE := 0;
signal rand_ident_ctr : natural range 0 to RAND_POOL_SIZE
:= apply_rand_seed(seed, NODE_COUNT+1);
-- Auxiliarly signal for identifier correction to have similiar identifiers
-- and interesting arbitration!
......@@ -405,7 +406,7 @@ architecture behavioral of sanity_test is
signal rand_ctr : inout natural range 0 to RAND_POOL_SIZE;
variable frame : inout SW_CAN_frame_type;
constant index : in natural;
signal com_id : in natural
constant com_id : in natural
)is
variable aux_vect : std_logic_vector(28 downto 0);
variable aux_common : std_logic_vector(28 downto 0);
......@@ -449,7 +450,7 @@ architecture behavioral of sanity_test is
mem_bus.clk_sys <= 'Z';
end procedure;
function bus_matrix_to_delay(signal bm : in real) return time is
function bus_matrix_to_delay(bm : in real) return time is
begin
return 10.0 * bm * 500 ps;
end function;
......@@ -503,15 +504,8 @@ begin
-- Clock generation
----------------------------------------------------------------------------
clock_generic : for i in 1 to NODE_COUNT generate
clock_gen_1 : process
constant period : natural := f100_Mhz;
constant duty : natural := 50;
variable epsilon : natural;
begin
epsilon := epsilon_v(i);
generate_clock(period, duty, epsilon, mem_aux_clk(i));
timestamp_v(i) <= std_logic_vector(unsigned(timestamp_v(i)) + 1);
end process;
clock_gen_proc(f100_mhz, 50, epsilon_v(i), mem_aux_clk(i));
timestamp_gen_proc(mem_aux_clk(i), timestamp_v(i));
end generate clock_generic;
......@@ -550,6 +544,7 @@ begin
-- 500 ps equals approximately 10 cm of conductor! This is coarse estimate,
-- however purpose of this simulation is not to examine exact propagation
-- delays via different physical channels! This simple estimate is enough.
-- TODO: may be optimized by eschewing bus_clk and delaying the signals directly
----------------------------------------------------------------------------
bus_clk_proc : process
begin
......@@ -617,35 +612,36 @@ begin
variable gaus_par : rand_distribution_par_type;
variable exp_par : rand_distribution_par_type;
begin
if (do_noise) then
-- Generate noise pulse with gauss distribution
gaus_par(GAUSS_iterations) := real(iter_am);
gaus_par(GAUSS_mean) := nw_mean;
gaus_par(GAUSS_variance) := nw_var;
rand_real_distr_v(rand_ctr, noise_time, GAUSS, gaus_par);
-- Generate noise pulse gap with exponential distribution
exp_par(EXPONENTIAL_mean) := ng_mean;
rand_real_distr_v(rand_ctr, noise_gap, EXPONENTIAL, exp_par);
noise_force <= (OTHERS => '0');
wait for integer(noise_gap) * 1 ns;
--Generate noise polarity and info whether noise
-- should be forced to the node
-- We cant put noise to all. That would be global
-- error at all times
rand_logic_vect_v(rand_ctr ,aux,0.5);
noise_reg <= aux;
rand_logic_vect_v(rand_ctr ,aux,0.5);
noise_force <= aux;
wait for integer(noise_time) * 1 ns;
else
wait for 10 ns;
end if;
apply_rand_seed(seed, 0, rand_ctr);
loop
if (do_noise) then
-- Generate noise pulse with gauss distribution
gaus_par(GAUSS_iterations) := real(iter_am);
gaus_par(GAUSS_mean) := nw_mean;
gaus_par(GAUSS_variance) := nw_var;
rand_real_distr_v(rand_ctr, noise_time, GAUSS, gaus_par);
-- Generate noise pulse gap with exponential distribution
exp_par(EXPONENTIAL_mean) := ng_mean;
rand_real_distr_v(rand_ctr, noise_gap, EXPONENTIAL, exp_par);
noise_force <= (OTHERS => '0');
wait for integer(noise_gap) * 1 ns;
--Generate noise polarity and info whether noise
-- should be forced to the node
-- We cant put noise to all. That would be global
-- error at all times
rand_logic_vect_v(rand_ctr ,aux,0.5);
noise_reg <= aux;
rand_logic_vect_v(rand_ctr ,aux,0.5);
noise_force <= aux;
wait for integer(noise_time) * 1 ns;
else
wait for 10 ns;
end if;
end loop;
end process;
----------------------------------------------------------------------------
......@@ -688,6 +684,7 @@ begin
variable fault_state : SW_fault_state;
begin
if (do_restart_mem_if(i)) then
apply_rand_seed(seed, i, rand_ctr_gen(i));
restart_mem_bus(mb_arr(i));
wait for 10 ns;
......@@ -939,58 +936,5 @@ begin
evaluate_test(error_tol, error_ctr, status);
end process;
errors <= error_ctr;
end architecture;
architecture sanity_test of CAN_test is
signal epsilon_v : epsilon_type := (0,150,300,450);
signal trv_del_v : trv_del_type := (5,5,5,5);
signal bus_matrix : bus_matrix_type := ( ( 0.0,10.0,20.0,30.0),
(10.0, 0.0,10.0,20.0),
(20.0,10.0, 0.0,10.0),
(30.0,20.0,10.0, 0.0));
---------------------
-- Noise parameters
---------------------
signal iter_am : natural := 40;
-- Noise pulse width mean in nanaoseconds
signal nw_mean : real := 15.0;
-- Noise pulse width variance
signal nw_var : real := 5.0;
-- Gap between two noise pulses mean in nanoseconds
signal ng_mean : real := 10000.0;
-- Gap variance
signal ng_var : real := 6000.0;
signal topology : string (1 to 50) :=
" ";
-- With 100 MHz this is 1Mbit(nom)/2Mbit(Data)
signal timing_config : bit_time_config_type := (5,5,9,5,5,3,3,3,3,2);
begin
i_st: entity work.sanity_test
port map(
run => run,
iterations => iterations,
log_level => log_level,
error_beh => error_beh,
error_tol => error_tol,
status => status,
errors => errors,
epsilon_v => epsilon_v,
trv_del_v => trv_del_v,
bus_matrix => bus_matrix,
iter_am => iter_am,
nw_mean => nw_mean,
nw_var => nw_var,
ng_mean => ng_mean,
ng_var => ng_var,
topology => topology,
timing_config => timing_config
);
errors <= error_ctr;
end architecture;
......@@ -73,6 +73,8 @@ entity tb_sanity is
-- Timeout in simulation time. 0 means no limit.
timeout : string := "0 ms";
seed : natural := 0;
topology : string;
bus_len_v : string; --bus_length_type;
trv_del_v : string; --anat_nc_t;
......@@ -221,14 +223,12 @@ begin
end generate;
t_sanity: entity work.sanity_test
port map (
generic map (
seed => seed,
iterations => iterations,
log_level => log_level,
error_beh => error_beh,
error_tol => error_tol,
errors => t_sanity_errors,
status => t_sanity_status,
run => t_sanity_run,
-- test params
epsilon_v => epsilon_v,
trv_del_v => decoded_trv_del_v,
......@@ -240,5 +240,10 @@ begin
ng_var => decoded_ng_var,
topology => padded_topology,
timing_config => decoded_timing_config
)
port map (
errors => t_sanity_errors,
status => t_sanity_status,
run => t_sanity_run
);
end architecture;
......@@ -6,13 +6,8 @@ import os
import re
import sys
from pathlib import Path
from .log import MyLogRecord
from . import vunit_ifc
from . import test_unit, test_sanity, test_feature
from vunit.ui import VUnit
from os.path import abspath
from .test_common import add_common_sources, add_flags
from .log import MyLogRecord
d = Path(abspath(__file__)).parent
......@@ -24,6 +19,16 @@ def setup_logging() -> None:
logging.config.dictConfig(cfg)
global log
log = logging.getLogger('fw')
setup_logging()
from . import vunit_ifc
from . import test_unit, test_sanity, test_feature
from vunit.ui import VUnit
from .test_common import add_common_sources, add_flags
#-------------------------------------------------------------------------------
......@@ -55,7 +60,6 @@ class AliasedGroup(click.Group):
@click.option('--compile', is_flag=True) #, hidden=True
@click.pass_context
def cli(ctx, compile):
setup_logging()
ctx.obj = {'compile': compile}
sys.argv[0] = abspath(sys.argv[0])
pass
......
......@@ -23,7 +23,8 @@ entity tb_{{test}} is generic (
log_level : log_lvl_type := info_l;
error_beh : err_beh_type := quit;
error_tol : natural := 0;
timeout : string := "0 ms"
timeout : string := "0 ms";
seed : natural := 0
); end entity;
architecture tb of tb_{{test}} is
component vunittb_wrapper is generic (
......@@ -32,7 +33,8 @@ architecture tb of tb_{{test}} is
log_level : log_lvl_type;
error_beh : err_beh_type;
error_tol : natural;
timeout : string
timeout : string;
seed : natural
); end component;
for all:vunittb_wrapper use configuration work.tbconf_{{test}};
begin
......@@ -42,7 +44,8 @@ begin
log_level => log_level,
error_beh => error_beh,
error_tol => error_tol,
timeout => timeout);
timeout => timeout,
seed => seed);
end architecture;
-- -----------------------------------------------------------------------------
{% endfor %}
......@@ -6,9 +6,11 @@ import logging
from pathlib import Path
from jinja2 import Environment, PackageLoader
from pprint import pprint
import random
__all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files',
'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options', 'TestsBase']
'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options',
'TestsBase', 'get_seed']
d = Path(abspath(__file__)).parent
log = logging.getLogger(__name__)
......@@ -88,6 +90,19 @@ def add_flags(ui, lib, build):
ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
def get_seed(cfg):
if 'seed' in cfg and 'randomize' in cfg:
log.warning('Both "seed" and "randomize" are set - seed takes precedence')
if 'seed' in cfg:
seed = cfg['seed']
elif cfg.get('randomize', False):
# only 31 bits
seed = int(random.random() * 2**31) & 0x7FFFFFFF
else:
seed = 0
return seed
def dict_merge(up, *lowers):
for lower in lowers:
for k, v in lower.items():
......
import logging
from pathlib import Path
from .test_common import add_sources, TestsBase, dict_merge, get_common_modelsim_init_files
from .test_common import add_sources, TestsBase, dict_merge, \
get_common_modelsim_init_files, get_seed
from textwrap import dedent
log = logging.getLogger(__name__)
......@@ -54,7 +55,8 @@ class FeatureTests(TestsBase):
'iterations' : cfg['iterations'],
'log_level' : cfg['log_level'] + '_l',
'error_tol' : cfg['error_tolerance'],
'test_name' : name
'test_name' : name,
'seed' : get_seed(cfg)
}
tb.add_config(name, generics=generics)
......
import logging
from .test_common import TestsBase, add_sources, dict_merge, vhdl_serialize
from .test_common import TestsBase, add_sources, dict_merge, vhdl_serialize, \
get_seed
log = logging.getLogger(__name__)
......@@ -56,6 +57,7 @@ class SanityTests(TestsBase):
'iterations' : cfg['iterations'],
'log_level' : cfg['log_level'] + '_l',
'error_tol' : cfg['error_tolerance'],
'seed' : get_seed(cfg),
'topology' : cfg['topology'],
#'bm' : vhdl_serialize(bm),
'bus_len_v' : vhdl_serialize(cfg['bus_len_v']),
......
import re
import logging
from textwrap import dedent
from .test_common import add_sources, dict_merge, TestsBase, get_common_modelsim_init_files
from .test_common import add_sources, dict_merge, TestsBase, \
get_common_modelsim_init_files, get_seed
from pprint import pprint
log = logging.getLogger(__name__)
......@@ -18,16 +19,19 @@ class UnitTests(TestsBase):
unit_tests = lib.get_test_benches('*_unit_test')
for name, cfg in config['tests'].items():
dict_merge(cfg, default)
tb = lib.get_test_benches('*tb_{}_unit_test'.format(name), allow_empty=True)
tb = lib.get_test_benches('*tb_{}_unit_test'.format(name),
allow_empty=True)
if not len(tb):
pprint([x.name for x in unit_tests])
raise RuntimeError('Testbench {}_unit_test does not exist (but specified in config).'.format(name))
raise RuntimeError('Testbench {}_unit_test does not exist'
+ ' (but specified in config).'.format(name))
assert len(tb) == 1
tb = tb[0]
tb.set_generic('timeout', cfg['timeout'])
tb.set_generic('iterations', cfg['iterations'])
tb.set_generic('log_level', cfg['log_level'] + '_l')
tb.set_generic('error_tol', cfg['error_tolerance'])
tb.set_generic('seed', get_seed(cfg))
# generate & set per-test modelsim tcl file
tcl = build / 'modelsim_init_{}.tcl'.format(name)
......
_default: &default
log_level: info
error_tolerance: 0
# seed: 0 # optional; use to reconstruct results from randomized runs
# randomize: false
unit:
default: