testfw: fix accumulating sim_options, extract common bits, cleanup

test/testfw/__init__.py

@@ -27,7 +27,7 @@ setup_logging()
 from . import vunit_ifc
 from . import test_unit, test_sanity, test_feature, test_reference
 from vunit.ui import VUnit
-from .test_common import add_common_sources, add_flags
+from .test_common import add_common_sources, get_compile_options
 
 
 #-------------------------------------------------------------------------------
@@ -150,7 +150,11 @@ def test(obj, *, config, strict, create_ghws, vunit_args):
 
     for t in tests:
         t.add_sources()
-    add_flags(ui, lib, build)
+
+    c = get_compile_options()
+    for k, v in c.items():
+        lib.set_compile_option(k, v)
+
     conf_ok = [t.configure() for t in tests]
 
     # check for unknown tests
@@ -198,27 +202,3 @@ def vunit_run(ui, build, out_basename) -> int:
             f.write(c)
         out.unlink()
     return res
-
-
-"""
-+ vunit configurations
-+ pass modelsim gui file via ui.set_sim_option("modelsim.init_file.gui", ...)
-+ include the standard library files in ui.set_sim_option("modelsim.init_files.after_load", [...])
-+ set TCOMP global variable
-
-- allow preprocessed calls to log()
-- use some log from vunit?
-- use random from unit?
-
-+ use per-test default configurations (with set tcl files etc.), different sanity configurations
-x pass encoded composite generics (sanity test)
-
-+ use watchdog - pass the time in config: test_runner_watchdog(runner, 10 ms);
-
-- bash completion for files & tests:
-    - click._bashcompletion.get_choices -> extend the if to check if the given argument is an instance of XXX
-      and implement completion method for that instance. Complete test names.
-
-- feature tests
-- sanity - optimize bus delay shift registers
-"""

test/testfw/test_common.py

@@ -8,11 +8,13 @@ from jinja2 import Environment, PackageLoader
 from pprint import pprint
 import random
 from .gtkwave import tcl2gtkw
-from typing import List
+from typing import List, Tuple
+import copy
+import re
 
-__all__ = ['add_sources', 'add_common_sources', 'get_common_modelsim_init_files',
-           'add_flags', 'dict_merge', 'vhdl_serialize', 'dump_sim_options',
-           'TestsBase', 'get_seed']
+__all__ = ['add_sources', 'add_common_sources',
+           'dict_merge', 'vhdl_serialize', 'dump_sim_options',
+           'TestsBase', 'get_seed', 'OptionsDict']
 
 d = Path(abspath(__file__)).parent
 log = logging.getLogger(__name__)
@@ -20,6 +22,43 @@ log = logging.getLogger(__name__)
 jinja_env = Environment(loader=PackageLoader(__package__, 'data'), autoescape=False)
 
 
+class OptionsDict(dict):
+    # def __getattr__(self, key):
+    #     return self[key]
+
+    def __iadd__(self, upper: dict):
+        self.__merge(self, upper)
+        return self
+
+    def __add__(self, upper: dict) -> 'OptionsDict':
+        res = copy.deepcopy(self)
+        res += upper
+        return res
+
+    def __radd__(self, lower: dict) -> 'OptionsDict':
+        res = copy.deepcopy(lower)
+        res += self
+        return res
+
+    @classmethod
+    def __merge(cls, lower, upper) -> None:
+        if isinstance(lower, OptionsDict):
+            if not isinstance(upper, OptionsDict):
+                raise TypeError('Cannot merge {} and {}'.format(type(lower), type('upper')))
+
+            for k, v in upper.items():
+                if k not in lower:
+                    lower[k] = v
+                else:
+                    cls.__merge(lower[k], v)
+        elif isinstance(lower, list):
+            if not isinstance(upper, list):
+                raise TypeError('Cannot merge {} and {}'.format(type(lower), type('upper')))
+            lower.extend(upper)
+        else:
+            raise TypeError('Cannot merge {} and {}'.format(type(lower), type('upper')))
+
+
 class TestsBase:
     def __init__(self, ui, lib, config, build, base, create_ghws: bool):
         self.ui = ui
@@ -43,9 +82,18 @@ class TestsBase:
 
         raise NotImplementedError()
 
-    def add_modelsim_gui_file(self, tb, cfg, name, tcl_init_files: List[str] = None) -> None:
-        if tcl_init_files is None:
-            tcl_init_files = get_common_modelsim_init_files()
+    def generate_init_tcl(self, fname: str, tcomp: str) -> OptionsDict:
+        tcl = self.build / fname
+        with tcl.open('wt', encoding='utf-8') as f:
+            print(dedent('''\
+                global TCOMP
+                set TCOMP {}
+                '''.format(tcomp)), file=f)
+        return OptionsDict({"modelsim.init_files.after_load": [str(tcl)]})
+
+    def add_modelsim_gui_file(self, tb, cfg, name, tcl_init_files: List[str]) -> OptionsDict:
+        """Return sim_options to add to the testcase."""
+        sim_options = OptionsDict({'ghdl.sim_flags': []})
         if 'wave' in cfg:
             tcl = self.base / cfg['wave']
             if not tcl.exists():
@@ -65,7 +113,7 @@ class TestsBase:
                     get_test_results
                     '''.format(name)), file=f)
 
-        tb.set_sim_option("modelsim.init_file.gui", str(tcl))
+        sim_options["modelsim.init_file.gui"] = str(tcl)
         if 'gtkw' in cfg:
             gtkw = self.base / cfg['gtkw']
             if not gtkw.exists():
@@ -79,9 +127,7 @@ class TestsBase:
             # the conversion now.
             if self.create_ghws:
                 log.info('Will generate {}'.format(ghw_file))
-                sim_flags = get_common_sim_flags()
-                sim_flags += ['--wave=' + str(ghw_file)]
-                tb.set_sim_option("ghdl.sim_flags", sim_flags)
+                sim_options["ghdl.sim_flags"] += ['--wave=' + str(ghw_file)]
             else:
                 if not ghw_file.exists():
                     log.warning("Cannot convert wave file {} to gtkw, because"
@@ -94,12 +140,32 @@ class TestsBase:
 
         if gtkw:
             try:
-                tb.set_sim_option("ghdl.gtkwave_flags", ['--save='+str(gtkw)])
+                tb.set_sim_option("ghdl.gtkwave_flags", [])
+                sim_options["ghdl.gtkwave_flags"] = ['--save='+str(gtkw)]
             except ValueError:
                 try:
-                    tb.set_sim_option("ghdl.gtkw_file", str(gtkw))
+                    tb.set_sim_option("ghdl.gtkw_file", "")
+                    sim_options["ghdl.gtkw_file"] = str(gtkw)
                 except ValueError:
                     log.warning('Setting GTKW file per test is not supported in this VUnit version.')
+        return OptionsDict(sim_options)
+
+    def get_default_sim_options(self) -> OptionsDict:
+        c, s = get_default_compile_and_sim_options()
+        return s
+
+    def add_psl_cov(self, name) -> OptionsDict:
+        name = re.sub(r'[^a-zA-Z0-9_-]', '_', name)
+        psl_path = self.build / "functional_coverage" / "coverage_data" \
+                    / "psl_cov_{}.json".format(name)
+        sim_flags = ["--psl-report={}".format(psl_path)]
+        return OptionsDict({"ghdl.sim_flags": sim_flags})
+
+    @staticmethod
+    def set_sim_options(tb, options: OptionsDict) -> None:
+        for k, v in options.items():
+            tb.set_sim_option(k, v)
+
 
 def add_sources(lib, patterns) -> None:
     for pattern in patterns:
@@ -109,6 +175,7 @@ def add_sources(lib, patterns) -> None:
             if f != "tb_wrappers.vhd":
                 lib.add_source_file(str(f))
 
+
 def add_common_sources(lib, ui) -> None:
     add_sources(lib, ['../src/**/*.vhd'])
     ui.enable_check_preprocessing()
@@ -116,38 +183,41 @@ def add_common_sources(lib, ui) -> None:
     add_sources(lib, ['*.vhd', 'lib/*.vhd', 'models/*.vhd'])
 
 
-def get_common_modelsim_init_files() -> List[str]:
-    modelsim_init_files = ['../lib/test_lib.tcl', 'modelsim_init.tcl']
-    modelsim_init_files = [str(d/x) for x in modelsim_init_files]
-    return modelsim_init_files
-
-def get_common_sim_flags() -> List[str]:
-    return ["--ieee-asserts=disable-at-0"]
-
-def add_flags(ui, lib, build) -> None:
-    unit_tests = lib.get_test_benches('*_unit_test', allow_empty=True)
-    for ut in unit_tests:
-        ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
-
-    reference_tests = lib.get_test_benches('*reference*', allow_empty=True)
-    for rt in reference_tests:
-        rt.scan_tests_from_file(str(build / "../reference/vunit_reference_wrapper.vhd"))
-
-    #lib.add_compile_option("ghdl.flags", ["-Wc,-g"])
-    lib.add_compile_option("ghdl.flags", ["-fprofile-arcs", "-ftest-coverage", "-fpsl", "-g"])
-
-    elab_flags = ["-Wl,-lgcov", "-g"]
-    elab_flags.append("-Wl,--coverage")
-    elab_flags.append("-Wl,-no-pie")
-    elab_flags.append("-fpsl")
-    ui.set_sim_option("ghdl.elab_flags", elab_flags)
-
-    # Global simulation flags
-    sim_flags = get_common_sim_flags()
-    ui.set_sim_option("ghdl.sim_flags", sim_flags)
-
-    modelsim_init_files = get_common_modelsim_init_files()
-    ui.set_sim_option("modelsim.init_files.after_load", modelsim_init_files)
+def get_default_compile_and_sim_options() -> Tuple[OptionsDict, OptionsDict]:
+    # TODO: move to config
+    debug = True
+    coverage = True
+    psl = True
+
+    compile_flags = []  # type: List[str]
+    elab_flags = ["-Wl,-no-pie"]
+
+    if debug:
+        compile_flags += ['-g']
+        elab_flags += ['-g']
+    if coverage:
+        compile_flags += ["-fprofile-arcs", "-ftest-coverage"]
+        elab_flags += ["-Wl,-lgcov", "-Wl,--coverage"]
+    if psl:
+        compile_flags += ['-fpsl']
+        elab_flags += ['-fpsl']
+
+    compile_options = OptionsDict()
+    compile_options["ghdl.flags"] = compile_flags
+
+    cmif = ['../lib/test_lib.tcl', 'modelsim_init.tcl']
+    common_modelsim_init_files = [str(d/x) for x in cmif]
+    sim_options = OptionsDict({
+        "ghdl.elab_flags": elab_flags,
+        "modelsim.init_files.after_load": common_modelsim_init_files,
+        "ghdl.sim_flags": ["--ieee-asserts=disable-at-0"],
+    })
+    return compile_options, sim_options
+
+
+def get_compile_options() -> OptionsDict:
+    c, s = get_default_compile_and_sim_options()
+    return c
 
 
 def get_seed(cfg) -> int:

test/testfw/test_feature.py

 import logging
 from pathlib import Path
 from .test_common import add_sources, TestsBase, dict_merge, \
-                         get_common_modelsim_init_files, get_seed
+                         get_seed, OptionsDict
 from textwrap import dedent
 import re
 
@@ -26,25 +26,14 @@ class FeatureTests(TestsBase):
         tb = self.lib.get_test_benches('*tb_feature')[0]
         tb.scan_tests_from_file(str(wrname))
 
-    def create_psl_cov_file_opt(self, name):
-        psl_path = "functional_coverage/coverage_data/psl_cov_feature_{}.json".format(name)
-        psl_flag = "--psl-report={}".format(psl_path)
-        return {"ghdl.sim_flags" : [psl_flag]}
-
     def configure(self) -> bool:
         tb = self.lib.get_test_benches('*tb_feature')[0]
         default = self.config['default']
 
+        sim_options = self.get_default_sim_options()
         # generate & set per-test modelsim tcl file
-        tcl = self.build / 'modelsim_init_feature.tcl'
-        with tcl.open('wt', encoding='utf-8') as f:
-            print(dedent('''\
-                global TCOMP
-                set TCOMP tb_feature/test_comp
-                '''), file=f)
-        init_files = get_common_modelsim_init_files()
-        init_files += [str(tcl)]
-        tb.set_sim_option("modelsim.init_files.after_load", init_files)
+        sim_options += self.generate_init_tcl('modelsim_init_feature.tcl', 'tb_feature/test_comp')
+        sim_options += self.add_modelsim_gui_file(tb, default, 'feature', sim_options['modelsim.init_files.after_load'])
 
         for name, cfg in self.config['tests'].items():
             if cfg is None:
@@ -64,12 +53,13 @@ class FeatureTests(TestsBase):
                 'seed'         : get_seed(cfg)
             }
 
-            if (cfg['psl_coverage']):
-                psl_opts = self.create_psl_cov_file_opt(name)
-                tb.add_config(name, generics=generics, sim_options=psl_opts)
-            else:
-                tb.add_config(name, generics=generics)
-        self.add_modelsim_gui_file(tb, default, 'feature', init_files)
+            local_sim_options = OptionsDict()
+            if cfg['psl_coverage']:
+                local_sim_options += self.add_psl_cov('{}.{}'.format(tb.name, name))
+
+            local_sim_options = sim_options + local_sim_options
+            tb.add_config(name, generics=generics, sim_options=local_sim_options)
+
         return self._check_for_unconfigured()
 
     def _check_for_unconfigured(self) -> bool:

test/testfw/test_reference.py

 import logging
 from pathlib import Path
 from .test_common import add_sources, TestsBase, dict_merge, \
-                         get_common_modelsim_init_files, get_seed
-from textwrap import dedent
+                         get_seed, OptionsDict
 
 log = logging.getLogger(__name__)
 
@@ -18,28 +17,20 @@ class ReferenceTests(TestsBase):
         sources.append('reference/vunit_reference_wrapper.vhd')
         add_sources(self.lib, sources)
 
-    def create_psl_cov_file_opt(self, name):
-        psl_path = "functional_coverage/coverage_data/psl_cov_reference_{}.json".format(name)
-        psl_flag = "--psl-report={}".format(psl_path)
-        return {"ghdl.sim_flags" : [psl_flag]}
-
     def configure(self) -> bool:
         tb = self.lib.get_test_benches('*reference*')[0]
         default = self.config['default']
 
-        tcl = self.build / 'modelsim_init_reference.tcl'
-        with tcl.open('wt', encoding='utf-8') as f:
-            print(dedent('''\
-                global TCOMP
-                set TCOMP tb_reference_wrapper/i_test
-                '''), file=f)
+        # TODO: is this necessary?
+        tb.scan_tests_from_file(str(self.base / "reference/vunit_reference_wrapper.vhd"))
 
-        init_files = get_common_modelsim_init_files()
-        init_files += [str(tcl)]
+        sim_options = self.get_default_sim_options()
+        # generate & set per-test modelsim tcl file
+        sim_options += self.generate_init_tcl('modelsim_init_reference.tcl', 'tb_reference_wrapper/i_test')
+        sim_options += self.add_modelsim_gui_file(tb, default, 'reference', sim_options['modelsim.init_files.after_load'])
 
         for data_set, cfg in self.config['tests'].items():
             dict_merge(cfg, default)
-            # bm = len_to_matrix(cfg['topology'], cfg['bus_len_v'])
             generics = {
                 'timeout'      : cfg['timeout'],
                 'iterations'   : cfg['iterations'],
@@ -48,13 +39,10 @@ class ReferenceTests(TestsBase):
                 'seed'         : get_seed(cfg),
                 'data_path'    : str(self.build) + '/../' + cfg['data_path'],
             }
+            local_sim_options = OptionsDict()
+            if cfg['psl_coverage']:
+                local_sim_options += self.add_psl_cov('{}.{}'.format(tb.name, data_set))
+            local_sim_options = sim_options + local_sim_options
+            tb.add_config(data_set, generics=generics, sim_options=local_sim_options)
 
-            if (cfg['psl_coverage']):
-                psl_opts = self.create_psl_cov_file_opt(data_set)
-                tb.add_config(data_set, generics=generics, sim_options=psl_opts)
-            else:
-                tb.add_config(data_set, generics=generics)
-
-        tb.set_sim_option("modelsim.init_files.after_load", init_files)
-        self.add_modelsim_gui_file(tb, default, 'reference', init_files)
         return True

test/testfw/test_sanity.py

 import logging
 from textwrap import dedent
 from .test_common import TestsBase, add_sources, dict_merge, vhdl_serialize, \
-                         get_seed, get_common_modelsim_init_files
+                         get_seed, OptionsDict
 
 log = logging.getLogger(__name__)
 
 
-def len_to_matrix(topology, bus_len):
-    l = bus_len
-    if topology == 'bus':
-        bm = [[0.0,            l[1],           l[1]+l[2],      l[1]+l[2]+l[3]],
-              [l[1],           0.0,            l[2],           l[2]+l[3]],
-              [l[1]+l[2],      l[2],           0.0,            l[3]],
-              [l[1]+l[2]+l[3], l[2]+l[3],      l[3],           0.0]]
-    elif topology == 'star':
-        bm = [[0.0,            l[1]+l[2],      l[1]+l[3],      l[1]+l[4]],
-              [l[1]+l[2],      0.0,            l[2]+l[3],      l[2]+l[4]],
-              [l[1]+l[3],      l[2]+l[3],      0.0,            l[3]+l[4]],
-              [l[1]+l[4],      l[2]+l[4],      l[3]+l[4],      0.0]]
-    elif topology == 'tree':
-        bm = [[0.0,            l[1]+l[2],      l[1]+l[3]+l[5], l[1]+l[4]+l[5]],
-              [l[1]+l[2],      0.0,            l[2]+l[3]+l[5], l[2]+l[4]+l[5]],
-              [l[1]+l[3]+l[5], l[2]+l[3]+l[5], 0.0,            l[3]+l[4]],
-              [l[1]+l[4]+l[5], l[2]+l[4]+l[5], l[3]+l[4],      0.0]]
-    elif topology == 'ring':
-        bm = [[0.0, min(l[1], l[2]+l[3]+l[4]), min(l[1]+l[2],l[3]+l[4]), min(l[4],l[1]+l[2]+l[3])],
-              [min(l[1], l[2]+l[3]+l[4]), 0.0, min(l[2], l[1]+l[3]+l[4]), min(l[2]+l[3],l[1]+l[4])],
-              [min(l[1]+l[2],l[3]+l[4]), min(l[2], l[1]+l[3]+l[4]), 0.0, min(l[3],l[1]+l[2]+l[4])],
-              [min(l[4],l[1]+l[2]+l[3]), min(l[2]+l[3],l[1]+l[4]), min(l[3],l[1]+l[2]+l[4]), 0.0]]
-    elif topology == 'custom':
-        bm = [[0.0,  l[1], l[2], l[3]],
-              [l[1], 0.0,  l[4], l[5]],
-              [l[2], l[4], 0.0,  l[6]],
-              [l[3], l[6], l[6], 0.0]]
-    else:
-        raise ValueError("Invalid bus topology.")
-
-    return bm
-
-
 class SanityTests(TestsBase):
     def add_sources(self):
         add_sources(self.lib, ['sanity/**/*.vhd'])
@@ -48,25 +15,14 @@ class SanityTests(TestsBase):
         valid_name = valid_name.replace("/", "_")
         return valid_name
 
-    def create_psl_cov_file_opt(self, name):
-        test_name = "psl_cov_sanity_{}.json".format(self.format_valid_test_name(name))
-        psl_path = "functional_coverage/coverage_data/{}".format(test_name)
-        psl_flag = "--psl-report={}".format(psl_path)
-        return {"ghdl.sim_flags" : [psl_flag]}
-
     def configure(self):
         tb = self.lib.get_test_benches('*tb_sanity')[0]
         default = self.config['default']
 
-        tcl = self.build / 'modelsim_init_sanity.tcl'
-        with tcl.open('wt', encoding='utf-8') as f:
-            print(dedent('''\
-                global TCOMP
-                set TCOMP tb_sanity
-                '''), file=f)
-
-        init_files = get_common_modelsim_init_files()
-        init_files += [str(tcl)]
+        sim_options = self.get_default_sim_options()
+        # generate & set per-test modelsim tcl file
+        sim_options += self.generate_init_tcl('modelsim_init_sanity.tcl', 'tb_sanity')
+        sim_options += self.add_modelsim_gui_file(tb, default, 'sanity', sim_options['modelsim.init_files.after_load'])
 
         for name, cfg in self.config['tests'].items():
             if 'wave' in cfg:
@@ -74,7 +30,6 @@ class SanityTests(TestsBase):
                          ' (set it in default instead)'.format(name))
 
             dict_merge(cfg, default)
-            # bm = len_to_matrix(cfg['topology'], cfg['bus_len_v'])
             generics = {
                 'timeout'      : cfg['timeout'],
                 'iterations'   : cfg['iterations'],
@@ -94,13 +49,9 @@ class SanityTests(TestsBase):
                 'gauss_iter'   : vhdl_serialize(cfg['gauss_iter']),
             }
 
-            sanity_cfg_name = name.replace(" ", "_").replace("/", "_").strip('"')
-
+            local_sim_options = OptionsDict()
             if cfg['psl_coverage']:
-                psl_opts = self.create_psl_cov_file_opt(name)
-                tb.add_config(name, generics=generics, sim_options=psl_opts)
-            else:
-                tb.add_config(name, generics=generics)
-
-        self.add_modelsim_gui_file(tb, default, 'sanity', init_files)
+                local_sim_options += self.add_psl_cov('{}.{}'.format(tb.name, name))
+            local_sim_options = sim_options + local_sim_options
+            tb.add_config(name, generics=generics, sim_options=local_sim_options)
         return True

test/testfw/test_unit.py

@@ -2,7 +2,7 @@ import re
 import logging
 from textwrap import dedent
 from .test_common import add_sources, dict_merge, TestsBase, \
-                         get_common_modelsim_init_files, get_seed
+                         get_seed, OptionsDict
 from pprint import pprint
 
 log = logging.getLogger(__name__)
@@ -13,15 +13,14 @@ class UnitTests(TestsBase):
         add_sources(self.lib, ['unit/**/*.vhd'])
         self._create_wrapper(self.build / "tb_wrappers.vhd")
 
-    def add_psl_cov_file(self, tb, name):
-        psl_path = "functional_coverage/coverage_data/psl_cov_unit_{}.json".format(name)
-        psl_flag = "--psl-report={}".format(psl_path)
-        tb.set_sim_option("ghdl.sim_flags", [psl_flag])
-
     def configure(self) -> bool:
         lib, config, build = self.lib, self.config, self.build
         default = config['default']
         unit_tests = lib.get_test_benches('*_unit_test')
+
+        for ut in unit_tests:
+            ut.scan_tests_from_file(str(build / "../unit/vunittb_wrapper.vhd"))
+
         for name, cfg in config['tests'].items():
             dict_merge(cfg, default)
             tb = lib.get_test_benches('*tb_{}_unit_test'.format(name),
@@ -39,19 +38,15 @@ class UnitTests(TestsBase):
             tb.set_generic('error_tol', cfg['error_tolerance'])
             tb.set_generic('seed', get_seed(cfg))
 
+            sim_options = self.get_default_sim_options()
             # generate & set per-test modelsim tcl file
-            tcl = build / 'modelsim_init_{}.tcl'.format(name)
-            with tcl.open('wt', encoding='utf-8') as f:
-                print(dedent('''\
-                    global TCOMP
-                    set TCOMP tb_{}_unit_test/tb/i_test
-                    '''.format(name)), file=f)
-            init_files = get_common_modelsim_init_files()
-            init_files += [str(tcl)]
-            tb.set_sim_option("modelsim.init_files.after_load", init_files)
-            if (cfg['psl_coverage']):
-                self.add_psl_cov_file(tb, name)
-            self.add_modelsim_gui_file(tb, cfg, name, tcl_init_files=init_files)
+            sim_options += self.generate_init_tcl('modelsim_init_{}.tcl'.format(name), 'tb_{}_unit_test/tb/i_test'.format(name))
+            sim_options += self.add_modelsim_gui_file(tb, cfg, name, sim_options['modelsim.init_files.after_load'])
+
+            if cfg['psl_coverage']:
+                sim_options += self.add_psl_cov(tb.name)
+            self.set_sim_options(tb, sim_options)
+
         return self._check_for_unconfigured()
 
     def _check_for_unconfigured(self) -> bool: