diff --git a/Makefile b/Makefile index 7aba09569..087204634 100644 --- a/Makefile +++ b/Makefile @@ -17,9 +17,7 @@ clean: clean-docs doc-srcs: $(GENERATED_DOC_SRCS) docs: doc-srcs - @if mkdocs build | grep -q "ERROR"; then \ - exit 1; \ - fi + mkdocs build clean-docs: rm -rf $(GENERATED_DOCS_DIR) diff --git a/apt-requirements.txt b/apt-requirements.txt index 5bb7b560d..15f12e8b7 100644 --- a/apt-requirements.txt +++ b/apt-requirements.txt @@ -6,8 +6,4 @@ clang-format device-tree-compiler graphviz -python3 -python3-pip -python3-setuptools -python3-wheel tar diff --git a/docs/requirements.txt b/docs/requirements.txt index 6a766858d..e913931e3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,7 +4,8 @@ # Keep sorted. mkdocs -# Last version compatible with python-3.6 (default on Ubuntu 18.04) -mkdocs-material <= 8.2.11 +mkdocs-material mkdocs-include-markdown-plugin -mkdocs-macros-plugin \ No newline at end of file +mkdocs-macros-plugin +mkdocstrings +mkdocstrings-python diff --git a/docs/rm/sim/Simulation.md b/docs/rm/sim/Simulation.md new file mode 100644 index 000000000..6671fb590 --- /dev/null +++ b/docs/rm/sim/Simulation.md @@ -0,0 +1 @@ +::: Simulation diff --git a/docs/rm/sim/Simulator.md b/docs/rm/sim/Simulator.md new file mode 100644 index 000000000..56f03482d --- /dev/null +++ b/docs/rm/sim/Simulator.md @@ -0,0 +1 @@ +::: Simulator diff --git a/docs/rm/sim/sim_utils.md b/docs/rm/sim/sim_utils.md new file mode 100644 index 000000000..876e5fac4 --- /dev/null +++ b/docs/rm/sim/sim_utils.md @@ -0,0 +1 @@ +::: sim_utils \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 3f9595b0a..70d213601 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,10 @@ markdown_extensions: emoji_generator: !!python/name:materialx.emoji.to_svg plugins: - include-markdown + - mkdocstrings: + handlers: + python: + paths: [util/sim] - macros: on_error_fail: true use_directory_urls: false @@ -49,10 +53,15 @@ nav: - Custom Instructions: rm/custom_instructions.md # - Solder: rm/solder.md - Software: - - Pages: runtime/Pages/index.md - - Files: runtime/Files/index.md - - Classes: runtime/Classes/index.md - - Examples: runtime/Examples/index.md - - Modules: runtime/Modules/index.md - - Namespaces: runtime/Namespaces/index.md + - Simulation Utilities: + - sim_utils: rm/sim/sim_utils.md + - rm/sim/Simulation.md + - rm/sim/Simulator.md + - Snitch Runtime: + - Pages: runtime/Pages/index.md + - Files: runtime/Files/index.md + - Classes: runtime/Classes/index.md + - Examples: runtime/Examples/index.md + - Modules: runtime/Modules/index.md + - Namespaces: runtime/Namespaces/index.md - Publications: publications.md diff --git a/target/common/common.mk b/target/common/common.mk index 4cdf731f1..0cf03c463 100644 --- a/target/common/common.mk +++ b/target/common/common.mk @@ -2,6 +2,10 @@ # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 +# Makefile invocation +DEBUG ?= OFF # ON to turn on wave logging + +# Directories LOGS_DIR ?= logs TB_DIR ?= $(SNITCH_ROOT)/target/common/test UTIL_DIR ?= $(SNITCH_ROOT)/util @@ -41,7 +45,13 @@ SED_SRCS := sed -e ${MATCH_END} -e ${MATCH_BGN} VSIM_BENDER += -t test -t rtl -t simulation -t vsim VSIM_SOURCES = $(shell ${BENDER} script flist ${VSIM_BENDER} | ${SED_SRCS}) VSIM_BUILDDIR ?= work-vsim +VSIM_FLAGS += -t 1ps +ifeq ($(DEBUG), ON) +VSIM_FLAGS += -do "log -r /*; run -a" VOPT_FLAGS = +acc +else +VSIM_FLAGS += -do "run -a" +endif # VCS_BUILDDIR should to be the same as the `DEFAULT : ./work-vcs` # in target/snitch_cluster/synopsys_sim.setup diff --git a/target/snitch_cluster/Makefile b/target/snitch_cluster/Makefile index 49223e8f2..037621213 100644 --- a/target/snitch_cluster/Makefile +++ b/target/snitch_cluster/Makefile @@ -9,7 +9,7 @@ # Makefile invocation # ####################### -DEBUG ?= OFF # ON to turn on debugging symbols +DEBUG ?= OFF # ON to turn on debugging symbols and wave logging CFG_OVERRIDE ?= # Override default config file SELECT_RUNTIME ?= # Select snRuntime implementation: "banshee" or "rtl" (default) @@ -68,8 +68,6 @@ QUESTA_64BIT = -64 VLOG_64BIT = -64 VSIM_FLAGS += ${QUESTA_64BIT} -VSIM_FLAGS += -t 1ps -VSIM_FLAGS += -do "log -r /*; run -a" VLOG_FLAGS += -svinputport=compat VLOG_FLAGS += -override_timescale 1ns/1ps diff --git a/target/snitch_cluster/run.py b/target/snitch_cluster/run.py index 334ef2479..bef478ef7 100755 --- a/target/snitch_cluster/run.py +++ b/target/snitch_cluster/run.py @@ -24,12 +24,12 @@ def main(): args = parser('vsim', SIMULATORS.keys()).parse_args() - simulations = get_simulations(args.testlist, SIMULATORS[args.simulator]) + simulations = get_simulations(args.testlist, SIMULATORS[args.simulator], args.run_dir) return run_simulations(simulations, n_procs=args.n_procs, - run_dir=Path(args.run_dir), dry_run=args.dry_run, - early_exit=args.early_exit) + early_exit=args.early_exit, + verbose=args.verbose) if __name__ == '__main__': diff --git a/target/snitch_cluster/sw/run.yaml b/target/snitch_cluster/sw/run.yaml index c5424cb1d..ce241a8d4 100644 --- a/target/snitch_cluster/sw/run.yaml +++ b/target/snitch_cluster/sw/run.yaml @@ -68,7 +68,7 @@ runs: - elf: tests/build/varargs_2.elf - elf: tests/build/zero_mem.elf - elf: tests/build/non_null_exitcode.elf - exit_code: 14 + retcode: 14 - elf: apps/blas/axpy/build/axpy.elf cmd: [../../../sw/blas/axpy/verify.py, "${sim_bin}", "${elf}"] - elf: apps/blas/gemm/build/gemm.elf diff --git a/util/container/Dockerfile b/util/container/Dockerfile index 9935cf862..d917a6790 100644 --- a/util/container/Dockerfile +++ b/util/container/Dockerfile @@ -7,7 +7,11 @@ # 1. Stage FROM ubuntu:18.04 AS builder ARG CMAKE_VERSION=3.19.4 +ARG PYTHON_VERSION=3.9.12 +# Run dpkg without interactive dialogue +ARG DEBIAN_FRONTEND=noninteractive +# Install APT requirements COPY apt-requirements.txt /tmp/apt-requirements.txt RUN apt-get update && \ sed 's/#.*//' /tmp/apt-requirements.txt \ @@ -20,8 +24,26 @@ RUN apt-get update && \ lsb-release \ software-properties-common \ unzip \ - wget \ - zlib1g-dev + wget +# Required to install Python +RUN apt-get update && apt-get install -y \ + zlib1g-dev \ + libreadline-gplv2-dev \ + libncursesw5-dev \ + libssl-dev \ + libsqlite3-dev \ + tk-dev \ + libgdbm-dev \ + libc6-dev \ + libbz2-dev \ + libffi-dev + +# Install Python +RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz +RUN tar xzf Python-${PYTHON_VERSION}.tgz +RUN cd Python-${PYTHON_VERSION} && \ + ./configure --enable-optimizations --prefix=/opt/python/ && \ + make install -j # Build Rust tools RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -37,6 +59,7 @@ RUN wget https://apt.llvm.org/llvm.sh RUN chmod +x llvm.sh RUN ./llvm.sh 12 +# Change working directory WORKDIR /tools # Install a newer version of cmake (we need this for banshee) @@ -73,9 +96,11 @@ RUN apt-get update && \ sed 's/#.*//' /tmp/apt-requirements.txt \ | xargs apt-get install -y && \ apt-get install -y --no-install-recommends \ + ca-certificates \ gnupg2 \ curl \ wget \ + build-essential \ git && \ apt-get clean ; \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* @@ -88,12 +113,6 @@ RUN echo 'deb http://download.opensuse.org/repositories/home:/phiwag:/edatools/x rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* ENV VLT_ROOT "/usr/share/verilator" -# Install Python requirements -COPY python-requirements.txt /tmp/python-requirements.txt -COPY docs/requirements.txt /tmp/docs/requirements.txt -COPY sw/dnn/requirements.txt /tmp/sw/dnn/requirements.txt -RUN pip3 install -r /tmp/python-requirements.txt - # Get the precompiled LLVM toolchain RUN latest_tag=`curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/pulp-platform/llvm-project/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` && \ echo "SNITCH_LLVM_VERSION=${SNITCH_LLVM_VERSION} LLVM_TAR=${LLVM_TAR} latest_tag=${latest_tag}" && \ @@ -120,6 +139,17 @@ RUN apt-get update && apt-get install software-properties-common -y && \ # Copy artifacts from stage 1. COPY --from=builder /root/.cargo/bin/bender bin/ COPY --from=builder /root/.cargo/bin/banshee bin/ +COPY --from=builder /opt/python /opt/python + +# Create and activate virtual environment +ENV VIRTUAL_ENV "/root/.venvs/snitch_cluster" +RUN /opt/python/bin/python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +# Install Python requirements +COPY python-requirements.txt /tmp/python-requirements.txt +COPY docs/requirements.txt /tmp/docs/requirements.txt +COPY sw/dnn/requirements.txt /tmp/sw/dnn/requirements.txt +RUN pip install -r /tmp/python-requirements.txt # Set locale to UTF-8, required because Python 3.6 defaults on ASCII encoding. # See https://click.palletsprojects.com/en/8.1.x/unicode-support/ diff --git a/util/sim/Simulation.py b/util/sim/Simulation.py index dc79ca3bd..3cc219389 100644 --- a/util/sim/Simulation.py +++ b/util/sim/Simulation.py @@ -13,50 +13,100 @@ class Simulation(object): + """Provides a common interface to manage simulations.""" LOG_FILE = 'sim.txt' - def __init__(self, elf=None): + def __init__(self, elf=None, dry_run=False, retcode=0, run_dir=None): + """Constructor for the Simulation class. + + A Simulation object is defined at a minimum by a software + binary to be simulated on the desired hardware. The hardware is + implicitly determined by the simulation command. + + Arguments: + elf: The software binary to simulate. + run_dir: The directory where to launch the simulation + command. If none is passed, the current working + directory is assumed. + dry_run: A preview of the simulation command will be + displayed without actually launching the simulation. + """ self.elf = elf + self.dry_run = dry_run + self.run_dir = run_dir if run_dir is not None else Path.cwd() self.testname = Path(self.elf).stem self.cmd = [] self.log = None self.process = None + self.expected_retcode = int(retcode) - def launch(self, run_dir=None, dry_run=False): - # Default to current working directory as simulation directory - if not run_dir: - run_dir = Path.cwd() + def launch(self, dry_run=None): + """Launch the simulation. + + Launch the simulation by invoking the command stored in the + `cmd` attribute of the class. Subclasses are required to define + a non-empty `cmd` attribute prior to invoking this method. + + Arguments: + dry_run: A preview of the simulation command is displayed + without actually launching the simulation. + """ + # Override dry_run setting at launch time + if dry_run is not None: + self.dry_run = dry_run # Print launch message and simulation command cprint(f'Run test {colored(self.elf, "cyan")}', attrs=["bold"]) cmd_string = ' '.join(self.cmd) - print(f'$ {cmd_string}', flush=True) + print(f'[{self.run_dir}]$ {cmd_string}', flush=True) # Launch simulation if not doing a dry run - if not dry_run: + if not self.dry_run: # Create run directory and log file - os.makedirs(run_dir, exist_ok=True) - self.log = run_dir / self.LOG_FILE + os.makedirs(self.run_dir, exist_ok=True) + self.log = self.run_dir / self.LOG_FILE # Launch simulation subprocess with open(self.log, 'w') as f: self.process = subprocess.Popen(self.cmd, stdout=f, stderr=subprocess.STDOUT, - cwd=run_dir, universal_newlines=True) + cwd=self.run_dir, universal_newlines=True) def completed(self): - if self.process: + """Return whether the simulation completed.""" + if self.dry_run: + return True + elif self.process: return self.process.poll() is not None else: return False + def get_retcode(self): + """Get the return code of the simulation.""" + if self.dry_run: + return 0 + else: + if self.process: + return int(self.process.returncode) + def successful(self): - return None + """Return whether the simulation was successful.""" + actual_retcode = self.get_retcode() + if actual_retcode is not None: + return int(actual_retcode) == int(self.expected_retcode) + else: + return False def print_log(self): + """Print a log of the simulation to stdout.""" with open(self.log, 'r') as f: print(f.read()) def print_status(self): + """Print a status message to stdout. + + The status message reports whether the test is still running + or, if it completed, whether it was successful or failed. + """ if self.completed(): if self.successful(): cprint(f'{self.elf} test passed', 'green', attrs=['bold'], flush=True) @@ -66,43 +116,43 @@ def print_status(self): cprint(f'{self.elf} test running', 'black', flush=True) -class BistSimulation(Simulation): - - def __init__(self, elf=None, retcode=0): - super().__init__(elf) - self.expected_retcode = retcode - self.actual_retcode = None - - def get_retcode(self): - return None - - def successful(self): - # Simulation is successful if it returned a return code, and - # the return code matches the expected value - self.actual_retcode = self.get_retcode() - if self.actual_retcode is not None: - return int(self.actual_retcode) == int(self.expected_retcode) - else: - return False +class RTLSimulation(Simulation): + """A simulation run on an RTL simulator. + An RTL simulation is launched through a simulation binary built + in advance from some RTL design. + """ -class RTLSimulation(BistSimulation): + def __init__(self, sim_bin=None, **kwargs): + """Constructor for the RTLSimulation class. - def __init__(self, elf=None, retcode=0, sim_bin=None): - super().__init__(elf, retcode) + Arguments: + sim_bin: The simulation binary. + kwargs: Arguments passed to the base class constructor. + """ + super().__init__(**kwargs) self.cmd = [str(sim_bin), str(self.elf)] class VerilatorSimulation(RTLSimulation): + """An RTL simulation running on Verilator. + + The return code of the simulation is returned directly as the + return code of the command launching the simulation. + """ def get_retcode(self): return self.process.returncode class QuestaVCSSimulation(RTLSimulation): + """An RTL simulation running on QuestaSim or VCS. - def get_retcode(self): + QuestaSim and VCS print out the simulation return code in the + simulation log. This is parsed to extract the return code. + """ + def get_retcode(self): # Extract the application's return code from the simulation log with open(self.log, 'r') as f: for line in f.readlines(): @@ -114,7 +164,7 @@ def get_retcode(self): regex_fail = r'\[FAILURE\] Finished with exit code\s+(\d+)' match = re.search(regex_fail, line) if match: - return match.group(1) + return int(match.group(1)) def successful(self): # Check that simulation return code matches expected value (in super class) @@ -127,38 +177,66 @@ def successful(self): class QuestaSimulation(QuestaVCSSimulation): + """An RTL simulation running on QuestaSim.""" - def __init__(self, elf=None, retcode=0, sim_bin=None): - super().__init__(elf, retcode, sim_bin) + def __init__(self, **kwargs): + super().__init__(**kwargs) self.cmd += ['', '-batch'] class VCSSimulation(QuestaVCSSimulation): + """An RTL simulation running on VCS.""" pass -class BansheeSimulation(BistSimulation): +class BansheeSimulation(Simulation): + """A simulation running on Banshee. + + The return code of the simulation is returned directly as the + return code of the command launching the simulation. + """ + + def __init__(self, banshee_cfg=None, **kwargs): + """Constructor for the BansheeSimulation class. - def __init__(self, elf=None, retcode=0, banshee_cfg=None): - super().__init__(elf, retcode) + Arguments: + banshee_cfg: A Banshee config file. + kwargs: Arguments passed to the base class constructor. + """ + super().__init__(**kwargs) self.cmd = ['banshee', '--no-opt-llvm', '--no-opt-jit', '--configuration', str(banshee_cfg), '--trace', str(self.elf)] - def get_retcode(self): - return self.process.returncode - class CustomSimulation(Simulation): - - def __init__(self, elf=None, sim_bin=None, cmd=None): - super().__init__(elf) - self.dynamic_args = {'sim_bin': str(sim_bin), 'elf': str(elf)} + """A simulation which is run through a custom command. + + The custom command generally invokes an RTL simulator binary behind + the scenes and executes some additional verification logic after + the end of the simulation. + + Custom simulations are considered unsuccessful if the return code + of the custom command is non-null. As a custom command can + implement any verification logic, there is no reason to implement + any additional logic here. + """ + + def __init__(self, sim_bin=None, cmd=None, **kwargs): + """Constructor for the CustomSimulation class. + + Arguments: + sim_bin: The simulation binary. + cmd: The custom command used to launch the simulation. + kwargs: Arguments passed to the base class constructor. + """ + super().__init__(**kwargs) + self.dynamic_args = { + 'sim_bin': str(sim_bin), + 'elf': str(self.elf), + 'run_dir': str(self.run_dir) + } self.cmd = cmd - def launch(self, run_dir=None, dry_run=False): - self.dynamic_args['run_dir'] = str(run_dir) + def launch(self, **kwargs): self.cmd = [Template(arg).render(**self.dynamic_args) for arg in self.cmd] - super().launch(run_dir, dry_run) - - def successful(self): - return self.process.returncode == 0 + super().launch(**kwargs) diff --git a/util/sim/Simulator.py b/util/sim/Simulator.py index a31207feb..3d3090573 100644 --- a/util/sim/Simulator.py +++ b/util/sim/Simulator.py @@ -9,60 +9,171 @@ class Simulator(object): + """An object capable of constructing Simulation objects. + + A simulator constructs a [Simulation][Simulation.Simulation] object + from a test object, as defined e.g. in a test suite specification + file. + + At minimum, a test is defined by a binary (`elf`) which is to be + simulated and a set of simulators it can be run on. A test could be + defined by a class of its own, but at the moment we assume a test + to be represented by a dictionary with the `elf` and `simulators` + keys at minimum. + """ def __init__(self, name, simulation_cls): + """Constructor for the Simulator class. + + A simulator must be identifiable by a unique identifier string + and construct at least one type of + [Simulation][Simulation.Simulation] object. + + Arguments: + name: The unique identifier of the simulator. + simulation_cls: One type of + [Simulation][Simulation.Simulation] object the + simulator can construct. + """ self.name = name self.simulation_cls = simulation_cls def supports(self, test): + """Check whether a certain test is supported by the simulator. + + Arguments: + test: The test to check. + """ return 'simulators' not in test or self.name in test['simulators'] - def get_simulation(self, test): - return self.simulation_cls(test['elf']) + def get_simulation(self, test, simulation_cls=None, **kwargs): + """Construct a Simulation object from the specified test. + + Arguments: + test: The test for which a Simulation object must be + constructed. + simulation_cls: Create a simulation instance of this + Simulation subclass. Use `self.simulation_cls` by + default. + """ + kwargs.update({key: test[key] for key in ['elf', 'run_dir', 'retcode'] if key in test}) + if simulation_cls is not None: + return simulation_cls(**kwargs) + else: + return self.simulation_cls(**kwargs) class RTLSimulator(Simulator): - - def __init__(self, name, simulation_cls, binary): - super().__init__(name, simulation_cls) + """Base class for RTL simulators. + + An RTL simulator requires a simulation binary built from an RTL + design to launch a simulation. + + A test may need to be run with a custom command, itself invoking + the simulation binary behind the scenes, e.g. for verification + purposes. Such a test carries the custom command (a list of args) + under the `cmd` key. In such case, the RTL simulator constructs a + [CustomSimulation][Simulation.CustomSimulation] object from the + given test, with the custom command and simulation binary. + """ + + def __init__(self, binary, **kwargs): + """Constructor for the RTLSimulator class. + + Arguments: + binary: The simulation binary. + kwargs: Arguments passed to the base class constructor. + """ + super().__init__(**kwargs) self.binary = binary def get_simulation(self, test): if 'cmd' in test: - return CustomSimulation(test['elf'], self.binary, test['cmd']) + return super().get_simulation( + test, + simulation_cls=CustomSimulation, + sim_bin=self.binary, + cmd=test['cmd']) else: - return self.simulation_cls( - test['elf'], - retcode=test['exit_code'] if 'exit_code' in test else 0, + return super().get_simulation( + test, sim_bin=self.binary ) class VCSSimulator(RTLSimulator): + """VCS simulator + + An [RTL simulator][Simulator.RTLSimulator], identified by the name + `vcs`, tailored to the creation of + [VCS simulations][Simulation.VCSSimulation]. + """ def __init__(self, binary): - super().__init__('vcs', VCSSimulation, binary) + """Constructor for the VCSSimulator class. + + Arguments: + binary: The VCS simulation binary. + """ + super().__init__(binary, name='vcs', simulation_cls=VCSSimulation) class QuestaSimulator(RTLSimulator): + """QuestaSim simulator + + An [RTL simulator][Simulator.RTLSimulator], identified by the name + `vsim`, tailored to the creation of + [QuestaSim simulations][Simulation.QuestaSimulation]. + """ def __init__(self, binary): - super().__init__('vsim', QuestaSimulation, binary) + """Constructor for the QuestaSimulator class. + + Arguments: + binary: The QuestaSim simulation binary. + """ + super().__init__(binary, name='vsim', simulation_cls=QuestaSimulation) class VerilatorSimulator(RTLSimulator): + """Verilator simulator + + An [RTL simulator][Simulator.RTLSimulator], identified by the name + `verilator`, tailored to the creation of + [Verilator simulations][Simulation.VerilatorSimulation]. + """ def __init__(self, binary): - super().__init__('verilator', VerilatorSimulation, binary) + """Constructor for the VerilatorSimulator class. + + Arguments: + binary: The Verilator simulation binary. + """ + super().__init__(binary, name='verilator', simulation_cls=VerilatorSimulation) class BansheeSimulator(Simulator): + """Banshee simulator + + A simulator, identified by the name `banshee`, tailored to the + creation of [Banshee simulations][Simulation.BansheeSimulation]. + """ def __init__(self, cfg): - super().__init__('banshee', BansheeSimulation) + """Constructor for the BansheeSimulator class. + + Arguments: + cfg: A Banshee config file. + """ + super().__init__(name='banshee', simulation_cls=BansheeSimulation) self.cfg = cfg def supports(self, test): + """See base class. + + The Banshee simulator does not support tests carrying a custom + command. + """ supported = super().supports(test) if 'cmd' in test: return False @@ -70,8 +181,7 @@ def supports(self, test): return supported def get_simulation(self, test): - return self.simulation_cls( - test['elf'], - retcode=test['exit_code'] if 'exit_code' in test else 0, + return super().get_simulation( + test, banshee_cfg=self.cfg ) diff --git a/util/sim/sim_utils.py b/util/sim/sim_utils.py index b4749bbf0..371d56b81 100755 --- a/util/sim/sim_utils.py +++ b/util/sim/sim_utils.py @@ -3,6 +3,49 @@ # SPDX-License-Identifier: Apache-2.0 # # Luca Colagrande +"""Convenience functions to set up a Python simulation framework. + +Such a framework enables you to transparently run a software test suite +on any simulator of choice, provided that the latter is supported by +the framework. It can be used in CIs, regression testing or to conduct +systematic evaluation experiments. + +Three interfaces are required to implement a common framework: + +1. a test suite specification interface to specify the software tests +2. a command-line interface used to launch the simulations +3. an interface to the simulators supported by the framework + +The framework can be divided into three components each managing one of +the defined interfaces: + +1. a test suite frontend +2. a command-line frontend +3. a simulation backend + +A fourth component, the core, serves to glue all other components +together. + +The [parser()][sim_utils.parser] function provides a minimum +command-line interface to control the tool. + +The [get_simulations()][sim_utils.get_simulations] function +provides a common means to implement the test suite frontend. At the +input interface it assumes a test suite specification file in YAML +syntax, and returns a list of simulation objects which implement a +common interface to the simulation backend. This interface is defined +by the [Simulation][Simulation.Simulation] class. + +The core logic of the framework is implemented in the +[run_simulations()][sim_utils.run_simulations] function. It takes +the output from [get_simulations()][sim_utils.get_simulations] and +launches the simulations through the interface to the simulation +backend. + +The simulation backend is implemented by the +[Simulation][Simulation.Simulation] and +[Simulator][Simulator.Simulator] classes and their subclasses. +""" import argparse from termcolor import colored, cprint @@ -17,6 +60,18 @@ def parser(default_simulator='vsim', simulator_choices=['vsim']): + """Default command-line parser for Python simulation frameworks. + + Returns a Python `argparse` parser with common options used to + simulate one or multiple binaries on an RTL design. Can be extended + by adding arguments to it. + + Args: + default_simulator: The simulator to be used when none is + specified on the command-line. + simulator_choices: All simulator choices which can be passed on + the command-line. + """ # Argument parsing parser = argparse.ArgumentParser() parser.add_argument( @@ -43,6 +98,10 @@ def parser(default_simulator='vsim', simulator_choices=['vsim']): '--early-exit', action='store_true', help='Exit as soon as any test fails') + parser.add_argument( + '--verbose', + action='store_true', + help='Activate verbose printing') parser.add_argument( '-j', action='store', @@ -57,9 +116,17 @@ def parser(default_simulator='vsim', simulator_choices=['vsim']): return parser -# Checks if a string s represents a valid relative path w.r.t. to a certain base_path and resolves -# it to an absolute path, if this is the case. Otherwise returns the original string. -def resolve_relative_path(base_path, s): +def _resolve_relative_path(base_path, s): + """Resolve a relative path string w.r.t. a ceratin base. + + Checks if an input string represents a valid relative path w.r.t. + to a certain base path and resolves it to an absolute path, if this + is the case. Otherwise returns the original string. + + Args: + s: The input string + base_path: The base path + """ try: base_path = Path(base_path).resolve() # Get the absolute path of the base directory input_path = Path(s) @@ -78,8 +145,24 @@ def resolve_relative_path(base_path, s): return s -# Create simulation objects from a test list file -def get_simulations(testlist, simulator): +def get_simulations(testlist, simulator, run_dir=None): + """Create simulation objects from a test list file. + + Args: + testlist: Path to a test list file. A test list file is a YAML + file describing a set of tests. + simulator: The simulator to use to run the tests. A test run on + a specific simulator defines a simulation. + run_dir: A directory under which all tests should be run. If + provided, a unique subdirectory for each test will be + created under this directory, based on the test name. + + Returns: + A list of `Simulation` objects. The list contains a + `Simulation` object for every test which supports the given + `simulator`. This object defines a simulation of the test on + that particular `simulator`. + """ # Get tests from test list file testlist_path = Path(testlist).absolute() with open(testlist_path, 'r') as f: @@ -88,13 +171,27 @@ def get_simulations(testlist, simulator): for test in tests: test['elf'] = testlist_path.parent / test['elf'] if 'cmd' in test: - test['cmd'] = [resolve_relative_path(testlist_path.parent, arg) for arg in test['cmd']] + test['cmd'] = [_resolve_relative_path(testlist_path.parent, arg) for arg in test['cmd']] # Create simulation object for every test which supports the specified simulator simulations = [simulator.get_simulation(test) for test in tests if simulator.supports(test)] + # Set simulation run directory + if run_dir is not None: + for sim in simulations: + sim.run_dir = Path(run_dir) / sim.testname return simulations def print_summary(failed_sims, early_exit=False, dry_run=False): + """Print a summary of the simulation suite's exit status. + + Args: + failed_sims: A list of failed simulations from the simulation + suite. + early_exit: Whether the simulation suite was configured to + terminate upon the first failing simulation. + dry_run: Whether the simulation suite was launched in dry run + mode. + """ if not dry_run: header = f'==== Test summary {"(early exit)" if early_exit else ""} ====' cprint(header, attrs=['bold']) @@ -104,8 +201,8 @@ def print_summary(failed_sims, early_exit=False, dry_run=False): print(f'{colored("All tests passed!", "green")}') -def terminate_simulations(): - print('Terminating simulations') +def terminate_processes(): + print('Terminate processes') # Get PID and PGID of parent process (current Python script) ppid = os.getpid() pgid = os.getpgid(0) @@ -116,32 +213,54 @@ def terminate_simulations(): os.kill(pid, signal.SIGKILL) -def run_simulations(simulations, n_procs=1, run_dir=None, dry_run=False, early_exit=False): +def get_unique_run_dir(sim, prefix=None): + """Get unique run directory for a simulation. + + If the simulation was already assigned a run directory at creation + time, None is returned. Otherwise, return a unique run directory + based on the testname under an optional prefix directory. + + Args: + sim: The simulation for which the run directory is + requested. + prefix: Get a unique run directory under a directory which + could be common to multiple simulations. We call this + a prefix. By default the current working directory is + assumed as the prefix. + """ + if sim.run_dir is None: + if prefix is None: + prefix = Path.cwd() + return prefix / sim.testname + + +def run_simulations(simulations, n_procs=1, dry_run=None, early_exit=False, + verbose=False): + """Run simulations defined by a list of `Simulation` objects. + + Args: + simulations: A list of `Simulation` objects as returned e.g. by + [sim_utils.get_simulations][]. + + Returns: + The number of failed simulations. + """ # Register SIGTERM handler, used to gracefully terminate all simulation subprocesses - signal.signal(signal.SIGTERM, lambda _, __: terminate_simulations()) + signal.signal(signal.SIGTERM, lambda _, __: terminate_processes()) # Spawn a process for every test, wait for all running tests to terminate and check results running_sims = [] failed_sims = [] early_exit_requested = False - uniquify_run_dir = len(simulations) > 1 try: while (len(simulations) or len(running_sims)) and not early_exit_requested: # If there are still simulations to run and there are less running simulations than # the maximum number of processes allowed in parallel, spawn new simulation if len(simulations) and len(running_sims) < n_procs: running_sims.append(simulations.pop(0)) - # Launch simulation in current working directory, by default - if run_dir is None: - run_dir = Path.cwd() - # Create unique subdirectory for each test under run directory, if multiple tests - if uniquify_run_dir: - unique_run_dir = run_dir / running_sims[-1].testname - else: - unique_run_dir = run_dir - running_sims[-1].launch(run_dir=unique_run_dir, dry_run=dry_run) + running_sims[-1].launch(dry_run=dry_run) # Remove completed sims from running sims list - idcs = [i for i, sim in enumerate(running_sims) if dry_run or sim.completed()] + idcs = [i for i, sim in enumerate(running_sims) if sim.completed()] completed_sims = [running_sims.pop(i) for i in sorted(idcs, reverse=True)] # Check completed sims and report status for sim in completed_sims: @@ -149,7 +268,8 @@ def run_simulations(simulations, n_procs=1, run_dir=None, dry_run=False, early_e sim.print_status() else: failed_sims.append(sim) - sim.print_log() + if verbose: + sim.print_log() sim.print_status() # If in early-exit mode, terminate as soon as any simulation fails if early_exit: @@ -161,7 +281,7 @@ def run_simulations(simulations, n_procs=1, run_dir=None, dry_run=False, early_e # Clean up after early exit if early_exit_requested: - terminate_simulations() + terminate_processes() # Print summary print_summary(failed_sims, early_exit_requested)