From 8839a9cfe86a033e97705fb53758881a1edccb84 Mon Sep 17 00:00:00 2001 From: Luca Colagrande Date: Thu, 7 Dec 2023 15:18:13 +0100 Subject: [PATCH] treewide: Refactor Python simulation framework and fix process leakage (#70) * target: Align with SEPP package recommendations * target: Switch to Questa's three-step flow * test/ipc: Retry on `ferror` * util/sim: Refactor Python simulation framework and fix process leakage * gitlab-ci: Enable colors * target: Remove redundant Make targets --- .github/workflows/ci.yml | 14 +- .gitlab-ci.yml | 31 +-- hw/mem_interface/util/compile.sh | 2 +- hw/mem_interface/util/run_vsim.sh | 2 +- hw/reqrsp_interface/util/compile.sh | 2 +- hw/reqrsp_interface/util/run_vsim.sh | 2 +- hw/snitch_cluster/util/compile.sh | 2 +- hw/snitch_cluster/util/run_vsim.sh | 2 +- hw/snitch_icache/util/compile.sh | 2 +- hw/snitch_icache/util/run_vsim.sh | 2 +- hw/snitch_ssr/util/compile.sh | 2 +- hw/tcdm_interface/util/compile.sh | 2 +- hw/tcdm_interface/util/run_vsim.sh | 2 +- python-requirements.txt | 1 + target/common/common.mk | 30 ++- target/common/test/ipc.cc | 104 +++++----- target/snitch_cluster/.gitignore | 3 +- target/snitch_cluster/Makefile | 20 +- target/snitch_cluster/run.py | 36 ++++ target/snitch_cluster/sw/run.yaml | 4 +- util/container/Dockerfile | 1 + util/sim/Simulation.py | 164 ++++++++++++++++ util/sim/Simulator.py | 77 ++++++++ util/sim/sim_utils.py | 168 ++++++++++++++++ util/sim/simulate.py | 281 --------------------------- 25 files changed, 562 insertions(+), 394 deletions(-) create mode 100755 target/snitch_cluster/run.py create mode 100644 util/sim/Simulation.py create mode 100644 util/sim/Simulator.py create mode 100755 util/sim/sim_utils.py delete mode 100755 util/sim/simulate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8df08c866..f8f87b3f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,13 +43,12 @@ jobs: make -C target/snitch_cluster bin/snitch_cluster.vlt - name: Run Tests working-directory: target/snitch_cluster - run: |- - ../../util/sim/simulate.py sw/run.yaml --simulator verilator -j \ - --verbose + run: | + ./run.py sw/run.yaml --simulator verilator -j - ############################################ + ######################################### # Build SW on Snitch Cluster w/ Banshee # - ############################################ + ######################################### sw-snitch-cluster-banshee: name: Simulate SW on Snitch Cluster w/ Banshee @@ -68,6 +67,5 @@ jobs: env: SNITCH_LOG: info working-directory: target/snitch_cluster - run: |- - ../../util/sim/simulate.py sw/run.yaml --simulator banshee -j \ - --verbose + run: | + ./run.py sw/run.yaml --simulator banshee -j diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 624c2a003..4035071d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,13 +5,17 @@ variables: GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive + # Enable colors in CI terminal + TERM: ansi + FORCE_COLOR: 1 + # Configure environment PYTHON: /usr/local/anaconda3-2022.05/bin/python3 BENDER: bender-0.27.1 CC: gcc-9.2.0 CXX: g++-9.2.0 - VCS: vcs-2020.12 - VERILATOR: verilator-4.110 - QUESTA: questa-2022.3 + VCS_SEPP: vcs-2020.12 + VERILATOR_SEPP: verilator-4.110 + QUESTA_SEPP: questa-2022.3 LLVM_BINROOT: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: /usr/pack/gcc-9.2.0-af/linux-x64/bin/gcc LLVM_SYS_120_PREFIX: /usr/pack/llvm-12.0.1-af @@ -79,8 +83,8 @@ snitch-ip-tests: - tcdm_interface script: - cd hw/$IP - - $QUESTA ./util/compile.sh - - $QUESTA ./util/run_vsim.sh + - ./util/compile.sh + - ./util/run_vsim.sh ######################## # Snitch cluster tests # @@ -89,29 +93,26 @@ snitch-ip-tests: # Verilator snitch-cluster-vlt: needs: [snitch-cluster-sw] - # yamllint disable rule:line-length script: - cd target/snitch_cluster - - $VERILATOR make bin/snitch_cluster.vlt - - $VERILATOR ../../util/sim/simulate.py sw/run.yaml --simulator verilator -j --verbose - # yamllint enable rule:line-length + - make bin/snitch_cluster.vlt + - ./run.py sw/run.yaml --simulator verilator -j --run-dir runs/vlt # VCS snitch-cluster-vcs: needs: [snitch-cluster-sw] script: - cd target/snitch_cluster - - $VCS make bin/snitch_cluster.vcs - - $VCS ../../util/sim/simulate.py sw/run.yaml --simulator vcs -j --verbose + - make bin/snitch_cluster.vcs + - ./run.py sw/run.yaml --simulator vcs -j --run-dir runs/vcs # Questa snitch-cluster-vsim: needs: [snitch-cluster-sw] script: - cd target/snitch_cluster - - $QUESTA make bin/snitch_cluster.vsim - - $QUESTA ../../util/sim/simulate.py sw/run.yaml --simulator vsim -j - --verbose + - make bin/snitch_cluster.vsim + - ./run.py sw/run.yaml --simulator vsim -j --run-dir runs/vsim # Banshee snitch-cluster-banshee: @@ -127,4 +128,4 @@ snitch-cluster-banshee: - cd banshee - cargo install --debug --path . - cd ../target/snitch_cluster - - ../../util/sim/simulate.py sw/run.yaml --simulator banshee -j --verbose + - ./run.py sw/run.yaml --simulator banshee -j --run-dir runs/banshee diff --git a/hw/mem_interface/util/compile.sh b/hw/mem_interface/util/compile.sh index 51814285a..1a3678cfa 100755 --- a/hw/mem_interface/util/compile.sh +++ b/hw/mem_interface/util/compile.sh @@ -17,4 +17,4 @@ $BENDER script vsim -t test \ --vlog-arg="+cover=sbecft" \ > compile.tcl echo 'return 0' >> compile.tcl -$VSIM -c -do 'exit -code [source compile.tcl]' +$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]' diff --git a/hw/mem_interface/util/run_vsim.sh b/hw/mem_interface/util/run_vsim.sh index e30929642..45a6b77e1 100755 --- a/hw/mem_interface/util/run_vsim.sh +++ b/hw/mem_interface/util/run_vsim.sh @@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) [ ! -z "$VSIM" ] || VSIM=vsim call_vsim() { - echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 + echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 grep "Errors: 0," vsim.log } diff --git a/hw/reqrsp_interface/util/compile.sh b/hw/reqrsp_interface/util/compile.sh index 84956acea..af966e202 100755 --- a/hw/reqrsp_interface/util/compile.sh +++ b/hw/reqrsp_interface/util/compile.sh @@ -17,4 +17,4 @@ $(BENDER) script vsim -t test \ --vlog-arg="+cover=sbecft" \ > compile.tcl echo 'return 0' >> compile.tcl -$VSIM -c -do 'exit -code [source compile.tcl]' +$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]' diff --git a/hw/reqrsp_interface/util/run_vsim.sh b/hw/reqrsp_interface/util/run_vsim.sh index e7fe59fb9..9eeee2e14 100755 --- a/hw/reqrsp_interface/util/run_vsim.sh +++ b/hw/reqrsp_interface/util/run_vsim.sh @@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) [ ! -z "$VSIM" ] || VSIM=vsim call_vsim() { - echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 + echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 grep "Errors: 0," vsim.log } diff --git a/hw/snitch_cluster/util/compile.sh b/hw/snitch_cluster/util/compile.sh index 51814285a..1a3678cfa 100755 --- a/hw/snitch_cluster/util/compile.sh +++ b/hw/snitch_cluster/util/compile.sh @@ -17,4 +17,4 @@ $BENDER script vsim -t test \ --vlog-arg="+cover=sbecft" \ > compile.tcl echo 'return 0' >> compile.tcl -$VSIM -c -do 'exit -code [source compile.tcl]' +$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]' diff --git a/hw/snitch_cluster/util/run_vsim.sh b/hw/snitch_cluster/util/run_vsim.sh index e9298efed..00d08aee3 100755 --- a/hw/snitch_cluster/util/run_vsim.sh +++ b/hw/snitch_cluster/util/run_vsim.sh @@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) [ ! -z "$VSIM" ] || VSIM=vsim call_vsim() { - echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 + echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 grep "Errors: 0," vsim.log } diff --git a/hw/snitch_icache/util/compile.sh b/hw/snitch_icache/util/compile.sh index 51814285a..1a3678cfa 100755 --- a/hw/snitch_icache/util/compile.sh +++ b/hw/snitch_icache/util/compile.sh @@ -17,4 +17,4 @@ $BENDER script vsim -t test \ --vlog-arg="+cover=sbecft" \ > compile.tcl echo 'return 0' >> compile.tcl -$VSIM -c -do 'exit -code [source compile.tcl]' +$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]' diff --git a/hw/snitch_icache/util/run_vsim.sh b/hw/snitch_icache/util/run_vsim.sh index 94671daf5..42cc47f94 100755 --- a/hw/snitch_icache/util/run_vsim.sh +++ b/hw/snitch_icache/util/run_vsim.sh @@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) [ ! -z "$VSIM" ] || VSIM=vsim call_vsim() { - echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 + echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 grep "Errors: 0," vsim.log } diff --git a/hw/snitch_ssr/util/compile.sh b/hw/snitch_ssr/util/compile.sh index 84956acea..af966e202 100755 --- a/hw/snitch_ssr/util/compile.sh +++ b/hw/snitch_ssr/util/compile.sh @@ -17,4 +17,4 @@ $(BENDER) script vsim -t test \ --vlog-arg="+cover=sbecft" \ > compile.tcl echo 'return 0' >> compile.tcl -$VSIM -c -do 'exit -code [source compile.tcl]' +$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]' diff --git a/hw/tcdm_interface/util/compile.sh b/hw/tcdm_interface/util/compile.sh index 51814285a..1a3678cfa 100755 --- a/hw/tcdm_interface/util/compile.sh +++ b/hw/tcdm_interface/util/compile.sh @@ -17,4 +17,4 @@ $BENDER script vsim -t test \ --vlog-arg="+cover=sbecft" \ > compile.tcl echo 'return 0' >> compile.tcl -$VSIM -c -do 'exit -code [source compile.tcl]' +$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]' diff --git a/hw/tcdm_interface/util/run_vsim.sh b/hw/tcdm_interface/util/run_vsim.sh index 078ae72a8..6f10155d0 100755 --- a/hw/tcdm_interface/util/run_vsim.sh +++ b/hw/tcdm_interface/util/run_vsim.sh @@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) [ ! -z "$VSIM" ] || VSIM=vsim call_vsim() { - echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 + echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1 grep "Errors: 0," vsim.log } diff --git a/python-requirements.txt b/python-requirements.txt index d426cf140..6db0bf03f 100644 --- a/python-requirements.txt +++ b/python-requirements.txt @@ -19,6 +19,7 @@ pytablewriter termcolor pandas pyelftools +psutil -r docs/requirements.txt -r sw/dnn/requirements.txt diff --git a/target/common/common.mk b/target/common/common.mk index 48b875f76..4cdf731f1 100644 --- a/target/common/common.mk +++ b/target/common/common.mk @@ -6,12 +6,22 @@ LOGS_DIR ?= logs TB_DIR ?= $(SNITCH_ROOT)/target/common/test UTIL_DIR ?= $(SNITCH_ROOT)/util +# SEPP packages +QUESTA_SEPP ?= +VCS_SEPP ?= +VERILATOR_SEPP ?= + # External executables BENDER ?= bender DASM ?= spike-dasm -VLT ?= verilator +VLT ?= $(VERILATOR_SEPP) verilator +VCS ?= $(VCS_SEPP) vcs VERIBLE_FMT ?= verible-verilog-format CLANG_FORMAT ?= clang-format +VSIM ?= $(QUESTA_SEPP) vsim +VOPT ?= $(QUESTA_SEPP) vopt +VLOG ?= $(QUESTA_SEPP) vlog +VLIB ?= $(QUESTA_SEPP) vlib # Internal executables GENTRACE_PY ?= $(UTIL_DIR)/trace/gen_trace.py @@ -21,7 +31,7 @@ PERF_CSV_PY ?= $(UTIL_DIR)/trace/perf_csv.py LAYOUT_EVENTS_PY ?= $(UTIL_DIR)/trace/layout_events.py EVENTVIS_PY ?= $(UTIL_DIR)/trace/eventvis.py -VERILATOR_ROOT ?= $(dir $(shell which $(VLT)))/../share/verilator +VERILATOR_ROOT ?= $(dir $(shell $(VERILATOR_SEPP) which verilator)).. VLT_ROOT ?= ${VERILATOR_ROOT} MATCH_END := '/+incdir+/ s/$$/\/*\/*/' @@ -31,6 +41,7 @@ SED_SRCS := sed -e ${MATCH_END} -e ${MATCH_BGN} VSIM_BENDER += -t test -t rtl -t simulation -t vsim VSIM_SOURCES = $(shell ${BENDER} script flist ${VSIM_BENDER} | ${SED_SRCS}) VSIM_BUILDDIR ?= work-vsim +VOPT_FLAGS = +acc # VCS_BUILDDIR should to be the same as the `DEFAULT : ./work-vcs` # in target/snitch_cluster/synopsys_sim.setup @@ -150,9 +161,14 @@ endef $(VSIM_BUILDDIR): mkdir -p $@ +# Expects vlog/vcom script in $< (e.g. as output by bender) +# Expects the top module name in $1 +# Produces a binary used to run the simulation at the path specified by $@ define QUESTASIM - ${VSIM} -c -do "source $<; quit" | tee $(dir $<)vsim.log - @! grep -P "Errors: [1-9]*," $(dir $<)vsim.log + ${VSIM} -c -do "source $<; quit" | tee $(dir $<)vlog.log + @! grep -P "Errors: [1-9]*," $(dir $<)vlog.log + $(VOPT) $(VOPT_FLAGS) -work $(VSIM_BUILDDIR) $1 -o $(1)_opt | tee $(dir $<)vopt.log + @! grep -P "Errors: [1-9]*," $(dir $<)vopt.log @mkdir -p $(dir $@) @echo "#!/bin/bash" > $@ @echo 'binary=$$(realpath $$1)' >> $@ @@ -160,7 +176,7 @@ define QUESTASIM @echo 'echo $$binary > $(LOGS_DIR)/.rtlbinary' >> $@ @echo '${VSIM} +permissive ${VSIM_FLAGS} $$3 -work ${MKFILE_DIR}/${VSIM_BUILDDIR} -c \ -ldflags "-Wl,-rpath,${FESVR}/lib -L${FESVR}/lib -lfesvr -lutil" \ - $1 +permissive-off ++$$binary ++$$2' >> $@ + $(1)_opt +permissive-off ++$$binary ++$$2' >> $@ @chmod +x $@ @echo "#!/bin/bash" > $@.gui @echo 'binary=$$(realpath $$1)' >> $@.gui @@ -168,7 +184,7 @@ define QUESTASIM @echo 'echo $$binary > $(LOGS_DIR)/.rtlbinary' >> $@.gui @echo '${VSIM} +permissive ${VSIM_FLAGS} -work ${MKFILE_DIR}/${VSIM_BUILDDIR} \ -ldflags "-Wl,-rpath,${FESVR}/lib -L${FESVR}/lib -lfesvr -lutil" \ - $1 +permissive-off ++$$binary ++$$2' >> $@.gui + $(1)_opt +permissive-off ++$$binary ++$$2' >> $@.gui @chmod +x $@.gui endef @@ -179,7 +195,7 @@ $(VCS_BUILDDIR)/compile.sh: mkdir -p $(VCS_BUILDDIR) ${BENDER} script vcs ${VCS_BENDER} --vlog-arg="${VLOGAN_FLAGS}" --vcom-arg="${VHDLAN_FLAGS}" > $@ chmod +x $@ - $@ > $(VCS_BUILDDIR)/compile.log + $(VCS_SEPP) $@ > $(VCS_BUILDDIR)/compile.log ######## # Util # diff --git a/target/common/test/ipc.cc b/target/common/test/ipc.cc index f4074a1ef..09188a7b3 100644 --- a/target/common/test/ipc.cc +++ b/target/common/test/ipc.cc @@ -19,63 +19,65 @@ void* IpcIface::ipc_thread_handle(void* in) { // Handle commands ipc_op_t op; - while (size_t num_messages_read = fread(&op, sizeof(ipc_op_t), 1, tx)) { - if (num_messages_read != 1) { + while (1) { + if (!fread(&op, sizeof(ipc_op_t), 1, tx)) { if (feof(tx)) { printf( "[IPC] All messages read. Closing FIFOs and joining main " "thread.\n"); - } else if (ferror(tx)) { - perror("[IPC] read from tx failed\n"); - } - break; - } - switch (op.opcode) { - case Read: - // Read full blocks until one full block or less left - printf("[IPC] Read from 0x%x len 0x%x ...\n", op.addr, op.len); - for (uint64_t i = op.len; i > IPC_BUF_SIZE; i -= IPC_BUF_SIZE) { - sim::MEM.read(op.addr, IPC_BUF_SIZE, buf_data); - fwrite(buf_data, IPC_BUF_SIZE, 1, rx); - op.addr += IPC_BUF_SIZE; - op.len -= IPC_BUF_SIZE; - } - sim::MEM.read(op.addr, op.len, buf_data); - fwrite(buf_data, op.len, 1, rx); - fflush(rx); - break; - case Write: - // Write full blocks until one full block or less left - printf("[IPC] Write to 0x%x len %d ...\n", op.addr, op.len); - for (uint64_t i = op.len; i > IPC_BUF_SIZE; i -= IPC_BUF_SIZE) { - fread(buf_data, IPC_BUF_SIZE, 1, tx); - sim::MEM.write(op.addr, IPC_BUF_SIZE, buf_data, buf_strb); - op.addr += IPC_BUF_SIZE; - op.len -= IPC_BUF_SIZE; - } - fread(buf_data, op.len, 1, tx); - sim::MEM.write(op.addr, op.len, buf_data, buf_strb); - break; - case Poll: - // Unpack 32b checking mask and expected value from length - uint32_t mask = op.len & 0xFFFFFFFF; - uint32_t expected = (op.len >> 32) & 0xFFFFFFFF; - printf("[IPC] Poll on 0x%x mask 0x%x expected 0x%x ...\n", - op.addr, mask, expected); - uint32_t read; - do { - sim::MEM.read(op.addr, sizeof(uint32_t), - (uint8_t*)(void*)&read); - nanosleep( - (const struct timespec[]){{0, IPC_POLL_PERIOD_NS}}, - NULL); - } while ((read & mask) == (expected & mask)); - // Send back read 32b word - fwrite(&read, sizeof(uint32_t), 1, rx); - fflush(rx); break; + } + } else { + switch (op.opcode) { + case Read: + // Read full blocks until one full block or less left + printf("[IPC] Read from 0x%x len 0x%x ...\n", op.addr, + op.len); + for (uint64_t i = op.len; i > IPC_BUF_SIZE; + i -= IPC_BUF_SIZE) { + sim::MEM.read(op.addr, IPC_BUF_SIZE, buf_data); + fwrite(buf_data, IPC_BUF_SIZE, 1, rx); + op.addr += IPC_BUF_SIZE; + op.len -= IPC_BUF_SIZE; + } + sim::MEM.read(op.addr, op.len, buf_data); + fwrite(buf_data, op.len, 1, rx); + fflush(rx); + break; + case Write: + // Write full blocks until one full block or less left + printf("[IPC] Write to 0x%x len %d ...\n", op.addr, op.len); + for (uint64_t i = op.len; i > IPC_BUF_SIZE; + i -= IPC_BUF_SIZE) { + fread(buf_data, IPC_BUF_SIZE, 1, tx); + sim::MEM.write(op.addr, IPC_BUF_SIZE, buf_data, + buf_strb); + op.addr += IPC_BUF_SIZE; + op.len -= IPC_BUF_SIZE; + } + fread(buf_data, op.len, 1, tx); + sim::MEM.write(op.addr, op.len, buf_data, buf_strb); + break; + case Poll: + // Unpack 32b checking mask and expected value from length + uint32_t mask = op.len & 0xFFFFFFFF; + uint32_t expected = (op.len >> 32) & 0xFFFFFFFF; + printf("[IPC] Poll on 0x%x mask 0x%x expected 0x%x ...\n", + op.addr, mask, expected); + uint32_t read; + do { + sim::MEM.read(op.addr, sizeof(uint32_t), + (uint8_t*)(void*)&read); + nanosleep( + (const struct timespec[]){{0, IPC_POLL_PERIOD_NS}}, + NULL); + } while ((read & mask) == (expected & mask)); + // Send back read 32b word + fwrite(&read, sizeof(uint32_t), 1, rx); + fflush(rx); + break; + } } - printf("[IPC] ... done\n"); } // TX FIFO closed at other end: close both FIFOs and join main thread diff --git a/target/snitch_cluster/.gitignore b/target/snitch_cluster/.gitignore index b7f1de414..f74d9fde4 100644 --- a/target/snitch_cluster/.gitignore +++ b/target/snitch_cluster/.gitignore @@ -6,4 +6,5 @@ /work-vsim/ /work-vlt/ /work-vcs/ -/*.log \ No newline at end of file +/*.log +/runs/ \ No newline at end of file diff --git a/target/snitch_cluster/Makefile b/target/snitch_cluster/Makefile index 7b38bbad6..49223e8f2 100644 --- a/target/snitch_cluster/Makefile +++ b/target/snitch_cluster/Makefile @@ -37,9 +37,6 @@ REGGEN ?= $(shell $(BENDER) path register_interface)/vendor/lowrisc_ope CLUSTER_GEN ?= $(ROOT)/util/clustergen.py CLUSTER_GEN_SRC ?= $(wildcard $(ROOT)/util/clustergen/*.py) -VSIM ?= vsim -VLOG ?= vlog - ######################### # Files and directories # ######################### @@ -72,7 +69,6 @@ VLOG_64BIT = -64 VSIM_FLAGS += ${QUESTA_64BIT} VSIM_FLAGS += -t 1ps -VSIM_FLAGS += -voptargs=+acc VSIM_FLAGS += -do "log -r /*; run -a" VLOG_FLAGS += -svinputport=compat @@ -245,7 +241,7 @@ clean-vsim: clean-work rm -rf bin/snitch_cluster.vsim bin/snitch_cluster.vsim.gui $(VSIM_BUILDDIR) vsim.wlf ${VSIM_BUILDDIR}/compile.vsim.tcl: - vlib $(dir $@) + $(VLIB) $(dir $@) ${BENDER} script vsim ${VSIM_BENDER} --vlog-arg="${VLOG_FLAGS} -work $(dir $@) " > $@ echo '${VLOG} -work $(dir $@) ${TB_CC_SOURCES} ${TB_ASM_SOURCES} -vv -ccflags "$(TB_CC_FLAGS)"' >> $@ echo 'return 0' >> $@ @@ -267,22 +263,10 @@ clean-vcs: clean-work # Build compilation script and compile all sources for VCS simulation bin/snitch_cluster.vcs: ${VCS_SOURCES} ${TB_SRCS} $(TB_CC_SOURCES) $(TB_ASM_SOURCES) $(VCS_BUILDDIR)/compile.sh work/lib/libfesvr.a mkdir -p bin - vcs -Mlib=$(VCS_BUILDDIR) -Mdir=$(VCS_BUILDDIR) -o bin/snitch_cluster.vcs -cc $(CC) -cpp $(CXX) \ + $(VCS) -Mlib=$(VCS_BUILDDIR) -Mdir=$(VCS_BUILDDIR) -o bin/snitch_cluster.vcs -cc $(CC) -cpp $(CXX) \ -assert disable_cover -override_timescale=1ns/1ps -full64 tb_bin $(TB_CC_SOURCES) $(TB_ASM_SOURCES) \ -CFLAGS "$(TB_CC_FLAGS)" -LDFLAGS "-L${FESVR}/lib" -lfesvr -########## -# Traces # -########## - -$(LOGS_DIR)/perf.csv: $(shell (ls $(LOGS_DIR)/trace_hart_*.dasm 2>/dev/null | sed 's/trace_hart/hart/' | sed 's/.dasm/_perf.json/')) \ - $(PERF_CSV_PY) - $(PYTHON) $(PERF_CSV_PY) -o $@ -i $(LOGS_DIR)/hart_*_perf.json - -$(LOGS_DIR)/event.csv: $(shell (ls $(LOGS_DIR)/trace_hart_*.dasm 2>/dev/null | sed 's/trace_hart/hart/' | sed 's/.dasm/_perf.json/')) \ - $(PERF_CSV_PY) - $(PYTHON) $(PERF_CSV_PY) -o $@ -i $(LOGS_DIR)/hart_*_perf.json --filter tstart tend - ######## # Util # ######## diff --git a/target/snitch_cluster/run.py b/target/snitch_cluster/run.py new file mode 100755 index 000000000..334ef2479 --- /dev/null +++ b/target/snitch_cluster/run.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# Copyright 2023 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Luca Colagrande + +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent / '../../util/sim')) +from sim_utils import parser, get_simulations, run_simulations # noqa: E402 +from Simulator import QuestaSimulator, VCSSimulator, VerilatorSimulator, \ + BansheeSimulator # noqa: E402 + + +SIMULATORS = { + 'vsim': QuestaSimulator(Path(__file__).parent.resolve() / 'bin/snitch_cluster.vsim'), + 'vcs': VCSSimulator(Path(__file__).parent.resolve() / 'bin/snitch_cluster.vcs'), + 'verilator': VerilatorSimulator(Path(__file__).parent.resolve() / 'bin/snitch_cluster.vlt'), + 'banshee': BansheeSimulator(Path(__file__).parent.resolve() / 'src/banshee.yaml') +} + + +def main(): + args = parser('vsim', SIMULATORS.keys()).parse_args() + simulations = get_simulations(args.testlist, SIMULATORS[args.simulator]) + return run_simulations(simulations, + n_procs=args.n_procs, + run_dir=Path(args.run_dir), + dry_run=args.dry_run, + early_exit=args.early_exit) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/target/snitch_cluster/sw/run.yaml b/target/snitch_cluster/sw/run.yaml index f25ea7641..c5424cb1d 100644 --- a/target/snitch_cluster/sw/run.yaml +++ b/target/snitch_cluster/sw/run.yaml @@ -70,9 +70,9 @@ runs: - elf: tests/build/non_null_exitcode.elf exit_code: 14 - elf: apps/blas/axpy/build/axpy.elf - cmd: ../../sw/blas/axpy/verify.py {sim_bin} {elf} + cmd: [../../../sw/blas/axpy/verify.py, "${sim_bin}", "${elf}"] - elf: apps/blas/gemm/build/gemm.elf - cmd: ../../sw/blas/gemm/verify.py {sim_bin} {elf} + cmd: [../../../sw/blas/gemm/verify.py, "${sim_bin}", "${elf}"] - elf: apps/dnn/batchnorm/build/batchnorm.elf - elf: apps/dnn/linear/build/linear.elf - elf: apps/dnn/maxpool/build/maxpool.elf diff --git a/util/container/Dockerfile b/util/container/Dockerfile index ea320f325..9935cf862 100644 --- a/util/container/Dockerfile +++ b/util/container/Dockerfile @@ -86,6 +86,7 @@ RUN echo 'deb http://download.opensuse.org/repositories/home:/phiwag:/edatools/x apt-get update && apt-get install -y verilator-${VERILATOR_VERSION} && \ apt-get clean ; \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* +ENV VLT_ROOT "/usr/share/verilator" # Install Python requirements COPY python-requirements.txt /tmp/python-requirements.txt diff --git a/util/sim/Simulation.py b/util/sim/Simulation.py new file mode 100644 index 000000000..dc79ca3bd --- /dev/null +++ b/util/sim/Simulation.py @@ -0,0 +1,164 @@ +# Copyright 2023 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Luca Colagrande + +from termcolor import colored, cprint +from pathlib import Path +import subprocess +import re +import os +from mako.template import Template + + +class Simulation(object): + + LOG_FILE = 'sim.txt' + + def __init__(self, elf=None): + self.elf = elf + self.testname = Path(self.elf).stem + self.cmd = [] + self.log = None + self.process = None + + def launch(self, run_dir=None, dry_run=False): + # Default to current working directory as simulation directory + if not run_dir: + run_dir = Path.cwd() + + # Print launch message and simulation command + cprint(f'Run test {colored(self.elf, "cyan")}', attrs=["bold"]) + cmd_string = ' '.join(self.cmd) + print(f'$ {cmd_string}', flush=True) + + # Launch simulation if not doing a dry run + if not dry_run: + # Create run directory and log file + os.makedirs(run_dir, exist_ok=True) + self.log = run_dir / self.LOG_FILE + # Launch simulation subprocess + with open(self.log, 'w') as f: + self.process = subprocess.Popen(self.cmd, stdout=f, stderr=subprocess.STDOUT, + cwd=run_dir, universal_newlines=True) + + def completed(self): + if self.process: + return self.process.poll() is not None + else: + return False + + def successful(self): + return None + + def print_log(self): + with open(self.log, 'r') as f: + print(f.read()) + + def print_status(self): + if self.completed(): + if self.successful(): + cprint(f'{self.elf} test passed', 'green', attrs=['bold'], flush=True) + else: + cprint(f'{self.elf} test failed', 'red', attrs=['bold'], flush=True) + else: + cprint(f'{self.elf} test running', 'black', flush=True) + + +class BistSimulation(Simulation): + + def __init__(self, elf=None, retcode=0): + super().__init__(elf) + self.expected_retcode = retcode + self.actual_retcode = None + + def get_retcode(self): + return None + + def successful(self): + # Simulation is successful if it returned a return code, and + # the return code matches the expected value + self.actual_retcode = self.get_retcode() + if self.actual_retcode is not None: + return int(self.actual_retcode) == int(self.expected_retcode) + else: + return False + + +class RTLSimulation(BistSimulation): + + def __init__(self, elf=None, retcode=0, sim_bin=None): + super().__init__(elf, retcode) + self.cmd = [str(sim_bin), str(self.elf)] + + +class VerilatorSimulation(RTLSimulation): + + def get_retcode(self): + return self.process.returncode + + +class QuestaVCSSimulation(RTLSimulation): + + def get_retcode(self): + + # Extract the application's return code from the simulation log + with open(self.log, 'r') as f: + for line in f.readlines(): + regex_success = r'\[SUCCESS\] Program finished successfully' + match_success = re.search(regex_success, line) + if match_success: + return 0 + else: + regex_fail = r'\[FAILURE\] Finished with exit code\s+(\d+)' + match = re.search(regex_fail, line) + if match: + return match.group(1) + + def successful(self): + # Check that simulation return code matches expected value (in super class) + # and that the simulation process terminated correctly + success = super().successful() + if self.process.returncode != 0: + return False + else: + return success + + +class QuestaSimulation(QuestaVCSSimulation): + + def __init__(self, elf=None, retcode=0, sim_bin=None): + super().__init__(elf, retcode, sim_bin) + self.cmd += ['', '-batch'] + + +class VCSSimulation(QuestaVCSSimulation): + pass + + +class BansheeSimulation(BistSimulation): + + def __init__(self, elf=None, retcode=0, banshee_cfg=None): + super().__init__(elf, retcode) + self.cmd = ['banshee', '--no-opt-llvm', '--no-opt-jit', '--configuration', + str(banshee_cfg), '--trace', str(self.elf)] + + def get_retcode(self): + return self.process.returncode + + +class CustomSimulation(Simulation): + + def __init__(self, elf=None, sim_bin=None, cmd=None): + super().__init__(elf) + self.dynamic_args = {'sim_bin': str(sim_bin), 'elf': str(elf)} + self.cmd = cmd + + def launch(self, run_dir=None, dry_run=False): + self.dynamic_args['run_dir'] = str(run_dir) + self.cmd = [Template(arg).render(**self.dynamic_args) for arg in self.cmd] + super().launch(run_dir, dry_run) + + def successful(self): + return self.process.returncode == 0 diff --git a/util/sim/Simulator.py b/util/sim/Simulator.py new file mode 100644 index 000000000..a31207feb --- /dev/null +++ b/util/sim/Simulator.py @@ -0,0 +1,77 @@ +# Copyright 2023 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Luca Colagrande + +from Simulation import QuestaSimulation, VCSSimulation, VerilatorSimulation, BansheeSimulation, \ + CustomSimulation + + +class Simulator(object): + + def __init__(self, name, simulation_cls): + self.name = name + self.simulation_cls = simulation_cls + + def supports(self, test): + return 'simulators' not in test or self.name in test['simulators'] + + def get_simulation(self, test): + return self.simulation_cls(test['elf']) + + +class RTLSimulator(Simulator): + + def __init__(self, name, simulation_cls, binary): + super().__init__(name, simulation_cls) + self.binary = binary + + def get_simulation(self, test): + if 'cmd' in test: + return CustomSimulation(test['elf'], self.binary, test['cmd']) + else: + return self.simulation_cls( + test['elf'], + retcode=test['exit_code'] if 'exit_code' in test else 0, + sim_bin=self.binary + ) + + +class VCSSimulator(RTLSimulator): + + def __init__(self, binary): + super().__init__('vcs', VCSSimulation, binary) + + +class QuestaSimulator(RTLSimulator): + + def __init__(self, binary): + super().__init__('vsim', QuestaSimulation, binary) + + +class VerilatorSimulator(RTLSimulator): + + def __init__(self, binary): + super().__init__('verilator', VerilatorSimulation, binary) + + +class BansheeSimulator(Simulator): + + def __init__(self, cfg): + super().__init__('banshee', BansheeSimulation) + self.cfg = cfg + + def supports(self, test): + supported = super().supports(test) + if 'cmd' in test: + return False + else: + return supported + + def get_simulation(self, test): + return self.simulation_cls( + test['elf'], + retcode=test['exit_code'] if 'exit_code' in test else 0, + banshee_cfg=self.cfg + ) diff --git a/util/sim/sim_utils.py b/util/sim/sim_utils.py new file mode 100755 index 000000000..b4749bbf0 --- /dev/null +++ b/util/sim/sim_utils.py @@ -0,0 +1,168 @@ +# Copyright 2023 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Luca Colagrande + +import argparse +from termcolor import colored, cprint +from pathlib import Path +import os +import time +import yaml +import signal +import psutil + +POLL_PERIOD = 0.2 + + +def parser(default_simulator='vsim', simulator_choices=['vsim']): + # Argument parsing + parser = argparse.ArgumentParser() + parser.add_argument( + 'testlist', + help='File specifying a list of apps to run') + parser.add_argument( + '--simulator', + action='store', + nargs='?', + default=default_simulator, + choices=simulator_choices, + help='Choose a simulator to run the test with') + parser.add_argument( + '--run-dir', + action='store', + default='runs', + nargs='?', + help='Parent directory of each test run directory') + parser.add_argument( + '--dry-run', + action='store_true', + help='Preview the simulation commands which will be run') + parser.add_argument( + '--early-exit', + action='store_true', + help='Exit as soon as any test fails') + parser.add_argument( + '-j', + action='store', + dest='n_procs', + nargs='?', + type=int, + default=1, + const=os.cpu_count(), + help=('Maximum number of tests to run in parallel. ' + 'One if the option is not present. Equal to the number of CPU cores ' + 'if the option is present but not followed by an argument.')) + return parser + + +# Checks if a string s represents a valid relative path w.r.t. to a certain base_path and resolves +# it to an absolute path, if this is the case. Otherwise returns the original string. +def resolve_relative_path(base_path, s): + try: + base_path = Path(base_path).resolve() # Get the absolute path of the base directory + input_path = Path(s) + if input_path.is_absolute() or not s.startswith(("./", "../")): + return s + else: + # Resolve the path against the base directory and check existence + absolute_path = (base_path / input_path).resolve() + return str(absolute_path) + except (TypeError, ValueError): + # Handle invalid base_path or s + return s + except Exception as e: + # Handle other exceptions like permission errors, etc. + print(f"An error occurred: {str(e)}") + return s + + +# Create simulation objects from a test list file +def get_simulations(testlist, simulator): + # Get tests from test list file + testlist_path = Path(testlist).absolute() + with open(testlist_path, 'r') as f: + tests = yaml.safe_load(f)['runs'] + # Convert relative paths in testlist file to absolute paths + for test in tests: + test['elf'] = testlist_path.parent / test['elf'] + if 'cmd' in test: + test['cmd'] = [resolve_relative_path(testlist_path.parent, arg) for arg in test['cmd']] + # Create simulation object for every test which supports the specified simulator + simulations = [simulator.get_simulation(test) for test in tests if simulator.supports(test)] + return simulations + + +def print_summary(failed_sims, early_exit=False, dry_run=False): + if not dry_run: + header = f'==== Test summary {"(early exit)" if early_exit else ""} ====' + cprint(header, attrs=['bold']) + if failed_sims: + [sim.print_status() for sim in failed_sims] + else: + print(f'{colored("All tests passed!", "green")}') + + +def terminate_simulations(): + print('Terminating simulations') + # Get PID and PGID of parent process (current Python script) + ppid = os.getpid() + pgid = os.getpgid(0) + # Kill processes in current process group, except parent process + for proc in psutil.process_iter(['pid', 'name']): + pid = proc.info['pid'] + if os.getpgid(pid) == pgid and pid != ppid: + os.kill(pid, signal.SIGKILL) + + +def run_simulations(simulations, n_procs=1, run_dir=None, dry_run=False, early_exit=False): + # Register SIGTERM handler, used to gracefully terminate all simulation subprocesses + signal.signal(signal.SIGTERM, lambda _, __: terminate_simulations()) + + # Spawn a process for every test, wait for all running tests to terminate and check results + running_sims = [] + failed_sims = [] + early_exit_requested = False + uniquify_run_dir = len(simulations) > 1 + try: + while (len(simulations) or len(running_sims)) and not early_exit_requested: + # If there are still simulations to run and there are less running simulations than + # the maximum number of processes allowed in parallel, spawn new simulation + if len(simulations) and len(running_sims) < n_procs: + running_sims.append(simulations.pop(0)) + # Launch simulation in current working directory, by default + if run_dir is None: + run_dir = Path.cwd() + # Create unique subdirectory for each test under run directory, if multiple tests + if uniquify_run_dir: + unique_run_dir = run_dir / running_sims[-1].testname + else: + unique_run_dir = run_dir + running_sims[-1].launch(run_dir=unique_run_dir, dry_run=dry_run) + # Remove completed sims from running sims list + idcs = [i for i, sim in enumerate(running_sims) if dry_run or sim.completed()] + completed_sims = [running_sims.pop(i) for i in sorted(idcs, reverse=True)] + # Check completed sims and report status + for sim in completed_sims: + if sim.successful(): + sim.print_status() + else: + failed_sims.append(sim) + sim.print_log() + sim.print_status() + # If in early-exit mode, terminate as soon as any simulation fails + if early_exit: + early_exit_requested = True + break + time.sleep(POLL_PERIOD) + except KeyboardInterrupt: + early_exit_requested = True + + # Clean up after early exit + if early_exit_requested: + terminate_simulations() + + # Print summary + print_summary(failed_sims, early_exit_requested) + return len(failed_sims) diff --git a/util/sim/simulate.py b/util/sim/simulate.py deleted file mode 100755 index 40d0a2b6e..000000000 --- a/util/sim/simulate.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Luca Colagrande - -# TODO colluca: timeout feature - -import argparse -import multiprocessing -from pathlib import Path -import subprocess -from termcolor import colored, cprint -import os -import re -import sys -import time -import yaml - - -BANSHEE_CFG = 'src/banshee.yaml' - -# Tool settings -SIMULATORS = ['vsim', 'banshee', 'verilator', 'vcs', 'other'] -DEFAULT_SIMULATOR = SIMULATORS[0] -SIMULATOR_BINS = { - 'vsim': 'bin/snitch_cluster.vsim', - 'banshee': 'banshee', - 'verilator': 'bin/snitch_cluster.vlt', - 'vcs': 'bin/snitch_cluster.vcs' -} -SIMULATOR_CMDS = { - 'vsim': '{sim_bin} {elf} "" -batch', - 'banshee': ('{{sim_bin}} --no-opt-llvm --no-opt-jit --configuration {cfg}' - ' --trace {{elf}} > /dev/null').format(cfg=BANSHEE_CFG), - 'verilator': '{sim_bin} {elf}', - 'vcs': '{sim_bin} {elf}' -} - - -def parser(): - # Argument parsing - parser = argparse.ArgumentParser() - parser.add_argument( - 'testlist', - help='File specifying a list of apps to run') - parser.add_argument( - '--simulator', - action='store', - nargs='?', - default=DEFAULT_SIMULATOR, - choices=SIMULATORS, - help='Choose a simulator to run the test with') - parser.add_argument( - '--sim-bin', - action='store', - nargs='?', - help='Override default path to simulator binary') - parser.add_argument( - '--dry-run', - action='store_true', - help='Preview the simulation commands which will be run') - parser.add_argument( - '--early-exit', - action='store_true', - help='Exit as soon as any test fails') - parser.add_argument( - '-j', - action='store', - dest='n_procs', - nargs='?', - type=int, - default=1, - const=os.cpu_count(), - help=('Maximum number of tests to run in parallel. ' - 'One if the option is not present. Equal to the number of CPU cores ' - 'if the option is present but not followed by an argument.')) - parser.add_argument( - '--verbose', - action='store_true', - help=('Option to print simulation logs when multiple tests are run in parallel.' - 'Logs are always printed when n_procs == 1')) - return parser - - -# Get tests from a test list file -def get_tests(testlist_path): - testlist_path = Path(testlist_path).absolute() - with open(testlist_path, 'r') as file: - tests = yaml.safe_load(file)['runs'] - return tests - - -def check_exit_code(test, exit_code): - if 'exit_code' in test: - return not (int(test['exit_code']) == int(exit_code)) - else: - return exit_code - - -def multiple_processes(args): - return args.n_procs != 1 - - -def run_simulation(cmd, simulator, test, quiet=False): - # Defaults - result = 1 - log = '' - - # Spawn simulation subprocess - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True) - - # Poll simulation subprocess and log its output - while p.poll() is None: - line = p.stdout.readline() - log += line - if not quiet: - print(line, end='', flush=True) - - # When simulating with vsim or vcs, we need to parse the simulation - # log to catch the application's return code - if simulator in ['vsim', 'vcs']: - # Capture success - regex_success = r'\[SUCCESS\] Program finished successfully' - match_success = re.search(regex_success, line) - if match_success: - result = 0 - else: - regex_fail = r'\[FAILURE\] Finished with exit code\s+(\d+)' - match = re.search(regex_fail, line) - if match: - exit_code = match.group(1) - result = check_exit_code(test, exit_code) - - # Check if the subprocess terminated correctly - exit_code = p.poll() - # In Banshee and Verilator the exit code of the Snitch binary is returned - # through the exit code of the simulation command - if simulator in ['banshee', 'verilator']: - result = check_exit_code(test, exit_code) - # For custom commands the return code is that of the command - elif simulator == 'other': - result = exit_code - # For standard simulation commands the simulated Snitch binary exit - # code is overriden only if the simulator failed - else: - if exit_code != 0: - result = exit_code - - return result, log - - -def run_test(test, args): - # Extract args - simulator = args.simulator - dry_run = args.dry_run - testlist = args.testlist - quiet = multiple_processes(args) - - # Simulator binary can be overriden on the command-line or test-wise - sim_bin = SIMULATOR_BINS[simulator] - if args.sim_bin: - sim_bin = args.sim_bin - if 'sim_bin' in test: - sim_bin = test['sim_bin'] - - # Check if simulator is supported for this test - if 'simulators' in test: - if simulator not in test['simulators']: - return (0, '') - - # Construct path to executable - elf = Path(test['elf']) - if testlist: - elf = Path(testlist).absolute().parent / elf - cprint(f'Run test {colored(elf, "cyan")}', attrs=["bold"]) - - # Construct simulation command (override only supported for RTL) - if 'cmd' in test and simulator != 'banshee': - cmd = test['cmd'] - cmd = cmd.format(sim_bin=sim_bin, elf=elf, simulator=simulator) - simulator = 'other' - else: - cmd = SIMULATOR_CMDS[simulator] - cmd = cmd.format(sim_bin=sim_bin, elf=elf) - - # Check if the simulation should be run in a specific directory. - # This is useful, e.g. to preserve the logs of multiple simulations - # which are executed in parallel - if 'rundir' in test: - cmd = f'cd {test["rundir"]} && {cmd}' - if not quiet or args.verbose: - print(f'$ {cmd}', flush=True) - - # Run simulation - result = 0 - log = '' - if not dry_run: - result, log = run_simulation(cmd, simulator, test, quiet) - - # Report failure or success - if result != 0: - cprint(f'{elf} test failed', 'red', attrs=['bold'], flush=True) - else: - cprint(f'{elf} test passed', 'green', attrs=['bold'], flush=True) - - return (result, log) - - -def print_failed_test(test): - print(f'{colored(test["elf"], "cyan")} test {colored("failed", "red")}') - - -def print_test_summary(failed_tests, args): - if not args.dry_run: - header = f'\n==== Test summary {"(early exit)" if args.early_exit else ""} ====' - cprint(header, attrs=['bold']) - if failed_tests: - for failed_test in failed_tests: - print_failed_test(failed_test) - else: - print(f'{colored("All tests passed!", "green")}') - - -def run_tests(tests, args): - - # Create a process Pool - with multiprocessing.Pool(args.n_procs) as pool: - - # Create a shared object which parent and child processes can access - # concurrently to terminate the pool early as soon as one process fails - exit_early = multiprocessing.Value('B') - exit_early.value = 0 - - # Define callback for early exit - def completion_callback(return_value): - result = return_value[0] - log = return_value[1] - if args.early_exit and result != 0: - exit_early.value = 1 - # Printing the log all at once here, rather than line-by-line - # in run_simulation, ensures that the logs of different processes - # are not interleaved in stdout. - # However, as we prefer line-by-line printing when a single process - # is used, we have to make sure we don't print twice. - if args.verbose and multiple_processes(args): - print(log) - - # Queue tests to process pool - results = [] - for test in tests: - result = pool.apply_async(run_test, args=(test, args), callback=completion_callback) - results.append(result) - - # Wait for all tests to complete - running = range(len(tests)) - while len(running) != 0 and not exit_early.value: - time.sleep(1) - running = [i for i in running if not results[i].ready()] - - # Query test results - failed_tests = [] - for test, result in zip(tests, results): - if result.ready() and result.get()[0] != 0: - failed_tests.append(test) - - print_test_summary(failed_tests, args) - - return len(failed_tests) - - -def main(): - args = parser().parse_args() - tests = get_tests(args.testlist) - return run_tests(tests, args) - - -if __name__ == '__main__': - sys.exit(main())