treewide: Refactor Python simulation framework and fix process leakage (

#70)

* target: Align with SEPP package recommendations

* target: Switch to Questa's three-step flow

* test/ipc: Retry on `ferror`

* util/sim: Refactor Python simulation framework and fix process leakage

* gitlab-ci: Enable colors

* target: Remove redundant Make targets

.github/workflows/ci.yml

@@ -43,13 +43,12 @@ jobs:
           make -C target/snitch_cluster bin/snitch_cluster.vlt
       - name: Run Tests
         working-directory: target/snitch_cluster
-        run: |-
-          ../../util/sim/simulate.py sw/run.yaml --simulator verilator -j \
-          --verbose
+        run: |
+          ./run.py sw/run.yaml --simulator verilator -j
 
-  ############################################
+  #########################################
   # Build SW on Snitch Cluster w/ Banshee #
-  ############################################
+  #########################################
 
   sw-snitch-cluster-banshee:
     name: Simulate SW on Snitch Cluster w/ Banshee
@@ -68,6 +67,5 @@ jobs:
         env:
           SNITCH_LOG: info
         working-directory: target/snitch_cluster
-        run: |-
-          ../../util/sim/simulate.py sw/run.yaml --simulator banshee -j \
-          --verbose
+        run: |
+          ./run.py sw/run.yaml --simulator banshee -j

.gitlab-ci.yml

@@ -5,13 +5,17 @@
 variables:
   GIT_STRATEGY: clone
   GIT_SUBMODULE_STRATEGY: recursive
+  # Enable colors in CI terminal
+  TERM: ansi
+  FORCE_COLOR: 1
+  # Configure environment
   PYTHON: /usr/local/anaconda3-2022.05/bin/python3
   BENDER: bender-0.27.1
   CC: gcc-9.2.0
   CXX: g++-9.2.0
-  VCS: vcs-2020.12
-  VERILATOR: verilator-4.110
-  QUESTA: questa-2022.3
+  VCS_SEPP: vcs-2020.12
+  VERILATOR_SEPP: verilator-4.110
+  QUESTA_SEPP: questa-2022.3
   LLVM_BINROOT: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin
   CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: /usr/pack/gcc-9.2.0-af/linux-x64/bin/gcc
   LLVM_SYS_120_PREFIX: /usr/pack/llvm-12.0.1-af
@@ -79,8 +83,8 @@ snitch-ip-tests:
           - tcdm_interface
   script:
     - cd hw/$IP
-    - $QUESTA ./util/compile.sh
-    - $QUESTA ./util/run_vsim.sh
+    - ./util/compile.sh
+    - ./util/run_vsim.sh
 
 ########################
 # Snitch cluster tests #
@@ -89,29 +93,26 @@ snitch-ip-tests:
 # Verilator
 snitch-cluster-vlt:
   needs: [snitch-cluster-sw]
-  # yamllint disable rule:line-length
   script:
     - cd target/snitch_cluster
-    - $VERILATOR make bin/snitch_cluster.vlt
-    - $VERILATOR ../../util/sim/simulate.py sw/run.yaml --simulator verilator -j --verbose
-  # yamllint enable rule:line-length
+    - make bin/snitch_cluster.vlt
+    - ./run.py sw/run.yaml --simulator verilator -j --run-dir runs/vlt
 
 # VCS
 snitch-cluster-vcs:
   needs: [snitch-cluster-sw]
   script:
     - cd target/snitch_cluster
-    - $VCS make bin/snitch_cluster.vcs
-    - $VCS ../../util/sim/simulate.py sw/run.yaml --simulator vcs -j --verbose
+    - make bin/snitch_cluster.vcs
+    - ./run.py sw/run.yaml --simulator vcs -j --run-dir runs/vcs
 
 # Questa
 snitch-cluster-vsim:
   needs: [snitch-cluster-sw]
   script:
     - cd target/snitch_cluster
-    - $QUESTA make bin/snitch_cluster.vsim
-    - $QUESTA ../../util/sim/simulate.py sw/run.yaml --simulator vsim -j
-      --verbose
+    - make bin/snitch_cluster.vsim
+    - ./run.py sw/run.yaml --simulator vsim -j --run-dir runs/vsim
 
 # Banshee
 snitch-cluster-banshee:
@@ -127,4 +128,4 @@ snitch-cluster-banshee:
     - cd banshee
     - cargo install --debug --path .
     - cd ../target/snitch_cluster
-    - ../../util/sim/simulate.py sw/run.yaml --simulator banshee -j --verbose
+    - ./run.py sw/run.yaml --simulator banshee -j --run-dir runs/banshee

hw/mem_interface/util/compile.sh

@@ -17,4 +17,4 @@ $BENDER script vsim -t test \
     --vlog-arg="+cover=sbecft" \
     > compile.tcl
 echo 'return 0' >> compile.tcl
-$VSIM -c -do 'exit -code [source compile.tcl]'
+$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]'

hw/mem_interface/util/run_vsim.sh

@@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 [ ! -z "$VSIM" ] || VSIM=vsim
 
 call_vsim() {
-    echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
+    echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
     grep "Errors: 0," vsim.log
 }
 

hw/reqrsp_interface/util/compile.sh

@@ -17,4 +17,4 @@ $(BENDER) script vsim -t test \
     --vlog-arg="+cover=sbecft" \
     > compile.tcl
 echo 'return 0' >> compile.tcl
-$VSIM -c -do 'exit -code [source compile.tcl]'
+$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]'

hw/reqrsp_interface/util/run_vsim.sh

@@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 [ ! -z "$VSIM" ] || VSIM=vsim
 
 call_vsim() {
-    echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
+    echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
     grep "Errors: 0," vsim.log
 }
 

hw/snitch_cluster/util/compile.sh

@@ -17,4 +17,4 @@ $BENDER script vsim -t test \
     --vlog-arg="+cover=sbecft" \
     > compile.tcl
 echo 'return 0' >> compile.tcl
-$VSIM -c -do 'exit -code [source compile.tcl]'
+$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]'

hw/snitch_cluster/util/run_vsim.sh

@@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 [ ! -z "$VSIM" ] || VSIM=vsim
 
 call_vsim() {
-    echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
+    echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
     grep "Errors: 0," vsim.log
 }
 

hw/snitch_icache/util/compile.sh

@@ -17,4 +17,4 @@ $BENDER script vsim -t test \
     --vlog-arg="+cover=sbecft" \
     > compile.tcl
 echo 'return 0' >> compile.tcl
-$VSIM -c -do 'exit -code [source compile.tcl]'
+$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]'

hw/snitch_icache/util/run_vsim.sh

@@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 [ ! -z "$VSIM" ] || VSIM=vsim
 
 call_vsim() {
-    echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
+    echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
     grep "Errors: 0," vsim.log
 }
 

hw/snitch_ssr/util/compile.sh

@@ -17,4 +17,4 @@ $(BENDER) script vsim -t test \
     --vlog-arg="+cover=sbecft" \
     > compile.tcl
 echo 'return 0' >> compile.tcl
-$VSIM -c -do 'exit -code [source compile.tcl]'
+$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]'

hw/tcdm_interface/util/compile.sh

@@ -17,4 +17,4 @@ $BENDER script vsim -t test \
     --vlog-arg="+cover=sbecft" \
     > compile.tcl
 echo 'return 0' >> compile.tcl
-$VSIM -c -do 'exit -code [source compile.tcl]'
+$QUESTA_SEPP $VSIM -c -do 'exit -code [source compile.tcl]'

hw/tcdm_interface/util/run_vsim.sh

@@ -12,7 +12,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 [ ! -z "$VSIM" ] || VSIM=vsim
 
 call_vsim() {
-    echo "log -r /*; run -all" | $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
+    echo "log -r /*; run -all" | $QUESTA_SEPP $VSIM -c -coverage -voptargs='+acc +cover=sbecft' "$@" | tee vsim.log 2>&1
     grep "Errors: 0," vsim.log
 }
 

python-requirements.txt

@@ -19,6 +19,7 @@ pytablewriter
 termcolor
 pandas
 pyelftools
+psutil
 
 -r docs/requirements.txt
 -r sw/dnn/requirements.txt

target/common/common.mk

@@ -6,12 +6,22 @@ LOGS_DIR ?= logs
 TB_DIR   ?= $(SNITCH_ROOT)/target/common/test
 UTIL_DIR ?= $(SNITCH_ROOT)/util
 
+# SEPP packages
+QUESTA_SEPP    ?=
+VCS_SEPP       ?=
+VERILATOR_SEPP ?=
+
 # External executables
 BENDER       ?= bender
 DASM         ?= spike-dasm
-VLT          ?= verilator
+VLT          ?= $(VERILATOR_SEPP) verilator
+VCS          ?= $(VCS_SEPP) vcs
 VERIBLE_FMT  ?= verible-verilog-format
 CLANG_FORMAT ?= clang-format
+VSIM         ?= $(QUESTA_SEPP) vsim
+VOPT         ?= $(QUESTA_SEPP) vopt
+VLOG         ?= $(QUESTA_SEPP) vlog
+VLIB         ?= $(QUESTA_SEPP) vlib
 
 # Internal executables
 GENTRACE_PY      ?= $(UTIL_DIR)/trace/gen_trace.py
@@ -21,7 +31,7 @@ PERF_CSV_PY      ?= $(UTIL_DIR)/trace/perf_csv.py
 LAYOUT_EVENTS_PY ?= $(UTIL_DIR)/trace/layout_events.py
 EVENTVIS_PY      ?= $(UTIL_DIR)/trace/eventvis.py
 
-VERILATOR_ROOT ?= $(dir $(shell which $(VLT)))/../share/verilator
+VERILATOR_ROOT ?= $(dir $(shell $(VERILATOR_SEPP) which verilator))..
 VLT_ROOT       ?= ${VERILATOR_ROOT}
 
 MATCH_END := '/+incdir+/ s/$$/\/*\/*/'
@@ -31,6 +41,7 @@ SED_SRCS  := sed -e ${MATCH_END} -e ${MATCH_BGN}
 VSIM_BENDER   += -t test -t rtl -t simulation -t vsim
 VSIM_SOURCES   = $(shell ${BENDER} script flist ${VSIM_BENDER} | ${SED_SRCS})
 VSIM_BUILDDIR ?= work-vsim
+VOPT_FLAGS     = +acc
 
 # VCS_BUILDDIR should to be the same as the `DEFAULT : ./work-vcs`
 # in target/snitch_cluster/synopsys_sim.setup
@@ -150,25 +161,30 @@ endef
 $(VSIM_BUILDDIR):
 	mkdir -p $@
 
+# Expects vlog/vcom script in $< (e.g. as output by bender)
+# Expects the top module name in $1
+# Produces a binary used to run the simulation at the path specified by $@
 define QUESTASIM
-	${VSIM} -c -do "source $<; quit" | tee $(dir $<)vsim.log
-	@! grep -P "Errors: [1-9]*," $(dir $<)vsim.log
+	${VSIM} -c -do "source $<; quit" | tee $(dir $<)vlog.log
+	@! grep -P "Errors: [1-9]*," $(dir $<)vlog.log
+	$(VOPT) $(VOPT_FLAGS) -work $(VSIM_BUILDDIR) $1 -o $(1)_opt | tee $(dir $<)vopt.log
+	@! grep -P "Errors: [1-9]*," $(dir $<)vopt.log
 	@mkdir -p $(dir $@)
 	@echo "#!/bin/bash" > $@
 	@echo 'binary=$$(realpath $$1)' >> $@
 	@echo 'mkdir -p $(LOGS_DIR)' >> $@
 	@echo 'echo $$binary > $(LOGS_DIR)/.rtlbinary' >> $@
 	@echo '${VSIM} +permissive ${VSIM_FLAGS} $$3 -work ${MKFILE_DIR}/${VSIM_BUILDDIR} -c \
 				-ldflags "-Wl,-rpath,${FESVR}/lib -L${FESVR}/lib -lfesvr -lutil" \
-				$1 +permissive-off ++$$binary ++$$2' >> $@
+				$(1)_opt +permissive-off ++$$binary ++$$2' >> $@
 	@chmod +x $@
 	@echo "#!/bin/bash" > $@.gui
 	@echo 'binary=$$(realpath $$1)' >> $@.gui
 	@echo 'mkdir -p $(LOGS_DIR)' >> $@.gui
 	@echo 'echo $$binary > $(LOGS_DIR)/.rtlbinary' >> $@.gui
 	@echo '${VSIM} +permissive ${VSIM_FLAGS} -work ${MKFILE_DIR}/${VSIM_BUILDDIR} \
 				-ldflags "-Wl,-rpath,${FESVR}/lib -L${FESVR}/lib -lfesvr -lutil" \
-				$1 +permissive-off ++$$binary ++$$2' >> $@.gui
+				$(1)_opt +permissive-off ++$$binary ++$$2' >> $@.gui
 	@chmod +x $@.gui
 endef
 
@@ -179,7 +195,7 @@ $(VCS_BUILDDIR)/compile.sh:
 	mkdir -p $(VCS_BUILDDIR)
 	${BENDER} script vcs ${VCS_BENDER} --vlog-arg="${VLOGAN_FLAGS}" --vcom-arg="${VHDLAN_FLAGS}" > $@
 	chmod +x $@
-	$@ > $(VCS_BUILDDIR)/compile.log
+	$(VCS_SEPP) $@ > $(VCS_BUILDDIR)/compile.log
 
 ########
 # Util #

target/common/test/ipc.cc

@@ -19,63 +19,65 @@ void* IpcIface::ipc_thread_handle(void* in) {
     // Handle commands
     ipc_op_t op;
 
-    while (size_t num_messages_read = fread(&op, sizeof(ipc_op_t), 1, tx)) {
-        if (num_messages_read != 1) {
+    while (1) {
+        if (!fread(&op, sizeof(ipc_op_t), 1, tx)) {
             if (feof(tx)) {
                 printf(
                     "[IPC] All messages read. Closing FIFOs and joining main "
                     "thread.\n");
-            } else if (ferror(tx)) {
-                perror("[IPC] read from tx failed\n");
-            }
-            break;
-        }
-        switch (op.opcode) {
-            case Read:
-                // Read full blocks until one full block or less left
-                printf("[IPC] Read from 0x%x len 0x%x ...\n", op.addr, op.len);
-                for (uint64_t i = op.len; i > IPC_BUF_SIZE; i -= IPC_BUF_SIZE) {
-                    sim::MEM.read(op.addr, IPC_BUF_SIZE, buf_data);
-                    fwrite(buf_data, IPC_BUF_SIZE, 1, rx);
-                    op.addr += IPC_BUF_SIZE;
-                    op.len -= IPC_BUF_SIZE;
-                }
-                sim::MEM.read(op.addr, op.len, buf_data);
-                fwrite(buf_data, op.len, 1, rx);
-                fflush(rx);
-                break;
-            case Write:
-                // Write full blocks until one full block or less left
-                printf("[IPC] Write to 0x%x len %d ...\n", op.addr, op.len);
-                for (uint64_t i = op.len; i > IPC_BUF_SIZE; i -= IPC_BUF_SIZE) {
-                    fread(buf_data, IPC_BUF_SIZE, 1, tx);
-                    sim::MEM.write(op.addr, IPC_BUF_SIZE, buf_data, buf_strb);
-                    op.addr += IPC_BUF_SIZE;
-                    op.len -= IPC_BUF_SIZE;
-                }
-                fread(buf_data, op.len, 1, tx);
-                sim::MEM.write(op.addr, op.len, buf_data, buf_strb);
-                break;
-            case Poll:
-                // Unpack 32b checking mask and expected value from length
-                uint32_t mask = op.len & 0xFFFFFFFF;
-                uint32_t expected = (op.len >> 32) & 0xFFFFFFFF;
-                printf("[IPC] Poll on 0x%x mask 0x%x expected 0x%x ...\n",
-                       op.addr, mask, expected);
-                uint32_t read;
-                do {
-                    sim::MEM.read(op.addr, sizeof(uint32_t),
-                                  (uint8_t*)(void*)&read);
-                    nanosleep(
-                        (const struct timespec[]){{0, IPC_POLL_PERIOD_NS}},
-                        NULL);
-                } while ((read & mask) == (expected & mask));
-                // Send back read 32b word
-                fwrite(&read, sizeof(uint32_t), 1, rx);
-                fflush(rx);
                 break;
+            }
+        } else {
+            switch (op.opcode) {
+                case Read:
+                    // Read full blocks until one full block or less left
+                    printf("[IPC] Read from 0x%x len 0x%x ...\n", op.addr,
+                           op.len);
+                    for (uint64_t i = op.len; i > IPC_BUF_SIZE;
+                         i -= IPC_BUF_SIZE) {
+                        sim::MEM.read(op.addr, IPC_BUF_SIZE, buf_data);
+                        fwrite(buf_data, IPC_BUF_SIZE, 1, rx);
+                        op.addr += IPC_BUF_SIZE;
+                        op.len -= IPC_BUF_SIZE;
+                    }
+                    sim::MEM.read(op.addr, op.len, buf_data);
+                    fwrite(buf_data, op.len, 1, rx);
+                    fflush(rx);
+                    break;
+                case Write:
+                    // Write full blocks until one full block or less left
+                    printf("[IPC] Write to 0x%x len %d ...\n", op.addr, op.len);
+                    for (uint64_t i = op.len; i > IPC_BUF_SIZE;
+                         i -= IPC_BUF_SIZE) {
+                        fread(buf_data, IPC_BUF_SIZE, 1, tx);
+                        sim::MEM.write(op.addr, IPC_BUF_SIZE, buf_data,
+                                       buf_strb);
+                        op.addr += IPC_BUF_SIZE;
+                        op.len -= IPC_BUF_SIZE;
+                    }
+                    fread(buf_data, op.len, 1, tx);
+                    sim::MEM.write(op.addr, op.len, buf_data, buf_strb);
+                    break;
+                case Poll:
+                    // Unpack 32b checking mask and expected value from length
+                    uint32_t mask = op.len & 0xFFFFFFFF;
+                    uint32_t expected = (op.len >> 32) & 0xFFFFFFFF;
+                    printf("[IPC] Poll on 0x%x mask 0x%x expected 0x%x ...\n",
+                           op.addr, mask, expected);
+                    uint32_t read;
+                    do {
+                        sim::MEM.read(op.addr, sizeof(uint32_t),
+                                      (uint8_t*)(void*)&read);
+                        nanosleep(
+                            (const struct timespec[]){{0, IPC_POLL_PERIOD_NS}},
+                            NULL);
+                    } while ((read & mask) == (expected & mask));
+                    // Send back read 32b word
+                    fwrite(&read, sizeof(uint32_t), 1, rx);
+                    fflush(rx);
+                    break;
+            }
         }
-        printf("[IPC] ... done\n");
     }
 
     // TX FIFO closed at other end: close both FIFOs and join main thread