treewide: Package Python utilities (#177)

Co-authored-by: Viviane Potocnik <vivianep@iis.ee.ethz.ch>
Co-authored-by: Luca Colagrande <luca.colagrande3@gmail.com>

.github/workflows/ci.yml

@@ -48,6 +48,8 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
+      - name: Install local Python packages
+        run: pip install .
       - name: Build Software
         working-directory: target/snitch_cluster
         run: |
@@ -81,6 +83,8 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
+      - name: Install local Python packages
+        run: pip install .
       - name: Build Software
         run: |
           bender vendor init

.github/workflows/lint.yml

@@ -48,6 +48,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+      - name: Install local Python packages
+        run: pip install .
       - name: Generate opcodes
         run: |
           ./util/generate-opcodes.sh

.gitignore

@@ -22,3 +22,5 @@ gmon.out
 # Installation directories
 /.venv/
 /tools/
+/build/
+/snitch.egg-info/

iis-setup.sh

@@ -23,6 +23,9 @@ TMPDIR=tmp pip install -r python-requirements.txt
 TMPDIR=tmp pip install -r $($BENDER path idma)/requirements.txt
 rm -rf tmp
 
+# Install local packages in editable mode.
+pip install -e .
+
 # Install spike-dasm
 mkdir tools/
 cd tools/

pyproject.toml

@@ -0,0 +1,19 @@
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Solderpad Hardware License, Version 0.51, see LICENSE for details.
+# SPDX-License-Identifier: SHL-0.51
+
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "snitch"
+authors = [
+    {name = "Luca Colagrande", email = "colluca@iis.ee.ethz.ch"}
+]
+dynamic = ["version"]
+
+[tool.setuptools.package-dir]
+"snitch.dnn" = "sw/dnn"
+"snitch.blas" = "sw/blas"
+"snitch.util" = "util"

sw/apps/atax/scripts/datagen.py

@@ -7,12 +7,9 @@
 #         Luca Colagrande <colluca@iis.ee.ethz.ch>
 
 import numpy as np
-import os
-import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-from data_utils import format_scalar_definition, format_array_definition, \
-                       format_array_declaration, format_ifdef_wrapper, DataGen  # noqa: E402
+from snitch.util.sim.data_utils import format_scalar_definition, format_array_definition, \
+    format_array_declaration, format_ifdef_wrapper, DataGen
 
 
 # AXI splits bursts crossing 4KB address boundaries. To minimize

sw/apps/atax/scripts/verify.py

@@ -7,11 +7,9 @@
 
 import numpy as np
 import sys
-from pathlib import Path
 from datagen import AtaxDataGen
 
-sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
 
 
 class AtaxVerifier(Verifier):

sw/apps/correlation/scripts/datagen.py

@@ -7,12 +7,9 @@
 #         Luca Colagrande <colluca@iis.ee.ethz.ch>
 
 import numpy as np
-import os
-import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-from data_utils import format_scalar_definition, format_array_definition, \
-                       format_array_declaration, format_ifdef_wrapper, DataGen  # noqa: E402
+from snitch.util.sim.data_utils import format_scalar_definition, format_array_definition, \
+    format_array_declaration, format_ifdef_wrapper, DataGen
 
 
 # AXI splits bursts crossing 4KB address boundaries. To minimize

sw/apps/correlation/scripts/verify.py

@@ -7,11 +7,9 @@
 
 import numpy as np
 import sys
-from pathlib import Path
 from datagen import CorrelationDataGen
 
-sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
 
 
 class CorrelationVerifier(Verifier):

sw/apps/covariance/scripts/datagen.py

@@ -7,12 +7,9 @@
 #         Luca Colagrande <colluca@iis.ee.ethz.ch>
 
 import numpy as np
-import os
-import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-from data_utils import format_scalar_definition, format_array_definition, \
-                       format_array_declaration, format_ifdef_wrapper, DataGen  # noqa: E402
+from snitch.util.sim.data_utils import format_scalar_definition, format_array_definition, \
+    format_array_declaration, format_ifdef_wrapper, DataGen
 
 
 # AXI splits bursts crossing 4KB address boundaries. To minimize

sw/apps/covariance/scripts/verify.py

@@ -7,11 +7,9 @@
 
 import numpy as np
 import sys
-from pathlib import Path
 from datagen import CovarianceDataGen
 
-sys.path.append(str(Path(__file__).parent / '../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
 
 
 class CovarianceVerifier(Verifier):

sw/blas/__init__.py

@@ -0,0 +1,9 @@
+# Copyright 2024 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Luca Colagrande <colluca@iis.ee.ethz.ch>
+
+from . import gemm
+
+__all__ = ['gemm']

sw/blas/axpy/scripts/datagen.py

@@ -6,12 +6,10 @@
 # Author: Luca Colagrande <colluca@iis.ee.ethz.ch>
 
 import numpy as np
-import os
 import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-from data_utils import format_scalar_definition, format_array_definition, \
-                       format_array_declaration, format_ifdef_wrapper, DataGen  # noqa: E402
+from snitch.util.sim.data_utils import format_scalar_definition, format_array_definition, \
+    format_array_declaration, format_ifdef_wrapper, DataGen
 
 
 class AxpyDataGen(DataGen):

sw/blas/axpy/scripts/verify.py

@@ -6,11 +6,9 @@
 # Luca Colagrande <colluca@iis.ee.ethz.ch>
 
 import sys
-from pathlib import Path
 from datagen import AxpyDataGen
 
-sys.path.append(str(Path(__file__).parent / '../../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
 
 
 class AxpyVerifier(Verifier):

sw/blas/dot/scripts/datagen.py

@@ -4,12 +4,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import numpy as np
-import os
 import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-from data_utils import format_scalar_definition, format_array_definition, \
-                       format_scalar_declaration, format_ifdef_wrapper, DataGen  # noqa: E402
+from snitch.util.sim.data_utils import format_scalar_definition, format_array_definition, \
+    format_scalar_declaration, format_ifdef_wrapper, DataGen
 
 
 class DotDataGen(DataGen):

sw/blas/dot/scripts/verify.py

@@ -4,11 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import sys
-from pathlib import Path
 from datagen import DotDataGen
 
-sys.path.append(str(Path(__file__).parent / '../../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
 
 
 class DotVerifier(Verifier):

sw/blas/gemm/__init__.py

@@ -4,6 +4,6 @@
 #
 # Luca Colagrande <colluca@iis.ee.ethz.ch>
 
-from .scripts import datagen
+from .scripts.datagen import GemmDataGen
 
-__all__ = ['datagen']
+__all__ = ['GemmDataGen']

sw/blas/gemm/scripts/datagen.py

@@ -9,15 +9,13 @@
 #          Luca Colagrande <colluca@iis.ee.ethz.ch>
 
 import numpy as np
-import os
 import re
 import pyflexfloat as ff
 import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-import data_utils  # noqa: E402
-from data_utils import DataGen, format_array_declaration, format_struct_definition, \
-                       format_array_definition, format_ifdef_wrapper  # noqa: E402
+from snitch.util.sim import data_utils
+from snitch.util.sim.data_utils import DataGen, format_array_declaration, \
+    format_struct_definition, format_array_definition, format_ifdef_wrapper
 
 
 np.random.seed(42)

sw/blas/gemm/scripts/verify.py

@@ -7,12 +7,10 @@
 
 import numpy as np
 import sys
-from pathlib import Path
 from datagen import GemmDataGen
 
-sys.path.append(str(Path(__file__).parent / '../../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
-from data_utils import ctype_from_precision_t  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
+from snitch.util.sim.data_utils import ctype_from_precision_t
 
 
 class GemmVerifier(Verifier):

sw/dnn/batchnorm/scripts/datagen.py

@@ -10,15 +10,11 @@
 import argparse
 import pathlib
 import json5
-import sys
-import os
 import torch
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-import data_utils  # noqa: E402
-from data_utils import emit_license, \
-                       format_struct_definition, format_array_definition, \
-                       format_array_declaration, format_ifdef_wrapper  # noqa: E402
+from snitch.util.sim import data_utils
+from snitch.util.sim.data_utils import emit_license, format_struct_definition, \
+    format_array_definition, format_array_declaration, format_ifdef_wrapper
 
 torch.manual_seed(42)
 

sw/dnn/concat/scripts/datagen.py

@@ -9,15 +9,11 @@
 import numpy as np
 import pathlib
 import json5
-import sys
-import os
 import torch
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-import data_utils  # noqa: E402
-from data_utils import emit_license, \
-                       format_struct_definition, format_array_definition, \
-                       format_array_declaration, format_ifdef_wrapper  # noqa: E402
+from snitch.util.sim import data_utils
+from snitch.util.sim.data_utils import emit_license, format_struct_definition, \
+    format_array_definition, format_array_declaration, format_ifdef_wrapper
 
 torch.manual_seed(42)
 

sw/dnn/concat/scripts/verify.py

@@ -7,12 +7,10 @@
 
 import sys
 import torch
-from pathlib import Path
 from datagen import golden_model
 
-sys.path.append(str(Path(__file__).parent / '../../../../util/sim/'))
-from verif_utils import Verifier  # noqa: E402
-from data_utils import ctype_from_precision_t  # noqa: E402
+from snitch.util.sim.verif_utils import Verifier
+from snitch.util.sim.data_utils import ctype_from_precision_t
 
 
 class ConcatVerifier(Verifier):

sw/dnn/conv2d/scripts/datagen.py

@@ -9,15 +9,11 @@
 import numpy as np
 import pathlib
 import hjson
-import sys
-import os
 import torch
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../../util/sim/"))
-import data_utils  # noqa: E402
-from data_utils import emit_license, \
-                       format_struct_definition, format_array_definition, \
-                       format_array_declaration  # noqa: E402
+from snitch.util.sim import data_utils
+from snitch.util.sim.data_utils import emit_license, format_struct_definition, \
+    format_array_definition, format_array_declaration
 
 torch.manual_seed(42)
 

sw/dnn/conv2d/scripts/verify.py

@@ -11,10 +11,9 @@
 import torch
 from data.datagen import golden_model
 
-sys.path.append(str(Path(__file__).parent / '../../../../util/sim/'))
-import verification  # noqa: E402
-from elf import Elf  # noqa: E402
-from data_utils import from_buffer, ctype_from_precision_t, check_result  # noqa: E402
+from snitch.util.sim import verification
+from snitch.util.sim.elf import Elf
+from snitch.util.sim.data_utils import from_buffer, ctype_from_precision_t, check_result
 
 
 ERR_THRESHOLD = 1E-6
@@ -77,8 +76,8 @@ def main():
     fail, rel_err = check_result(output_golden, output_actual, rtol=ERR_THRESHOLD)
     if fail:
         verification.dump_results_to_csv(
-                    [output_golden, output_actual, rel_err],
-                    Path.cwd() / 'results.csv')
+            [output_golden, output_actual, rel_err],
+            Path.cwd() / 'results.csv')
         print('Maximum relative error:', np.max(rel_err))
 
     return int(fail)