diff --git a/.github/actions/boost-cache/action.yml b/.github/actions/boost-cache/action.yml new file mode 100644 index 0000000..8d5da1f --- /dev/null +++ b/.github/actions/boost-cache/action.yml @@ -0,0 +1,35 @@ +name: Cache Boost +description: Caches Boost for an OS and a Boost minor version +inputs: + boost_minor: + required: true + os: + required: true +outputs: + cache-hit: + description: "Indicates if caching was successful" + value: ${{ steps.cache-boost.outputs.cache-hit }} + location: + description: "The path where Boost ends up" + value: "${{ github.workspace }}/boost" + +runs: + using: "composite" + steps: + - name: Cache Boost + uses: actions/cache@v3 + id: cache-boost + env: + cache-name: cache-boost + with: + path: ${{ github.workspace }}/boost + key: ${{ inputs.os }}-boost-${{ inputs.boost_minor }} + + - name: Install Boost + if: steps.cache-boost.outputs.cache-hit != 'true' + shell: bash + run: | + wget -qO- https://boostorg.jfrog.io/artifactory/main/release/1.${{ inputs.boost_minor }}.0/source/boost_1_${{ inputs.boost_minor }}_0.tar.bz2 | tar xjf - + cd boost_1_${{ inputs.boost_minor }}_0 + ./bootstrap.sh + ./b2 --prefix=../boost --with-serialization --with-filesystem --with-test install diff --git a/.github/actions/mpi-setup/action.yml b/.github/actions/mpi-setup/action.yml new file mode 100644 index 0000000..e6a0ec3 --- /dev/null +++ b/.github/actions/mpi-setup/action.yml @@ -0,0 +1,14 @@ +name: Set up MPI +description: Allows to use MPI in Github Actions by configuring oversubscription + +runs: + using: "composite" + steps: + - name: Set up MPI for Github Actions + shell: bash + run: | + mkdir -p "$HOME/.openmpi" + cat < "$HOME/.openmpi/mca-params.conf" + rmaps_base_oversubscribe = true + rmaps_default_mapping_policy = :oversubscribe + EOF diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml new file mode 100644 index 0000000..6200f67 --- /dev/null +++ b/.github/workflows/ctest.yml @@ -0,0 +1,45 @@ +name: Run C++ tests + +on: + workflow_call: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "macos-14"] + boost_minor: [85] + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Install Ubuntu system dependencies + if: startsWith(matrix.os, 'ubuntu-') + run: | + sudo apt-get update + sudo apt-get install ninja-build libopenmpi-dev + + - name: Install MacOs system dependencies + if: startsWith(matrix.os, 'macos-') + run: | + brew install ninja openmpi + + - uses: ./.github/actions/boost-cache + id: cache-boost + with: + os: ${{ matrix.os }} + boost_minor: ${{ matrix.boost_minor }} + + - name: Configure and build + run: | + cmake -B build -G Ninja -DCMAKE_PREFIX_PATH=${{ steps.cache-boost.outputs.location }} . + cmake --build build + + - uses: ./.github/actions/mpi-setup + + - name: Run CTest + run: | + ctest --test-dir build --output-on-failure diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..1a97d0a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,39 @@ +name: Run all tests + +on: + push: + branches: ['main'] + tags: ['v?[0-9]+.[0-9]+.[0-9]+'] + pull_request: + +jobs: + cache_boost: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "macos-14"] + boost_minor: [85] + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/boost-cache + id: cache-boost + with: + os: ${{ matrix.os }} + boost_minor: ${{ matrix.boost_minor }} + + test_cxx: + uses: ./.github/workflows/ctest.yml + secrets: inherit + needs: [cache_boost] + + test_python: + uses: ./.github/workflows/test.yml + secrets: inherit + needs: [cache_boost] + + publish: + uses: ./.github/workflows/publish.yml + secrets: inherit + needs: [cache_boost] diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..cb98da5 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,142 @@ +name: Publish wheels and sdist tarball to PyPi + +on: + workflow_call: + +jobs: + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + + - name: Build a source tarball + run: | + python -m pip install build + python -m build -s + + - name: Store sdist as artifact + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist/*.tar.gz + + test_sdist: + name: Test source distribution + + runs-on: ubuntu-latest + needs: [build_sdist] + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + + - name: Download artifacts produced during the build_wheels and build_sdist jobs + uses: actions/download-artifact@v3 + with: + name: dist + path: dist/ + + - uses: ./.github/actions/boost-cache + id: cache-boost + with: + os: ubuntu-latest + boost_minor: 85 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libopenmpi-dev + + - uses: ./.github/actions/mpi-setup + + - name: Install package, clean local directory + run: | + export CMAKE_PREFIX_PATH=${{ steps.cache-boost.outputs.location }} + export SKBUILD_CMAKE_DEFINE="CMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" + python -m pip install dist/* + python -m pip install mock mpi4py pytest pytest-mpi pytest-xdist + + - name: Run tests + run: | + pytest -n 3 tests + + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + # Used to host cibuildwheel + - uses: actions/setup-python@v5 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.18.1 + + - name: Build wheels + run: python -m cibuildwheel --output-dir dist + env: + CIBW_TEST_COMMAND: "pytest {project}/tests" + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./dist/*.whl + + publish: + name: Publish package to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + + runs-on: ubuntu-latest + needs: [build_wheels, build_sdist, test_sdist] + + environment: + name: publish_pypi + url: https://pypi.org/p/brain-indexer + + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Download artifacts produced by the build_sdist job + uses: actions/download-artifact@v3 + with: + name: dist + path: dist/ + + - name: Download artifacts produced by the build_wheels job + uses: actions/download-artifact@v3 + with: + pattern: cibw-wheels-* + path: dist/ + merge-multiple: true + + - name: Display structure of downloaded files + run: ls -R + working-directory: dist + + - name: Publish source distribution package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: dist/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..2efb673 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,56 @@ +name: Run Python tests and lint + +on: + workflow_call: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "macos-14"] + boost_minor: [85] + python-version: ['3.11', '3.12'] + toxenv: ['py3'] + include: + - os: 'ubuntu-latest' + python-version: '3.12' + toxenv: 'flake8' + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Ubuntu system dependencies + if: startsWith(matrix.os, 'ubuntu-') + run: | + sudo apt-get update + sudo apt-get install libopenmpi-dev + + - name: Install MacOs system dependencies + if: startsWith(matrix.os, 'macos-') + run: | + brew install openmpi + + - uses: ./.github/actions/boost-cache + id: cache-boost + with: + os: ${{ matrix.os }} + boost_minor: ${{ matrix.boost_minor }} + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + + - name: Test with tox + run: | + export CMAKE_PREFIX_PATH=${{ steps.cache-boost.outputs.location }} + export SKBUILD_CMAKE_DEFINE="CMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" + tox -e ${{ matrix.toxenv }} diff --git a/pyproject.toml b/pyproject.toml index 1b0ab5c..0134904 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "brain-indexer" description = "A spatial index implementation for spheres, morphologies and synapses" authors = [{ name="Blue Brain Project", email = "bbp-ou-hpc@epfl.ch" } ] -license = {text = "Blue Brain Project Licence"} +license = {"file" = "LICENSE.txt"} # maintainer is the field chosen for docs `contributors` maintainers = [{name="Fernando Pereira"}, {name="Antonio Bellotta"}, {name="Luc Dominic Grosheintz-Laval"}] dynamic = ["version"] @@ -68,7 +68,7 @@ SI_UNIT_TESTS="OFF" [tool.setuptools_scm] [tool.cibuildwheel] -skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64"] +skip = ["cp3{6,7,8,9}-*", "pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64"] test-requires = ["pytest", "pytest-mpi"] test-command = ["python -m pytest {project}/tests"] @@ -76,6 +76,19 @@ test-command = ["python -m pytest {project}/tests"] # for redistributable wheels, MPI support should not be enabled (until MPI settles on a standard ABI: https://github.com/mpi-forum/mpi-issues/issues/751) "cmake.define.SI_MPI"="OFF" +[tool.cibuildwheel.linux] +before-all = """ +yum install -y wget +wget -qO- https://boostorg.jfrog.io/artifactory/main/release/1.85.0/source/boost_1_85_0.tar.bz2 | tar xjf - +cd boost_1_85_0 +./bootstrap.sh +./b2 --prefix=/opt/boost --with-serialization --with-filesystem --with-test install +""" + +[tool.cibuildwheel.linux.environment] +CMAKE_PREFIX_PATH = "/opt/boost" +SKBUILD_CMAKE_DEFINE = "CMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" + [tool.cibuildwheel.macos] environment = { MACOSX_DEPLOYMENT_TARGET = "10.15" } diff --git a/tox.ini b/tox.ini index e2477a7..16dc85e 100644 --- a/tox.ini +++ b/tox.ini @@ -7,14 +7,13 @@ envlist = flake8, py3 [testenv] # Let setup.py install test deps so we dont maintain two dep lists # Preinstall a few deps which are problematic when getting automatically with setup.py -usedevelop = True deps = pytest pytest-mpi pytest-xdist passenv = CMAKE_PREFIX_PATH - BOOST_ROOT + SKBUILD_CMAKE_DEFINE # This variable is required on BB5. It fixes errors related to "undefined # symbol: omp_get_num_threads" MKL_THREADING_LAYER @@ -36,13 +35,13 @@ max-line-length = 90 [testenv:flake8] changedir = {toxinidir} +usedevelop = True deps = flake8 skip_install = True commands = flake8 [testenv:docs] changedir = {toxinidir} -usedevelop = True deps = Jinja2~=3.0.0 sphinx<5.1.0,>=5.0.0 @@ -54,9 +53,6 @@ deps = # https://github.com/sphinx-doc/sphinx/issues/9841 docutils<0.18 setuptools -passenv = - CMAKE_PREFIX_PATH - BOOST_ROOT setenv = PIP_INDEX_URL = https://bbpteam.epfl.ch/repository/devpi/simple commands = sphinx-build -W docs docs/_build