diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..684c9f6 --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,330 @@ +PROJECT := python_native +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +SHA1 := $(shell git rev-parse --verify HEAD) + +# General commands +.PHONY: help +BOLD=\e[1m +RESET=\e[0m + +help: + @echo -e "${BOLD}SYNOPSIS${RESET}" + @echo -e "\tmake [NOCACHE=1]" + @echo + @echo -e "${BOLD}DESCRIPTION${RESET}" + @echo -e "\tTools to generate various wheel packages." + @echo + @echo -e "${BOLD}MAIN TARGETS${RESET}" + @echo -e "\t${BOLD}help${RESET}: display this help and exit." + @echo -e "\t${BOLD}python${RESET}: Build musllinux and manylinux python '${PROJECT}' wheel packages (3.8+)." + @echo -e "\t${BOLD}python_${RESET}: Build all python '${PROJECT}' wheel packages (3.8+) for a specific platform." + @echo -e "\t${BOLD}python__${RESET}: Build all python '${PROJECT}' wheel packages (3.8+) for a specific platform." + @echo -e "\t${BOLD}python__${RESET}: Build python '${PROJECT}' wheel packages (3.8+) for a specific target." + @echo -e "\t${BOLD}save_python_${RESET}: Save python '${PROJECT}' image." + @echo -e "\t${BOLD}clean_python_${RESET}: Clean manylinux and musllinux python '${PROJECT}' wheel packages." + @echo -e "\t${BOLD}sh_python_${RESET}: Run a container using the python '${PROJECT}' image." + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}amd64${RESET}" + @echo -e "\t\t${BOLD}arm64${RESET}" + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}_${RESET}" + @echo -e "\t\t${BOLD}_manylinux_cp${RESET}" + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}manylinux${RESET} (manylinux_2_28)" + @echo -e "\t\t${BOLD}musllinux${RESET} (musllinux_1_2)" + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}38${RESET} Python3.8" + @echo -e "\t\t${BOLD}39${RESET} Python3.9" + @echo -e "\t\t${BOLD}310${RESET} Python3.10" + @echo -e "\t\t${BOLD}311${RESET} Python3.11" + @echo -e "\t\t${BOLD}312${RESET} Python3.12" + @echo -e "\t\t${BOLD}313${RESET} Python3.13" + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}env${RESET}" + @echo -e "\t\t${BOLD}devel${RESET}" + @echo -e "\t\t${BOLD}build${RESET}" + @echo -e "\t\t${BOLD}test${RESET}" + @echo -e "\t\t${BOLD}export${RESET}" + @echo -e "\te.g. 'make python_amd64_manylinux_cp39_export'" + @echo -e "\te.g. 'make python_arm64_musllinux_export'" + @echo + @echo -e "\t${BOLD}NOCACHE=1${RESET}: use 'docker build --no-cache' when building container (default use cache)." + @echo -e "\t${BOLD}VERBOSE=1${RESET}: use 'docker build --progress=plain' when building container." + @echo + @echo -e "${BOLD}NOTES${RESET}" + @echo -e "\tAll generated code will be located in the export/ folder, use target ${BOLD}distclean${RESET} to remove it." + @echo + +# Delete all implicit rules to speed up makefile +.SUFFIXES: +# Remove some rules from gmake that .SUFFIXES does not remove. +SUFFIXES := +# keep all intermediate files e.g. export/docker_*.tar +# src: https://www.gnu.org/software/make/manual/html_node/Special-Targets.html +.SECONDARY: + +$(info branch: ${BRANCH}) +$(info SHA1: ${SHA1}) + +DOCKER_BUILD_CMD := docker build +DOCKER_BUILDX_CMD := docker buildx build +ifdef NOCACHE +DOCKER_BUILD_CMD := ${DOCKER_BUILD_CMD} --no-cache +DOCKER_BUILDX_CMD := ${DOCKER_BUILDX_CMD} --no-cache +endif +ifdef VERBOSE +DOCKER_BUILD_CMD := ${DOCKER_BUILD_CMD} --progress=plain +DOCKER_BUILDX_CMD := ${DOCKER_BUILDX_CMD} --progress=plain +endif +DOCKER_RUN_CMD := docker run --rm --init --net=host + +################# +### DELIVERY ## +################# +.PHONY: delivery +delivery: python archives + +.PHONY: test_delivery +test_delivery: test_archives + +############### +### PYTHON ## +############### +# $* stem +# $< first prerequist +# $@ target name +PYTHON_PLATFORMS := amd64 arm64 +PYTHON_DISTROS := manylinux musllinux +PYTHON_STAGES := env devel build test + +export: + -mkdir $@ + +cache: + -mkdir $@ + +## MANYLINUX ## +PYTHON_VERSIONS := 38 39 310 311 312 313 + +export/manylinux: | export + -mkdir -p $@ + +export/manylinux/build-manylinux.sh: build-manylinux.sh | export/manylinux + cp $< $@ + +define manylinux_inner = +#$$(info manylinux_inner: PLATFORM:'$1' VERSION:'$2' STAGE:'$3') + +.PHONY: python_$1_manylinux_cp$2_$3 +python_$1_manylinux_cp$2_$3: $1/manylinux.Dockerfile export/manylinux/build-manylinux.sh + @docker image rm -f ${PROJECT}:$$@ 2>/dev/null + ${DOCKER_BUILDX_CMD} --platform linux/$1 \ + --tag ${PROJECT}:$$@ \ + --build-arg GIT_BRANCH=${BRANCH} \ + --build-arg GIT_SHA1=${SHA1} \ + --build-arg PYTHON_VERSION=$2 \ + --target=$3 \ + -f $$< \ + export/manylinux + +.PHONY: save_python_$1_manylinux_cp$2_$3 +save_python_$1_manylinux_cp$2_$3: cache/docker_$1_manylinux_cp$2_$3.tar +cache/docker_$1_manylinux_cp$2_$3.tar: python_$1_manylinux_cp$2_$3 | cache + @rm -f $$@ + docker save ${PROJECT}:$$< -o $$@ + +.PHONY: clean_python_$1_manylinux_cp$2_$3 +clean_python_$1_manylinux_cp$2_$3: $1/manylinux.Dockerfile export/manylinux/build-manylinux.sh + docker image rm -f ${PROJECT}:python_$1_manylinux_cp$2_$3 2>/dev/null + rm -f cache/docker_$1_manylinux_cp$2_$3.tar + +# Debug purpose +.PHONY: sh_python_$1_manylinux_cp$2_$3 +sh_python_$1_manylinux_cp$2_$3: python_$1_manylinux_cp$2_$3 + ${DOCKER_RUN_CMD} \ + -v `pwd`/export:/export \ + -it \ + --name ${PROJECT}_$$< \ + ${PROJECT}:$$< +endef + +define manylinux_outer = +#$$(info manylinux_outer: PLATFORM: '$1' VERSION: '$2') + +$$(foreach stage,${PYTHON_STAGES},$$(eval $$(call manylinux_inner,$1,$2,$${stage}))) + +.PHONY: python_$1_manylinux_cp$2_export +python_$1_manylinux_cp$2_export: python_$1_manylinux_cp$2_build + ${DOCKER_RUN_CMD} \ + -v `pwd`/export:/export \ + -it \ + --name ${PROJECT}_$$< \ + ${PROJECT}:$$< \ + "cp build*/python/dist/*-many*.whl /export" +endef + +$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,amd64,${version}))) +$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,arm64,${version}))) + +# Merge +define manylinux_merge = +#$$(info manylinux_merge: PLATFORM:'$1' STAGE:'$2') + +.PHONY: python_$1_manylinux_$2 +python_$1_manylinux_$2: $(addprefix python_$1_manylinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +.PHONY: save_python_$1_manylinux_$2 +save_python_$1_manylinux_$2: $(addprefix save_python_$1_manylinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +.PHONY: clean_python_$1_manylinux_$2 +clean_python_$1_manylinux_$2: $(addprefix clean_python_$1_manylinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +endef + +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,amd64,${stage}))) +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,arm64,${stage}))) + +## MUSLLINUX ## +export/musllinux: | export + -mkdir -p $@ + +export/musllinux/build-musllinux.sh: build-musllinux.sh | export/musllinux + cp $< $@ + +define musllinux_inner = +#$$(info musllinux_inner: PLATFORM:'$1' VERSION:'$2' STAGE:'$3') + +.PHONY: python_$1_musllinux_cp$2_$3 +python_$1_musllinux_cp$2_$3: $1/musllinux.Dockerfile | export/musllinux/build-musllinux.sh + @docker image rm -f ${PROJECT}:$$@ 2>/dev/null + ${DOCKER_BUILDX_CMD} --platform linux/$1 \ + --tag ${PROJECT}:$$@ \ + --build-arg GIT_BRANCH=${BRANCH} \ + --build-arg GIT_SHA1=${SHA1} \ + --build-arg PYTHON_VERSION=$2 \ + --target=$3 \ + -f $$< \ + export/musllinux + +.PHONY: save_python_$1_musllinux_cp$2_$3 +save_python_$1_musllinux_cp$2_$3: cache/docker_$1_musllinux_cp$2_$3.tar +cache/docker_$1_musllinux_cp$2_$3.tar: python_$1_musllinux_cp$2_$3 | cache + @rm -f $$@ + docker save ${PROJECT}:$$< -o $$@ + +.PHONY: clean_python_$1_musllinux_cp$2_$3 +clean_python_$1_musllinux_cp$2_$3: $1/musllinux.Dockerfile | export/musllinux/build-musllinux.sh + docker image rm -f ${PROJECT}:python_$1_musllinux_cp$2_$3 2>/dev/null + rm -f cache/docker_$1_musllinux_cp$2_$3.tar + +# Debug purpose +.PHONY: sh_python_$1_musllinux_cp$2_$3 +sh_python_$1_musllinux_cp$2_$3: python_$1_musllinux_cp$2_$3 + ${DOCKER_RUN_CMD} \ + -v `pwd`/export:/export \ + -it \ + --name ${PROJECT}_$$< \ + ${PROJECT}:$$< +endef + +define musllinux_outer = +#$$(info musllinux_outer: PLATFORM: '$1' VERSION: '$2') + +$$(foreach stage,${PYTHON_STAGES},$$(eval $$(call musllinux_inner,$1,$2,$${stage}))) + +.PHONY: python_$1_musllinux_cp$2_export +python_$1_musllinux_cp$2_export: python_$1_musllinux_cp$2_build + ${DOCKER_RUN_CMD} \ + -v `pwd`/export:/export \ + -it \ + --name ${PROJECT}_$$< \ + ${PROJECT}:$$< \ + "cp build*/python/dist/*-musl*.whl /export" +endef + +$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,amd64,${version}))) +$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,arm64,${version}))) + +# Merge +define musllinux_merge = +#$$(info musllinux_merge: PLATFORM:'$1' STAGE:'$2') + +.PHONY: python_$1_musllinux_$2 +python_$1_musllinux_$2: $(addprefix python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +.PHONY: save_python_$1_musllinux_$2 +save_python_$1_musllinux_$2: $(addprefix save_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +.PHONY: clean_python_$1_musllinux_$2 +clean_python_$1_musllinux_$2: $(addprefix clean_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS})) +endef + +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,amd64,${stage}))) +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,arm64,${stage}))) + +## MERGE DISTRO ## +define python_distro_merge = +#$$(info python_distro_merge: PLATFORM:'$1' STAGE:'$2') + +.PHONY: python_$1_$2 +python_$1_$2: $(addprefix python_$1_, $(addsuffix _$2, ${PYTHON_DISTROS})) +.PHONY: save_python_$1_$2 +save_python_$1_$2: $(addprefix save_python_$1_, $(addsuffix _$2, ${PYTHON_DISTROS})) +.PHONY: clean_python_$1_$2 +clean_python_$1_$2: $(addprefix clean_python_$1_, $(addsuffix _$2, ${PYTHON_DISTROS})) +endef + +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,amd64,${stage}))) +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,arm64,${stage}))) + +## MERGE PLATFORM ## +define clean_python_platform = +#$$(info clean_python_platform: PLATFORM:'$1') + +.PHONY: clean_python_$1 +clean_python_$1: $(addprefix clean_python_$1_, ${PYTHON_STAGES}) +endef + +$(foreach platform,${PYTHON_PLATFORMS},$(eval $(call clean_python_platform,${platform}))) + + +define python_platform_merge = +#$$(info python_platform_merge: STAGE:'$1') + +.PHONY: python_$1 +python_$1: $(addprefix python_, $(addsuffix _$1, ${PYTHON_PLATFORMS})) +.PHONY: save_python_$1 +save_python_$1: $(addprefix save_python_, $(addsuffix _$1, ${PYTHON_PLATFORMS})) +.PHONY: clean_python_$1 +clean_python_$1: $(addprefix clean_python_, $(addsuffix _$1, ${PYTHON_PLATFORMS})) +endef + +$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_platform_merge,${stage}))) + +# Alias +.PHONY: python +python: python_amd64_export + +.PHONY: clean_python +clean_python: $(addprefix clean_python_, ${PYTHON_PLATFORMS}) + -rm -rf cache/* + -rm -rf export/* + +############# +## CLEAN ## +############# +.PHONY: clean +clean: clean_python + -docker container prune -f + -docker image prune -f + -rm -rf cache + +.PHONY: distclean +distclean: clean + -docker container ls -a + -docker container prune -f + -docker image ls -a + -docker image prune -a -f + -docker system df + -docker system prune -a -f + -rm -rf export diff --git a/tools/amd64/manylinux.Dockerfile b/tools/amd64/manylinux.Dockerfile new file mode 100644 index 0000000..dd3de9f --- /dev/null +++ b/tools/amd64/manylinux.Dockerfile @@ -0,0 +1,51 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64:latest AS env +# note: Almalinux:8 based image with +# CMake 3.31.2 and SWIG 4.3.0 already installed + +RUN dnf -y update \ +&& dnf -y install \ + curl wget \ + git patch \ + which pkgconfig autoconf libtool \ + make gcc-c++ \ + redhat-lsb openssl-devel pcre2-devel \ + zlib-devel unzip zip \ +&& dnf clean all \ +&& rm -rf /var/cache/dnf +ENTRYPOINT ["/usr/bin/bash", "-c"] +CMD ["/usr/bin/bash"] + +#ENV TZ=America/Los_Angeles +#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +##################### +## PYTHON-NATIVE ## +##################### +FROM env AS devel +ENV GIT_URL=https://github.com/Mizux/python-native + +ARG GIT_BRANCH +ENV GIT_BRANCH=${GIT_BRANCH:-main} +ARG GIT_SHA1 +ENV GIT_SHA1=${GIT_SHA1:-unknown} + +# Download sources +# use GIT_SHA1 to modify the command +# i.e. avoid docker reusing the cache when new commit is pushed +RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ +&& cd /project \ +&& git reset --hard "${GIT_SHA1}" +WORKDIR /project + +# Copy build script and setup env +ENV PLATFORM=x86_64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION:-3} +COPY build-manylinux.sh . +RUN chmod a+x "build-manylinux.sh" + +FROM devel AS build +RUN ./build-manylinux.sh build + +FROM build AS test +RUN ./build-manylinux.sh test diff --git a/tools/amd64/musllinux.Dockerfile b/tools/amd64/musllinux.Dockerfile new file mode 100644 index 0000000..554e847 --- /dev/null +++ b/tools/amd64/musllinux.Dockerfile @@ -0,0 +1,46 @@ +FROM quay.io/pypa/musllinux_1_2_x86_64:latest AS env +# CMake 3.31.2 and SWIG 4.3.0 already installed + +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["/bin/sh"] + +## Python +#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \ +# py3-numpy py3-pandas py3-matplotlib +#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \ +#&& python3 -m pip install absl-py mypy mypy-protobuf + +##################### +## PYTHON-NATIVE ## +##################### +FROM env AS devel +ENV GIT_URL=https://github.com/Mizux/python-native + +ARG GIT_BRANCH +ENV GIT_BRANCH=${GIT_BRANCH:-main} +ARG GIT_SHA1 +ENV GIT_SHA1=${GIT_SHA1:-unknown} + +# Download sources +# use GIT_SHA1 to modify the command +# i.e. avoid docker reusing the cache when new commit is pushed +RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ +&& cd /project \ +&& git reset --hard "${GIT_SHA1}" +WORKDIR /project + +# Copy build script and setup env +ENV PLATFORM=x86_64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION:-3} +COPY build-musllinux.sh . +RUN chmod a+x "build-musllinux.sh" + +FROM devel AS build +RUN ./build-musllinux.sh build + +FROM build AS test +RUN ./build-musllinux.sh test diff --git a/tools/arm64/manylinux.Dockerfile b/tools/arm64/manylinux.Dockerfile new file mode 100644 index 0000000..0d41f3f --- /dev/null +++ b/tools/arm64/manylinux.Dockerfile @@ -0,0 +1,53 @@ +# To build it on x86_64 please read +# https://github.com/multiarch/qemu-user-static#getting-started +FROM quay.io/pypa/manylinux_2_28_aarch64:latest AS env +# note: Almalinux:8 based image with +# CMake 3.31.2 and SWIG 4.3.0 already installed + +RUN dnf -y update \ +&& dnf -y install \ + curl wget \ + git patch \ + which pkgconfig autoconf libtool \ + make gcc-c++ \ + redhat-lsb openssl-devel pcre2-devel \ + zlib-devel unzip zip \ +&& dnf clean all \ +&& rm -rf /var/cache/dnf +ENTRYPOINT ["/usr/bin/bash", "-c"] +CMD ["/usr/bin/bash"] + +#ENV TZ=America/Los_Angeles +#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +##################### +## PYTHON-NATIVE ## +##################### +FROM env AS devel +ENV GIT_URL=https://github.com/Mizux/python-native + +ARG GIT_BRANCH +ENV GIT_BRANCH=${GIT_BRANCH:-main} +ARG GIT_SHA1 +ENV GIT_SHA1=${GIT_SHA1:-unknown} + +# Download sources +# use GIT_SHA1 to modify the command +# i.e. avoid docker reusing the cache when new commit is pushed +RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \ +&& cd /project \ +&& git reset --hard "${GIT_SHA1}" +WORKDIR /project + +# Copy build script and setup env +ENV PLATFORM=aarch64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION:-3} +COPY build-manylinux.sh . +RUN chmod a+x "build-manylinux.sh" + +FROM devel AS build +RUN ./build-manylinux.sh build + +FROM build AS test +RUN ./build-manylinux.sh test diff --git a/tools/arm64/musllinux.Dockerfile b/tools/arm64/musllinux.Dockerfile new file mode 100644 index 0000000..c9f9f57 --- /dev/null +++ b/tools/arm64/musllinux.Dockerfile @@ -0,0 +1,46 @@ +FROM quay.io/pypa/musllinux_1_2_aarch64:latest AS env +# CMake 3.31.2 and SWIG 4.3.0 already installed + +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["/bin/sh"] + +## Python +#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \ +# py3-numpy py3-pandas py3-matplotlib +#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \ +#&& python3 -m pip install absl-py mypy mypy-protobuf + +##################### +## PYTHON-NATIVE ## +##################### +FROM env AS devel +ENV GIT_URL=https://github.com/Mizux/python-native + +ARG GIT_BRANCH +ENV GIT_BRANCH=${GIT_BRANCH:-main} +ARG GIT_SHA1 +ENV GIT_SHA1=${GIT_SHA1:-unknown} + +# Download sources +# use GIT_SHA1 to modify the command +# i.e. avoid docker reusing the cache when new commit is pushed +RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \ +&& cd /project \ +&& git reset --hard "${GIT_SHA1}" +WORKDIR /project + +# Copy build script and setup env +ENV PLATFORM=aarch64 +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION:-3} +COPY build-musllinux.sh . +RUN chmod a+x "build-musllinux.sh" + +FROM devel AS build +RUN ./build-musllinux.sh build + +FROM build AS test +RUN ./build-musllinux.sh test diff --git a/tools/build-manylinux.sh b/tools/build-manylinux.sh new file mode 100755 index 0000000..2357361 --- /dev/null +++ b/tools/build-manylinux.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash + +# Build all the wheel artifacts for the platforms supported by manylinux2014 and +# export them to the specified location. +set -exo pipefail + +function assert_defined(){ + if [[ -z "${!1}" ]]; then + >&2 echo "Variable '${1}' must be defined" + exit 1 + fi +} + +function usage() { + local -r NAME=$(basename "$0") + echo -e "$NAME - Build using a cross toolchain. + +SYNOPSIS +\t$NAME [-h|--help] [build|test|all] + +DESCRIPTION +\tBuild wheel artifacts. + +\tYou MUST define the following variables before running this script: +\t* PLATFORM: x86_64 aarch64 +\t* PYTHON_VERSION: 3 38 39 310 311 312 313 +note: PYTHON_VERSION=3 will generate for all pythons which could take time... + +OPTIONS +\t-h --help: show this help text +\tbuild: build the project using each python (note: remove previous build dir) +\ttest: install each wheel in a venv then test it (note: don't build !) +\tall: build + test (default) + +EXAMPLES +* Using export +export PLATFORM=x86_64 +export PYTHON_VERSION=39 +$0 build + +* One-liner: +PLATFORM=x86_64 PYTHON_VERSION=39 $0 build" +} + +function contains_element() { + # Look for the presence of an element in an array. Echoes '0' if found, + # '1' otherwise. + # Arguments: + # $1 the element to be searched + # $2 the array to search into + local e match="$1" + shift + for e; do + [[ "$e" == "$match" ]] && return 0 + done + return 1 +} + +function build_wheel() { + assert_defined BUILD_DIR + assert_defined VENV_DIR + # Build the wheel artifact + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Create and activate virtualenv + # this is needed so protoc can call the correct python executable + local -r PYBIN="$1/bin" + "${PYBIN}/pip" install virtualenv + "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${VENV_DIR}" + # shellcheck source=/dev/null + source "${VENV_DIR}/bin/activate" + pip install -U pip setuptools wheel absl-py # absl-py is needed by make test_python + pip install -U mypy mypy-protobuf # need to generate protobuf mypy files + + echo "current dir: $(pwd)" + + if [[ ! -e "CMakeLists.txt" ]] || [[ ! -d "cmake" ]]; then + >&2 echo "Can't find project's CMakeLists.txt or cmake" + exit 2 + fi + cmake -S. -B"${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DPython3_ROOT_DIR="$1" \ + -DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_EXAMPLES=OFF #--debug-find + cmake --build "${BUILD_DIR}" -v -j4 + + # Restore environment + deactivate +} + +function check_wheel() { + assert_defined BUILD_DIR + assert_defined VENV_DIR + # Check the wheel artifact + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # shellcheck source=/dev/null + source "${VENV_DIR}/bin/activate" + pip install -U auditwheel + + # Check mypy files + declare -a MYPY_FILES=( + "pythonnative/bar/pyBar.py" + "pythonnative/foo/pyFoo.py" + "pythonnative/foobar/pyFooBar.py" + ) + for FILE in "${MYPY_FILES[@]}"; do + if [[ ! -f "${BUILD_DIR}/python/${FILE}" ]]; then + echo "error: ${FILE} missing in the python project" + exit 1 + fi + done + + # Check all generated wheel packages + ROOT_DIR=$(pwd) + pushd "${BUILD_DIR}/python/dist" + for FILE in *.whl; do + # if no files found do nothing + [[ -e "$FILE" ]] || continue + python -m auditwheel show "$FILE" || true + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${ROOT_DIR}/${BUILD_DIR}/lib64" python -m auditwheel repair --plat "manylinux_2_28_$PLATFORM" "$FILE" -w . + #python -m auditwheel -v repair --plat manylinux_2_28_x86_64 "$FILE" -w . + #python -m auditwheel -v repair --plat manylinux_2_28_aarch64 "$FILE" -w . + done + popd + + # Restore environment + deactivate +} + +function test_wheel() { + assert_defined BUILD_DIR + assert_defined TEST_DIR + # Test the wheel artifacts + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Create and activate virtualenv + local -r PYBIN="$1/bin" + "${PYBIN}/pip" install virtualenv + "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${TEST_DIR}" + + # shellcheck source=/dev/null + source "${TEST_DIR}/bin/activate" + pip install -U pip setuptools wheel + + # Install the wheel artifact + #pwd + local -r WHEEL_FILE=$(find "${BUILD_DIR}"/python/dist/*.whl | head -1) + echo "WHEEL file: ${WHEEL_FILE}" + pip install --no-cache-dir "$WHEEL_FILE" + pip show pythonnative + + # Python scripts to be used as tests for the installed wheel. This list of files + # has been taken from the 'test_python' make target. + declare -a TESTS=( + "pythonnative/algorithms/samples/simple_knapsack_program.py" + "pythonnative/graph/samples/simple_max_flow_program.py" + "pythonnative/graph/samples/simple_min_cost_flow_program.py" + "pythonnative/linear_solver/samples/simple_lp_program.py" + "pythonnative/linear_solver/samples/simple_mip_program.py" + "pythonnative/sat/samples/simple_sat_program.py" + "pythonnative/constraint_solver/samples/tsp.py" + "pythonnative/constraint_solver/samples/vrp.py" + "pythonnative/constraint_solver/samples/cvrptw_break.py" + ) + + # Run all the specified test scripts using the current environment. + local -r ROOT_DIR=$(pwd) + pushd "$(mktemp -d)" # ensure we are not importing something from $PWD + python --version + for TEST in "${TESTS[@]}"; do + python "${ROOT_DIR}/${TEST}" + done + popd + + # Restore environment + deactivate +} + +function build() { + # For each python platform provided by manylinux, build and test artifacts. + for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do + # shellcheck disable=SC2155 + PYTAG=$(basename "$PYROOT") + echo "Python: $PYTAG" + + # Check for platforms to be skipped + if contains_element "$PYTAG" "${SKIPS[@]}"; then + >&2 echo "skipping deprecated platform $PYTAG" + continue + fi + + BUILD_DIR="build_${PYTAG}" + VENV_DIR="env_${PYTAG}" + build_wheel "$PYROOT" + check_wheel "$PYROOT" + done +} + +function tests() { + # For each python platform provided by manylinux, build and test artifacts. + for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do + # shellcheck disable=SC2155 + PYTAG=$(basename "$PYROOT") + echo "Python: $PYTAG" + + # Check for platforms to be skipped + if contains_element "$PYTAG" "${SKIPS[@]}"; then + >&2 echo "skipping deprecated platform $PYTAG" + continue + fi + + BUILD_DIR="build_${PYTAG}" + TEST_DIR="test_${PYTAG}" + test_wheel "$PYROOT" + done +} + +# Main +function main() { + case ${1} in + -h | --help) + usage; exit ;; + esac + + assert_defined PLATFORM + assert_defined PYTHON_VERSION + + # Setup + declare -a SKIPS=( "pp37-pypy37_pp73" ) + + case ${1} in + build) + build ;; + test) + tests ;; + *) + build + tests ;; + esac +} + +main "${1:-all}" diff --git a/tools/build-musllinux.sh b/tools/build-musllinux.sh new file mode 100755 index 0000000..181f93e --- /dev/null +++ b/tools/build-musllinux.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash + +# Build all the wheel artifacts for the platforms supported by musllinux_1_2 and +# export them to the specified location. +set -exo pipefail + +function assert_defined(){ + if [[ -z "${!1}" ]]; then + >&2 echo "Variable '${1}' must be defined" + exit 1 + fi +} + +function usage() { + local -r NAME=$(basename "$0") + echo -e "$NAME - Build using a cross toolchain. + +SYNOPSIS +\t$NAME [-h|--help] [build|test|all] + +DESCRIPTION +\tBuild wheel artifacts. + +\tYou MUST define the following variables before running this script: +\t* PLATFORM: x86_64 aarch64 +\t* PYTHON_VERSION: 3 38 39 310 311 312 313 +note: PYTHON_VERSION=3 will generate for all pythons which could take time... + +OPTIONS +\t-h --help: show this help text +\tbuild: build the project using each python (note: remove previous build dir) +\ttest: install each wheel in a venv then test it (note: don't build !) +\tall: build + test (default) + +EXAMPLES +* Using export +export PLATFORM=x86_64 +export PYTHON_VERSION=39 +$0 build + +* One-liner: +PLATFORM=x86_64 PYTHON_VERSION=39 $0 build" +} + +function contains_element() { + # Look for the presence of an element in an array. Echoes '0' if found, + # '1' otherwise. + # Arguments: + # $1 the element to be searched + # $2 the array to search into + local e match="$1" + shift + for e; do + [[ "$e" == "$match" ]] && return 0 + done + return 1 +} + +function build_wheel() { + assert_defined BUILD_DIR + assert_defined VENV_DIR + # Build the wheel artifact + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Create and activate virtualenv + # this is needed so protoc can call the correct python executable + local -r PYBIN="$1/bin" + "${PYBIN}/pip" install virtualenv + "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${VENV_DIR}" + # shellcheck source=/dev/null + source "${VENV_DIR}/bin/activate" + pip install -U pip setuptools wheel absl-py # absl-py is needed by make test_python + pip install -U mypy mypy-protobuf # need to generate protobuf mypy files + + echo "current dir: $(pwd)" + + if [[ ! -e "CMakeLists.txt" ]] || [[ ! -d "cmake" ]]; then + >&2 echo "Can't find project's CMakeLists.txt or cmake" + exit 2 + fi + cmake -S. -B"${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DPython3_ROOT_DIR="$1" \ + -DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_EXAMPLES=OFF #--debug-find + cmake --build "${BUILD_DIR}" -v -j4 + + # Restore environment + deactivate +} + +function check_wheel() { + assert_defined BUILD_DIR + assert_defined VENV_DIR + # Check the wheel artifact + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # shellcheck source=/dev/null + source "${VENV_DIR}/bin/activate" + pip install -U auditwheel + + # Check mypy files + declare -a MYPY_FILES=( + "pythonnative/bar/pyBar.py" + "pythonnative/foo/pyFoo.py" + "pythonnative/foobar/pyFooBar.py" + ) + for FILE in "${MYPY_FILES[@]}"; do + if [[ ! -f "${BUILD_DIR}/python/${FILE}" ]]; then + echo "error: ${FILE} missing in the python project" + exit 1 + fi + done + + # Check all generated wheel packages + ROOT_DIR=$(pwd) + pushd "${BUILD_DIR}/python/dist" + for FILE in *.whl; do + # if no files found do nothing + [[ -e "$FILE" ]] || continue + auditwheel show "$FILE" || true + auditwheel -v repair --plat "musllinux_1_2_$PLATFORM" "$FILE" -w . + #auditwheel -v repair --plat musllinux_1_2_x86_64 "$FILE" -w . + #auditwheel -v repair --plat musllinux_1_2_aarch64 "$FILE" -w . + done + popd + + # Restore environment + deactivate +} + +function test_wheel() { + assert_defined BUILD_DIR + assert_defined TEST_DIR + # Test the wheel artifacts + # Arguments: + # $1 the python root directory + if [[ "$#" -ne 1 ]]; then + echo "$0 called with an illegal number of parameters" + exit 1 + fi + + # Create and activate virtualenv + local -r PYBIN="$1/bin" + "${PYBIN}/pip" install virtualenv + "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${TEST_DIR}" + + # shellcheck source=/dev/null + source "${TEST_DIR}/bin/activate" + pip install -U pip setuptools wheel + + # Install the wheel artifact + #pwd + local -r WHEEL_FILE=$(find "${BUILD_DIR}"/python/dist/*.whl | head -1) + echo "WHEEL file: ${WHEEL_FILE}" + pip install --no-cache-dir "$WHEEL_FILE" + pip show pythonnative + + # Python scripts to be used as tests for the installed wheel. This list of files + # has been taken from the 'test_python' make target. + declare -a TESTS=( + "pythonnative/algorithms/samples/simple_knapsack_program.py" + "pythonnative/graph/samples/simple_max_flow_program.py" + "pythonnative/graph/samples/simple_min_cost_flow_program.py" + "pythonnative/linear_solver/samples/simple_lp_program.py" + "pythonnative/linear_solver/samples/simple_mip_program.py" + "pythonnative/sat/samples/simple_sat_program.py" + "pythonnative/constraint_solver/samples/tsp.py" + "pythonnative/constraint_solver/samples/vrp.py" + "pythonnative/constraint_solver/samples/cvrptw_break.py" + ) + + # Run all the specified test scripts using the current environment. + local -r ROOT_DIR=$(pwd) + pushd "$(mktemp -d)" # ensure we are not importing something from $PWD + python --version + for TEST in "${TESTS[@]}"; do + python "${ROOT_DIR}/${TEST}" + done + popd + + # Restore environment + deactivate +} + +function build() { + # For each python platform provided by musllinux, build and test artifacts. + for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do + # shellcheck disable=SC2155 + PYTAG=$(basename "$PYROOT") + echo "Python: $PYTAG" + + # Check for platforms to be skipped + if contains_element "$PYTAG" "${SKIPS[@]}"; then + >&2 echo "skipping deprecated platform $PYTAG" + continue + fi + + BUILD_DIR="build_${PYTAG}" + VENV_DIR="env_${PYTAG}" + build_wheel "$PYROOT" + check_wheel "$PYROOT" + done +} + +function tests() { + # For each python platform provided by musllinux, build and test artifacts. + for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do + # shellcheck disable=SC2155 + PYTAG=$(basename "$PYROOT") + echo "Python: $PYTAG" + + # Check for platforms to be skipped + if contains_element "$PYTAG" "${SKIPS[@]}"; then + >&2 echo "skipping deprecated platform $PYTAG" + continue + fi + + BUILD_DIR="build_${PYTAG}" + TEST_DIR="test_${PYTAG}" + test_wheel "$PYROOT" + done +} + +# Main +function main() { + case ${1} in + -h | --help) + usage; exit ;; + esac + + assert_defined PLATFORM + assert_defined PYTHON_VERSION + + # Setup + declare -a SKIPS=( "cp36-cp36m" "cp37-cp37m" ) + + case ${1} in + build) + build ;; + test) + tests ;; + *) + build + tests ;; + esac +} + +main "${1:-all}"