From 562db869a59ad0c080bc72ea1ffedab1247f6ee0 Mon Sep 17 00:00:00 2001 From: Silas Date: Sat, 27 Jul 2024 13:34:42 +0200 Subject: [PATCH] fix cross platform build (#2) * build: try fixing * build: fix vpl for linux-x86 * build: fix zimg for linux-x86 * build: fix zimg for linux-x86 * build: fix setup vulkan source * build: use platform from property for ci * chore: update jni-utils * chore: update jni-utils * ci: cleanup some stuff * build: rename linux-armhf target to linux-arm (still armhf architecture, but now matches jni-utils naming) * build: fix linux-x86 target * build: fix linux-x86 target --- .github/actions/deploy-ubuntu/action.yml | 79 +++++++++++------------- .github/workflows/ffmpeg.yml | 6 +- ffmpeg/build.gradle.kts | 9 ++- ffmpeg/cppbuild.sh | 8 +-- gradle/libs.versions.toml | 2 +- 5 files changed, 49 insertions(+), 55 deletions(-) diff --git a/.github/actions/deploy-ubuntu/action.yml b/.github/actions/deploy-ubuntu/action.yml index f0349b3..9d96b9a 100644 --- a/.github/actions/deploy-ubuntu/action.yml +++ b/.github/actions/deploy-ubuntu/action.yml @@ -32,25 +32,25 @@ runs: echo "HOME=$HOME" >> $GITHUB_ENV fi cd $HOME - + # Free disk space by removing some unused system components $SUDO rm -rf /usr/local/lib/android $SUDO rm -rf /usr/share/dotnet - + # Allocate a swapfile on Linux as it's not enabled by default. Needed for pytorch and mkl. $SUDO fallocate -l 4GB /swapfile $SUDO chmod 600 /swapfile $SUDO mkswap /swapfile $SUDO swapon /swapfile - + mkdir -p .ccache echo "max_size = 2.0G" > .ccache/ccache.conf echo "hash_dir = false" >> .ccache/ccache.conf echo "sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros" >> .ccache/ccache.conf - + export ARCH=amd64 export PREFIX=x86_64-linux-gnu - if [[ "$CI_DEPLOY_PLATFORM" == "linux-armhf" ]]; then + if [[ "$CI_DEPLOY_PLATFORM" == "linux-arm" ]]; then export ARCH=armhf export PREFIX=arm-linux-gnueabihf export USERLAND_BUILDME="buildme" @@ -84,79 +84,70 @@ runs: fi echo "ARCH=$ARCH" >> $GITHUB_ENV echo "PREFIX=$PREFIX" >> $GITHUB_ENV - + $SUDO dpkg --list - + $SUDO apt-get update $SUDO apt-get -y install gnupg source /etc/os-release export CODENAME=$UBUNTU_CODENAME - + if [[ ! "$ARCH" == "amd64" ]]; then - # https://github.com/actions/runner-images/issues/675 + # https://github.com/actions/runner-images/issues/675 $SUDO gem install apt-spy2 $SUDO apt-spy2 check $SUDO apt-spy2 fix --commit $SUDO sed -i 's/azure\.//' /etc/apt/apt-mirrors.txt /etc/apt/sources.list $SUDO cat /etc/apt/apt-mirrors.txt /etc/apt/sources.list $SUDO apt-get update - - # https://github.com/actions/runner-images/issues/4589 + + # https://github.com/actions/runner-images/issues/4589 $SUDO apt-add-repository -y ppa:ondrej/php $SUDO apt-get -y install ppa-purge $SUDO ppa-purge -y ppa:ondrej/php fi - + if [[ "$ARCH" == "i386" ]]; then $SUDO dpkg --add-architecture $ARCH TOOLCHAIN="gcc-$PREFIX g++-$PREFIX gfortran-$PREFIX" elif [[ ! "$ARCH" == "amd64" ]]; then + echo "Adding $ARCH architecture" $SUDO dpkg --add-architecture $ARCH $SUDO sed -i 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list + $SUDO sed -i 's/deb mirror/deb [arch=amd64] mirror/g' /etc/apt/sources.list $SUDO echo deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports $CODENAME main restricted universe multiverse | $SUDO tee -a /etc/apt/sources.list $SUDO echo deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports $CODENAME-updates main restricted universe multiverse | $SUDO tee -a /etc/apt/sources.list $SUDO echo deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports $CODENAME-backports main restricted universe multiverse | $SUDO tee -a /etc/apt/sources.list $SUDO echo deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports $CODENAME-security main restricted universe multiverse | $SUDO tee -a /etc/apt/sources.list TOOLCHAIN="gcc-$PREFIX g++-$PREFIX gfortran-$PREFIX linux-libc-dev-$ARCH-cross binutils-multiarch" fi - # $SUDO echo deb [arch=amd64] https://ppa.launchpad.net/openjdk-r/ppa/ubuntu $CODENAME main | $SUDO tee -a /etc/apt/sources.list - # $SUDO echo deb [arch=amd64] https://ppa.launchpad.net/deadsnakes/ppa/ubuntu $CODENAME main | $SUDO tee -a /etc/apt/sources.list - # $SUDO apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB9B1D8886F44E2A - # $SUDO apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA6932366A755776 + echo "sources.list:" + cat /etc/apt/sources.list + for f in /etc/apt/sources.list.d/*.list; do echo "$f:"; cat $f; done $SUDO apt-get update - $SUDO apt-get -y install gcc-multilib g++-multilib gfortran-multilib python3 python2.7 python3-minimal python2.7-minimal rpm libasound2-dev:$ARCH freeglut3-dev:$ARCH libfontconfig-dev:$ARCH libgtk2.0-dev:$ARCH libusb-dev:$ARCH libusb-1.0-0-dev:$ARCH libffi-dev:$ARCH libbz2-dev:$ARCH zlib1g-dev:$ARCH libxcb1-dev:$ARCH libpulse-dev:$ARCH - $SUDO apt-get -y install pkg-config ccache clang $TOOLCHAIN openjdk-8-jdk ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake cmake make libtool bison flex perl nasm ragel curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev libdrm-dev libva-dev - + $SUDO apt-get -y install gcc-multilib g++-multilib gfortran-multilib python3 python2.7 python3-minimal python2.7-minimal rpm libasound2-dev:$ARCH freeglut3-dev:$ARCH libfontconfig-dev:$ARCH libgtk2.0-dev:$ARCH libusb-dev:$ARCH libusb-1.0-0-dev:$ARCH libffi-dev:$ARCH libbz2-dev:$ARCH zlib1g-dev:$ARCH libxcb1-dev:$ARCH libpulse-dev:$ARCH libdrm-dev:$ARCH libva-dev:$ARCH + $SUDO apt-get -y install pkgconf ccache clang $TOOLCHAIN openjdk-8-jdk ant python2 python3-pip swig git file wget unzip tar bzip2 gzip patch autoconf-archive autogen automake cmake make libtool bison flex perl nasm ragel curl libcurl4-openssl-dev libssl-dev libffi-dev libbz2-dev zlib1g-dev rapidjson-dev + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV ln -sf $JAVA_HOME /usr/lib/jvm/default-java find /usr/lib/jvm/default-java/ - # curl -LO https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.tar.gz - curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz - curl -LO https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.18.3.tar.gz - curl -LO https://services.gradle.org/distributions/gradle-7.1.1-bin.zip - # $SUDO tar -xzf cmake-3.24.2-linux-x86_64.tar.gz -C /opt/ - $SUDO tar -xzf apache-maven-3.6.3-bin.tar.gz -C /opt/ - $SUDO tar -xzf git-2.18.3.tar.gz - $SUDO unzip gradle-7.1.1-bin.zip -d /opt/ - # pushd git-2.18.3; make -j2 prefix=/usr/local/; $SUDO make -j2 prefix=/usr/local/ install; popd - # $SUDO ln -sf /opt/cmake-3.24.2-linux-x86_64/bin/* /usr/bin/ - $SUDO ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn - $SUDO ln -sf /opt/gradle-7.1.1/bin/gradle /usr/bin/gradle - - $SUDO python3 -m pip install gdown || $SUDO python3 -m pip install gdown - + # install vulkan sdk echo Installing Vulkan-SDK if [[ "$CODENAME" == "jammy" ]]; then curl -s https://packages.lunarg.com/lunarg-signing-key-pub.asc | $SUDO tee /etc/apt/trusted.gpg.d/lunarg.asc >/dev/null $SUDO curl -s -o /etc/apt/sources.list.d/lunarg-vulkan-1.3.280-jammy.list https://packages.lunarg.com/vulkan/1.3.280/lunarg-vulkan-1.3.280-jammy.list + $SUDO sed -i 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list.d/lunarg-vulkan-1.3.280-jammy.list + $SUDO sed -i 's/deb-src http/deb-src [arch=amd64] http/g' /etc/apt/sources.list.d/lunarg-vulkan-1.3.280-jammy.list else curl -s https://packages.lunarg.com/lunarg-signing-key-pub.asc | $SUDO apt-key add - $SUDO curl -s -o /etc/apt/sources.list.d/lunarg-vulkan-1.3.280-focal.list https://packages.lunarg.com/vulkan/1.3.280/lunarg-vulkan-1.3.280-focal.list + $SUDO sed -i 's/deb http/deb [arch=amd64] http/g' /etc/apt/sources.list.d/lunarg-vulkan-1.3.280-focal.list + $SUDO sed -i 's/deb-src http/deb-src [arch=amd64] http/g' /etc/apt/sources.list.d/lunarg-vulkan-1.3.280-focal.list fi $SUDO apt-get update && $SUDO apt-get install -y libvulkan1 libvulkan-dev vulkan-sdk - + if [[ -n ${USERLAND_BUILDME:-} ]]; then curl -LOJ https://github.com/raspberrypi/userland/archive/master.tar.gz tar -xzf userland-master.tar.gz @@ -166,7 +157,7 @@ runs: export USERLAND_PATH=$(pwd)/userland-master/ echo "USERLAND_PATH=$USERLAND_PATH" >> $GITHUB_ENV fi - + if [[ $CI_DEPLOY_PLATFORM == android-* ]]; then curl -LO https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip unzip -qq android-ndk-r21e-linux-x86_64.zip @@ -175,7 +166,7 @@ runs: else export MAVEN_OPTIONS="-Djavacpp.platform.compiler=$PREFIX-g++ -Djava.library.path=/usr/$PREFIX/lib/:/usr/lib/$PREFIX/:$USERLAND_PATH/build/lib/" fi - + if [[ -n ${ARCH_CUDA:-} ]] && [[ -n ${CI_DEPLOY_NEED_CUDA:-} ]]; then echo Installing CUDA, cuDNN, nvCOMP, etc curl -LO https://developer.download.nvidia.com/compute/cuda/12.3.2/local_installers/$CUDA @@ -183,7 +174,7 @@ runs: curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel8/$ARCH_CUDA/libcudnn8-devel-$CUDNN.rpm curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel8/$ARCH_CUDA/libnccl-$NCCL.rpm curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel8/$ARCH_CUDA/libnccl-devel-$NCCL.rpm - + $SUDO rpm -i --force --ignorearch --nodeps $CUDA libcudnn*.rpm libnccl*.rpm rm -f *.rpm *.tgz *.txz *.tar.* pushd /var/cuda-repo-rhel8-12-3-local/; $SUDO rpm -i --force --ignorearch --nodeps cuda*.rpm libc*.rpm libn*.rpm; $SUDO rm *.rpm; popd @@ -192,15 +183,15 @@ runs: $SUDO ln -sf /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/local/cuda/lib64/libnvidia-ml.so $SUDO mv /usr/include/cudnn* /usr/include/nccl* /usr/local/cuda/include/ $SUDO mv /usr/lib64/libcudnn* /usr/lib64/libnccl* /usr/local/cuda/lib64/ - + if [[ -n ${NVCOMP:-} ]]; then curl -LO https://developer.download.nvidia.com/compute/nvcomp/3.0.5/local_installers/$NVCOMP.tgz $SUDO tar -xvf $NVCOMP.tgz -C /usr/local/cuda/lib64/ --strip-components=1 lib/ || $SUDO tar -xvf $NVCOMP.tgz -C /usr/local/cuda/lib64/ --strip-components=2 nvcomp-3.0.5-ctk-12.2/lib/ $SUDO tar -xvf $NVCOMP.tgz -C /usr/local/cuda/include/ --strip-components=1 include/ || $SUDO tar -xvf $NVCOMP.tgz -C /usr/local/cuda/include/ --strip-components=2 nvcomp-3.0.5-ctk-12.2/include/ rm -f $NVCOMP.tgz fi - - # Work around issues with CUDA 10.2/11.x + + # Work around issues with CUDA 10.2/11.x $SUDO mv /usr/include/cublas* /usr/include/nvblas* /usr/local/cuda/include/ || true $SUDO mv /usr/lib64/libcublas* /usr/lib64/libnvblas* /usr/local/cuda/lib64/ || true for f in /usr/local/cuda/include/*_v8.h; do $SUDO ln -sf $f ${f:0:${#f}-5}.h; done @@ -224,7 +215,7 @@ runs: $SUDO sed -i /device_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh $SUDO sed -i /device_segmented_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh $SUDO sed -i /warp_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh - + # Remove downloaded archives and unused libraries to avoid running out of disk space $SUDO rm -f $(find /usr/local/cuda/ -name '*.a' -and -not -name libcudart_static.a -and -not -name libcudadevrt.a -and -not -name libnvcomp_device.a) fi @@ -242,9 +233,9 @@ runs: - name: Publish release shell: bash if: "github.event_name == 'release'" - run: ./gradlew build publish -Pdeploy.kotlin=false -Pdeploy.native=true -Pffmpeg.gpl=${{ matrix.gpl }} -Pdeploy.version=${{ github.event.release.tag_name }} --console=plain --info --configure-on-demand --parallel --build-cache + run: ./gradlew build publish -Pdeploy.kotlin=false -Pdeploy.native=true -Pffmpeg.platform=${{ github.job }} -Pffmpeg.gpl=${{ matrix.gpl }} -Pdeploy.version=${{ github.event.release.tag_name }} --console=plain --info --configure-on-demand --parallel --build-cache - name: Publish snapshot shell: bash if: "github.event_name != 'release'" - run: ./gradlew build publish -Pdeploy.kotlin=false -Pdeploy.native=true -Pffmpeg.gpl=${{ matrix.gpl }} -Pdeploy.version=${{ steps.vars.outputs.sha_short }}-dev --console=plain --info --configure-on-demand --parallel --build-cache + run: ./gradlew build publish -Pdeploy.kotlin=false -Pdeploy.native=true -Pffmpeg.platform=${{ github.job }} -Pffmpeg.gpl=${{ matrix.gpl }} -Pdeploy.version=${{ steps.vars.outputs.sha_short }}-dev --console=plain --info --configure-on-demand --parallel --build-cache diff --git a/.github/workflows/ffmpeg.yml b/.github/workflows/ffmpeg.yml index b7084fe..a989552 100644 --- a/.github/workflows/ffmpeg.yml +++ b/.github/workflows/ffmpeg.yml @@ -56,12 +56,12 @@ jobs: # steps: # - uses: actions/checkout@v4 # - uses: ./.github/actions/deploy-ubuntu - linux-armhf: + linux-arm: runs-on: ubuntu-22.04 strategy: matrix: gpl: ["false", "true"] - name: linux-armhf (${{ matrix.gpl == 'true' && 'GPL' || 'LGPL' }}) + name: linux-arm (${{ matrix.gpl == 'true' && 'GPL' || 'LGPL' }}) steps: - uses: actions/checkout@v4 - uses: ./.github/actions/deploy-ubuntu @@ -136,7 +136,7 @@ jobs: # - uses: actions/checkout@v4 # - uses: ./.github/actions/deploy-windows # redeploy: -# needs: [android-arm, android-arm64, android-x86, android-x86_64, linux-armhf, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-arm64, macosx-x86_64, windows-x86, windows-x86_64] +# needs: [android-arm, android-arm64, android-x86, android-x86_64, linux-arm, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-arm64, macosx-x86_64, windows-x86, windows-x86_64] # needs: [android-arm64, android-x86_64, linux-arm64, linux-ppc64le, linux-x86_64, macosx-arm64, macosx-x86_64, windows-x86_64] # runs-on: ubuntu-22.04 # steps: diff --git a/ffmpeg/build.gradle.kts b/ffmpeg/build.gradle.kts index f15add5..68e21ed 100644 --- a/ffmpeg/build.gradle.kts +++ b/ffmpeg/build.gradle.kts @@ -1,4 +1,5 @@ import dev.silenium.libs.jni.NativePlatform +import dev.silenium.libs.jni.Platform buildscript { repositories { @@ -23,9 +24,10 @@ val deployNative = (findProperty("deploy.native") as String?)?.toBoolean() ?: tr val withGPL: Boolean = findProperty("ffmpeg.gpl").toString().toBoolean() val platformExtension = "-gpl".takeIf { withGPL }.orEmpty() -val platform = NativePlatform.platform(platformExtension) +val platformString = findProperty("ffmpeg.platform")?.toString() +val platform = platformString?.let { Platform(it, platformExtension) } ?: NativePlatform.platform(platformExtension) -val compileNative = if(deployNative) { +val compileNative = if (deployNative) { tasks.register("compileNative") { enabled = deployNative commandLine( @@ -49,7 +51,8 @@ val compileNative = if(deployNative) { val nativesJar = if (deployNative) { tasks.register("nativesJar") { - val platform = NativePlatform.platform(platformExtension) + // Required for configuration cache + val platform = platformString?.let { Platform(it, platformExtension) } ?: NativePlatform.platform(platformExtension) from(compileNative!!.get().outputs.files) { include("lib/*.so") diff --git a/ffmpeg/cppbuild.sh b/ffmpeg/cppbuild.sh index da31784..d590b12 100755 --- a/ffmpeg/cppbuild.sh +++ b/ffmpeg/cppbuild.sh @@ -751,7 +751,7 @@ EOF echo "" cd zimg-release-$ZIMG_VERSION autoreconf -iv - ./configure --prefix=$INSTALL_PATH --disable-shared --with-pic --host=i686-linux + ./configure --prefix=$INSTALL_PATH --disable-shared --with-pic CFLAGS="-m32" CXXFLAGS="-m32" make -j $MAKEJ V=0 make install echo "" @@ -862,7 +862,7 @@ EOF LIBS= if [[ ! -z $(ldconfig -p | grep libva-drm) ]]; then cd ../libvpl-$VPL_VERSION - PKG_CONFIG_PATH=../lib/pkgconfig cmake -B _build -DCMAKE_INSTALL_PREFIX=$INSTALL_PATH -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-m64" -DCMAKE_CXX_FLAGS="-m64" + PKG_CONFIG_PATH=../lib/pkgconfig cmake -B _build -DCMAKE_INSTALL_PREFIX=$INSTALL_PATH -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-m32" -DCMAKE_CXX_FLAGS="-m32" cmake --build _build cmake --install _build ENABLE="$ENABLE --enable-libvpl" @@ -885,7 +885,7 @@ EOF make install cd .. cd ../ffmpeg-$FFMPEG_VERSION - LDEXEFLAGS='-Wl,-rpath,\$$ORIGIN/' PKG_CONFIG_PATH=../lib/pkgconfig/ ./configure --prefix=.. $DISABLE $ENABLE $ENABLE_VULKAN --enable-libdrm --enable-cuda --enable-cuvid --enable-nvenc --enable-pthreads --enable-libxcb --enable-libpulse --cc="gcc -m32 -D__ILP32__" --extra-cflags="-I../include/ -I../include/libxml2 -I../include/mfx -I../include/svt-av1" --extra-ldflags="-L../lib/" --extra-libs="-lstdc++ -lpthread -ldl -lz -lm $LIBS" || cat ffbuild/config.log + LDEXEFLAGS='-Wl,-rpath,\$$ORIGIN/' PKG_CONFIG_PATH=../lib/pkgconfig/:/usr/lib/i386-linux-gnu/pkgconfig ./configure --prefix=.. $DISABLE $ENABLE $ENABLE_VULKAN --enable-libdrm --enable-cuda --enable-cuvid --enable-nvenc --enable-pthreads --enable-libxcb --enable-libpulse --cc="gcc -m32 -D__ILP32__" --extra-cflags="-I../include/ -I../include/libxml2 -I../include/mfx -I../include/svt-av1" --extra-ldflags="-L../lib/" --extra-libs="-lstdc++ -lpthread -ldl -lz -lm $LIBS" || cat ffbuild/config.log make -j $MAKEJ make install ;; @@ -1039,7 +1039,7 @@ EOF make install ;; - linux-armhf) + linux-arm) tar --totals -xjf ../alsa-lib-$ALSA_VERSION.tar.bz2 export CFLAGS="-I$INSTALL_PATH/include -L$INSTALL_PATH/lib" diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4383bba..aa38b94 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,6 @@ [versions] kotlin = "2.0.0" -jni-utils = "0.1.2" +jni-utils = "0.1.5" [libraries] jni-utils = { group = "dev.silenium.libs.jni", name = "jni-utils", version.ref = "jni-utils" }