From b808df3cb46b9fcb1ab14cac416663060180db5c Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Wed, 31 Jul 2024 15:50:39 -0400 Subject: [PATCH] Automate generation of Windows bindings --- .github/workflows/sys-bindings-generator.yml | 88 ++++++++++++++++-- aws-lc-sys/Cargo.toml | 4 +- aws-lc-sys/builder/cmake_builder.rs | 8 +- aws-lc-sys/builder/main.rs | 93 ++++++++++++++------ scripts/build/collect_symbols.sh | 45 ++++++---- 5 files changed, 177 insertions(+), 61 deletions(-) diff --git a/.github/workflows/sys-bindings-generator.yml b/.github/workflows/sys-bindings-generator.yml index 6f903ac8399..00fa51ae29f 100644 --- a/.github/workflows/sys-bindings-generator.yml +++ b/.github/workflows/sys-bindings-generator.yml @@ -31,7 +31,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-go@v4 with: - go-version: '>=1.18' + go-version: '>=1.20' - name: Install current Bash on macOS if: runner.os == 'macOS' run: brew install bash coreutils @@ -39,12 +39,50 @@ jobs: env: AWS_LC_SYS_NO_PREFIX: "1" run: | - cargo test -p aws-lc-sys --features bindgen,ssl + cargo test -p aws-lc-sys --features bindgen - name: Collect symbols run: | ./scripts/build/collect_symbols.sh -c aws-lc-sys - name: Commit & Push changes run: ./scripts/ci/ci_add_commit_rebase_push.sh "Symbols from ${{ matrix.os }}" + collect-windows-symbols-and-commit: + if: github.repository == 'aws/aws-lc-rs' + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + target: + - 'aarch64-pc-windows-msvc' + - 'x86_64-pc-windows-msvc' + - 'x86_64-pc-windows-gnu' + - 'i686-pc-windows-msvc' + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + ref: ${{ github.ref_name }} + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: stable + target: ${{ matrix.target }} + - if: contains(matrix.target, 'x86') || contains(matrix.target, 'i686') + uses: ilammy/setup-nasm@v1 + - uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + - name: No-prefix build for ${{ matrix.target }} + env: + AWS_LC_SYS_NO_PREFIX: "1" + run: | + cargo ${{ (matrix.target == 'aarch64-pc-windows-msvc' && 'build') || 'test' }} -p aws-lc-sys --features bindgen --target ${{ matrix.target }} + - name: Collect symbols + shell: bash + run: | + ./scripts/build/collect_symbols.sh -c aws-lc-sys -t ${{ matrix.target }} + - name: Commit & Push changes + shell: bash + run: ./scripts/ci/ci_add_commit_rebase_push.sh "Symbols from ${{ matrix.target }}" collect-cross-symbols-and-commit: if: github.repository == 'aws/aws-lc-rs' runs-on: ubuntu-latest @@ -69,21 +107,21 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-go@v4 with: - go-version: '>=1.18' + go-version: '>=1.20' - name: Install cross run: cargo install cross --locked --git https://github.com/cross-rs/cross - name: No-prefix build for ${{ matrix.target }} env: AWS_LC_SYS_NO_PREFIX: "1" run: | - cross test -p aws-lc-sys --features bindgen,ssl --target ${{ matrix.target }} + cross test -p aws-lc-sys --features bindgen --target ${{ matrix.target }} - name: Collect symbols run: | ./scripts/build/collect_symbols.sh -c aws-lc-sys -t ${{ matrix.target }} - name: Commit & Push changes run: ./scripts/ci/ci_add_commit_rebase_push.sh "Symbols for ${{ matrix.target }}" generate-headers-and-commit: - needs: [ collect-cross-symbols-and-commit, collect-symbols-and-commit ] + needs: [ collect-cross-symbols-and-commit, collect-symbols-and-commit, collect-windows-symbols-and-commit ] if: github.repository == 'aws/aws-lc-rs' runs-on: ubuntu-latest steps: @@ -97,7 +135,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-go@v4 with: - go-version: '>=1.18' + go-version: '>=1.20' - name: Generate Prefix Headers run: ./scripts/generate/_generate_prefix_headers.sh -c aws-lc-sys - name: Update sys-crate metadata @@ -125,9 +163,43 @@ jobs: env: AWS_LC_SYS_INTERNAL_BINDGEN: "1" run: | - cargo test -p aws-lc-sys --features bindgen,ssl + cargo test -p aws-lc-sys --features bindgen - name: Commit & Push changes run: ./scripts/ci/ci_add_commit_rebase_push.sh "Generated bindings from ${{ matrix.os }}" + generate-windows-bindings-and-commit: + needs: generate-headers-and-commit + if: github.repository == 'aws/aws-lc-rs' + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + target: + - 'aarch64-pc-windows-msvc' + - 'x86_64-pc-windows-msvc' + - 'x86_64-pc-windows-gnu' + - 'i686-pc-windows-msvc' + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + ref: ${{ github.ref_name }} + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: stable + target: ${{ matrix.target }} + - if: contains(matrix.target, 'x86') || contains(matrix.target, 'i686') + uses: ilammy/setup-nasm@v1 + - uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + - name: Generate bindings for ${{ matrix.target }} + env: + AWS_LC_SYS_INTERNAL_BINDGEN: "1" + run: | + cargo test -p aws-lc-sys --features bindgen --target ${{ matrix.target }} + - name: Commit & Push changes + run: ./scripts/ci/ci_add_commit_rebase_push.sh "Generated bindings for ${{ matrix.target }}" generate-cross-bindings-and-commit: needs: generate-headers-and-commit if: github.repository == 'aws/aws-lc-rs' @@ -148,7 +220,7 @@ jobs: env: AWS_LC_SYS_INTERNAL_BINDGEN: "1" run: | - cross test -p aws-lc-sys --features bindgen,ssl --target ${{ matrix.target }} + cross test -p aws-lc-sys --features bindgen --target ${{ matrix.target }} - name: Commit & Push changes run: ./scripts/ci/ci_add_commit_rebase_push.sh "Generated bindings for ${{ matrix.target }}" collect-src-and-commit: diff --git a/aws-lc-sys/Cargo.toml b/aws-lc-sys/Cargo.toml index 6a46fc5d37f..fbed1ddcb22 100644 --- a/aws-lc-sys/Cargo.toml +++ b/aws-lc-sys/Cargo.toml @@ -56,10 +56,10 @@ dunce = "1.0" fs_extra = "1.3" cc = { version = "1.0.83", features = ["parallel"] } -[target.'cfg(any(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos"), any(target_env = "gnu", target_env = "musl", target_env = "")), all(target_arch = "x86", target_os = "linux", target_env = "gnu")))'.build-dependencies] +[target.'cfg(all(not(feature="ssl"),any(all(any(target_arch="x86_64",target_arch="aarch64"),any(target_os="linux",target_os="macos",target_os="windows"),any(target_env="gnu",target_env="musl",target_env="msvc",target_env="")),all(target_arch="x86",target_os="windows",target_env="msvc"),all(target_arch="x86",target_os="linux",target_env="gnu"))))'.build-dependencies] bindgen = { version = "0.69.2", optional = true } -[target.'cfg(not(any(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos"), any(target_env = "gnu", target_env = "musl", target_env = "")), all(target_arch = "x86", target_os = "linux", target_env = "gnu"))))'.build-dependencies] +[target.'cfg(not(all(not(feature="ssl"),any(all(any(target_arch="x86_64",target_arch="aarch64"),any(target_os="linux",target_os="macos",target_os="windows"),any(target_env="gnu",target_env="musl",target_env="msvc",target_env="")),all(target_arch="x86",target_os="windows",target_env="msvc"),all(target_arch="x86",target_os="linux",target_env="gnu")))))'.build-dependencies] bindgen = { version = "0.69.2" } [dependencies] diff --git a/aws-lc-sys/builder/cmake_builder.rs b/aws-lc-sys/builder/cmake_builder.rs index ed05a2c997d..5dd0bbc98fa 100644 --- a/aws-lc-sys/builder/cmake_builder.rs +++ b/aws-lc-sys/builder/cmake_builder.rs @@ -155,7 +155,7 @@ impl CmakeBuilder { // See issue: https://github.com/aws/aws-lc-rs/issues/453 if target_os() == "windows" { - Self::configure_windows(&mut cmake_cfg); + self.configure_windows(&mut cmake_cfg); } // If the build environment vendor is Apple @@ -200,13 +200,13 @@ impl CmakeBuilder { } if target_env() == "ohos" { - Self::configure_open_harmony(&mut cmake_cfg); + self.configure_open_harmony(&mut cmake_cfg); } cmake_cfg } - fn configure_windows(cmake_cfg: &mut cmake::Config) { + fn configure_windows(&self, cmake_cfg: &mut cmake::Config) { match (target_env().as_str(), target_arch().as_str()) { ("msvc", "aarch64") => { cmake_cfg.generator_toolset(format!( @@ -238,7 +238,7 @@ impl CmakeBuilder { } } - fn configure_open_harmony(cmake_cfg: &mut cmake::Config) { + fn configure_open_harmony(&self, cmake_cfg: &mut cmake::Config) { const OHOS_NDK_HOME: &str = "OHOS_NDK_HOME"; if let Ok(ndk) = env::var(OHOS_NDK_HOME) { cmake_cfg.define( diff --git a/aws-lc-sys/builder/main.rs b/aws-lc-sys/builder/main.rs index 4bf220138b0..5ac5364a831 100644 --- a/aws-lc-sys/builder/main.rs +++ b/aws-lc-sys/builder/main.rs @@ -12,13 +12,22 @@ use cmake_builder::CmakeBuilder; #[cfg(any( feature = "bindgen", - not(any( - all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos"), - any(target_env = "gnu", target_env = "musl", target_env = "") - ), - all(target_arch = "x86", target_os = "linux", target_env = "gnu") + not(all( + not(feature = "ssl"), + any( + all( + any(target_arch = "x86_64", target_arch = "aarch64"), + any(target_os = "linux", target_os = "macos", target_os = "windows"), + any( + target_env = "gnu", + target_env = "musl", + target_env = "msvc", + target_env = "" + ) + ), + all(target_arch = "x86", target_os = "windows", target_env = "msvc"), + all(target_arch = "x86", target_os = "linux", target_env = "gnu") + ) )) ))] mod bindgen; @@ -170,13 +179,22 @@ fn execute_command(executable: &OsStr, args: &[&OsStr]) -> TestCommandResult { #[cfg(any( feature = "bindgen", - not(any( - all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos"), - any(target_env = "gnu", target_env = "musl", target_env = "") - ), - all(target_arch = "x86", target_os = "linux", target_env = "gnu") + not(all( + not(feature = "ssl"), + any( + all( + any(target_arch = "x86_64", target_arch = "aarch64"), + any(target_os = "linux", target_os = "macos", target_os = "windows"), + any( + target_env = "gnu", + target_env = "musl", + target_env = "msvc", + target_env = "" + ) + ), + all(target_arch = "x86", target_os = "windows", target_env = "msvc"), + all(target_arch = "x86", target_os = "linux", target_env = "gnu") + ) )) ))] fn generate_bindings(manifest_dir: &Path, prefix: &Option, bindings_path: &PathBuf) { @@ -327,13 +345,17 @@ fn initialize() { if !is_external_bindgen() && (is_internal_bindgen() || !has_bindgen_feature()) { let target = target(); let supported_platform = match target.as_str() { - "i686-unknown-linux-gnu" - | "x86_64-unknown-linux-gnu" + "aarch64-apple-darwin" + | "aarch64-pc-windows-msvc" | "aarch64-unknown-linux-gnu" - | "x86_64-unknown-linux-musl" | "aarch64-unknown-linux-musl" + | "i686-pc-windows-msvc" + | "i686-unknown-linux-gnu" | "x86_64-apple-darwin" - | "aarch64-apple-darwin" => Some(target), + | "x86_64-pc-windows-gnu" + | "x86_64-pc-windows-msvc" + | "x86_64-unknown-linux-gnu" + | "x86_64-unknown-linux-musl" => Some(target), _ => None, }; if let Some(platform) = supported_platform { @@ -382,13 +404,17 @@ fn prepare_cargo_cfg() { // Also remove `#![allow(unexpected_cfgs)]` from src/lib.rs /* println!("cargo::rustc-check-cfg=cfg(use_bindgen_generated)"); - println!("cargo::rustc-check-cfg=cfg(i686_unknown_linux_gnu)"); - println!("cargo::rustc-check-cfg=cfg(x86_64_unknown_linux_gnu)"); + println!("cargo::rustc-check-cfg=cfg(aarch64_apple_darwin)"); + println!("cargo::rustc-check-cfg=cfg(aarch64_pc_windows_msvc)"); println!("cargo::rustc-check-cfg=cfg(aarch64_unknown_linux_gnu)"); - println!("cargo::rustc-check-cfg=cfg(x86_64_unknown_linux_musl)"); println!("cargo::rustc-check-cfg=cfg(aarch64_unknown_linux_musl)"); + println!("cargo::rustc-check-cfg=cfg(i686_pc_windows_msvc)"); + println!("cargo::rustc-check-cfg=cfg(i686_unknown_linux_gnu)"); println!("cargo::rustc-check-cfg=cfg(x86_64_apple_darwin)"); - println!("cargo::rustc-check-cfg=cfg(aarch64_apple_darwin)"); + println!("cargo::rustc-check-cfg=cfg(x86_64_pc-windows-gnu)"); + println!("cargo::rustc-check-cfg=cfg(x86_64_pc_windows_msvc)"); + println!("cargo::rustc-check-cfg=cfg(x86_64_unknown_linux_gnu)"); + println!("cargo::rustc-check-cfg=cfg(x86_64_unknown_linux_musl)"); */ } @@ -427,13 +453,22 @@ fn main() { } else if is_bindgen_required() { #[cfg(any( feature = "bindgen", - not(any( - all( - any(target_arch = "x86_64", target_arch = "aarch64"), - any(target_os = "linux", target_os = "macos"), - any(target_env = "gnu", target_env = "musl", target_env = "") - ), - all(target_arch = "x86", target_os = "linux", target_env = "gnu") + not(all( + not(feature = "ssl"), + any( + all( + any(target_arch = "x86_64", target_arch = "aarch64"), + any(target_os = "linux", target_os = "macos", target_os = "windows"), + any( + target_env = "gnu", + target_env = "musl", + target_env = "msvc", + target_env = "" + ) + ), + all(target_arch = "x86", target_os = "windows", target_env = "msvc"), + all(target_arch = "x86", target_os = "linux", target_env = "gnu") + ) )) ))] if !is_external_bindgen() { diff --git a/scripts/build/collect_symbols.sh b/scripts/build/collect_symbols.sh index 43defea1747..75737744196 100755 --- a/scripts/build/collect_symbols.sh +++ b/scripts/build/collect_symbols.sh @@ -57,11 +57,11 @@ if [[ ! -d "${AWS_LC_DIR}" ]]; then fi function filter_symbols() { - grep -v -E "^bignum_" | grep -v "curve25519_x25519" | grep -v "edwards25519_" + grep -E '^\w*$' | grep -v -E "^bignum_" | grep -v "curve25519_x25519" | grep -v "edwards25519_" } function filter_nm_symbols() { - grep -v -E '^_Z' | grep -v 'BORINGSSL_bcm_' | grep -v 'BORINGSSL_integrity_test' + grep -v -E '^_Z' | grep -v -E '^\?' | grep -v 'BORINGSSL_bcm_' | grep -v 'BORINGSSL_integrity_test' } function filter_macho_symbols() { @@ -69,11 +69,7 @@ function filter_macho_symbols() { } function find_libcrypto() { - find "${REPO_ROOT}/target" -type f \( -name "lib*crypto.a" -o -name "lib*crypto.so" -o -name "lib*crypto.dylib" \) | grep "${CRATE_NAME}" -} - -function find_libssl() { - find "${REPO_ROOT}/target" -type f \( -name "lib*ssl.a" -o -name "lib*ssl.so" -o -name "lib*ssl.dylib" \) | grep "${CRATE_NAME}" + find "${REPO_ROOT}/target" -type f \( -name "*crypto.lib" -o -name "lib*crypto.a" -o -name "lib*crypto.so" -o -name "lib*crypto.dylib" \) | grep "${CRATE_NAME}" } LIBCRYPTO_PATH="$(find_libcrypto)" @@ -82,22 +78,35 @@ if [[ "${?}" -ne 0 ]]; then exit 1 fi -LIBSSL_PATH="$(find_libssl)" -if [[ "${?}" -ne 0 ]]; then - echo "Unable to find libssl" - exit 1 -fi - mkdir -p "$(dirname "${SYMBOLS_FILE}")" echo Writing symbols to: ${SYMBOLS_FILE} -if [[ "${LIBCRYPTO_PATH}" = *.dylib ]]; then - nm --extern-only --defined-only -j "${LIBCRYPTO_PATH}" "${LIBSSL_PATH}" | grep -v "${REPO_ROOT}" | sort | uniq | filter_macho_symbols | filter_nm_symbols | filter_symbols >"${SYMBOLS_FILE}" -elif [[ "${LIBCRYPTO_PATH}" = *.so ]]; then - nm --extern-only --defined-only --format=just-symbols "${LIBCRYPTO_PATH}" "${LIBSSL_PATH}" | grep -v "${REPO_ROOT}" | sort | uniq | filter_nm_symbols | filter_symbols >"${SYMBOLS_FILE}" +if [[ "${PLATFORM}" = *-msvc ]]; then + if [[ "${PLATFORM}" = aarch64-* ]]; then + MSVC_ARCH=arm64 + elif [[ "${PLATFORM}" = i686-* ]]; then + MSVC_ARCH=x86 + else + MSVC_ARCH=x64 + fi + PFx86=$(printenv "ProgramFiles(x86)") + VS_INSTALL_PATH="$("$(echo "${PFx86//\\/\/}//Microsoft Visual Studio/Installer/vswhere.exe")" | grep 'resolvedInstallationPath:' | sed -e 's/[^:]*: \(.*\)$/\1/')" + + DUMPBIN="$(ls -1 "${VS_INSTALL_PATH//\\/\/}"/VC/Tools/MSVC/*/bin/Hostx64/${MSVC_ARCH}/dumpbin.exe | tail -n 1)" + PATH="$(dirname "${DUMPBIN/C:/\/c}")":"${PATH}" +if [[ "${MSVC_ARCH}" = x* ]]; then + dumpbin //EXPORTS //SYMBOLS "${LIBCRYPTO_PATH}" | grep External | grep -v UNDEF | sed -e 's/.*External\s*|\s*\(.*\)$/\1/' | grep -v -E '^snprintf$' | grep -E '^\w*$' | sort | uniq >"${SYMBOLS_FILE}" +else + dumpbin //EXPORTS //SYMBOLS "${LIBCRYPTO_PATH}" | grep External | grep -v UNDEF | sed -e 's/.*External\s*|\s*\(.*\)$/\1/' | sort | uniq | filter_symbols>"${SYMBOLS_FILE}" +fi + echo "dumpbin pipes: ${PIPESTATUS[@]}" +elif [[ "${LIBCRYPTO_PATH}" = *.dylib ]]; then + nm --extern-only --defined-only -j "${LIBCRYPTO_PATH}" | grep -v "${REPO_ROOT}" | sort | uniq | filter_macho_symbols | filter_nm_symbols | filter_symbols >"${SYMBOLS_FILE}" +elif [[ "${LIBCRYPTO_PATH}" = *.so || "${LIBCRYPTO_PATH}" = *.lib ]]; then + nm --extern-only --defined-only --format=just-symbols "${LIBCRYPTO_PATH}" | sort | uniq | filter_nm_symbols | filter_symbols >"${SYMBOLS_FILE}" else pushd "${AWS_LC_DIR}" - go run -mod readonly "${AWS_LC_DIR}"/util/read_symbols.go "${LIBCRYPTO_PATH}" "${LIBSSL_PATH}" | filter_symbols >"${SYMBOLS_FILE}" + go run -mod readonly "${AWS_LC_DIR}"/util/read_symbols.go "${LIBCRYPTO_PATH}" | filter_symbols >"${SYMBOLS_FILE}" popd fi