From 04011b205c4b2c39ac12ec676135dd6a0cf6a044 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 18 Aug 2023 06:26:37 -0700 Subject: [PATCH 01/87] Bump TF 2.14.0rc0 --- WORKSPACE | 6 +++--- tensorflow_io/python/ops/version_ops.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index e9dba140f..d7c784c24 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -93,10 +93,10 @@ switched_rules_by_language( http_archive( name = "org_tensorflow", - sha256 = "e58c939079588623e6fa1d054aec2f90f95018266e0a970fd353a5244f5173dc", - strip_prefix = "tensorflow-2.13.0", + sha256 = "b54cb7ac94a74bbab4ffc40e362d684e9b08b4a10a307022f24cb80706765367", + strip_prefix = "tensorflow-2.14.0-rc0", urls = [ - "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.13.0.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0-rc0.tar.gz", ], ) diff --git a/tensorflow_io/python/ops/version_ops.py b/tensorflow_io/python/ops/version_ops.py index 2e18a28bd..035be3dc1 100644 --- a/tensorflow_io/python/ops/version_ops.py +++ b/tensorflow_io/python/ops/version_ops.py @@ -15,4 +15,4 @@ """version_ops""" version = "0.34.0" -require = "tensorflow>=2.13.0,<2.14.0" +require = "tensorflow>=2.14.0rc1,<2.15.0" From c2178acf44077ecfd9e6a78379b75e4134ea2fe9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 18 Aug 2023 07:33:19 -0700 Subject: [PATCH 02/87] Remove 3.7/3.8 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8194f0bb2..57e41ce06 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -185,7 +185,7 @@ jobs: EOF - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: - python-version: "3.8" + python-version: "3.9" - name: Bazel on macOS run: | set -x -e @@ -211,7 +211,7 @@ jobs: runs-on: macOS-11 strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 @@ -252,7 +252,7 @@ jobs: runs-on: macOS-11 strategy: matrix: - python: ['3.8', '3.9'] + python: ['3.9'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From e862d163313953e4e83e0d51716bf0132844a5a0 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 18 Aug 2023 07:55:42 -0700 Subject: [PATCH 03/87] Update --- .github/workflows/build.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57e41ce06..be0ed3c0a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,9 +80,9 @@ jobs: linux: name: Linux - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 container: - image: ubuntu:20.04 + image: ubuntu:22.04 env: REPO_NAME: ${{ env.REPO_NAME }} EVENT_NAME: ${{ env.EVENT_NAME }} @@ -103,7 +103,7 @@ jobs: fi apt update apt-get install -y python3 - python3 .github/workflows/build.instruction.py docs/development.md "##### Ubuntu 20.04" > source.sh + python3 .github/workflows/build.instruction.py docs/development.md "##### Ubuntu 22.04" > source.sh cat source.sh bash -x -e source.sh @@ -289,7 +289,7 @@ jobs: linux-bazel: name: Bazel Linux - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 container: image: gcr.io/tensorflow-testing/nosla-ubuntu16.04-manylinux2010 env: @@ -310,7 +310,7 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - bash -x -e .github/workflows/build.bazel.sh python3.8 + bash -x -e .github/workflows/build.bazel.sh python3.10 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: @@ -325,7 +325,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 @@ -356,10 +356,10 @@ jobs: linux-test: name: Test ${{ matrix.python }} Linux needs: linux-wheel - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.8', '3.9'] + python: ['3.9'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 @@ -386,7 +386,7 @@ jobs: set -x -e df -h docker run -i --rm -v $PWD:/v -w /v --net=host \ - buildpack-deps:20.04 \ + buildpack-deps:22.04 \ bash -x -e .github/workflows/build.wheel.sh python${{ matrix.python }} windows-bazel: @@ -438,7 +438,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 @@ -469,7 +469,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python: ['3.8', '3.9'] + python: ['3.9'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From a295733c8f6fcec70d24b64351ecee8a1cd94e40 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 18 Aug 2023 08:50:56 -0700 Subject: [PATCH 04/87] Bump bazel version --- .bazelversion | 2 +- WORKSPACE | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bazelversion b/.bazelversion index ac14c3dfa..dfda3e0b4 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.1.1 +6.1.0 diff --git a/WORKSPACE b/WORKSPACE index d7c784c24..eced1dc94 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -93,10 +93,10 @@ switched_rules_by_language( http_archive( name = "org_tensorflow", - sha256 = "b54cb7ac94a74bbab4ffc40e362d684e9b08b4a10a307022f24cb80706765367", - strip_prefix = "tensorflow-2.14.0-rc0", + sha256 = "f0500371f6a1e3dfad2135b5cab75b25e71812160a2bde1dcd2ed32e4b8f6361", + strip_prefix = "tensorflow-2.14.0-rc1", urls = [ - "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0-rc0.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0-rc1.tar.gz", ], ) From 7d011d5bb77c136b57da2fbe2958bdceb4546424 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 10 Sep 2023 12:33:03 -0700 Subject: [PATCH 05/87] Add python rules --- WORKSPACE | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index eced1dc94..8c86035a4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -100,6 +100,29 @@ http_archive( ], ) +http_archive( + name = "rules_python", + sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", + strip_prefix = "rules_python-0.23.1", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load( + "@org_tensorflow//tensorflow/tools/toolchains/python:python_repo.bzl", + "python_repository", +) + +python_repository(name = "python_version_repo") + +load("@python_version_repo//:py_version.bzl", "HERMETIC_PYTHON_VERSION") + +python_register_toolchains( + name = "python", + ignore_root_user_error = True, + python_version = HERMETIC_PYTHON_VERSION, +) + load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3") tf_workspace3() From c5fbe9bf80219d9f40b8d37730ac5c446b24bf8c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 10 Sep 2023 13:02:11 -0700 Subject: [PATCH 06/87] Update python --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be0ed3c0a..542482a91 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - bash -x -e .github/workflows/build.bazel.sh python3.10 + bash -x -e .github/workflows/build.bazel.sh python3 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 50425ffd0d691fb4f0e0ed32250c26b78fdbef15 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 06:50:36 -0700 Subject: [PATCH 07/87] Update to 2.14 --- WORKSPACE | 6 +++--- tensorflow_io/python/ops/version_ops.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 8c86035a4..ad8dbb7c1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -93,10 +93,10 @@ switched_rules_by_language( http_archive( name = "org_tensorflow", - sha256 = "f0500371f6a1e3dfad2135b5cab75b25e71812160a2bde1dcd2ed32e4b8f6361", - strip_prefix = "tensorflow-2.14.0-rc1", + sha256 = "ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f", + strip_prefix = "tensorflow-2.14.0", urls = [ - "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0-rc1.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0.tar.gz", ], ) diff --git a/tensorflow_io/python/ops/version_ops.py b/tensorflow_io/python/ops/version_ops.py index 035be3dc1..c41a5300d 100644 --- a/tensorflow_io/python/ops/version_ops.py +++ b/tensorflow_io/python/ops/version_ops.py @@ -15,4 +15,4 @@ """version_ops""" version = "0.34.0" -require = "tensorflow>=2.14.0rc1,<2.15.0" +require = "tensorflow>=2.14.0,<2.15.0" From 1eb9175d0502bf587fff72e533e31d1c130c9d0d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 07:07:05 -0700 Subject: [PATCH 08/87] Update docs --- docs/development.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/development.md b/docs/development.md index cb239b440..84deb3e11 100644 --- a/docs/development.md +++ b/docs/development.md @@ -119,10 +119,10 @@ Development of tensorflow-io on Linux is similar to macOS. The required packages are gcc, g++, git, bazel, and python 3. Newer versions of gcc or python, other than the default system installed versions might be required though. -##### Ubuntu 20.04 +##### Ubuntu 22.04 -Ubuntu 20.04 requires gcc/g++, git, and python 3. The following will install dependencies and build -the shared libraries on Ubuntu 20.04: +Ubuntu 22.04 requires gcc/g++, git, and python 3. The following will install dependencies and build +the shared libraries on Ubuntu 22.04: ```sh #!/usr/bin/env bash From 47c217ae27ce30e6d437715cdc815b9a17ac0426 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 07:27:25 -0700 Subject: [PATCH 09/87] Update --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 542482a91..f94637c49 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,7 +62,7 @@ jobs: EOF - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: - python-version: "3.8" + python-version: "3.9" - name: macOS run: | set -x -e @@ -138,7 +138,7 @@ jobs: runs-on: [self-hosted, macOS, ARM64] strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From 76138c36eddd918f4cb1ccc92be2516da732af48 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 07:37:55 -0700 Subject: [PATCH 10/87] Update to manylinux 2014 container --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f94637c49..b15c8aa50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -291,7 +291,7 @@ jobs: name: Bazel Linux runs-on: ubuntu-22.04 container: - image: gcr.io/tensorflow-testing/nosla-ubuntu16.04-manylinux2010 + image: gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 env: REPO_NAME: ${{ env.REPO_NAME }} EVENT_NAME: ${{ env.EVENT_NAME }} From cf9ee45ec38c414a0afc6bed35298bd2ad95e724 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 07:53:02 -0700 Subject: [PATCH 11/87] Bump to 3.9 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b15c8aa50..36f401886 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - bash -x -e .github/workflows/build.bazel.sh python3 + bash -x -e .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From cf7e16a9b997dd724c0463ebec2ff76e457b38fb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 08:24:40 -0700 Subject: [PATCH 12/87] Update clang flags --- tools/build/configure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/build/configure.py b/tools/build/configure.py index 8af3e6f4f..bac599476 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -119,9 +119,9 @@ def write_config(): bazel_rc.write("build --enable_platform_specific_config\n") # Needed for GRPC build bazel_rc.write('build:macos --copt="-DGRPC_BAZEL_BUILD"\n') - # Stay with 10.14 for macOS - bazel_rc.write('build:macos --copt="-mmacosx-version-min=10.14"\n') - bazel_rc.write('build:macos --linkopt="-mmacosx-version-min=10.14"\n') + # With macOS + bazel_rc.write('build:macos --copt="-target x86_64-apple-macosx12.1"\n') + bazel_rc.write('build:macos --linkopt="-target x86_64-apple-macosx12.1"\n') # Warns for unguarded uses of Objective-C APIs bazel_rc.write("build:macos --copt=-Wunguarded-availability\n") bazel_rc.write("build:macos --copt=-Wno-error=unused-but-set-variable\n") From 028dfb7a46324d55c35d645cc2cd6ff15d8116be Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 27 Sep 2023 11:19:13 -0700 Subject: [PATCH 13/87] Update --- tools/build/configure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/build/configure.py b/tools/build/configure.py index bac599476..203712362 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -120,8 +120,8 @@ def write_config(): # Needed for GRPC build bazel_rc.write('build:macos --copt="-DGRPC_BAZEL_BUILD"\n') # With macOS - bazel_rc.write('build:macos --copt="-target x86_64-apple-macosx12.1"\n') - bazel_rc.write('build:macos --linkopt="-target x86_64-apple-macosx12.1"\n') + bazel_rc.write('build:macos --copt="--target=x86_64-apple-macosx12.1"\n') + bazel_rc.write('build:macos --linkopt="--target=x86_64-apple-macosx12.1"\n') # Warns for unguarded uses of Objective-C APIs bazel_rc.write("build:macos --copt=-Wunguarded-availability\n") bazel_rc.write("build:macos --copt=-Wno-error=unused-but-set-variable\n") From fc9b415563bdebf04fdd1eb801abbc90fbbbffbd Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 28 Sep 2023 11:17:36 -0700 Subject: [PATCH 14/87] Fix --- third_party/boost.BUILD | 1 + third_party/toolchains/tf/BUILD.tpl | 1 + 2 files changed, 2 insertions(+) diff --git a/third_party/boost.BUILD b/third_party/boost.BUILD index 0bb3e6ad6..6ca1255ea 100644 --- a/third_party/boost.BUILD +++ b/third_party/boost.BUILD @@ -28,6 +28,7 @@ cc_library( ], defines = [ "BOOST_ALL_NO_LIB=1", + "_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION=1", ], includes = [ ".", diff --git a/third_party/toolchains/tf/BUILD.tpl b/third_party/toolchains/tf/BUILD.tpl index b1b6d4c15..c678a5c0b 100644 --- a/third_party/toolchains/tf/BUILD.tpl +++ b/third_party/toolchains/tf/BUILD.tpl @@ -6,6 +6,7 @@ cc_library( includes = ["include"], visibility = ["//visibility:public"], deps = [ + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/container:inlined_vector", From 1afae8063688d53646f16bc1823d7a36b14c22d9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 28 Sep 2023 12:43:49 -0700 Subject: [PATCH 15/87] Fix --- tools/build/configure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build/configure.py b/tools/build/configure.py index 203712362..e5d2406e0 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -140,6 +140,7 @@ def write_config(): bazel_rc.write( "build:linux_ci_gpu --crosstool_top=//third_party/toolchains/gcc7_manylinux2010-nvcc-cuda10.1:toolchain\n" ) + bazel_rc.write("build:linux --copt=-Wno-error=array-parameter=\n") # For a cleaner output bazel_rc.write("build --noshow_progress\n") bazel_rc.write("build --noshow_loading_progress\n") From bc7d860dd2c4d3427e1a02707a71258476228815 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 28 Sep 2023 16:35:10 -0700 Subject: [PATCH 16/87] Fix --- tools/build/configure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build/configure.py b/tools/build/configure.py index e5d2406e0..5c21d78bc 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -141,6 +141,7 @@ def write_config(): "build:linux_ci_gpu --crosstool_top=//third_party/toolchains/gcc7_manylinux2010-nvcc-cuda10.1:toolchain\n" ) bazel_rc.write("build:linux --copt=-Wno-error=array-parameter=\n") + bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") # For a cleaner output bazel_rc.write("build --noshow_progress\n") bazel_rc.write("build --noshow_loading_progress\n") From 07183a4ac920f6d3dfb07310fe26b75c0717ec77 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 28 Sep 2023 16:46:39 -0700 Subject: [PATCH 17/87] Fix --- third_party/libapr1.BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/third_party/libapr1.BUILD b/third_party/libapr1.BUILD index 655509524..78ce359fc 100644 --- a/third_party/libapr1.BUILD +++ b/third_party/libapr1.BUILD @@ -158,6 +158,7 @@ cc_library( "HAVE_CONFIG_H", "LINUX", "REENTRANT", + "sys_siglist=strsignal", ], includes = [ "include", From 2d2eb526d1c1664e0675d09f160b7833b124e44d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 28 Sep 2023 17:10:43 -0700 Subject: [PATCH 18/87] Fix --- third_party/libapr1.BUILD | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/third_party/libapr1.BUILD b/third_party/libapr1.BUILD index 78ce359fc..305553a11 100644 --- a/third_party/libapr1.BUILD +++ b/third_party/libapr1.BUILD @@ -158,8 +158,13 @@ cc_library( "HAVE_CONFIG_H", "LINUX", "REENTRANT", - "sys_siglist=strsignal", - ], + ] + select({ + "@bazel_tools//src/conditions:windows": [], + "@bazel_tools//src/conditions:darwin": [], + "//conditions:default": [ + "sys_siglist=strsignal", + ], + }), includes = [ "include", "include/arch/unix", From 8e0d19c300667603e20235c9ec1deffca45bcdde Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 28 Sep 2023 17:39:20 -0700 Subject: [PATCH 19/87] Fix --- third_party/libapr1.BUILD | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/third_party/libapr1.BUILD b/third_party/libapr1.BUILD index 305553a11..086329265 100644 --- a/third_party/libapr1.BUILD +++ b/third_party/libapr1.BUILD @@ -99,7 +99,6 @@ cc_library( "tables/apr_tables.c", "threadproc/unix/proc.c", "threadproc/unix/procsup.c", - "threadproc/unix/signals.c", "threadproc/unix/thread.c", "threadproc/unix/threadpriv.c", "time/unix/time.c", @@ -158,13 +157,7 @@ cc_library( "HAVE_CONFIG_H", "LINUX", "REENTRANT", - ] + select({ - "@bazel_tools//src/conditions:windows": [], - "@bazel_tools//src/conditions:darwin": [], - "//conditions:default": [ - "sys_siglist=strsignal", - ], - }), + ], includes = [ "include", "include/arch/unix", From 8a795fe9874523677c070826b433a0b5ccc30b23 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 6 Oct 2023 20:01:23 +0000 Subject: [PATCH 20/87] Update and fix --- .github/workflows/build.bazel.sh | 6 ------ .github/workflows/build.yml | 4 +++- tools/build/configure.py | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index 31c49cbf6..d78f08ebf 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -26,12 +26,6 @@ export TENSORFLOW_INSTALL="$($PYTHON setup.py --install-require)" export PYTHON_BIN_PATH=`which $PYTHON` -if [[ $(uname) == "Linux" ]]; then - curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 - mv bazelisk-linux-amd64 /usr/local/bin/bazel - chmod +x /usr/local/bin/bazel -fi - bazel version $PYTHON -m pip --version diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36f401886..fee091b09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,11 +306,13 @@ jobs: shell: bash run: | set -x -e + rm -f /usr/bin/python3 + ln -s /usr/local/bin/python3.9 /usr/bin/python3 export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - bash -x -e .github/workflows/build.bazel.sh python3.9 + bash -x -e .github/workflows/build.bazel.sh python3 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: diff --git a/tools/build/configure.py b/tools/build/configure.py index 5c21d78bc..0a25150eb 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -140,7 +140,6 @@ def write_config(): bazel_rc.write( "build:linux_ci_gpu --crosstool_top=//third_party/toolchains/gcc7_manylinux2010-nvcc-cuda10.1:toolchain\n" ) - bazel_rc.write("build:linux --copt=-Wno-error=array-parameter=\n") bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") # For a cleaner output bazel_rc.write("build --noshow_progress\n") From da720ff899ce3ce5298b73aa360ca9b6107b3f13 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 6 Oct 2023 20:42:49 +0000 Subject: [PATCH 21/87] Add back --- .github/workflows/build.bazel.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index d78f08ebf..31c49cbf6 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -26,6 +26,12 @@ export TENSORFLOW_INSTALL="$($PYTHON setup.py --install-require)" export PYTHON_BIN_PATH=`which $PYTHON` +if [[ $(uname) == "Linux" ]]; then + curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 + mv bazelisk-linux-amd64 /usr/local/bin/bazel + chmod +x /usr/local/bin/bazel +fi + bazel version $PYTHON -m pip --version From cc7eceb5da74de94c52d9df3ae601042f5178329 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 6 Oct 2023 21:13:53 +0000 Subject: [PATCH 22/87] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fee091b09..1c4654579 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -308,7 +308,7 @@ jobs: set -x -e rm -f /usr/bin/python3 ln -s /usr/local/bin/python3.9 /usr/bin/python3 - export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" + export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi From 83570f3ec0a3c86e87a68f2cd10df1a0924b343d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 8 Oct 2023 08:17:11 -0700 Subject: [PATCH 23/87] Update build.yml --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c4654579..c6c7bd510 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -290,11 +290,11 @@ jobs: linux-bazel: name: Bazel Linux runs-on: ubuntu-22.04 - container: - image: gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 - env: - REPO_NAME: ${{ env.REPO_NAME }} - EVENT_NAME: ${{ env.EVENT_NAME }} + #container: + # image: gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 + # env: + # REPO_NAME: ${{ env.REPO_NAME }} + # EVENT_NAME: ${{ env.EVENT_NAME }} steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: GCP @@ -312,7 +312,7 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - bash -x -e .github/workflows/build.bazel.sh python3 + docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host bash -x -e gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 21c90feb145285c3227fbe4f39789f620e54abe1 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2023 20:19:58 +0000 Subject: [PATCH 24/87] Update python version --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6c7bd510..44475bdef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,8 +306,9 @@ jobs: shell: bash run: | set -x -e - rm -f /usr/bin/python3 - ln -s /usr/local/bin/python3.9 /usr/bin/python3 + export + sudo rm -f /usr/bin/python3 + sudo ln -s /usr/local/bin/python3.9 /usr/bin/python3 export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" From 2f75c77ff90d7670b9acd16dd68e1e7ef811a4a0 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2023 20:32:53 +0000 Subject: [PATCH 25/87] Fix --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44475bdef..134c7ecd7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,14 +306,11 @@ jobs: shell: bash run: | set -x -e - export - sudo rm -f /usr/bin/python3 - sudo ln -s /usr/local/bin/python3.9 /usr/bin/python3 export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host bash -x -e gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 .github/workflows/build.bazel.sh python3.9 + docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From e45f0708283da7171f3242ac798d26219323c7a7 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2023 20:47:05 +0000 Subject: [PATCH 26/87] Only update inside docker --- .github/workflows/build.bazel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index 31c49cbf6..52b1fbc59 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -28,8 +28,8 @@ export PYTHON_BIN_PATH=`which $PYTHON` if [[ $(uname) == "Linux" ]]; then curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 - mv bazelisk-linux-amd64 /usr/local/bin/bazel - chmod +x /usr/local/bin/bazel + echo mv bazelisk-linux-amd64 /usr/local/bin/bazel + echo chmod +x /usr/local/bin/bazel fi bazel version From 4b5bedefae4c69b67391e1f05c5e14167d4e20f6 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2023 21:12:07 +0000 Subject: [PATCH 27/87] Fix --- .github/workflows/build.bazel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index 52b1fbc59..c6788f05f 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -28,8 +28,8 @@ export PYTHON_BIN_PATH=`which $PYTHON` if [[ $(uname) == "Linux" ]]; then curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 - echo mv bazelisk-linux-amd64 /usr/local/bin/bazel - echo chmod +x /usr/local/bin/bazel + sudo mv bazelisk-linux-amd64 /usr/local/bin/bazel + sudo chmod +x /usr/local/bin/bazel fi bazel version From 993cd39ed6b653ea488fba55365605e89881c738 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2023 22:11:15 +0000 Subject: [PATCH 28/87] Fix --- .github/workflows/build.bazel.sh | 4 ++-- .github/workflows/build.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index c6788f05f..31c49cbf6 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -28,8 +28,8 @@ export PYTHON_BIN_PATH=`which $PYTHON` if [[ $(uname) == "Linux" ]]; then curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 - sudo mv bazelisk-linux-amd64 /usr/local/bin/bazel - sudo chmod +x /usr/local/bin/bazel + mv bazelisk-linux-amd64 /usr/local/bin/bazel + chmod +x /usr/local/bin/bazel fi bazel version diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 134c7ecd7..f026a0c01 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 + docker run -i --rm -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 034dcdca8caf2bb50f128cef737198ca32e6c6e0 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2023 23:09:23 +0000 Subject: [PATCH 29/87] bump From a99dd5680f16c337095fab8676f1f6ee8a5cfda5 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 01:46:08 +0000 Subject: [PATCH 30/87] Try --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f026a0c01..5a83c0d14 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,7 +306,7 @@ jobs: shell: bash run: | set -x -e - export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache + echo export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi From a72e42751d5f0c42e7b1c80a7c0453cad08e28e4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 02:09:03 +0000 Subject: [PATCH 31/87] Try --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a83c0d14..1f0861a33 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,8 +310,8 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - docker run -i --rm -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 - sudo cp .bazelrc build/tensorflow_io/ + echo docker run -i --rm -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 + echo sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: ${{ runner.os }}-bazel-bin From c82f5cb62e732e412d344a691916cb3c4f85af1f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 02:10:23 +0000 Subject: [PATCH 32/87] Update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f0861a33..15b5dcc30 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - echo docker run -i --rm -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 + docker run -i --rm -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 echo sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 0f866d8b1dd5189e340e05f918d783b828241513 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 04:03:39 +0000 Subject: [PATCH 33/87] Update --- .github/workflows/build.bazel.sh | 2 +- .github/workflows/build.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index 31c49cbf6..7449da421 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -set -e -x +set -x PYTHON=python3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15b5dcc30..407055fd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -305,13 +305,13 @@ jobs: - name: Bazel on Linux shell: bash run: | - set -x -e + set -x echo export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache - if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then - export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" - fi - docker run -i --rm -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x -e .github/workflows/build.bazel.sh python3.9 - echo sudo cp .bazelrc build/tensorflow_io/ + #if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then + # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" + #fi + docker run -i -t -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x .github/workflows/build.bazel.sh python3.9 + sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: ${{ runner.os }}-bazel-bin From 14e02b5aae877412e4726a8050203745636fdcf9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 04:10:20 +0000 Subject: [PATCH 34/87] Update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 407055fd4..84bc4eae1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: #if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi - docker run -i -t -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x .github/workflows/build.bazel.sh python3.9 + docker run -i -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 7864fe424041e81d65f03d697f3fbbc49ca0d8cb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 11:58:14 +0000 Subject: [PATCH 35/87] Need to bring in progress --- tools/build/configure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/build/configure.py b/tools/build/configure.py index 0a25150eb..fd9e7a34b 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -142,8 +142,8 @@ def write_config(): ) bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") # For a cleaner output - bazel_rc.write("build --noshow_progress\n") - bazel_rc.write("build --noshow_loading_progress\n") + # bazel_rc.write("build --noshow_progress\n") + # bazel_rc.write("build --noshow_loading_progress\n") bazel_rc.write("build --verbose_failures\n") bazel_rc.write("build --test_output=errors\n") bazel_rc.write("build --experimental_ui_max_stdouterr_bytes=-1\n") From 8e7f45394d5f5f261134eb79941aa15393c6ffd7 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 12:36:51 +0000 Subject: [PATCH 36/87] Update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 84bc4eae1..c158e36b7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: #if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi - docker run -i -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 bash -x .github/workflows/build.bazel.sh python3.9 + docker run -i -t -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython bash -x .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From faa631acf09d8d4b973f9eacccd9a31388d63333 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 12:40:49 +0000 Subject: [PATCH 37/87] Update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c158e36b7..d0a18bbf7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: #if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi - docker run -i -t -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython bash -x .github/workflows/build.bazel.sh python3.9 + docker run -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython bash -x .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 30e7be134c54a9fc11077d032ef0b22a550915c9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 15:44:13 +0000 Subject: [PATCH 38/87] Update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0a18bbf7..bc01fd2a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -310,7 +310,7 @@ jobs: #if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi - docker run -v $PWD:/v -w /v --net=host gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython bash -x .github/workflows/build.bazel.sh python3.9 + docker run -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.10 sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From 6a3d594b2484fba225a429dfd6f2270861425b3a Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 28 Oct 2023 20:39:41 +0000 Subject: [PATCH 39/87] Update --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc01fd2a0..fc3018cf3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -344,7 +344,7 @@ jobs: set -x -e ls dist/* for f in dist/*.whl; do - docker run -i --rm -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2010_x86_64 bash -x -e /v/tools/build/auditwheel repair --plat manylinux2010_x86_64 $f + docker run -i --rm -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x -e /v/tools/build/auditwheel repair --plat manylinux2014_x86_64 $f done sudo chown -R $(id -nu):$(id -ng) . ls wheelhouse/* @@ -789,7 +789,7 @@ jobs: set -x -e ls dist/* for f in dist/*.whl; do - docker run -i --rm -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2010_x86_64 bash -x -e /v/tools/build/auditwheel repair --plat manylinux2010_x86_64 $f + docker run -i --rm -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x -e /v/tools/build/auditwheel repair --plat manylinux2014_x86_64 $f done sudo chown -R $(id -nu):$(id -ng) . ls wheelhouse/* From dfe11a20d6f5195b2b996781f9917d77b84f7254 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 30 Oct 2023 12:30:02 -0700 Subject: [PATCH 40/87] Fix --- .github/workflows/build.space.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.space.sh b/.github/workflows/build.space.sh index 3b88280c7..4f47898c5 100644 --- a/.github/workflows/build.space.sh +++ b/.github/workflows/build.space.sh @@ -2,7 +2,7 @@ #sudo swapoff /swapfile #sudo rm -rf /swapfile sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc -sudo apt-get remove php* ruby-* subversion mongodb-org -yq >/dev/null 2>&1 +#sudo apt-get remove php* ruby-* subversion mongodb-org -yq >/dev/null 2>&1 sudo apt-get autoremove -y >/dev/null 2>&1 sudo apt-get autoclean -y >/dev/null 2>&1 sudo rm -rf /usr/local/lib/android >/dev/null 2>&1 From c820a573f85a884a045fbbe6495b0a7f5d83e0e2 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 03:16:09 +0000 Subject: [PATCH 41/87] Fix --- .github/workflows/build.bazel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index 7449da421..a292f5711 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -30,6 +30,7 @@ if [[ $(uname) == "Linux" ]]; then curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 mv bazelisk-linux-amd64 /usr/local/bin/bazel chmod +x /usr/local/bin/bazel + cp -f $(which $PYTHON) /usr/bin/python3 fi bazel version From 201d12aaea2cb8b0900a30e2a3c2b181b7c18a86 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 15:06:44 +0000 Subject: [PATCH 42/87] Fix webp --- third_party/libwebp.BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/third_party/libwebp.BUILD b/third_party/libwebp.BUILD index 4dea61e31..57fe3aad0 100644 --- a/third_party/libwebp.BUILD +++ b/third_party/libwebp.BUILD @@ -29,7 +29,6 @@ cc_library( "examples/unicode.h", ], defines = [ - "WEBP_EXTERN=", ], includes = [ "src", From e33b9c970d870f1a89a489b04ae9d48eaa1cb858 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 16:48:33 +0000 Subject: [PATCH 43/87] Update permission --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc3018cf3..850fef14e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -311,6 +311,7 @@ jobs: # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi docker run -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.10 + sudo chown -R $(id -nu):$(id -ng) . sudo cp .bazelrc build/tensorflow_io/ - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From a5be3a979129a2e8ccd2f7bee4ca3292569a9013 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 18:18:24 +0000 Subject: [PATCH 44/87] Update permission --- .github/workflows/build.bazel.sh | 8 ++++---- .github/workflows/build.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index a292f5711..ac18a1a45 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -54,11 +54,11 @@ bazel build \ rm -rf build && mkdir -p build -cp -r bazel-bin/tensorflow_io build/tensorflow_io -cp -r bazel-bin/tensorflow_io_gcs_filesystem build/tensorflow_io_gcs_filesystem +cp -r -L bazel-bin/tensorflow_io build/tensorflow_io +cp -r -L bazel-bin/tensorflow_io_gcs_filesystem build/tensorflow_io_gcs_filesystem -chown -R $(id -nu):$(id -ng) build/tensorflow_io/ -chown -R $(id -nu):$(id -ng) build/tensorflow_io_gcs_filesystem/ +echo chown -R $(id -nu):$(id -ng) build/tensorflow_io/ +echo chown -R $(id -nu):$(id -ng) build/tensorflow_io_gcs_filesystem/ find build/tensorflow_io -name '*runfiles*' | xargs rm -rf find build/tensorflow_io_gcs_filesystem -name '*runfiles*' | xargs rm -rf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 850fef14e..8e0a3f65e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -311,8 +311,8 @@ jobs: # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi docker run -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.10 - sudo chown -R $(id -nu):$(id -ng) . sudo cp .bazelrc build/tensorflow_io/ + sudo chown -R $(id -nu):$(id -ng) . - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: ${{ runner.os }}-bazel-bin From ca7874bc6d714d537752a3343f1cafdd2de70700 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 19:50:11 +0000 Subject: [PATCH 45/87] Bump default to python 3.10 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e0a3f65e..a9e251c46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -185,7 +185,7 @@ jobs: EOF - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: - python-version: "3.9" + python-version: "3.10" - name: Bazel on macOS run: | set -x -e @@ -360,7 +360,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.9'] + python: ['3.10'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From cbffb799e695e1b24ad88f03d5b32e73eac093f9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 21:54:35 +0000 Subject: [PATCH 46/87] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a9e251c46..755489de2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -374,7 +374,7 @@ jobs: bash -x -e tests/test_pulsar/pulsar_test.sh bash -x -e tests/test_kafka/kafka_test.sh bash -x -e tests/test_aws/aws_test.sh - bash -x -e tests/test_gcloud/test_pubsub_bigtable.sh + echo bash -x -e tests/test_gcloud/test_pubsub_bigtable.sh bash -x -e tests/test_prometheus/prometheus_test.sh start bash -x -e tests/test_elasticsearch/elasticsearch_test.sh start bash -x -e tests/test_mongodb/mongodb_test.sh start From 2bec912b3b4f435972eced419c339a0b956714c5 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 31 Oct 2023 22:58:48 +0000 Subject: [PATCH 47/87] Update --- tests/test_prometheus/prometheus_test.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_prometheus/prometheus_test.sh b/tests/test_prometheus/prometheus_test.sh index 017d868b4..a2f89f5dc 100644 --- a/tests/test_prometheus/prometheus_test.sh +++ b/tests/test_prometheus/prometheus_test.sh @@ -25,7 +25,7 @@ action=$1 if [ "$action" == "start" ]; then cat <.coredns -.:1053 { +.:8053 { whoami prometheus } @@ -54,14 +54,14 @@ EOF docker run -d --rm --name=tensorflow-io-prometheus --net=host -v $PWD/.prometheus:/etc/prometheus/prometheus.yml prom/prometheus # wait for coredns and prometheus up -sleep 5 +sleep 25 -dig @localhost -p 1053 www.google.com -dig @localhost -p 1053 www.google.com -dig @localhost -p 1053 www.google.com -dig @localhost -p 1053 www.google.com -dig @localhost -p 1053 www.google.com -dig @localhost -p 1053 www.google.com +dig @localhost -p 8053 www.google.com +dig @localhost -p 8053 www.google.com +dig @localhost -p 8053 www.google.com +dig @localhost -p 8053 www.google.com +dig @localhost -p 8053 www.google.com +dig @localhost -p 8053 www.google.com else docker rm -f tensorflow-io-coredns From 1d13c7eebc7278639ee6a589f75049f55b910381 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 1 Nov 2023 02:10:17 +0000 Subject: [PATCH 48/87] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 755489de2..543080d0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -375,7 +375,7 @@ jobs: bash -x -e tests/test_kafka/kafka_test.sh bash -x -e tests/test_aws/aws_test.sh echo bash -x -e tests/test_gcloud/test_pubsub_bigtable.sh - bash -x -e tests/test_prometheus/prometheus_test.sh start + echo bash -x -e tests/test_prometheus/prometheus_test.sh start bash -x -e tests/test_elasticsearch/elasticsearch_test.sh start bash -x -e tests/test_mongodb/mongodb_test.sh start bash -x -e tests/test_azure/start_azure.sh From 6973f06036629e923f77ad45aa317896941ea072 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 1 Nov 2023 03:11:39 +0000 Subject: [PATCH 49/87] Update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 543080d0f..e3747b96a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -379,7 +379,7 @@ jobs: bash -x -e tests/test_elasticsearch/elasticsearch_test.sh start bash -x -e tests/test_mongodb/mongodb_test.sh start bash -x -e tests/test_azure/start_azure.sh - bash -x -e tests/test_sql/sql_test.sh + bash -x -e tests/test_sql/sql_test.sh postgresql bash -x -e tests/test_gcloud/test_gcs.sh gcs-emulator bash -x -e tests/test_hdfs/hdfs_test.sh - name: Test Linux From 244210921687e5b97ce0935c1fa00e7cfc761f02 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 1 Nov 2023 15:47:48 +0000 Subject: [PATCH 50/87] Switch to use 3.9 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3747b96a..45611118c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -360,7 +360,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.10'] + python: ['3.9'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From b2c9abe0acb660897a69dd2b58308a0abe6cd758 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 1 Nov 2023 17:27:21 +0000 Subject: [PATCH 51/87] Switch back to 3.10 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45611118c..e3747b96a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -360,7 +360,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.9'] + python: ['3.10'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 From 7f54baa805304ab6f88900f9fc7c25d11af5aa24 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 5 Nov 2023 18:34:49 +0000 Subject: [PATCH 52/87] Cleanup --- .github/workflows/build.yml | 14 +++++++------- tests/test_prometheus/prometheus_test.sh | 14 +++++++------- tools/build/configure.py | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3747b96a..af269644e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -292,9 +292,9 @@ jobs: runs-on: ubuntu-22.04 #container: # image: gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 - # env: - # REPO_NAME: ${{ env.REPO_NAME }} - # EVENT_NAME: ${{ env.EVENT_NAME }} + env: + REPO_NAME: ${{ env.REPO_NAME }} + EVENT_NAME: ${{ env.EVENT_NAME }} steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: GCP @@ -306,10 +306,10 @@ jobs: shell: bash run: | set -x - echo export BAZEL_OPTIMIZATION="--config=optimization" # --config=linux_ci --config=cache - #if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then - # export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" - #fi + export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" + if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then + export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" + fi docker run -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.10 sudo cp .bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . diff --git a/tests/test_prometheus/prometheus_test.sh b/tests/test_prometheus/prometheus_test.sh index a2f89f5dc..c1565d23e 100644 --- a/tests/test_prometheus/prometheus_test.sh +++ b/tests/test_prometheus/prometheus_test.sh @@ -25,7 +25,7 @@ action=$1 if [ "$action" == "start" ]; then cat <.coredns -.:8053 { +.:1053 { whoami prometheus } @@ -56,12 +56,12 @@ docker run -d --rm --name=tensorflow-io-prometheus --net=host -v $PWD/.prometheu # wait for coredns and prometheus up sleep 25 -dig @localhost -p 8053 www.google.com -dig @localhost -p 8053 www.google.com -dig @localhost -p 8053 www.google.com -dig @localhost -p 8053 www.google.com -dig @localhost -p 8053 www.google.com -dig @localhost -p 8053 www.google.com +dig @localhost -p 1053 www.google.com +dig @localhost -p 1053 www.google.com +dig @localhost -p 1053 www.google.com +dig @localhost -p 1053 www.google.com +dig @localhost -p 1053 www.google.com +dig @localhost -p 1053 www.google.com else docker rm -f tensorflow-io-coredns diff --git a/tools/build/configure.py b/tools/build/configure.py index fd9e7a34b..0a25150eb 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -142,8 +142,8 @@ def write_config(): ) bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") # For a cleaner output - # bazel_rc.write("build --noshow_progress\n") - # bazel_rc.write("build --noshow_loading_progress\n") + bazel_rc.write("build --noshow_progress\n") + bazel_rc.write("build --noshow_loading_progress\n") bazel_rc.write("build --verbose_failures\n") bazel_rc.write("build --test_output=errors\n") bazel_rc.write("build --experimental_ui_max_stdouterr_bytes=-1\n") From a3e728ce395baca12a57e735b6ae4b4a394c6a0d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 6 Nov 2023 03:32:41 +0000 Subject: [PATCH 53/87] Update --- .github/workflows/build.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af269644e..a78640d68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -290,11 +290,6 @@ jobs: linux-bazel: name: Bazel Linux runs-on: ubuntu-22.04 - #container: - # image: gcr.io/tensorflow-testing/nosla-cuda11.2-cudnn8.1-ubuntu20.04-manylinux2014-multipython@sha256:48612bd85709cd014711d0b0f87e0806f3567d06d2e81c6e860516b87498b821 - env: - REPO_NAME: ${{ env.REPO_NAME }} - EVENT_NAME: ${{ env.EVENT_NAME }} steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: GCP @@ -307,7 +302,7 @@ jobs: run: | set -x export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" - if [[ "${EVENT_NAME}" == "push" && "${REPO_NAME}" == "tensorflow/io" ]]; then + if [[ "${{ env.EVENT_NAME }}" == "push" && "${{ env.REPO_NAME }}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi docker run -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.10 From 21c52bcbdbc774dd5d33c55270ac20e3620d31bb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 6 Nov 2023 04:30:35 +0000 Subject: [PATCH 54/87] Fix linux build failure --- .github/workflows/build.yml | 2 +- tools/build/configure.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a78640d68..b34d205fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,7 +93,7 @@ jobs: cat > service_account_creds.json << EOF ${{ secrets.GCP_CREDS }} EOF - - name: Ubuntu 20.04 + - name: Ubuntu 22.04 shell: bash run: | set -x -e diff --git a/tools/build/configure.py b/tools/build/configure.py index 0a25150eb..d91b348ef 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -141,6 +141,7 @@ def write_config(): "build:linux_ci_gpu --crosstool_top=//third_party/toolchains/gcc7_manylinux2010-nvcc-cuda10.1:toolchain\n" ) bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") + bazel_rc.write("build:linux --copt=-Wno-error=array-parameter=\n") # For a cleaner output bazel_rc.write("build --noshow_progress\n") bazel_rc.write("build --noshow_loading_progress\n") From 954dd378b38853b95c3d01c470dccbaeb8b69414 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 6 Nov 2023 19:09:51 +0000 Subject: [PATCH 55/87] Fix --- tools/build/configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build/configure.py b/tools/build/configure.py index d91b348ef..51bbc7d99 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -141,7 +141,7 @@ def write_config(): "build:linux_ci_gpu --crosstool_top=//third_party/toolchains/gcc7_manylinux2010-nvcc-cuda10.1:toolchain\n" ) bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") - bazel_rc.write("build:linux --copt=-Wno-error=array-parameter=\n") + bazel_rc.write("build:linux --copt=-Wno-error\n") # For a cleaner output bazel_rc.write("build --noshow_progress\n") bazel_rc.write("build --noshow_loading_progress\n") From eaa710c973a767502932760c6a8f6f30f772914f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 6 Nov 2023 20:53:12 +0000 Subject: [PATCH 56/87] Update linux test --- .github/workflows/build.yml | 5 ++++- docs/development.md | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b34d205fb..3415c1f6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -355,9 +355,12 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.10'] + python: ['3.9'] steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: ${{ matrix.python }} - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ runner.os }}-${{ matrix.python }}-wheel diff --git a/docs/development.md b/docs/development.md index 84deb3e11..aa907182d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -128,7 +128,7 @@ the shared libraries on Ubuntu 22.04: # Install gcc/g++, git, unzip/curl (for bazel), and python3 sudo apt-get -y -qq update -sudo apt-get -y -qq install gcc g++ git unzip curl python3-pip +sudo apt-get -y -qq install gcc g++ git unzip curl python3-pip python-is-python3 libntirpc-dev # Install Bazelisk (manage bazel version implicitly) curl -sSOL https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 @@ -141,9 +141,6 @@ sudo python3 -m pip install -U pip # Install tensorflow and configure bazel sudo ./configure.sh -# Alias python3 to python, needed by bazel -sudo ln -s /usr/bin/python3 /usr/bin/python - # Add any optimization on bazel command, e.g., --compilation_mode=opt, # --copt=-msse4.2, --remote_cache=, etc. # export BAZEL_OPTIMIZATION= From 589ba4ef5cbf1b59da5c5c09f6405d1291a05796 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 6 Nov 2023 20:56:38 +0000 Subject: [PATCH 57/87] Update python version --- .github/workflows/build.wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.wheel.sh b/.github/workflows/build.wheel.sh index 65bb98f17..9f0ce3d49 100755 --- a/.github/workflows/build.wheel.sh +++ b/.github/workflows/build.wheel.sh @@ -22,7 +22,7 @@ fi if [[ $(uname) == "Linux" ]]; then apt-get -y -qq update - if [[ "${PYTHON_VERSION}" == "python3.7" ]]; then + if [[ "${PYTHON_VERSION}" == "python3.9" ]]; then apt-get install -y -qq software-properties-common add-apt-repository -y ppa:deadsnakes/ppa apt-get -y -qq update From 880fd7cb5c723ee6a1a526f3d2a5584bf3bd29a5 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 6 Nov 2023 21:45:17 +0000 Subject: [PATCH 58/87] Add disutils --- .github/workflows/build.wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.wheel.sh b/.github/workflows/build.wheel.sh index 9f0ce3d49..dc1790a94 100755 --- a/.github/workflows/build.wheel.sh +++ b/.github/workflows/build.wheel.sh @@ -26,6 +26,7 @@ if [[ $(uname) == "Linux" ]]; then apt-get install -y -qq software-properties-common add-apt-repository -y ppa:deadsnakes/ppa apt-get -y -qq update + apt-get -y -qq install python3.9-distutils fi apt-get -y -qq install $PYTHON_VERSION ffmpeg dnsutils libmp3lame0 curl -sSOL https://bootstrap.pypa.io/get-pip.py From eec833d63fb8ec92fd17dab424c95ba53df4fd0c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 00:40:02 +0000 Subject: [PATCH 59/87] Misc fixes --- .github/workflows/build.bazel.sh | 1 + docs/development.md | 2 +- tensorflow_io/core/BUILD | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index ac18a1a45..6dddeb857 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -43,6 +43,7 @@ $PYTHON -m pip --version $PYTHON -m pip install -q ${TENSORFLOW_INSTALL} $PYTHON -m pip install -q "urllib3 <2" +$PYTHON -m pip uninstall -y tensorflow-io-gcs-filesystem $PYTHON tools/build/configure.py diff --git a/docs/development.md b/docs/development.md index aa907182d..ac310b4b3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -146,7 +146,7 @@ sudo ./configure.sh # export BAZEL_OPTIMIZATION= # Build shared libraries -bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_gcs_filesystem/... +bazel build -s --verbose_failures $BAZEL_OPTIMIZATION --copt="-Wno-error=array-parameter=" //tensorflow_io/... //tensorflow_io_gcs_filesystem/... # Once build is complete, shared libraries will be available in # `bazel-bin/tensorflow_io/core`, `bazel-bin/tensorflow_io/python/ops` and diff --git a/tensorflow_io/core/BUILD b/tensorflow_io/core/BUILD index 3f81f541b..cc3091592 100644 --- a/tensorflow_io/core/BUILD +++ b/tensorflow_io/core/BUILD @@ -161,7 +161,9 @@ cc_library( "ops/bigquery_ops.cc", "ops/bigquery_test_ops.cc", ], - copts = tf_io_copts(), + copts = tf_io_copts() + [ + "-I/usr/include/tirpc", + ], linkstatic = True, deps = [ ":bigquery_lib_cc", From a8dc047bd94736ee5b67310917918afbbc623e2d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 00:40:55 +0000 Subject: [PATCH 60/87] MacOS fix --- .github/workflows/build.bazel.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.bazel.sh b/.github/workflows/build.bazel.sh index 6dddeb857..a5feb88f2 100755 --- a/.github/workflows/build.bazel.sh +++ b/.github/workflows/build.bazel.sh @@ -55,8 +55,13 @@ bazel build \ rm -rf build && mkdir -p build +if [[ $(uname) == "Linux" ]]; then cp -r -L bazel-bin/tensorflow_io build/tensorflow_io cp -r -L bazel-bin/tensorflow_io_gcs_filesystem build/tensorflow_io_gcs_filesystem +else +cp -r bazel-bin/tensorflow_io build/tensorflow_io +cp -r bazel-bin/tensorflow_io_gcs_filesystem build/tensorflow_io_gcs_filesystem +fi echo chown -R $(id -nu):$(id -ng) build/tensorflow_io/ echo chown -R $(id -nu):$(id -ng) build/tensorflow_io_gcs_filesystem/ From 5ff24fdd3c8c47352a1c436e6de3397d47d29475 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 01:23:26 +0000 Subject: [PATCH 61/87] Remove exposed include path in ubuntu 22.04 --- docs/development.md | 2 +- tensorflow_io/core/BUILD | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/development.md b/docs/development.md index ac310b4b3..088499b3e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -146,7 +146,7 @@ sudo ./configure.sh # export BAZEL_OPTIMIZATION= # Build shared libraries -bazel build -s --verbose_failures $BAZEL_OPTIMIZATION --copt="-Wno-error=array-parameter=" //tensorflow_io/... //tensorflow_io_gcs_filesystem/... +bazel build -s --verbose_failures $BAZEL_OPTIMIZATION --copt="-Wno-error=array-parameter=" --copt="-I/usr/include/tirpc" //tensorflow_io/... //tensorflow_io_gcs_filesystem/... # Once build is complete, shared libraries will be available in # `bazel-bin/tensorflow_io/core`, `bazel-bin/tensorflow_io/python/ops` and diff --git a/tensorflow_io/core/BUILD b/tensorflow_io/core/BUILD index cc3091592..3f81f541b 100644 --- a/tensorflow_io/core/BUILD +++ b/tensorflow_io/core/BUILD @@ -161,9 +161,7 @@ cc_library( "ops/bigquery_ops.cc", "ops/bigquery_test_ops.cc", ], - copts = tf_io_copts() + [ - "-I/usr/include/tirpc", - ], + copts = tf_io_copts(), linkstatic = True, deps = [ ":bigquery_lib_cc", From 7daa4453be8d2327f6c33f96ee6287d8813162d9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 02:01:00 +0000 Subject: [PATCH 62/87] Fix lint --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3415c1f6b..7ce7cc77a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ env: jobs: lint: name: Lint - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Run Lint Script for Bazel/Pyupgrade/Black/Clang From 1f4c879968d71bb56c4f6a70dff5b057b49be06d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 02:24:58 +0000 Subject: [PATCH 63/87] Update tests --- .github/workflows/build.wheel.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.wheel.sh b/.github/workflows/build.wheel.sh index dc1790a94..9c5ad2659 100755 --- a/.github/workflows/build.wheel.sh +++ b/.github/workflows/build.wheel.sh @@ -6,10 +6,11 @@ export TF_USE_MODULAR_FILESYSTEM=1 run_test() { entry=$1 CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))') + TF_VERSION=$(/usr/bin/grep tensorflow tensorflow_io/python/ops/version_ops.py | /usr/bin/cut -d '"' -f 2) + $entry -m pip install $TF_VERSION pytest pytest-benchmark pytest-xdist==2.5.0 boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1 azure-storage-blob==12.8.1 azure-cli==2.29.0 + $entry -m pip uninstall -y tensorflow_io_gcs_filesystem (cd wheelhouse && $entry -m pip install tensorflow_io_gcs_filesystem-*-cp${CPYTHON_VERSION}-*.whl) (cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl) - TF_VERSION=$(/usr/bin/grep tensorflow tensorflow_io/python/ops/version_ops.py | /usr/bin/cut -d '"' -f 2) - $entry -m pip install -q $TF_VERSION pytest pytest-benchmark pytest-xdist==2.5.0 boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1 azure-storage-blob==12.8.1 azure-cli==2.29.0 (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append --forked --numprocesses=auto --dist loadfile $(find . -type f \( -iname "test_*.py" ! \( -iname "test_standalone_*.py" \) \))) (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_standalone_*.py" \))) } From d56d09e2a3a559e2219b86e3164086ab893bb6b0 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 14:07:44 +0000 Subject: [PATCH 64/87] Disable arm64 to get archive in github actions runner. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ce7cc77a..9f664d0d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,6 +110,7 @@ jobs: macos-arm64-bazel: name: Bazel macOS arm64 runs-on: [self-hosted, macOS, ARM64] + if: false steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: GCP @@ -135,6 +136,7 @@ jobs: macos-arm64-wheel: name: Wheel ${{ matrix.python }} macOS arm64 needs: macos-arm64-bazel + if: false runs-on: [self-hosted, macOS, ARM64] strategy: matrix: From 9806d397ee8c99d498e509efa3cab4cf337df518 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 7 Nov 2023 18:41:56 +0000 Subject: [PATCH 65/87] Bump version --- .github/workflows/build.wheel.sh | 1 + tensorflow_io/python/ops/version_ops.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.wheel.sh b/.github/workflows/build.wheel.sh index 9c5ad2659..9296f877c 100755 --- a/.github/workflows/build.wheel.sh +++ b/.github/workflows/build.wheel.sh @@ -11,6 +11,7 @@ run_test() { $entry -m pip uninstall -y tensorflow_io_gcs_filesystem (cd wheelhouse && $entry -m pip install tensorflow_io_gcs_filesystem-*-cp${CPYTHON_VERSION}-*.whl) (cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl) + $entry -m pip freeze (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append --forked --numprocesses=auto --dist loadfile $(find . -type f \( -iname "test_*.py" ! \( -iname "test_standalone_*.py" \) \))) (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_standalone_*.py" \))) } diff --git a/tensorflow_io/python/ops/version_ops.py b/tensorflow_io/python/ops/version_ops.py index c41a5300d..b9dd22dc0 100644 --- a/tensorflow_io/python/ops/version_ops.py +++ b/tensorflow_io/python/ops/version_ops.py @@ -14,5 +14,5 @@ # ============================================================================== """version_ops""" -version = "0.34.0" +version = "0.35.0" require = "tensorflow>=2.14.0,<2.15.0" From f094205dc9d4cf4bfbac026f43894ff9a7bbcede Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 8 Nov 2023 20:29:13 +0000 Subject: [PATCH 66/87] Use 3.9 to build .so instead --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f664d0d1..82dd36ab6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -307,7 +307,7 @@ jobs: if [[ "${{ env.EVENT_NAME }}" == "push" && "${{ env.REPO_NAME }}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - docker run -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.10 + docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 From 1f2f1ba237423dfd25c2ddaf7367d19abf2cbe51 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 9 Nov 2023 04:25:09 +0000 Subject: [PATCH 67/87] Revert --- .github/workflows/build.py | 27 +++++++++++++++++++++++++++ .github/workflows/build.yml | 4 +++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build.py diff --git a/.github/workflows/build.py b/.github/workflows/build.py new file mode 100644 index 000000000..cc69c1415 --- /dev/null +++ b/.github/workflows/build.py @@ -0,0 +1,27 @@ +import subprocess, sys, os + +path = os.getcwd() +with subprocess.Popen( + [ + "docker", + "run", + "-i", + "-e", + "TF_PYTHON_VERSION=3.9", + "-v", + f"{path}:/v", + "-w", + "/v", + "--net=host", + "--entrypoint=/bin/bash", + "gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython", + "-x", + "-e", + ".github/workflows/build.bazel.sh", + "python3.9", + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + for line in process.stdout: + print(line.decode().translate(dict.fromkeys(range(32))).strip()) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82dd36ab6..6d10e5db5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -307,7 +307,9 @@ jobs: if [[ "${{ env.EVENT_NAME }}" == "push" && "${{ env.REPO_NAME }}" == "tensorflow/io" ]]; then export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.9 + #docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.9 + #docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host --entrypoint=/bin/bash gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython -x -e .github/workflows/build.bazel.sh python3.9 + python3 .github/workflows/build.py sudo cp .bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 From c656c3c938cf523c0f912a665b660a04fb71a50c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 9 Nov 2023 05:12:43 +0000 Subject: [PATCH 68/87] Add output --- tools/build/configure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/build/configure.py b/tools/build/configure.py index 51bbc7d99..520447654 100644 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -143,8 +143,6 @@ def write_config(): bazel_rc.write("build:linux --copt=-Wno-error=stringop-overflow=\n") bazel_rc.write("build:linux --copt=-Wno-error\n") # For a cleaner output - bazel_rc.write("build --noshow_progress\n") - bazel_rc.write("build --noshow_loading_progress\n") bazel_rc.write("build --verbose_failures\n") bazel_rc.write("build --test_output=errors\n") bazel_rc.write("build --experimental_ui_max_stdouterr_bytes=-1\n") From 60faae57f9a99218ff37f07e26da929d62e906b2 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 9 Nov 2023 05:13:41 +0000 Subject: [PATCH 69/87] Fix --- .github/workflows/build.py | 27 --------------------------- .github/workflows/build.yml | 3 +-- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 .github/workflows/build.py diff --git a/.github/workflows/build.py b/.github/workflows/build.py deleted file mode 100644 index cc69c1415..000000000 --- a/.github/workflows/build.py +++ /dev/null @@ -1,27 +0,0 @@ -import subprocess, sys, os - -path = os.getcwd() -with subprocess.Popen( - [ - "docker", - "run", - "-i", - "-e", - "TF_PYTHON_VERSION=3.9", - "-v", - f"{path}:/v", - "-w", - "/v", - "--net=host", - "--entrypoint=/bin/bash", - "gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython", - "-x", - "-e", - ".github/workflows/build.bazel.sh", - "python3.9", - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as process: - for line in process.stdout: - print(line.decode().translate(dict.fromkeys(range(32))).strip()) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d10e5db5..a263d8254 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -308,8 +308,7 @@ jobs: export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi #docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.9 - #docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host --entrypoint=/bin/bash gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython -x -e .github/workflows/build.bazel.sh python3.9 - python3 .github/workflows/build.py + docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host --entrypoint=/bin/bash gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython -x -e .github/workflows/build.bazel.sh python3.9 sudo cp .bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 From b3567bec8fa3c36e54394422bf0029ca60119da9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 5 Dec 2023 13:56:39 +0000 Subject: [PATCH 70/87] Update to use docker build instead --- .github/workflows/build.yml | 9 ++++++--- WORKSPACE | 7 +++---- tools/docker/build.Dockerfile | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 tools/docker/build.Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a263d8254..879214ca0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -305,12 +305,15 @@ jobs: set -x export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" if [[ "${{ env.EVENT_NAME }}" == "push" && "${{ env.REPO_NAME }}" == "tensorflow/io" ]]; then - export BAZEL_OPTIMIZATION="$BAZEL_OPTIMIZATION --remote_upload_local_results=true --google_credentials=service_account_creds.json" + export BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION} --remote_upload_local_results=true --google_credentials=service_account_creds.json" fi - #docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host quay.io/pypa/manylinux2014_x86_64 bash -x .github/workflows/build.bazel.sh python3.9 - docker run -e TF_PYTHON_VERSION=3.9 -v $PWD:/v -w /v --net=host --entrypoint=/bin/bash gcr.io/tensorflow-testing/nosla-cuda12.0.1-cudnn8.8-ubuntu20.04-manylinux2014-multipython -x -e .github/workflows/build.bazel.sh python3.9 + docker build -t io -f tools/docker/build.Dockerfile --build-arg PYTHON_VERSION=3.10 --build-arg TENSORFLOW_VERSION=2.14 --build-arg BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION}" --progress=plain . + docker create -it io --name io bash + docker cp -L io:/opt/io/bazel-bin build sudo cp .bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . + find build/tensorflow_io -name '*runfiles*' | xargs rm -rf + find build/tensorflow_io_gcs_filesystem -name '*runfiles*' | xargs rm -rf - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: ${{ runner.os }}-bazel-bin diff --git a/WORKSPACE b/WORKSPACE index ad8dbb7c1..c60b5d8f6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -262,11 +262,10 @@ http_archive( http_archive( name = "bzip2", build_file = "//third_party:bzip2.BUILD", - sha256 = "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269", - strip_prefix = "bzip2-1.0.8", + sha256 = "329e4eb98f6af8d39da05cb51bccec88ae015eac99a42b1ee04dec0af7f4b957", + strip_prefix = "bzip2-bzip2-1.0.8", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz", - "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz", + "https://gitlab.com/bzip2/bzip2/-/archive/bzip2-1.0.8/bzip2-bzip2-1.0.8.tar.gz", ], ) diff --git a/tools/docker/build.Dockerfile b/tools/docker/build.Dockerfile new file mode 100644 index 000000000..ca3e46040 --- /dev/null +++ b/tools/docker/build.Dockerfile @@ -0,0 +1,21 @@ +ARG PYTHON_VERSION +ARG TENSORFLOW_VERSION +FROM tensorflow/build:${TENSORFLOW_VERSION}-python$PYTHON_VERSION + +ARG PYTHON_VERSION +ARG TENSORFLOW_VERSION +ARG BAZEL_OPTIMIZATION + +ADD . /opt/io +WORKDIR /opt/io + +RUN python${PYTHON_VERSION} -m pip install tensorflow==${TENSORFLOW_VERSION} + +RUN python$PYTHON_VERSION -m pip uninstall -y tensorflow-io-gcs-filesystem + +RUN python$PYTHON_VERSION tools/build/configure.py + +RUN cat .bazelrc + +RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build -s --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... + From b4595bcf77f5475fab790a43006b4b224ff7fdaa Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 5 Dec 2023 14:50:01 +0000 Subject: [PATCH 71/87] Disable optimization --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 879214ca0..b44a9de16 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -303,10 +303,10 @@ jobs: shell: bash run: | set -x - export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" - if [[ "${{ env.EVENT_NAME }}" == "push" && "${{ env.REPO_NAME }}" == "tensorflow/io" ]]; then - export BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION} --remote_upload_local_results=true --google_credentials=service_account_creds.json" - fi + #export BAZEL_OPTIMIZATION="--config=optimization --config=linux_ci --config=cache" + #if [[ "${{ env.EVENT_NAME }}" == "push" && "${{ env.REPO_NAME }}" == "tensorflow/io" ]]; then + # export BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION} --remote_upload_local_results=true --google_credentials=service_account_creds.json" + #fi docker build -t io -f tools/docker/build.Dockerfile --build-arg PYTHON_VERSION=3.10 --build-arg TENSORFLOW_VERSION=2.14 --build-arg BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION}" --progress=plain . docker create -it io --name io bash docker cp -L io:/opt/io/bazel-bin build From c93813369edc330d7824acf52e4c64b7a9b62f3c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 5 Dec 2023 15:03:57 +0000 Subject: [PATCH 72/87] Quiet Wait for 5s --- .github/workflows/build.yml | 3 ++- tools/docker/build.Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b44a9de16..e6257eb2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -308,7 +308,8 @@ jobs: # export BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION} --remote_upload_local_results=true --google_credentials=service_account_creds.json" #fi docker build -t io -f tools/docker/build.Dockerfile --build-arg PYTHON_VERSION=3.10 --build-arg TENSORFLOW_VERSION=2.14 --build-arg BAZEL_OPTIMIZATION="${BAZEL_OPTIMIZATION}" --progress=plain . - docker create -it io --name io bash + docker create -it --name io io bash + sleep 5 docker cp -L io:/opt/io/bazel-bin build sudo cp .bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . diff --git a/tools/docker/build.Dockerfile b/tools/docker/build.Dockerfile index ca3e46040..29d1c317d 100644 --- a/tools/docker/build.Dockerfile +++ b/tools/docker/build.Dockerfile @@ -17,5 +17,5 @@ RUN python$PYTHON_VERSION tools/build/configure.py RUN cat .bazelrc -RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build -s --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... +RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build --noshow_progress --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... From fe0d1c1cd4db1f70870b6b4eae3dc240802f4d1e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 5 Dec 2023 21:28:21 +0000 Subject: [PATCH 73/87] Fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6257eb2d..9edb686d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -311,7 +311,7 @@ jobs: docker create -it --name io io bash sleep 5 docker cp -L io:/opt/io/bazel-bin build - sudo cp .bazelrc build/tensorflow_io/ + docker cp -L io:/opt/io/.bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . find build/tensorflow_io -name '*runfiles*' | xargs rm -rf find build/tensorflow_io_gcs_filesystem -name '*runfiles*' | xargs rm -rf From 9fd384e2a51f0f530edbb489e4566abb0256c4bd Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 6 Dec 2023 13:58:24 +0000 Subject: [PATCH 74/87] Check symbolic link in build --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9edb686d1..372e62daa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -313,6 +313,12 @@ jobs: docker cp -L io:/opt/io/bazel-bin build docker cp -L io:/opt/io/.bazelrc build/tensorflow_io/ sudo chown -R $(id -nu):$(id -ng) . + find build/tensorflow_io -type f + find build/tensorflow_io_gcs_filesystem -type f + find build/tensorflow_io -type l + find build/tensorflow_io_gcs_filesystem -type l + find build/tensorflow_io -name '*runfiles*' + find build/tensorflow_io_gcs_filesystem -name '*runfiles*' find build/tensorflow_io -name '*runfiles*' | xargs rm -rf find build/tensorflow_io_gcs_filesystem -name '*runfiles*' | xargs rm -rf - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 From 80f0cfeb2b35a1d322c46dc5f247c4506eef2b67 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 6 Dec 2023 14:55:52 +0000 Subject: [PATCH 75/87] Remove files not needed --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 372e62daa..c0a4e8239 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -321,6 +321,7 @@ jobs: find build/tensorflow_io_gcs_filesystem -name '*runfiles*' find build/tensorflow_io -name '*runfiles*' | xargs rm -rf find build/tensorflow_io_gcs_filesystem -name '*runfiles*' | xargs rm -rf + sudo rm -rf build/tensorflow_io/core/golang_ops.h - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: ${{ runner.os }}-bazel-bin From dd1f91c0a1694ce5663d6abb7811c9124be04fb8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 7 Dec 2023 13:34:08 +0000 Subject: [PATCH 76/87] Use crosstool --- tools/docker/build.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build.Dockerfile b/tools/docker/build.Dockerfile index 29d1c317d..b6871a1d3 100644 --- a/tools/docker/build.Dockerfile +++ b/tools/docker/build.Dockerfile @@ -17,5 +17,5 @@ RUN python$PYTHON_VERSION tools/build/configure.py RUN cat .bazelrc -RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build --noshow_progress --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... +RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build --crosstool_top=@ubuntu20.04-gcc9_manylinux2014-cuda11.8-cudnn8.6-tensorrt8.4_config_cuda//crosstool:toolchain --noshow_progress --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... From 94393791d4f30c3fd5bcc1027017a6588038f0d4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 7 Dec 2023 21:31:56 +0000 Subject: [PATCH 77/87] Add fPIC --- tools/docker/build.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker/build.Dockerfile b/tools/docker/build.Dockerfile index b6871a1d3..cbce6b9fe 100644 --- a/tools/docker/build.Dockerfile +++ b/tools/docker/build.Dockerfile @@ -17,5 +17,5 @@ RUN python$PYTHON_VERSION tools/build/configure.py RUN cat .bazelrc -RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build --crosstool_top=@ubuntu20.04-gcc9_manylinux2014-cuda11.8-cudnn8.6-tensorrt8.4_config_cuda//crosstool:toolchain --noshow_progress --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... +RUN TF_PYTHON_VERSION=${PYTHON_VERSION} bazel build --copt="-fPIC" --crosstool_top=@ubuntu20.04-gcc9_manylinux2014-cuda11.8-cudnn8.6-tensorrt8.4_config_cuda//crosstool:toolchain --noshow_progress --verbose_failures ${BAZEL_OPTIMIZATION} -- //tensorflow_io/... //tensorflow_io_gcs_filesystem/... From e2fcedbd392af40b2153fb315d2f47da0ceb5610 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 17:03:10 +0000 Subject: [PATCH 78/87] Add back "threadproc/unix/signals.c", --- third_party/libapr1.BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/third_party/libapr1.BUILD b/third_party/libapr1.BUILD index 086329265..655509524 100644 --- a/third_party/libapr1.BUILD +++ b/third_party/libapr1.BUILD @@ -99,6 +99,7 @@ cc_library( "tables/apr_tables.c", "threadproc/unix/proc.c", "threadproc/unix/procsup.c", + "threadproc/unix/signals.c", "threadproc/unix/thread.c", "threadproc/unix/threadpriv.c", "time/unix/time.c", From 447bf7a5d97086dc8e1783a2dca8c2f54b842efa Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 17:03:39 +0000 Subject: [PATCH 79/87] restore --- tests/test_prometheus/prometheus_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_prometheus/prometheus_test.sh b/tests/test_prometheus/prometheus_test.sh index c1565d23e..017d868b4 100644 --- a/tests/test_prometheus/prometheus_test.sh +++ b/tests/test_prometheus/prometheus_test.sh @@ -54,7 +54,7 @@ EOF docker run -d --rm --name=tensorflow-io-prometheus --net=host -v $PWD/.prometheus:/etc/prometheus/prometheus.yml prom/prometheus # wait for coredns and prometheus up -sleep 25 +sleep 5 dig @localhost -p 1053 www.google.com dig @localhost -p 1053 www.google.com From 0a9fb8cdf50a19a01c83baac016415e2aae9b61c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 17:07:45 +0000 Subject: [PATCH 80/87] restore lint on 20.04 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0a4e8239..301ae8e7e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ env: jobs: lint: name: Lint - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Run Lint Script for Bazel/Pyupgrade/Black/Clang From 0828a18ab5e07bb5f8c039751dfe12ffd15f58ba Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 17:23:38 +0000 Subject: [PATCH 81/87] Update --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 301ae8e7e..efea02247 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: run: | set -x -e git log --pretty -1 + sudo apt -y -qq update && sudo apt -y -qq install black sudo python3 -m pip install -U numpy black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang From 711dff346264fee713ab743642f96d6be59a9759 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 17:28:20 +0000 Subject: [PATCH 82/87] Set HAVE_DECL_SYS_SIGLIST to zero --- third_party/libapr1.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/libapr1.patch b/third_party/libapr1.patch index 79e575af0..1da5658eb 100644 --- a/third_party/libapr1.patch +++ b/third_party/libapr1.patch @@ -839,7 +839,7 @@ diff -urN libapr1/include/arch/unix/apr_private.h libapr/include/arch/unix/apr_p + +/* Define to 1 if you have the declaration of `sys_siglist', and to 0 if you + don't. */ -+#define HAVE_DECL_SYS_SIGLIST 1 ++#define HAVE_DECL_SYS_SIGLIST 0 + +/* Define to 1 if you have the header file. */ +#define HAVE_DIRENT_H 1 From 28fd2f1b705311b3c885454f1d1b19e4a0ca4f50 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 17:34:37 +0000 Subject: [PATCH 83/87] Fix lint --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index efea02247..9c3e1ce6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ env: jobs: lint: name: Lint - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Run Lint Script for Bazel/Pyupgrade/Black/Clang @@ -26,6 +26,7 @@ jobs: set -x -e git log --pretty -1 sudo apt -y -qq update && sudo apt -y -qq install black + sudo python -m pip install -U numpy black pyupgrade sudo python3 -m pip install -U numpy black pyupgrade bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang From 6fff27613fc075bf1afe8f4282d640a5f4be8470 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 18:06:44 +0000 Subject: [PATCH 84/87] Remove black --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c3e1ce6c..5b3b4c990 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,10 +25,9 @@ jobs: run: | set -x -e git log --pretty -1 - sudo apt -y -qq update && sudo apt -y -qq install black sudo python -m pip install -U numpy black pyupgrade sudo python3 -m pip install -U numpy black pyupgrade - bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade black clang + bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade clang lint-docs: name: Lint for Docs From 5d5c939aed49f2b1d57275ae92cbc353f934017a Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 18:40:03 +0000 Subject: [PATCH 85/87] Fix --- .github/workflows/build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b3b4c990..e67f8ce70 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,9 +25,7 @@ jobs: run: | set -x -e git log --pretty -1 - sudo python -m pip install -U numpy black pyupgrade - sudo python3 -m pip install -U numpy black pyupgrade - bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel pyupgrade clang + TF_PYTHON_VERSION=3.10 bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel clang lint-docs: name: Lint for Docs From 6849aa959eaa39f766f5975c2afe0d0337ad5052 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 18:46:43 +0000 Subject: [PATCH 86/87] Add missing libtinfo5 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e67f8ce70..3f6f34df0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: run: | set -x -e git log --pretty -1 + sudo apt update -y -qq && sudo apt install -y -qq libtinfo5 TF_PYTHON_VERSION=3.10 bazel run -s --verbose_failures --experimental_repo_remote_exec //tools/lint:check -- bazel clang lint-docs: From 0b918e63d2a5d27e4f0718eb3ea9b4240e8c1ab4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 17 Dec 2023 20:43:19 +0000 Subject: [PATCH 87/87] Re-enable arm64 build --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3f6f34df0..0714add92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,6 @@ jobs: macos-arm64-bazel: name: Bazel macOS arm64 runs-on: [self-hosted, macOS, ARM64] - if: false steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: GCP @@ -136,7 +135,6 @@ jobs: macos-arm64-wheel: name: Wheel ${{ matrix.python }} macOS arm64 needs: macos-arm64-bazel - if: false runs-on: [self-hosted, macOS, ARM64] strategy: matrix: