Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Apache ORC build sequence with a sample reader file #1373

Merged
merged 18 commits into from
Apr 28, 2021
13 changes: 13 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -1121,3 +1121,16 @@ http_archive(
"https://github.com/tinyobjloader/tinyobjloader/archive/v2.0.0rc8.tar.gz",
],
)

http_archive(
name = "liborc",
build_file = "//third_party:liborc.BUILD",
patch_cmds = [
"tar -xzf c++/libs/libhdfspp/libhdfspp.tar.gz -C c++/libs/libhdfspp",
],
sha256 = "abdffe48b8d2e7776c3b541ee2241401e49774941ca4a8c759e5d795daec8a45",
strip_prefix = "orc-rel-release-1.6.7",
urls = [
"https://github.com/apache/orc/archive/refs/tags/rel/release-1.6.7.tar.gz",
],
)
14 changes: 14 additions & 0 deletions tensorflow_io/core/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,19 @@ cc_library(
alwayslink = 1,
)

cc_library(
name = "orc_ops",
srcs = [
],
copts = tf_io_copts(),
linkstatic = True,
deps = [
"//tensorflow_io/core:dataset_ops",
"@liborc",
],
alwayslink = 1,
)

cc_library(
name = "numpy_ops",
srcs = [
Expand Down Expand Up @@ -720,6 +733,7 @@ cc_binary(
"//tensorflow_io/bigquery:bigquery_ops",
"//tensorflow_io/core:audio_video_ops",
"//tensorflow_io/core:avro_ops",
"//tensorflow_io/core:orc_ops",
"//tensorflow_io/core:cpuinfo",
"//tensorflow_io/core:file_ops",
"//tensorflow_io/core:grpc_ops",
Expand Down
130 changes: 130 additions & 0 deletions third_party/liborc.BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package(default_visibility = ["//visibility:public"])

licenses(["notice"]) # Apache 2.0

exports_files(["LICENSE"])

# Note: orc-proto-wrapper.cc includes orc_proto.pb.cc
# and prefix with Adaptor.hh. The Adaptor.hh
# was supposed to capture platform discrepancies.
# However, since orc_proto.pb.cc can be compiled
# with cc_proto_library successfully, there is no need
# for orc-proto-wrapper.cc
cc_library(
name = "liborc",
srcs = glob(
[
"c++/src/*.cc",
"c++/src/*.hh",
"c++/src/io/*.cc",
"c++/src/io/*.hh",
"c++/src/wrap/*.cc",
"c++/src/wrap/*.hh",
"c++/src/wrap/*.h",
"c++/include/orc/*.hh",
],
exclude = [
"c++/src/wrap/orc-proto-wrapper.cc",
"c++/src/OrcHdfsFile.cc",
],
) + select({
"@bazel_tools//src/conditions:windows": [],
"//conditions:default": [
"c++/src/OrcHdfsFile.cc",
],
}),
hdrs = [
"c++/include/orc/orc-config.hh",
"c++/src/Adaptor.hh",
],
copts = [],
defines = [],
includes = [
"c++/include",
"c++/src",
"c++/src/io",
"c++/src/wrap",
"proto",
],
linkopts = [],
visibility = ["//visibility:public"],
deps = [
":libhdfspp",
":orc_cc_proto",
"@lz4",
"@snappy",
"@zlib",
"@zstd",
],
)

cc_library(
name = "libhdfspp",
srcs = glob(
[
"c++/libs/libhdfspp/include/hdfspp/*.h",
],
exclude = [
],
),
hdrs = [
],
copts = [],
defines = [],
includes = [
"c++/libs/libhdfspp/include",
],
deps = [],
)

proto_library(
name = "orc_proto",
srcs = ["proto/orc_proto.proto"],
)

cc_proto_library(
name = "orc_cc_proto",
deps = [":orc_proto"],
)

genrule(
name = "orc-config_hh",
srcs = ["c++/include/orc/orc-config.hh.in"],
outs = ["c++/include/orc/orc-config.hh"],
cmd = ("sed " +
"-e 's/@ORC_VERSION@/1.6.7/g' " +
"-e 's/cmakedefine/define/g' " +
"$< >$@"),
)

genrule(
name = "Adaptor_hh",
srcs = ["c++/src/Adaptor.hh.in"],
outs = ["c++/src/Adaptor.hh"],
cmd = select({
"@bazel_tools//src/conditions:windows": (
"sed " +
"-e 's/cmakedefine HAS_PREAD/undef HAS_PREAD/g' " +
"-e 's/cmakedefine NEEDS_REDUNDANT_MOVE/undef NEEDS_REDUNDANT_MOVE/g' " +
"-e 's/cmakedefine NEEDS_Z_PREFIX/undef NEEDS_Z_PREFIX/g' " +
"-e 's/cmakedefine/define/g' " +
"$< >$@"
),
"@bazel_tools//src/conditions:darwin": (
"sed " +
"-e 's/cmakedefine NEEDS_REDUNDANT_MOVE/undef NEEDS_REDUNDANT_MOVE/g' " +
"-e 's/cmakedefine NEEDS_Z_PREFIX/undef NEEDS_Z_PREFIX/g' " +
"-e 's/cmakedefine/define/g' " +
"$< >$@"
),
"//conditions:default": (
"sed " +
"-e 's/cmakedefine INT64_IS_LL/undef INT64_IS_LL/g' " +
"-e 's/cmakedefine HAS_POST_2038/undef HAS_POST_2038/g' " +
"-e 's/cmakedefine NEEDS_REDUNDANT_MOVE/undef NEEDS_REDUNDANT_MOVE/g' " +
"-e 's/cmakedefine NEEDS_Z_PREFIX/undef NEEDS_Z_PREFIX/g' " +
"-e 's/cmakedefine/define/g' " +
"$< >$@"
),
}),
)