From 67cdc1d130383d94941e320e15d4487e0e5c474c Mon Sep 17 00:00:00 2001 From: lipeidian Date: Mon, 1 Apr 2024 17:34:31 +0800 Subject: [PATCH] feat(gvfs): support gvfs --- WORKSPACE | 35 ++++++++++++++----- .../core/filesystems/filesystem_plugins.cc | 3 +- .../filesystems/hdfs/hadoop_filesystem.cc | 5 +++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index e232897fb..4c8387e2d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -880,23 +880,42 @@ http_archive( ) # Needed for llvm_toolchain and Golang +# http_archive( +# name = "com_grail_bazel_toolchain", +# sha256 = "9e6065ded4b7453143e1586d6819729a63cd233114b72bf85ff3435367b02c90", +# strip_prefix = "bazel-toolchain-edd07e96a2ecaa131af9234d6582875d980c0ac7", +# urls = [ +# "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz", +# "https://github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz", +# ], +# ) + +# load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") + +# llvm_toolchain( +# name = "llvm_toolchain", +# llvm_version = "9.0.0", +# ) + http_archive( name = "com_grail_bazel_toolchain", - sha256 = "9e6065ded4b7453143e1586d6819729a63cd233114b72bf85ff3435367b02c90", - strip_prefix = "bazel-toolchain-edd07e96a2ecaa131af9234d6582875d980c0ac7", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz", - "https://github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz", - ], + urls = ["https://github.com/bazel-contrib/toolchains_llvm/archive/refs/tags/0.9.tar.gz"], + sha256 = "b2d168315dd0785f170b2b306b86e577c36e812b8f8b05568f9403141f2c24dd", + strip_prefix = "toolchains_llvm-0.9", ) -load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") +load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies") + +bazel_toolchain_dependencies() +load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") llvm_toolchain( name = "llvm_toolchain", - llvm_version = "9.0.0", + llvm_version = "0.9.0", ) + + # Golang related ruls, consider removal after switching to C++/C client for prometheus http_archive( name = "io_bazel_rules_go", diff --git a/tensorflow_io/core/filesystems/filesystem_plugins.cc b/tensorflow_io/core/filesystems/filesystem_plugins.cc index 5db773bf3..0ff43df0e 100644 --- a/tensorflow_io/core/filesystems/filesystem_plugins.cc +++ b/tensorflow_io/core/filesystems/filesystem_plugins.cc @@ -29,7 +29,7 @@ limitations under the License. TFIO_PLUGIN_EXPORT void TF_InitPlugin(TF_FilesystemPluginInfo* info) { info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate; info->plugin_memory_free = tensorflow::io::plugin_memory_free; - info->num_schemes = 7; + info->num_schemes = 8; info->ops = static_cast( tensorflow::io::plugin_memory_allocate(info->num_schemes * sizeof(info->ops[0]))); @@ -40,4 +40,5 @@ TFIO_PLUGIN_EXPORT void TF_InitPlugin(TF_FilesystemPluginInfo* info) { tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "hdfs"); tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "viewfs"); tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[6], "har"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[7], "gvfs"); } diff --git a/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc b/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc index 3d5a41b84..d9aab25d1 100644 --- a/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc +++ b/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc @@ -541,6 +541,11 @@ hdfsFS Connect(tf_hdfs_filesystem::HadoopFileSystemImplementation* hadoop_file, std::string path_har = path; SplitArchiveNameAndPath(&path_har, &namenode, status); if (TF_GetCode(status) != TF_OK) return nullptr; + } else if (scheme == "gvfs") { + std::string dfsPath = path; + size_t pos = dfsPath.find(hdfs_path); + dfsPath.replace(pos, dfsPath.length(), ""); + namenode = dfsPath + "/"; } else { if (namenode.empty()) { namenode = "default";