feat(hadoop_filesystem): support gravitino gvfs

tensorflow · Apr 1, 2024 · 008999f · 008999f
1 parent 4d70341
commit 008999f
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 8 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -878,23 +878,42 @@ http_archive(
 )
 
 # Needed for llvm_toolchain and Golang
+# http_archive(
+# name = "com_grail_bazel_toolchain",
+# sha256 = "9e6065ded4b7453143e1586d6819729a63cd233114b72bf85ff3435367b02c90",
+# strip_prefix = "bazel-toolchain-edd07e96a2ecaa131af9234d6582875d980c0ac7",
+# urls = [
+# "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
+# "https://github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
+# ],
+# )
+
+# load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+
+# llvm_toolchain(
+# name = "llvm_toolchain",
+# llvm_version = "9.0.0",
+# )
+
 http_archive(
  name = "com_grail_bazel_toolchain",
- sha256 = "9e6065ded4b7453143e1586d6819729a63cd233114b72bf85ff3435367b02c90",
- strip_prefix = "bazel-toolchain-edd07e96a2ecaa131af9234d6582875d980c0ac7",
- urls = [
- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
- "https://github.com/grailbio/bazel-toolchain/archive/edd07e96a2ecaa131af9234d6582875d980c0ac7.tar.gz",
- ],
+ urls = ["https://github.com/bazel-contrib/toolchains_llvm/archive/refs/tags/0.9.tar.gz"],
+ sha256 = "b2d168315dd0785f170b2b306b86e577c36e812b8f8b05568f9403141f2c24dd",
+ strip_prefix = "toolchains_llvm-0.9",
 )
 
-load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
+
+bazel_toolchain_dependencies()
 
+load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
 llvm_toolchain(
  name = "llvm_toolchain",
  llvm_version = "9.0.0",
 )
 
+
+
 # Golang related ruls, consider removal after switching to C++/C client for prometheus
 http_archive(
  name = "io_bazel_rules_go",

diff --git a/tensorflow_io/core/filesystems/filesystem_plugins.cc b/tensorflow_io/core/filesystems/filesystem_plugins.cc
@@ -29,10 +29,11 @@ limitations under the License.
 TFIO_PLUGIN_EXPORT void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
  info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate;
  info->plugin_memory_free = tensorflow::io::plugin_memory_free;
- info->num_schemes = 7;
+
 #if !defined(_MSC_VER)
  info->num_schemes = 8;
 #endif
+
  info->ops = static_cast<TF_FilesystemPluginOps*>(
  tensorflow::io::plugin_memory_allocate(info->num_schemes *
  sizeof(info->ops[0])));
@@ -43,7 +44,11 @@ TFIO_PLUGIN_EXPORT void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "hdfs");
  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "viewfs");
  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[6], "har");
+<<<<<<< HEAD
 #if !defined(_MSC_VER)
  tensorflow::io::oss::ProvideFilesystemSupportFor(&info->ops[7], "oss");
 #endif
+=======
+ tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[7], "gvfs");
+>>>>>>> feat(gvfs): support gvfs
 }
diff --git a/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc b/tensorflow_io/core/filesystems/hdfs/hadoop_filesystem.cc
@@ -541,6 +541,11 @@ hdfsFS Connect(tf_hdfs_filesystem::HadoopFileSystemImplementation* hadoop_file,
  std::string path_har = path;
  SplitArchiveNameAndPath(&path_har, &namenode, status);
  if (TF_GetCode(status) != TF_OK) return nullptr;
+ } else if (scheme == "gvfs") {
+ std::string dfsPath = path;
+ size_t pos = dfsPath.find(hdfs_path);
+ dfsPath.replace(pos, dfsPath.length(), "");
+ namenode = dfsPath + "/";
  } else {
  if (namenode.empty()) {
  namenode = "default";