arrow/filesystem/hdfs.h

0001 // Licensed to the Apache Software Foundation (ASF) under one
0002 // or more contributor license agreements.  See the NOTICE file
0003 // distributed with this work for additional information
0004 // regarding copyright ownership.  The ASF licenses this file
0005 // to you under the Apache License, Version 2.0 (the
0006 // "License"); you may not use this file except in compliance
0007 // with the License.  You may obtain a copy of the License at
0008 //
0009 //   http://www.apache.org/licenses/LICENSE-2.0
0010 //
0011 // Unless required by applicable law or agreed to in writing,
0012 // software distributed under the License is distributed on an
0013 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
0014 // KIND, either express or implied.  See the License for the
0015 // specific language governing permissions and limitations
0016 // under the License.
0017
0018 #pragma once
0019
0020 #include <memory>
0021 #include <string>
0022 #include <vector>
0023
0024 #include "arrow/filesystem/filesystem.h"
0025 #include "arrow/io/hdfs.h"
0026 #include "arrow/util/uri.h"
0027
0028 namespace arrow::fs {
0029
0030 /// Options for the HDFS implementation.
0031 struct ARROW_EXPORT HdfsOptions {
0032   HdfsOptions() = default;
0033   ~HdfsOptions() = default;
0034
0035   /// Hdfs configuration options, contains host, port, driver
0036   io::HdfsConnectionConfig connection_config;
0037
0038   /// Used by Hdfs OpenWritable Interface.
0039   int32_t buffer_size = 0;
0040   int16_t replication = 3;
0041   int64_t default_block_size = 0;
0042
0043   void ConfigureEndPoint(std::string host, int port);
0044   void ConfigureReplication(int16_t replication);
0045   void ConfigureUser(std::string user_name);
0046   void ConfigureBufferSize(int32_t buffer_size);
0047   void ConfigureBlockSize(int64_t default_block_size);
0048   void ConfigureKerberosTicketCachePath(std::string path);
0049   void ConfigureExtraConf(std::string key, std::string val);
0050
0051   bool Equals(const HdfsOptions& other) const;
0052
0053   static Result<HdfsOptions> FromUri(const ::arrow::util::Uri& uri);
0054   static Result<HdfsOptions> FromUri(const std::string& uri);
0055 };
0056
0057 /// HDFS-backed FileSystem implementation.
0058 ///
0059 /// implementation notes:
0060 /// - This is a wrapper of arrow/io/hdfs, so we can use FileSystem API to handle hdfs.
0061 class ARROW_EXPORT HadoopFileSystem : public FileSystem {
0062  public:
0063   ~HadoopFileSystem() override;
0064
0065   std::string type_name() const override { return "hdfs"; }
0066   HdfsOptions options() const;
0067   bool Equals(const FileSystem& other) const override;
0068   Result<std::string> PathFromUri(const std::string& uri_string) const override;
0069
0070   /// \cond FALSE
0071   using FileSystem::CreateDir;
0072   using FileSystem::DeleteDirContents;
0073   using FileSystem::GetFileInfo;
0074   using FileSystem::OpenAppendStream;
0075   using FileSystem::OpenOutputStream;
0076   /// \endcond
0077
0078   Result<FileInfo> GetFileInfo(const std::string& path) override;
0079   Result<std::vector<FileInfo>> GetFileInfo(const FileSelector& select) override;
0080
0081   Status CreateDir(const std::string& path, bool recursive) override;
0082
0083   Status DeleteDir(const std::string& path) override;
0084
0085   Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
0086
0087   Status DeleteRootDirContents() override;
0088
0089   Status DeleteFile(const std::string& path) override;
0090
0091   Status Move(const std::string& src, const std::string& dest) override;
0092
0093   Status CopyFile(const std::string& src, const std::string& dest) override;
0094
0095   Result<std::shared_ptr<io::InputStream>> OpenInputStream(
0096       const std::string& path) override;
0097   Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
0098       const std::string& path) override;
0099   Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
0100       const std::string& path,
0101       const std::shared_ptr<const KeyValueMetadata>& metadata) override;
0102   Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
0103       const std::string& path,
0104       const std::shared_ptr<const KeyValueMetadata>& metadata) override;
0105
0106   /// Create a HdfsFileSystem instance from the given options.
0107   static Result<std::shared_ptr<HadoopFileSystem>> Make(
0108       const HdfsOptions& options, const io::IOContext& = io::default_io_context());
0109
0110  protected:
0111   HadoopFileSystem(const HdfsOptions& options, const io::IOContext&);
0112
0113   class Impl;
0114   std::unique_ptr<Impl> impl_;
0115 };
0116
0117 }  // namespace arrow::fs