arrow/filesystem/localfs.h

0001 // Licensed to the Apache Software Foundation (ASF) under one
0002 // or more contributor license agreements.  See the NOTICE file
0003 // distributed with this work for additional information
0004 // regarding copyright ownership.  The ASF licenses this file
0005 // to you under the Apache License, Version 2.0 (the
0006 // "License"); you may not use this file except in compliance
0007 // with the License.  You may obtain a copy of the License at
0008 //
0009 //   http://www.apache.org/licenses/LICENSE-2.0
0010 //
0011 // Unless required by applicable law or agreed to in writing,
0012 // software distributed under the License is distributed on an
0013 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
0014 // KIND, either express or implied.  See the License for the
0015 // specific language governing permissions and limitations
0016 // under the License.
0017
0018 #pragma once
0019
0020 #include <memory>
0021 #include <string>
0022 #include <vector>
0023
0024 #include "arrow/filesystem/filesystem.h"
0025
0026 namespace arrow {
0027 namespace internal {
0028
0029 class Uri;
0030
0031 }
0032
0033 namespace fs {
0034
0035 /// Options for the LocalFileSystem implementation.
0036 struct ARROW_EXPORT LocalFileSystemOptions {
0037   static constexpr int32_t kDefaultDirectoryReadahead = 16;
0038   static constexpr int32_t kDefaultFileInfoBatchSize = 1000;
0039
0040   /// Whether OpenInputStream and OpenInputFile return a mmap'ed file,
0041   /// or a regular one.
0042   bool use_mmap = false;
0043
0044   /// Options related to `GetFileInfoGenerator` interface.
0045
0046   /// EXPERIMENTAL: The maximum number of directories processed in parallel
0047   /// by `GetFileInfoGenerator`.
0048   int32_t directory_readahead = kDefaultDirectoryReadahead;
0049
0050   /// EXPERIMENTAL: The maximum number of entries aggregated into each
0051   /// FileInfoVector chunk by `GetFileInfoGenerator`.
0052   ///
0053   /// Since each FileInfo entry needs a separate `stat` system call, a
0054   /// directory with a very large number of files may take a lot of time to
0055   /// process entirely. By generating a FileInfoVector after this chunk
0056   /// size is reached, we ensure FileInfo entries can start being consumed
0057   /// from the FileInfoGenerator with less initial latency.
0058   int32_t file_info_batch_size = kDefaultFileInfoBatchSize;
0059
0060   /// \brief Initialize with defaults
0061   static LocalFileSystemOptions Defaults();
0062
0063   bool Equals(const LocalFileSystemOptions& other) const;
0064
0065   static Result<LocalFileSystemOptions> FromUri(const ::arrow::util::Uri& uri,
0066                                                 std::string* out_path);
0067 };
0068
0069 /// \brief A FileSystem implementation accessing files on the local machine.
0070 ///
0071 /// This class handles only `/`-separated paths.  If desired, conversion
0072 /// from Windows backslash-separated paths should be done by the caller.
0073 /// Details such as symlinks are abstracted away (symlinks are always
0074 /// followed, except when deleting an entry).
0075 class ARROW_EXPORT LocalFileSystem : public FileSystem {
0076  public:
0077   explicit LocalFileSystem(const io::IOContext& = io::default_io_context());
0078   explicit LocalFileSystem(const LocalFileSystemOptions&,
0079                            const io::IOContext& = io::default_io_context());
0080   ~LocalFileSystem() override;
0081
0082   std::string type_name() const override { return "local"; }
0083
0084   Result<std::string> NormalizePath(std::string path) override;
0085   Result<std::string> PathFromUri(const std::string& uri_string) const override;
0086   Result<std::string> MakeUri(std::string path) const override;
0087
0088   bool Equals(const FileSystem& other) const override;
0089
0090   LocalFileSystemOptions options() const { return options_; }
0091
0092   /// \cond FALSE
0093   using FileSystem::CreateDir;
0094   using FileSystem::DeleteDirContents;
0095   using FileSystem::GetFileInfo;
0096   using FileSystem::OpenAppendStream;
0097   using FileSystem::OpenOutputStream;
0098   /// \endcond
0099
0100   Result<FileInfo> GetFileInfo(const std::string& path) override;
0101   Result<std::vector<FileInfo>> GetFileInfo(const FileSelector& select) override;
0102   FileInfoGenerator GetFileInfoGenerator(const FileSelector& select) override;
0103
0104   Status CreateDir(const std::string& path, bool recursive) override;
0105
0106   Status DeleteDir(const std::string& path) override;
0107   Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
0108   Status DeleteRootDirContents() override;
0109
0110   Status DeleteFile(const std::string& path) override;
0111
0112   Status Move(const std::string& src, const std::string& dest) override;
0113
0114   Status CopyFile(const std::string& src, const std::string& dest) override;
0115
0116   Result<std::shared_ptr<io::InputStream>> OpenInputStream(
0117       const std::string& path) override;
0118   Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
0119       const std::string& path) override;
0120   Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
0121       const std::string& path,
0122       const std::shared_ptr<const KeyValueMetadata>& metadata) override;
0123   Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
0124       const std::string& path,
0125       const std::shared_ptr<const KeyValueMetadata>& metadata) override;
0126
0127  protected:
0128   LocalFileSystemOptions options_;
0129 };
0130
0131 }  // namespace fs
0132 }  // namespace arrow