arrow/util/io_util.h

0001 // Licensed to the Apache Software Foundation (ASF) under one
0002 // or more contributor license agreements.  See the NOTICE file
0003 // distributed with this work for additional information
0004 // regarding copyright ownership.  The ASF licenses this file
0005 // to you under the Apache License, Version 2.0 (the
0006 // "License"); you may not use this file except in compliance
0007 // with the License.  You may obtain a copy of the License at
0008 //
0009 //   http://www.apache.org/licenses/LICENSE-2.0
0010 //
0011 // Unless required by applicable law or agreed to in writing,
0012 // software distributed under the License is distributed on an
0013 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
0014 // KIND, either express or implied.  See the License for the
0015 // specific language governing permissions and limitations
0016 // under the License.
0017
0018 #pragma once
0019
0020 #ifndef _WIN32
0021 #  define ARROW_HAVE_SIGACTION 1
0022 #endif
0023
0024 #include <atomic>
0025 #include <memory>
0026 #include <optional>
0027 #include <string>
0028 #include <utility>
0029 #include <vector>
0030
0031 #if ARROW_HAVE_SIGACTION
0032 #  include <csignal>  // Needed for struct sigaction
0033 #endif
0034
0035 #include "arrow/result.h"
0036 #include "arrow/status.h"
0037 #include "arrow/type_fwd.h"
0038 #include "arrow/util/macros.h"
0039 #include "arrow/util/windows_fixup.h"
0040
0041 namespace arrow::internal {
0042
0043 // NOTE: 8-bit path strings on Windows are encoded using UTF-8.
0044 // Using MBCS would fail encoding some paths.
0045
0046 #if defined(_WIN32)
0047 using NativePathString = std::wstring;
0048 #else
0049 using NativePathString = std::string;
0050 #endif
0051
0052 class ARROW_EXPORT PlatformFilename {
0053  public:
0054   struct Impl;
0055
0056   ~PlatformFilename();
0057   PlatformFilename();
0058   PlatformFilename(const PlatformFilename&);
0059   PlatformFilename(PlatformFilename&&);
0060   PlatformFilename& operator=(const PlatformFilename&);
0061   PlatformFilename& operator=(PlatformFilename&&);
0062   explicit PlatformFilename(NativePathString path);
0063   explicit PlatformFilename(const NativePathString::value_type* path);
0064
0065   const NativePathString& ToNative() const;
0066   std::string ToString() const;
0067
0068   PlatformFilename Parent() const;
0069   Result<PlatformFilename> Real() const;
0070
0071   // These functions can fail for character encoding reasons.
0072   static Result<PlatformFilename> FromString(std::string_view file_name);
0073   Result<PlatformFilename> Join(std::string_view child_name) const;
0074
0075   PlatformFilename Join(const PlatformFilename& child_name) const;
0076
0077   bool operator==(const PlatformFilename& other) const;
0078   bool operator!=(const PlatformFilename& other) const;
0079
0080   // Made public to avoid the proliferation of friend declarations.
0081   const Impl* impl() const { return impl_.get(); }
0082
0083  private:
0084   std::unique_ptr<Impl> impl_;
0085
0086   explicit PlatformFilename(Impl impl);
0087 };
0088
0089 /// Create a directory if it doesn't exist.
0090 ///
0091 /// Return whether the directory was created.
0092 ARROW_EXPORT
0093 Result<bool> CreateDir(const PlatformFilename& dir_path);
0094
0095 /// Create a directory and its parents if it doesn't exist.
0096 ///
0097 /// Return whether the directory was created.
0098 ARROW_EXPORT
0099 Result<bool> CreateDirTree(const PlatformFilename& dir_path);
0100
0101 /// Delete a directory's contents (but not the directory itself) if it exists.
0102 ///
0103 /// Return whether the directory existed.
0104 ARROW_EXPORT
0105 Result<bool> DeleteDirContents(const PlatformFilename& dir_path,
0106                                bool allow_not_found = true);
0107
0108 /// Delete a directory tree if it exists.
0109 ///
0110 /// Return whether the directory existed.
0111 ARROW_EXPORT
0112 Result<bool> DeleteDirTree(const PlatformFilename& dir_path, bool allow_not_found = true);
0113
0114 // Non-recursively list the contents of the given directory.
0115 // The returned names are the children's base names, not including dir_path.
0116 ARROW_EXPORT
0117 Result<std::vector<PlatformFilename>> ListDir(const PlatformFilename& dir_path);
0118
0119 /// Delete a file if it exists.
0120 ///
0121 /// Return whether the file existed.
0122 ARROW_EXPORT
0123 Result<bool> DeleteFile(const PlatformFilename& file_path, bool allow_not_found = true);
0124
0125 /// Return whether a file exists.
0126 ARROW_EXPORT
0127 Result<bool> FileExists(const PlatformFilename& path);
0128
0129 // TODO expose this more publicly to make it available from io/file.h?
0130 /// A RAII wrapper for a file descriptor.
0131 ///
0132 /// The underlying file descriptor is automatically closed on destruction.
0133 /// Moving is supported with well-defined semantics.
0134 /// Furthermore, closing is idempotent.
0135 class ARROW_EXPORT FileDescriptor {
0136  public:
0137   FileDescriptor() = default;
0138   explicit FileDescriptor(int fd) : fd_(fd) {}
0139   FileDescriptor(FileDescriptor&&);
0140   FileDescriptor& operator=(FileDescriptor&&);
0141
0142   ~FileDescriptor();
0143
0144   Status Close();
0145
0146   /// May return -1 if closed or default-initialized
0147   int fd() const { return fd_.load(); }
0148
0149   /// Detach and return the underlying file descriptor
0150   int Detach();
0151
0152   bool closed() const { return fd_.load() == -1; }
0153
0154  protected:
0155   static void CloseFromDestructor(int fd);
0156
0157   std::atomic<int> fd_{-1};
0158 };
0159
0160 /// Open a file for reading and return a file descriptor.
0161 ARROW_EXPORT
0162 Result<FileDescriptor> FileOpenReadable(const PlatformFilename& file_name);
0163
0164 /// Open a file for writing and return a file descriptor.
0165 ARROW_EXPORT
0166 Result<FileDescriptor> FileOpenWritable(const PlatformFilename& file_name,
0167                                         bool write_only = true, bool truncate = true,
0168                                         bool append = false);
0169
0170 /// Read from current file position.  Return number of bytes read.
0171 ARROW_EXPORT
0172 Result<int64_t> FileRead(int fd, uint8_t* buffer, int64_t nbytes);
0173 /// Read from given file position.  Return number of bytes read.
0174 ARROW_EXPORT
0175 Result<int64_t> FileReadAt(int fd, uint8_t* buffer, int64_t position, int64_t nbytes);
0176
0177 ARROW_EXPORT
0178 Status FileWrite(int fd, const uint8_t* buffer, const int64_t nbytes);
0179 ARROW_EXPORT
0180 Status FileTruncate(int fd, const int64_t size);
0181
0182 ARROW_EXPORT
0183 Status FileSeek(int fd, int64_t pos);
0184 ARROW_EXPORT
0185 Status FileSeek(int fd, int64_t pos, int whence);
0186 ARROW_EXPORT
0187 Result<int64_t> FileTell(int fd);
0188 ARROW_EXPORT
0189 Result<int64_t> FileGetSize(int fd);
0190
0191 ARROW_EXPORT
0192 Status FileClose(int fd);
0193
0194 struct Pipe {
0195   FileDescriptor rfd;
0196   FileDescriptor wfd;
0197
0198   Status Close() { return rfd.Close() & wfd.Close(); }
0199 };
0200
0201 ARROW_EXPORT
0202 Result<Pipe> CreatePipe();
0203
0204 ARROW_EXPORT
0205 Status SetPipeFileDescriptorNonBlocking(int fd);
0206
0207 class ARROW_EXPORT SelfPipe {
0208  public:
0209   static Result<std::shared_ptr<SelfPipe>> Make(bool signal_safe);
0210   virtual ~SelfPipe();
0211
0212   /// \brief Wait for a wakeup.
0213   ///
0214   /// Status::Invalid is returned if the pipe has been shutdown.
0215   /// Otherwise the next sent payload is returned.
0216   virtual Result<uint64_t> Wait() = 0;
0217
0218   /// \brief Wake up the pipe by sending a payload.
0219   ///
0220   /// This method is async-signal-safe if `signal_safe` was set to true.
0221   virtual void Send(uint64_t payload) = 0;
0222
0223   /// \brief Wake up the pipe and shut it down.
0224   virtual Status Shutdown() = 0;
0225 };
0226
0227 ARROW_EXPORT
0228 int64_t GetPageSize();
0229
0230 struct MemoryRegion {
0231   void* addr;
0232   size_t size;
0233 };
0234
0235 ARROW_EXPORT
0236 Status MemoryMapRemap(void* addr, size_t old_size, size_t new_size, int fildes,
0237                       void** new_addr);
0238 ARROW_EXPORT
0239 Status MemoryAdviseWillNeed(const std::vector<MemoryRegion>& regions);
0240
0241 ARROW_EXPORT
0242 Result<std::string> GetEnvVar(const char* name);
0243 ARROW_EXPORT
0244 Result<std::string> GetEnvVar(const std::string& name);
0245 ARROW_EXPORT
0246 Result<NativePathString> GetEnvVarNative(const char* name);
0247 ARROW_EXPORT
0248 Result<NativePathString> GetEnvVarNative(const std::string& name);
0249
0250 ARROW_EXPORT
0251 Status SetEnvVar(const char* name, const char* value);
0252 ARROW_EXPORT
0253 Status SetEnvVar(const std::string& name, const std::string& value);
0254 ARROW_EXPORT
0255 Status DelEnvVar(const char* name);
0256 ARROW_EXPORT
0257 Status DelEnvVar(const std::string& name);
0258
0259 ARROW_EXPORT
0260 std::string ErrnoMessage(int errnum);
0261 #if _WIN32
0262 ARROW_EXPORT
0263 std::string WinErrorMessage(int errnum);
0264 #endif
0265
0266 ARROW_EXPORT
0267 std::shared_ptr<StatusDetail> StatusDetailFromErrno(int errnum);
0268 ARROW_EXPORT
0269 std::optional<int> ErrnoFromStatusDetail(const StatusDetail& detail);
0270 #if _WIN32
0271 ARROW_EXPORT
0272 std::shared_ptr<StatusDetail> StatusDetailFromWinError(int errnum);
0273 #endif
0274 ARROW_EXPORT
0275 std::shared_ptr<StatusDetail> StatusDetailFromSignal(int signum);
0276
0277 template <typename... Args>
0278 Status StatusFromErrno(int errnum, StatusCode code, Args&&... args) {
0279   return Status::FromDetailAndArgs(code, StatusDetailFromErrno(errnum),
0280                                    std::forward<Args>(args)...);
0281 }
0282
0283 template <typename... Args>
0284 Status IOErrorFromErrno(int errnum, Args&&... args) {
0285   return StatusFromErrno(errnum, StatusCode::IOError, std::forward<Args>(args)...);
0286 }
0287
0288 #if _WIN32
0289 template <typename... Args>
0290 Status StatusFromWinError(int errnum, StatusCode code, Args&&... args) {
0291   return Status::FromDetailAndArgs(code, StatusDetailFromWinError(errnum),
0292                                    std::forward<Args>(args)...);
0293 }
0294
0295 template <typename... Args>
0296 Status IOErrorFromWinError(int errnum, Args&&... args) {
0297   return StatusFromWinError(errnum, StatusCode::IOError, std::forward<Args>(args)...);
0298 }
0299 #endif
0300
0301 template <typename... Args>
0302 Status StatusFromSignal(int signum, StatusCode code, Args&&... args) {
0303   return Status::FromDetailAndArgs(code, StatusDetailFromSignal(signum),
0304                                    std::forward<Args>(args)...);
0305 }
0306
0307 template <typename... Args>
0308 Status CancelledFromSignal(int signum, Args&&... args) {
0309   return StatusFromSignal(signum, StatusCode::Cancelled, std::forward<Args>(args)...);
0310 }
0311
0312 ARROW_EXPORT
0313 int ErrnoFromStatus(const Status&);
0314
0315 // Always returns 0 on non-Windows platforms (for Python).
0316 ARROW_EXPORT
0317 int WinErrorFromStatus(const Status&);
0318
0319 ARROW_EXPORT
0320 int SignalFromStatus(const Status&);
0321
0322 class ARROW_EXPORT TemporaryDir {
0323  public:
0324   ~TemporaryDir();
0325
0326   /// '/'-terminated path to the temporary dir
0327   const PlatformFilename& path() { return path_; }
0328
0329   /// Create a temporary subdirectory in the system temporary dir,
0330   /// named starting with `prefix`.
0331   static Result<std::unique_ptr<TemporaryDir>> Make(const std::string& prefix);
0332
0333  private:
0334   PlatformFilename path_;
0335
0336   explicit TemporaryDir(PlatformFilename&&);
0337 };
0338
0339 class ARROW_EXPORT SignalHandler {
0340  public:
0341   using Callback = void (*)(int);
0342
0343   SignalHandler();
0344   explicit SignalHandler(Callback cb);
0345 #if ARROW_HAVE_SIGACTION
0346   explicit SignalHandler(const struct sigaction& sa);
0347 #endif
0348
0349   Callback callback() const;
0350 #if ARROW_HAVE_SIGACTION
0351   const struct sigaction& action() const;
0352 #endif
0353
0354  protected:
0355 #if ARROW_HAVE_SIGACTION
0356   // Storing the full sigaction allows to restore the entire signal handling
0357   // configuration.
0358   struct sigaction sa_;
0359 #else
0360   Callback cb_;
0361 #endif
0362 };
0363
0364 /// \brief Return the current handler for the given signal number.
0365 ARROW_EXPORT
0366 Result<SignalHandler> GetSignalHandler(int signum);
0367
0368 /// \brief Set a new handler for the given signal number.
0369 ///
0370 /// The old signal handler is returned.
0371 ARROW_EXPORT
0372 Result<SignalHandler> SetSignalHandler(int signum, const SignalHandler& handler);
0373
0374 /// \brief Reinstate the signal handler
0375 ///
0376 /// For use in signal handlers.  This is needed on platforms without sigaction()
0377 /// such as Windows, as the default signal handler is restored there as
0378 /// soon as a signal is raised.
0379 ARROW_EXPORT
0380 void ReinstateSignalHandler(int signum, SignalHandler::Callback handler);
0381
0382 /// \brief Send a signal to the current process
0383 ///
0384 /// The thread which will receive the signal is unspecified.
0385 ARROW_EXPORT
0386 Status SendSignal(int signum);
0387
0388 /// \brief Send a signal to the given thread
0389 ///
0390 /// This function isn't supported on Windows.
0391 ARROW_EXPORT
0392 Status SendSignalToThread(int signum, uint64_t thread_id);
0393
0394 /// \brief Get an unpredictable random seed
0395 ///
0396 /// This function may be slightly costly, so should only be used to initialize
0397 /// a PRNG, not to generate a large amount of random numbers.
0398 /// It is better to use this function rather than std::random_device, unless
0399 /// absolutely necessary (e.g. to generate a cryptographic secret).
0400 ARROW_EXPORT
0401 int64_t GetRandomSeed();
0402
0403 /// \brief Get the current thread id
0404 ///
0405 /// In addition to having the same properties as std::thread, the returned value
0406 /// is a regular integer value, which is more convenient than an opaque type.
0407 ARROW_EXPORT
0408 uint64_t GetThreadId();
0409
0410 /// \brief Get the current memory used by the current process in bytes
0411 ///
0412 /// This function supports Windows, Linux, and Mac and will return 0 otherwise
0413 ARROW_EXPORT
0414 int64_t GetCurrentRSS();
0415
0416 /// \brief Get the total memory available to the system in bytes
0417 ///
0418 /// This function supports Windows, Linux, and Mac and will return 0 otherwise
0419 ARROW_EXPORT
0420 int64_t GetTotalMemoryBytes();
0421
0422 /// \brief Load a dynamic library
0423 ///
0424 /// This wraps dlopen() except on Windows, where LoadLibrary() is called.
0425 /// These two platforms handle absolute paths consistently; relative paths
0426 /// or the library's bare name may be handled but inconsistently.
0427 ///
0428 /// \return An opaque handle for the dynamic library, which can be used for
0429 ///         subsequent symbol lookup. Nullptr will never be returned; instead
0430 ///         an error will be raised.
0431 ARROW_EXPORT Result<void*> LoadDynamicLibrary(const PlatformFilename& path);
0432
0433 /// \brief Load a dynamic library
0434 ///
0435 /// An overload taking null terminated string.
0436 ARROW_EXPORT Result<void*> LoadDynamicLibrary(const char* path);
0437
0438 /// \brief Retrieve a symbol by name from a library handle.
0439 ///
0440 /// This wraps dlsym() except on Windows, where GetProcAddress() is called.
0441 ///
0442 /// \return The address associated with the named symbol. Nullptr will never be
0443 ///         returned; instead an error will be raised.
0444 ARROW_EXPORT Result<void*> GetSymbol(void* handle, const char* name);
0445
0446 template <typename T>
0447 Result<T*> GetSymbolAs(void* handle, const char* name) {
0448   ARROW_ASSIGN_OR_RAISE(void* sym, GetSymbol(handle, name));
0449   return reinterpret_cast<T*>(sym);
0450 }
0451
0452 }  // namespace arrow::internal