arrow/filesystem/test_util.h

0001 // Licensed to the Apache Software Foundation (ASF) under one
0002 // or more contributor license agreements.  See the NOTICE file
0003 // distributed with this work for additional information
0004 // regarding copyright ownership.  The ASF licenses this file
0005 // to you under the Apache License, Version 2.0 (the
0006 // "License"); you may not use this file except in compliance
0007 // with the License.  You may obtain a copy of the License at
0008 //
0009 //   http://www.apache.org/licenses/LICENSE-2.0
0010 //
0011 // Unless required by applicable law or agreed to in writing,
0012 // software distributed under the License is distributed on an
0013 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
0014 // KIND, either express or implied.  See the License for the
0015 // specific language governing permissions and limitations
0016 // under the License.
0017
0018 #pragma once
0019
0020 #include <chrono>
0021 #include <memory>
0022 #include <string>
0023 #include <vector>
0024
0025 #include "arrow/filesystem/filesystem.h"
0026 #include "arrow/filesystem/mockfs.h"
0027 #include "arrow/testing/visibility.h"
0028 #include "arrow/util/counting_semaphore.h"
0029
0030 namespace arrow {
0031 namespace fs {
0032
0033 static constexpr double kTimeSlack = 2.0;  // In seconds
0034
0035 static inline FileInfo File(std::string path) {
0036   return FileInfo(std::move(path), FileType::File);
0037 }
0038
0039 static inline FileInfo Dir(std::string path) {
0040   return FileInfo(std::move(path), FileType::Directory);
0041 }
0042
0043 // A subclass of MockFileSystem that blocks operations until an unlock method is
0044 // called.
0045 //
0046 // This is intended for testing fine-grained ordering of filesystem operations.
0047 //
0048 // N.B. Only OpenOutputStream supports gating at the moment but this is simply because
0049 //      it is all that has been needed so far.  Feel free to add support for more methods
0050 //      as required.
0051 class ARROW_TESTING_EXPORT GatedMockFilesystem : public internal::MockFileSystem {
0052  public:
0053   GatedMockFilesystem(TimePoint current_time,
0054                       const io::IOContext& = io::default_io_context());
0055   ~GatedMockFilesystem() override;
0056
0057   Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
0058       const std::string& path,
0059       const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
0060
0061   // Wait until at least num_waiters are waiting on OpenOutputStream
0062   Status WaitForOpenOutputStream(uint32_t num_waiters);
0063   // Unlock `num_waiters` individual calls to OpenOutputStream
0064   Status UnlockOpenOutputStream(uint32_t num_waiters);
0065
0066  private:
0067   util::CountingSemaphore open_output_sem_;
0068 };
0069
0070 ARROW_TESTING_EXPORT
0071 void CreateFile(FileSystem* fs, const std::string& path, const std::string& data);
0072
0073 // Sort a vector of FileInfo by lexicographic path order
0074 ARROW_TESTING_EXPORT
0075 void SortInfos(FileInfoVector* infos);
0076
0077 // Create a copy of a FileInfo vector sorted by lexicographic path order
0078 ARROW_TESTING_EXPORT
0079 FileInfoVector SortedInfos(const FileInfoVector& infos);
0080
0081 ARROW_TESTING_EXPORT
0082 void CollectFileInfoGenerator(FileInfoGenerator gen, FileInfoVector* out_infos);
0083
0084 ARROW_TESTING_EXPORT
0085 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type);
0086
0087 ARROW_TESTING_EXPORT
0088 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
0089                     TimePoint mtime);
0090
0091 ARROW_TESTING_EXPORT
0092 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
0093                     TimePoint mtime, int64_t size);
0094
0095 ARROW_TESTING_EXPORT
0096 void AssertFileInfo(const FileInfo& info, const std::string& path, FileType type,
0097                     int64_t size);
0098
0099 ARROW_TESTING_EXPORT
0100 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type);
0101
0102 ARROW_TESTING_EXPORT
0103 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type,
0104                     TimePoint mtime);
0105
0106 ARROW_TESTING_EXPORT
0107 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type,
0108                     TimePoint mtime, int64_t size);
0109
0110 ARROW_TESTING_EXPORT
0111 void AssertFileInfo(FileSystem* fs, const std::string& path, FileType type, int64_t size);
0112
0113 ARROW_TESTING_EXPORT
0114 void AssertFileContents(FileSystem* fs, const std::string& path,
0115                         const std::string& expected_data);
0116
0117 template <typename Duration>
0118 void AssertDurationBetween(Duration d, double min_secs, double max_secs) {
0119   auto seconds = std::chrono::duration_cast<std::chrono::duration<double>>(d);
0120   ASSERT_GE(seconds.count(), min_secs);
0121   ASSERT_LE(seconds.count(), max_secs);
0122 }
0123
0124 // Generic tests for FileSystem implementations.
0125 // To use this class, subclass both from it and ::testing::Test,
0126 // implement GetEmptyFileSystem(), and use GENERIC_FS_TEST_FUNCTIONS()
0127 // to define the various tests.
0128 class ARROW_TESTING_EXPORT GenericFileSystemTest {
0129  public:
0130   virtual ~GenericFileSystemTest();
0131
0132   void TestEmpty();
0133   void TestNormalizePath();
0134   void TestCreateDir();
0135   void TestDeleteDir();
0136   void TestDeleteDirContents();
0137   void TestDeleteRootDirContents();
0138   void TestDeleteFile();
0139   void TestDeleteFiles();
0140   void TestMoveFile();
0141   void TestMoveDir();
0142   void TestCopyFile();
0143   void TestCopyFiles();
0144   void TestGetFileInfo();
0145   void TestGetFileInfoVector();
0146   void TestGetFileInfoSelector();
0147   void TestGetFileInfoSelectorWithRecursion();
0148   void TestGetFileInfoAsync();
0149   void TestGetFileInfoGenerator();
0150   void TestOpenOutputStream();
0151   void TestOpenAppendStream();
0152   void TestOpenInputStream();
0153   void TestOpenInputStreamWithFileInfo();
0154   void TestOpenInputStreamAsync();
0155   void TestOpenInputFile();
0156   void TestOpenInputFileWithFileInfo();
0157   void TestOpenInputFileAsync();
0158   void TestSpecialChars();
0159
0160  protected:
0161   // This function should return the filesystem under test.
0162   virtual std::shared_ptr<FileSystem> GetEmptyFileSystem() = 0;
0163
0164   // Override the following functions to specify deviations from expected
0165   // filesystem semantics.
0166   // - Whether the filesystem may "implicitly" create intermediate directories
0167   virtual bool have_implicit_directories() const { return false; }
0168   // - Whether the filesystem may allow writing a file "over" a directory
0169   virtual bool allow_write_file_over_dir() const { return false; }
0170   // - Whether the filesystem may allow writing a directory "over" a file,
0171   //   for example copying file "A" to "B/C" while "B" exists and is a file.
0172   virtual bool allow_write_implicit_dir_over_file() const { return false; }
0173   // - Whether the filesystem allows reading a directory
0174   virtual bool allow_read_dir_as_file() const { return false; }
0175   // - Whether the filesystem allows moving a file
0176   virtual bool allow_move_file() const { return true; }
0177   // - Whether the filesystem allows moving a directory
0178   virtual bool allow_move_dir() const { return true; }
0179   // - Whether the filesystem allows moving a directory "over" a non-empty destination
0180   virtual bool allow_move_dir_over_non_empty_dir() const { return false; }
0181   // - Whether the filesystem allows appending to a file
0182   virtual bool allow_append_to_file() const { return true; }
0183   // - Whether the filesystem allows appending to a nonexistent file
0184   virtual bool allow_append_to_new_file() const { return true; }
0185   // - Whether the filesystem supports directory modification times
0186   virtual bool have_directory_mtimes() const { return true; }
0187   // - Whether some directory tree deletion tests may fail randomly
0188   virtual bool have_flaky_directory_tree_deletion() const { return false; }
0189   // - Whether the filesystem stores some metadata alongside files
0190   virtual bool have_file_metadata() const { return false; }
0191   // - Whether the filesystem has a false positive memory leak with generator
0192   virtual bool have_false_positive_memory_leak_with_generator() const { return false; }
0193   // - Whether the filesystem has a false positive memory leak in async close
0194   virtual bool have_false_positive_memory_leak_with_async_close() const { return false; }
0195
0196   void TestEmpty(FileSystem* fs);
0197   void TestNormalizePath(FileSystem* fs);
0198   void TestCreateDir(FileSystem* fs);
0199   void TestDeleteDir(FileSystem* fs);
0200   void TestDeleteDirContents(FileSystem* fs);
0201   void TestDeleteRootDirContents(FileSystem* fs);
0202   void TestDeleteFile(FileSystem* fs);
0203   void TestDeleteFiles(FileSystem* fs);
0204   void TestMoveFile(FileSystem* fs);
0205   void TestMoveDir(FileSystem* fs);
0206   void TestCopyFile(FileSystem* fs);
0207   void TestCopyFiles(FileSystem* fs);
0208   void TestGetFileInfo(FileSystem* fs);
0209   void TestGetFileInfoVector(FileSystem* fs);
0210   void TestGetFileInfoSelector(FileSystem* fs);
0211   void TestGetFileInfoSelectorWithRecursion(FileSystem* fs);
0212   void TestGetFileInfoAsync(FileSystem* fs);
0213   void TestGetFileInfoGenerator(FileSystem* fs);
0214   void TestOpenOutputStream(FileSystem* fs);
0215   void TestOpenAppendStream(FileSystem* fs);
0216   void TestOpenInputStream(FileSystem* fs);
0217   void TestOpenInputStreamWithFileInfo(FileSystem* fs);
0218   void TestOpenInputStreamAsync(FileSystem* fs);
0219   void TestOpenInputFile(FileSystem* fs);
0220   void TestOpenInputFileWithFileInfo(FileSystem* fs);
0221   void TestOpenInputFileAsync(FileSystem* fs);
0222   void TestSpecialChars(FileSystem* fs);
0223 };
0224
0225 #define GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, NAME) \
0226   TEST_MACRO(TEST_CLASS, NAME) { this->Test##NAME(); }
0227
0228 #define GENERIC_FS_TEST_FUNCTIONS_MACROS(TEST_MACRO, TEST_CLASS)                     \
0229   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, Empty)                            \
0230   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, NormalizePath)                    \
0231   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, CreateDir)                        \
0232   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteDir)                        \
0233   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteDirContents)                \
0234   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteRootDirContents)            \
0235   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteFile)                       \
0236   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, DeleteFiles)                      \
0237   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, MoveFile)                         \
0238   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, MoveDir)                          \
0239   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, CopyFile)                         \
0240   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, CopyFiles)                        \
0241   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfo)                      \
0242   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoVector)                \
0243   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoSelector)              \
0244   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoSelectorWithRecursion) \
0245   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoAsync)                 \
0246   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, GetFileInfoGenerator)             \
0247   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenOutputStream)                 \
0248   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenAppendStream)                 \
0249   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputStream)                  \
0250   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputStreamWithFileInfo)      \
0251   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputStreamAsync)             \
0252   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputFile)                    \
0253   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputFileWithFileInfo)        \
0254   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, OpenInputFileAsync)               \
0255   GENERIC_FS_TEST_FUNCTION(TEST_MACRO, TEST_CLASS, SpecialChars)
0256
0257 #define GENERIC_FS_TEST_FUNCTIONS(TEST_CLASS) \
0258   GENERIC_FS_TEST_FUNCTIONS_MACROS(TEST_F, TEST_CLASS)
0259
0260 #define GENERIC_FS_TYPED_TEST_FUNCTIONS(TEST_CLASS) \
0261   GENERIC_FS_TEST_FUNCTIONS_MACROS(TYPED_TEST, TEST_CLASS)
0262
0263 }  // namespace fs
0264 }  // namespace arrow