EIC code displayed by LXR master/​include/​arrow/​filesystem/​azurefs.h	Source navigation Diff markup Identifier search General search
	Version: master test Revert   Change

File indexing completed on 2025-08-28 08:26:58

 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
 // to you under the Apache License, Version 2.0 (the
 // "License"); you may not use this file except in compliance
 // with the License.  You may obtain a copy of the License at
 //
 //   http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing,
 // software distributed under the License is distributed on an
 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
 
 #pragma once
 
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "arrow/filesystem/filesystem.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/uri.h"
 
 namespace Azure::Core::Credentials {
 class TokenCredential;
 }
 
 namespace Azure::Storage {
 class StorageSharedKeyCredential;
 }
 
 namespace Azure::Storage::Blobs {
 class BlobServiceClient;
 }
 
 namespace Azure::Storage::Files::DataLake {
 class DataLakeFileSystemClient;
 class DataLakeServiceClient;
 }  // namespace Azure::Storage::Files::DataLake
 
 namespace arrow::fs {
 
 class TestAzureFileSystem;
 class TestAzureOptions;
 
 /// Options for the AzureFileSystem implementation.
 ///
 /// By default, authentication is handled by the Azure SDK's credential chain
 /// which may read from multiple environment variables, such as:
 /// - `AZURE_TENANT_ID`
 /// - `AZURE_CLIENT_ID`
 /// - `AZURE_CLIENT_SECRET`
 /// - `AZURE_AUTHORITY_HOST`
 /// - `AZURE_CLIENT_CERTIFICATE_PATH`
 /// - `AZURE_FEDERATED_TOKEN_FILE`
 ///
 /// Functions are provided for explicit configuration of credentials if that is preferred.
 struct ARROW_EXPORT AzureOptions {
   friend class TestAzureOptions;
 
   /// \brief The name of the Azure Storage Account being accessed.
   ///
   /// All service URLs will be constructed using this storage account name.
   /// `ConfigureAccountKeyCredential` assumes the user wants to authenticate
   /// this account.
   std::string account_name;
 
   /// \brief hostname[:port] of the Azure Blob Storage Service.
   ///
   /// If the hostname is a relative domain name (one that starts with a '.'), then storage
   /// account URLs will be constructed by prepending the account name to the hostname.
   /// If the hostname is a fully qualified domain name, then the hostname will be used
   /// as-is and the account name will follow the hostname in the URL path.
   ///
   /// Default: ".blob.core.windows.net"
   std::string blob_storage_authority = ".blob.core.windows.net";
 
   /// \brief hostname[:port] of the Azure Data Lake Storage Gen 2 Service.
   ///
   /// If the hostname is a relative domain name (one that starts with a '.'), then storage
   /// account URLs will be constructed by prepending the account name to the hostname.
   /// If the hostname is a fully qualified domain name, then the hostname will be used
   /// as-is and the account name will follow the hostname in the URL path.
   ///
   /// Default: ".dfs.core.windows.net"
   std::string dfs_storage_authority = ".dfs.core.windows.net";
 
   /// \brief Azure Blob Storage connection transport.
   ///
   /// Default: "https"
   std::string blob_storage_scheme = "https";
 
   /// \brief Azure Data Lake Storage Gen 2 connection transport.
   ///
   /// Default: "https"
   std::string dfs_storage_scheme = "https";
 
   // TODO(GH-38598): Add support for more auth methods.
   // std::string connection_string;
   // std::string sas_token;
 
   /// \brief Default metadata for OpenOutputStream.
   ///
   /// This will be ignored if non-empty metadata is passed to OpenOutputStream.
   std::shared_ptr<const KeyValueMetadata> default_metadata;
 
   /// Whether OutputStream writes will be issued in the background, without blocking.
   bool background_writes = true;
 
  private:
   enum class CredentialKind {
     kDefault,
     kAnonymous,
     kStorageSharedKey,
     kSASToken,
     kClientSecret,
     kManagedIdentity,
     kCLI,
     kWorkloadIdentity,
     kEnvironment,
   } credential_kind_ = CredentialKind::kDefault;
 
   std::shared_ptr<Azure::Storage::StorageSharedKeyCredential>
       storage_shared_key_credential_;
   std::string sas_token_;
   mutable std::shared_ptr<Azure::Core::Credentials::TokenCredential> token_credential_;
 
  public:
   AzureOptions();
   ~AzureOptions();
 
  private:
   void ExtractFromUriSchemeAndHierPart(const Uri& uri, std::string* out_path);
   Status ExtractFromUriQuery(const Uri& uri);
 
  public:
   /// \brief Construct a new AzureOptions from an URI.
   ///
   /// Supported formats:
   ///
   /// 1. abfs[s]://\<account\>.blob.core.windows.net[/\<container\>[/\<path\>]]
   /// 2. abfs[s]://\<container\>\@\<account\>.dfs.core.windows.net[/path]
   /// 3. abfs[s]://[\<account@]\<host[.domain]\>[\<:port\>][/\<container\>[/path]]
   /// 4. abfs[s]://[\<account@]\<container\>[/path]
   ///
   /// (1) and (2) are compatible with the Azure Data Lake Storage Gen2 URIs
   /// [1], (3) is for Azure Blob Storage compatible service including Azurite,
   /// and (4) is a shorter version of (1) and (2).
   ///
   /// Note that there is no difference between abfs and abfss. HTTPS is
   /// used with abfs by default. You can force to use HTTP by specifying
   /// "enable_tls=false" query.
   ///
   /// Supported query parameters:
   ///
   /// * blob_storage_authority: Set AzureOptions::blob_storage_authority
   /// * dfs_storage_authority: Set AzureOptions::dfs_storage_authority
   /// * enable_tls: If it's "false" or "0", HTTP not HTTPS is used.
   /// * credential_kind: One of "default", "anonymous", "workload_identity",
   ///   "environment" or "cli". If "default" is specified, it's
   ///   just ignored.  If "anonymous" is specified,
   ///   AzureOptions::ConfigureAnonymousCredential() is called. If
   ///   "workload_identity" is specified,
   ///   AzureOptions::ConfigureWorkloadIdentityCredential() is called. If
   ///   "environment" is specified,
   ///   AzureOptions::ConfigureEnvironmentCredential() is called. If "cli" is
   ///   specified, AzureOptions::ConfigureCLICredential() is called.
   /// * tenant_id: You must specify "client_id" and "client_secret"
   ///   too. AzureOptions::ConfigureClientSecretCredential() is called.
   /// * client_id: If you don't specify "tenant_id" and
   ///   "client_secret",
   ///   AzureOptions::ConfigureManagedIdentityCredential() is
   ///   called. If you specify "tenant_id" and "client_secret" too,
   ///   AzureOptions::ConfigureClientSecretCredential() is called.
   /// * client_secret: You must specify "tenant_id" and "client_id"
   ///   too. AzureOptions::ConfigureClientSecretCredential() is called.
   /// * A SAS token is made up of several query parameters. Appending a SAS
   ///   token to the URI configures SAS token auth by calling
   ///   AzureOptions::ConfigureSASCredential().
   ///
   /// [1]:
   /// https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction-abfs-uri
   static Result<AzureOptions> FromUri(const Uri& uri, std::string* out_path);
   static Result<AzureOptions> FromUri(const std::string& uri, std::string* out_path);
 
   Status ConfigureDefaultCredential();
   Status ConfigureAnonymousCredential();
   Status ConfigureAccountKeyCredential(const std::string& account_key);
   Status ConfigureSASCredential(const std::string& sas_token);
   Status ConfigureClientSecretCredential(const std::string& tenant_id,
                                          const std::string& client_id,
                                          const std::string& client_secret);
   Status ConfigureManagedIdentityCredential(const std::string& client_id = std::string());
   Status ConfigureCLICredential();
   Status ConfigureWorkloadIdentityCredential();
   Status ConfigureEnvironmentCredential();
 
   bool Equals(const AzureOptions& other) const;
 
   std::string AccountBlobUrl(const std::string& account_name) const;
   std::string AccountDfsUrl(const std::string& account_name) const;
 
   Result<std::unique_ptr<Azure::Storage::Blobs::BlobServiceClient>>
   MakeBlobServiceClient() const;
 
   Result<std::unique_ptr<Azure::Storage::Files::DataLake::DataLakeServiceClient>>
   MakeDataLakeServiceClient() const;
 };
 
 /// \brief FileSystem implementation backed by Azure Blob Storage (ABS) [1] and
 /// Azure Data Lake Storage Gen2 (ADLS Gen2) [2].
 ///
 /// ADLS Gen2 isn't a dedicated service or account type. It's a set of capabilities that
 /// support high throughput analytic workloads, built on Azure Blob Storage. All the data
 /// ingested via the ADLS Gen2 APIs is persisted as blobs in the storage account.
 /// ADLS Gen2 provides filesystem semantics, file-level security, and Hadoop
 /// compatibility. ADLS Gen1 exists as a separate object that will retired on 2024-02-29
 /// and new ADLS accounts use Gen2 instead.
 ///
 /// ADLS Gen2 and Blob APIs can operate on the same data, but there are
 /// some limitations [3]. The ones that are relevant to this
 /// implementation are listed here:
 ///
 /// - You can't use Blob APIs, and ADLS APIs to write to the same instance of a file. If
 ///   you write to a file by using ADLS APIs then that file's blocks won't be visible
 ///   to calls to the GetBlockList Blob API. The only exception is when you're
 ///   overwriting.
 /// - When you use the ListBlobs operation without specifying a delimiter, the results
 ///   include both directories and blobs. If you choose to use a delimiter, use only a
 ///   forward slash (/) \--- the only supported delimiter.
 /// - If you use the DeleteBlob API to delete a directory, that directory is deleted only
 ///   if it's empty. This means that you can't use the Blob API delete directories
 ///   recursively.
 ///
 /// [1]: https://azure.microsoft.com/en-us/products/storage/blobs
 /// [2]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage
 /// [3]:
 /// https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-known-issues
 class ARROW_EXPORT AzureFileSystem : public FileSystem {
  private:
   class Impl;
   std::unique_ptr<Impl> impl_;
 
   explicit AzureFileSystem(std::unique_ptr<Impl>&& impl);
 
   friend class TestAzureFileSystem;
   void ForceCachedHierarchicalNamespaceSupport(int hns_support);
 
  public:
   ~AzureFileSystem() override = default;
 
   static Result<std::shared_ptr<AzureFileSystem>> Make(
       const AzureOptions& options, const io::IOContext& = io::default_io_context());
 
   std::string type_name() const override { return "abfs"; }
 
   /// Return the original Azure options when constructing the filesystem
   const AzureOptions& options() const;
 
   bool Equals(const FileSystem& other) const override;
 
   /// \cond FALSE
   using FileSystem::CreateDir;
   using FileSystem::DeleteDirContents;
   using FileSystem::GetFileInfo;
   using FileSystem::OpenAppendStream;
   using FileSystem::OpenOutputStream;
   /// \endcond
 
   Result<FileInfo> GetFileInfo(const std::string& path) override;
 
   Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
   Status CreateDir(const std::string& path, bool recursive) override;
 
   /// \brief Delete a directory and its contents recursively.
   ///
   /// Atomicity is guaranteed only on Hierarchical Namespace Storage accounts.
   Status DeleteDir(const std::string& path) override;
 
   /// \brief Non-atomically deletes the contents of a directory.
   ///
   /// This function can return a bad Status after only partially deleting the
   /// contents of the directory.
   Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
   /// \brief Deletion of all the containers in the storage account (not
   /// implemented for safety reasons).
   ///
   /// \return Status::NotImplemented
   Status DeleteRootDirContents() override;
 
   /// \brief Deletes a file.
   ///
   /// Supported on both flat namespace and Hierarchical Namespace storage
   /// accounts. A check is made to guarantee the parent directory doesn't
   /// disappear after the blob is deleted and while this operation is running,
   /// no other client can delete the parent directory due to the use of leases.
   ///
   /// This means applications can safely retry this operation without coordination to
   /// guarantee only one client/process is trying to delete the same file.
   Status DeleteFile(const std::string& path) override;
 
   /// \brief Move/rename a file or directory.
   ///
   /// There are no files immediately at the root directory, so paths like
   /// "/segment" always refer to a container of the storage account and are
   /// treated as directories.
   ///
   /// If `dest` exists but the operation fails for some reason, `Move`
   /// guarantees `dest` is not lost.
   ///
   /// Conditions for a successful move:
   ///
   /// 1. `src` must exist.
   /// 2. `dest` can't contain a strict path prefix of `src`. More generally,
   ///    a directory can't be made a subdirectory of itself.
   /// 3. If `dest` already exists and it's a file, `src` must also be a file.
   ///    `dest` is then replaced by `src`.
   /// 4. All components of `dest` must exist, except for the last.
   /// 5. If `dest` already exists and it's a directory, `src` must also be a
   ///    directory and `dest` must be empty. `dest` is then replaced by `src`
   ///    and its contents.
   ///
   /// Leases are used to guarantee the pre-condition checks and the rename
   /// operation are atomic: other clients can't invalidate the pre-condition in
   /// the time between the checks and the actual rename operation.
   ///
   /// This is possible because Move() is only support on storage accounts with
   /// Hierarchical Namespace Support enabled.
   ///
   /// ## Limitations
   ///
   /// - Moves are not supported on storage accounts without
   ///   Hierarchical Namespace support enabled
   /// - Moves across different containers are not supported
   /// - Moving a path of the form `/container` is not supported as it would
   ///   require moving all the files in a container to another container.
   ///   The only exception is a `Move("/container_a", "/container_b")` where
   ///   both containers are empty or `container_b` doesn't even exist.
   ///   The atomicity of the emptiness checks followed by the renaming operation
   ///   is guaranteed by the use of leases.
   Status Move(const std::string& src, const std::string& dest) override;
 
   Status CopyFile(const std::string& src, const std::string& dest) override;
 
   Result<std::shared_ptr<io::InputStream>> OpenInputStream(
       const std::string& path) override;
 
   Result<std::shared_ptr<io::InputStream>> OpenInputStream(const FileInfo& info) override;
 
   Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
       const std::string& path) override;
 
   Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
       const FileInfo& info) override;
 
   Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
       const std::string& path,
       const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
   Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
       const std::string& path,
       const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
   Result<std::string> PathFromUri(const std::string& uri_string) const override;
 };
 
 }  // namespace arrow::fs

This page was automatically generated by the 2.3.7 LXR engine. The LXR team