source/include/H5FDsubfiling.h

0001 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
0002  * Copyright by The HDF Group.                                               *
0003  * All rights reserved.                                                      *
0004  *                                                                           *
0005  * This file is part of HDF5.  The full HDF5 copyright notice, including     *
0006  * terms governing use, modification, and redistribution, is contained in    *
0007  * the COPYING file, which can be found at the root of the source code       *
0008  * distribution tree, or in https://www.hdfgroup.org/licenses.               *
0009  * If you do not have access to either file, you may request a copy from     *
0010  * help@hdfgroup.org.                                                        *
0011  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
0012
0013 /* Purpose: The public header file for the subfiling driver. */
0014 #ifndef H5FDsubfiling_H
0015 #define H5FDsubfiling_H
0016
0017 #ifdef H5_HAVE_SUBFILING_VFD
0018 /**
0019  * \def H5FD_SUBFILING
0020  * Macro that returns the identifier for the #H5FD_SUBFILING driver. \hid_t{file driver}
0021  */
0022 #define H5FD_SUBFILING (H5FDperform_init(H5FD_subfiling_init))
0023 #else
0024 #define H5FD_SUBFILING (H5I_INVALID_HID)
0025 #endif
0026
0027 /**
0028  * \def H5FD_SUBFILING_NAME
0029  * The canonical name for the #H5FD_SUBFILING driver
0030  */
0031 #define H5FD_SUBFILING_NAME "subfiling"
0032
0033 #ifdef H5_HAVE_SUBFILING_VFD
0034
0035 #ifndef H5FD_SUBFILING_FAPL_MAGIC
0036 /**
0037  * \def H5FD_SUBFILING_CURR_FAPL_VERSION
0038  * The version number of the H5FD_subfiling_config_t configuration
0039  * structure for the #H5FD_SUBFILING driver
0040  */
0041 #define H5FD_SUBFILING_CURR_FAPL_VERSION 1
0042 /**
0043  * \def H5FD_SUBFILING_FAPL_MAGIC
0044  * Unique number used to distinguish the #H5FD_SUBFILING driver from other HDF5 file drivers
0045  */
0046 #define H5FD_SUBFILING_FAPL_MAGIC 0xFED01331
0047 #endif
0048
0049 /**
0050  * \def H5FD_SUBFILING_DEFAULT_STRIPE_SIZE
0051  * The default stripe size (in bytes) for data stripes in subfiles
0052  */
0053 #define H5FD_SUBFILING_DEFAULT_STRIPE_SIZE (32 * 1024 * 1024)
0054
0055 /**
0056  * \def H5FD_SUBFILING_DEFAULT_STRIPE_COUNT
0057  * Macro for the default Subfiling stripe count value. The default
0058  * is currently to use one subfile per node.
0059  */
0060 #define H5FD_SUBFILING_DEFAULT_STRIPE_COUNT -1
0061
0062 /**
0063  * \def H5FD_SUBFILING_FILENAME_TEMPLATE
0064  * The basic printf-style template for a #H5FD_SUBFILING driver
0065  * subfile filename. The format specifiers correspond to:
0066  *
0067  * \par \%s
0068  *   base filename, e.g. "file.h5"
0069  *
0070  * \par \%PRIu64
0071  *   file inode, e.g. 11273556
0072  *
0073  * \par \%0*d
0074  *   number (starting at 1) signifying the Nth (out of
0075  *   total number of subfiles) subfile. Zero-padded
0076  *   according to the number of digits in the number of
0077  *   subfiles (calculated by <tt>log10(num_subfiles) + 1)</tt>
0078  *
0079  * \par \%d
0080  *   number of subfiles
0081  *
0082  * yielding filenames such as:
0083  *
0084  * file.h5.subfile_11273556_01_of_10 \n
0085  * file.h5.subfile_11273556_02_of_10 \n
0086  * file.h5.subfile_11273556_10_of_10 \n
0087  */
0088 #define H5FD_SUBFILING_FILENAME_TEMPLATE "%s.subfile_%" PRIu64 "_%0*d_of_%d"
0089
0090 /**
0091  * \def H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE
0092  * The basic printf-style template for a #H5FD_SUBFILING driver
0093  * configuration filename. The format specifiers correspond to:
0094  *
0095  * \par \%s
0096  *   base filename, e.g. "file.h5"
0097  *
0098  * \par \%PRIu64
0099  *   file inode, e.g. 11273556
0100  *
0101  * yielding a filename such as:
0102  *
0103  * file.h5.subfile_11273556.config
0104  */
0105 #define H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE "%s.subfile_%" PRIu64 ".config"
0106
0107 /*
0108  * Environment variables interpreted by the HDF5 Subfiling feature
0109  */
0110
0111 /**
0112  * \def H5FD_SUBFILING_STRIPE_SIZE
0113  * Macro for name of the environment variable that specifies the size
0114  * (in bytes) for data stripes in subfiles
0115  *
0116  * The value set for this environment variable is interpreted as a
0117  * long long value and must be > 0.
0118  */
0119 #define H5FD_SUBFILING_STRIPE_SIZE "H5FD_SUBFILING_STRIPE_SIZE"
0120 /**
0121  * \def H5FD_SUBFILING_IOC_PER_NODE
0122  * Macro for name of the environment variable that specifies the number
0123  * of MPI ranks per node to use as I/O concentrators
0124  *
0125  * The value set for this environment variable is interpreted as a
0126  * long value and must be > 0.
0127  */
0128 #define H5FD_SUBFILING_IOC_PER_NODE "H5FD_SUBFILING_IOC_PER_NODE"
0129 /**
0130  * \def H5FD_SUBFILING_IOC_SELECTION_CRITERIA
0131  * Macro for name of the environment variable that provides information
0132  * for selection MPI ranks as I/O concentrators
0133  *
0134  * The value set for this environment variable is interpreted differently,
0135  * depending on the IOC selection method chosen.
0136  *
0137  * For #SELECT_IOC_ONE_PER_NODE, this value is ignored.
0138  *
0139  * For #SELECT_IOC_EVERY_NTH_RANK, this value is interpreted as a
0140  *     long value and must be > 0. The value will correspond to the
0141  *     `N` value when selecting every `N`-th MPI rank as an I/O
0142  *     concentrator.
0143  *
0144  * For #SELECT_IOC_WITH_CONFIG, this value is ignored as that particular
0145  *     IOC selection method is not currently supported.
0146  *
0147  * For #SELECT_IOC_TOTAL, this value is interpreted as a long value
0148  *     and must be > 0. The value will correspond to the total number
0149  *     of I/O concentrators to be used.
0150  */
0151 #define H5FD_SUBFILING_IOC_SELECTION_CRITERIA "H5FD_SUBFILING_IOC_SELECTION_CRITERIA"
0152 /**
0153  * \def H5FD_SUBFILING_SUBFILE_PREFIX
0154  * Macro for name of the environment variable that specifies a prefix
0155  * to apply to the filenames generated for subfiles
0156  *
0157  * The value set for this environment variable is interpreted as a
0158  * pathname.
0159  */
0160 #define H5FD_SUBFILING_SUBFILE_PREFIX "H5FD_SUBFILING_SUBFILE_PREFIX"
0161 /**
0162  * \def H5FD_SUBFILING_CONFIG_FILE_PREFIX
0163  * Macro for name of the environment variable that specifies a prefix
0164  * to apply to the subfiling configuration filename. Useful for cases
0165  * where the application wants to place the configuration file in a
0166  * different directory than the default of putting it alongside the
0167  * generated subfiles. For example, when writing to node-local storage
0168  * one may wish to place the configuration file on a scratch file
0169  * system readable by all nodes, while the subfiles are initially
0170  * written to the node-local storage.
0171  *
0172  * The value set for this environment variable is interpreted as a
0173  * pathname that must already exist.
0174  *
0175  * NOTE: As this prefix string will be encoded in the driver info
0176  *       message that gets written to the file, there is an upper
0177  *       limit of about ~900 single-byte characters for this string,
0178  *       though possibly less due to other information the driver
0179  *       may encode. Avoid long prefix names where possible.
0180  */
0181 #define H5FD_SUBFILING_CONFIG_FILE_PREFIX "H5FD_SUBFILING_CONFIG_FILE_PREFIX"
0182
0183 /**
0184  * \enum H5FD_subfiling_ioc_select_t
0185  * This enum defines the various constants to allow different
0186  * allocations of MPI ranks as I/O concentrators.
0187  *
0188  * \var SELECT_IOC_ONE_PER_NODE
0189  *      Default selection method. One MPI rank per node is used as an
0190  *      I/O concentrator. If this selection method is used, the number
0191  *      of I/O concentrators per node can be adjusted with the
0192  *      #H5FD_SUBFILING_IOC_PER_NODE environment variable.
0193  *
0194  * \var SELECT_IOC_EVERY_NTH_RANK
0195  *      Starting with MPI rank 0, a stride of 'N' is applied to the MPI
0196  *      rank values to determine the next I/O concentrator. The
0197  *      #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variable must
0198  *      be set to the value desired for 'N'.
0199  *
0200  * \var SELECT_IOC_WITH_CONFIG
0201  *      Currently unsupported. Use a configuration file to determine
0202  *      the mapping from MPI ranks to I/O concentrators. The
0203  *      #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variable must
0204  *      be set to the path to the configuration file.
0205  *
0206  * \var SELECT_IOC_TOTAL
0207  *      Specifies that a total of 'N' I/O concentrators should be used.
0208  *      Starting with MPI rank 0, a stride of 'MPI comm size' / 'N' is
0209  *      applied to the MPI rank values to determine the next I/O
0210  *      concentrator. The #H5FD_SUBFILING_IOC_SELECTION_CRITERIA
0211  *      environment variable must be set to the value desired for 'N'.
0212  *
0213  * \var ioc_selection_options
0214  *      Unused. Sentinel value
0215  */
0216 typedef enum {
0217     SELECT_IOC_ONE_PER_NODE = 0, /* Default                              */
0218     SELECT_IOC_EVERY_NTH_RANK,   /* Starting at rank 0, select-next += N */
0219     SELECT_IOC_WITH_CONFIG,      /* NOT IMPLEMENTED: Read-from-file      */
0220     SELECT_IOC_TOTAL,            /* Starting at rank 0, mpi_size / total */
0221     ioc_selection_options        /* Sentinel value                       */
0222     /* NOTE: Add to the Fortran constants (H5f90global.F90)  when adding new entries */
0223 } H5FD_subfiling_ioc_select_t;
0224
0225 /**
0226  * \struct H5FD_subfiling_params_t
0227  * \brief Subfiling parameter structure that is shared between the #H5FD_SUBFILING
0228  *        and #H5FD_IOC drivers
0229  *
0230  * \var H5FD_subfiling_ioc_select_t H5FD_subfiling_params_t::ioc_selection
0231  *      The method to use for selecting MPI ranks to be I/O concentrators. The
0232  *      current default is to select one MPI rank per node to be an I/O concentrator.
0233  *      Refer to #H5FD_subfiling_ioc_select_t for a description of the algorithms
0234  *      available for use.
0235  *
0236  * \var int64_t H5FD_subfiling_params_t::stripe_size
0237  *      The stripe size defines the size (in bytes) of the data stripes in the
0238  *      subfiles for the logical HDF5 file. Data is striped across the subfiles
0239  *      in a round-robin wrap-around fashion in segments equal to the stripe size.
0240  *
0241  *      For example, in an HDF5 file consisting of four subfiles with a 1MiB stripe
0242  *      size, the first and fifth 1MiB of data would reside in the first subfile,
0243  *      the second and sixth 1MiB of data would reside in the second subfile and so
0244  *      on.
0245  *
0246  *      This value can also be set or adjusted with the #H5FD_SUBFILING_STRIPE_SIZE
0247  *      environment variable.
0248  *
0249  * \var int32_t H5FD_subfiling_params_t::stripe_count
0250  *      The target number of subfiles to use for the logical HDF5 file. The current
0251  *      default is to use one subfile per node, but it can be useful to set a
0252  *      different target number of subfiles, especially if the HDF5 application will
0253  *      pre-create the HDF5 file on a single MPI rank. In that particular case, the
0254  *      single rank will need to know how many subfiles the logical HDF5 file will
0255  *      consist of in order to properly pre-create the file.
0256  *
0257  *      This value is used in conjunction with the IOC selection method to determine
0258  *      which MPI ranks will be assigned as I/O concentrators. Alternatively, the
0259  *      mapping between MPI ranks and I/O concentrators can be set or adjusted with a
0260  *      combination of the #ioc_selection field and the #H5FD_SUBFILING_IOC_PER_NODE
0261  *      and #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variables.
0262  */
0263 typedef struct H5FD_subfiling_params_t {
0264     H5FD_subfiling_ioc_select_t ioc_selection; /* Method to select I/O concentrators          */
0265     int64_t                     stripe_size;   /* Size (in bytes) of data stripes in subfiles */
0266     int32_t                     stripe_count;  /* Target number of subfiles to use            */
0267 } H5FD_subfiling_params_t;
0268
0269 //! <!-- [H5FD_subfiling_config_t_snip] -->
0270 /**
0271  * \struct H5FD_subfiling_config_t
0272  * \brief Configuration structure for H5Pset_fapl_subfiling() / H5Pget_fapl_subfiling()
0273  *
0274  * \details H5FD_subfiling_config_t is a public structure that is used to pass
0275  *          subfiling configuration data to the #H5FD_SUBFILING driver via
0276  *          a File Access Property List. A pointer to an instance of this structure
0277  *          is a parameter to H5Pset_fapl_subfiling() and H5Pget_fapl_subfiling().
0278  *
0279  * \var uint32_t H5FD_subfiling_config_t::magic
0280  *      A somewhat unique number which distinguishes the #H5FD_SUBFILING driver
0281  *      from other drivers. Used in combination with a version number, it can
0282  *      help to validate a user-generated File Access Property List. This field
0283  *      should be set to #H5FD_SUBFILING_FAPL_MAGIC.
0284  *
0285  * \var uint32_t H5FD_subfiling_config_t::version
0286  *      Version number of the H5FD_subfiling_config_t structure. Any instance
0287  *      passed to H5Pset_fapl_subfiling() / H5Pget_fapl_subfiling() must have
0288  *      a recognized version number or an error will be raised. Currently, this
0289  *      field should be set to #H5FD_SUBFILING_CURR_FAPL_VERSION.
0290  *
0291  * \var hid_t H5FD_subfiling_config_t::ioc_fapl_id
0292  *      The File Access Property List which is setup with the file driver that
0293  *      the #H5FD_SUBFILING driver will use for servicing I/O requests to the
0294  *      subfiles. Currently, the File Access Property List must be setup with
0295  *      the #H5FD_IOC driver by calling H5Pset_fapl_ioc(), but future development
0296  *      may allow other file drivers to be used.
0297  *
0298  * \var bool H5FD_subfiling_config_t::require_ioc
0299  *      A boolean flag which indicates whether the #H5FD_SUBFILING driver should
0300  *      use the #H5FD_IOC driver for its I/O operations. This field should currently
0301  *      always be set to true.
0302  *
0303  * \var H5FD_subfiling_params_t H5FD_subfiling_config_t::shared_cfg
0304  *      A structure which contains the subfiling parameters that are shared between
0305  *      the #H5FD_SUBFILING and #H5FD_IOC drivers. This includes the subfile stripe
0306  *      size, stripe count, IOC selection method, etc.
0307  *
0308  */
0309 typedef struct H5FD_subfiling_config_t {
0310     uint32_t magic;                     /* Must be set to H5FD_SUBFILING_FAPL_MAGIC                         */
0311     uint32_t version;                   /* Must be set to H5FD_SUBFILING_CURR_FAPL_VERSION                  */
0312     hid_t    ioc_fapl_id;               /* The FAPL setup with the stacked VFD to use for I/O concentrators */
0313     bool     require_ioc;               /* Whether to use the IOC VFD (currently must always be true)       */
0314     H5FD_subfiling_params_t shared_cfg; /* Subfiling/IOC parameters (stripe size, stripe count, etc.)       */
0315 } H5FD_subfiling_config_t;
0316 //! <!-- [H5FD_subfiling_config_t_snip] -->
0317
0318 #ifdef __cplusplus
0319 extern "C" {
0320 #endif
0321
0322 /**
0323  * \brief Internal routine to initialize #H5FD_SUBFILING driver. Not meant to be
0324  *        called directly by an HDF5 application
0325  */
0326 H5_DLL hid_t H5FD_subfiling_init(void);
0327 /**
0328  * \ingroup FAPL
0329  *
0330  * \brief Modifies the specified File Access Property List to use the #H5FD_SUBFILING driver
0331  *
0332  * \fapl_id
0333  * \param[in] vfd_config Pointer to #H5FD_SUBFILING driver configuration structure. May be NULL.
0334  * \returns \herr_t
0335  *
0336  * \details H5Pset_fapl_subfiling() modifies the File Access Property List to use the
0337  *          #H5FD_SUBFILING driver.
0338  *
0339  *          The #H5FD_SUBFILING driver is an MPI-based file driver that allows an
0340  *          HDF5 application to distribute a logical HDF5 file across a collection
0341  *          of "subfiles" in equal-sized data segment "stripes". I/O to the logical
0342  *          HDF5 file is then directed to the appropriate "subfile" according to the
0343  *          #H5FD_SUBFILING configuration and a system of I/O concentrators, which
0344  *          are MPI ranks operating worker threads.
0345  *
0346  *          By allowing a configurable stripe size, number of I/O concentrators and
0347  *          method for selecting MPI ranks as I/O concentrators, the #H5FD_SUBFILING
0348  *          driver aims to enable an HDF5 application to find a middle ground between
0349  *          the single shared file and file-per-process approaches to parallel file I/O
0350  *          for the particular machine the application is running on. In general, the
0351  *          goal is to avoid some of the complexity of the file-per-process approach
0352  *          while also minimizing the locking issues of the single shared file approach
0353  *          on a parallel file system.
0354  *
0355  * \note Since the #H5FD_SUBFILING driver is an MPI-based file driver, the HDF5
0356  *       application should ensure that H5Pset_mpi_params() is called before this
0357  *       routine so that the appropriate MPI communicator and info objects will be
0358  *       setup for use by the #H5FD_SUBFILING and #H5FD_IOC drivers.
0359  *
0360  * \note The current architecture of the #H5FD_SUBFILING driver requires that the
0361  *       HDF5 application must have been initialized with MPI_Init_thread() using
0362  *       a value of MPI_THREAD_MULTIPLE for the thread support level.
0363  *
0364  * \note The \p vfd_config parameter may be NULL. In this case, the reference
0365  *       implementation I/O concentrator VFD will be used with the default settings
0366  *       of one I/O concentrator per node and a stripe size of 32MiB. Refer to the
0367  *       H5FD_subfiling_config_t documentation for information about configuration
0368  *       for the #H5FD_SUBFILING driver.
0369  *
0370  * \since 1.14.0
0371  *
0372  */
0373 H5_DLL herr_t H5Pset_fapl_subfiling(hid_t fapl_id, const H5FD_subfiling_config_t *vfd_config);
0374 /**
0375  * \ingroup FAPL
0376  *
0377  * \brief Queries a File Access Property List for #H5FD_SUBFILING file driver properties
0378  *
0379  * \fapl_id
0380  * \param[out] config_out Pointer to H5FD_subfiling_config_t structure through which the
0381  *                        #H5FD_SUBFILING file driver properties will be returned.
0382  *
0383  * \returns \herr_t
0384  *
0385  * \details H5Pget_fapl_subfiling() queries the specified File Access Property List for
0386  *          #H5FD_SUBFILING driver properties as set by H5Pset_fapl_subfiling(). If the
0387  *          #H5FD_SUBFILING driver has not been set on the File Access Property List, a
0388  *          default configuration is returned. An HDF5 application may use this
0389  *          functionality to manually configure the #H5FD_SUBFILING driver by calling
0390  *          H5Pget_fapl_subfiling() on a newly-created File Access Property List, adjusting
0391  *          the default values and then calling H5Pset_fapl_subfiling() with the configured
0392  *          H5FD_subfiling_config_t structure.
0393  *
0394  * \note H5Pget_fapl_subfiling() returns the #H5FD_SUBFILING driver properties as they
0395  *       were initially set for the File Access Property List using H5Pset_fapl_subfiling().
0396  *       Alternatively, the driver properties can be modified at runtime according to values
0397  *       set for the #H5FD_SUBFILING_STRIPE_SIZE, #H5FD_SUBFILING_IOC_PER_NODE and
0398  *       #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variables. However, driver
0399  *       properties set through environment variables will not be reflected in what is
0400  *       returned by H5Pget_fapl_subfiling(), so an application may need to check those
0401  *       environment variables to get accurate values for the #H5FD_SUBFILING driver
0402  *       properties.
0403  *
0404  * \since 1.14.0
0405  *
0406  */
0407 H5_DLL herr_t H5Pget_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *config_out);
0408
0409 #ifdef __cplusplus
0410 }
0411 #endif
0412
0413 #endif /* H5_HAVE_SUBFILING_VFD */
0414
0415 #endif /* H5FDsubfiling_H */