mca/filem/filem.h

0001 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
0002 /*
0003  * Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
0004  *                         University Research and Technology
0005  *                         Corporation.  All rights reserved.
0006  * Copyright (c) 2004-2005 The University of Tennessee and The University
0007  *                         of Tennessee Research Foundation.  All rights
0008  *                         reserved.
0009  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
0010  *                         University of Stuttgart.  All rights reserved.
0011  * Copyright (c) 2004-2005 The Regents of the University of California.
0012  *                         All rights reserved.
0013  * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
0014  *                         reserved
0015  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
0016  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
0017  * Copyright (c) 2021-2022 Nanook Consulting.  All rights reserved.
0018  * $COPYRIGHT$
0019  *
0020  * Additional copyrights may follow
0021  *
0022  * $HEADER$
0023  */
0024 /**
0025  * @file
0026  *
0027  * Remote File Management (FileM) Interface
0028  *
0029  */
0030
0031 #ifndef MCA_FILEM_H
0032 #define MCA_FILEM_H
0033
0034 #include "prte_config.h"
0035 #include "constants.h"
0036 #include "types.h"
0037
0038 #include "src/pmix/pmix-internal.h"
0039 #include "src/mca/base/pmix_base.h"
0040 #include "src/mca/mca.h"
0041
0042 #include "src/class/pmix_object.h"
0043
0044 #include "src/runtime/prte_globals.h"
0045 BEGIN_C_DECLS
0046
0047 /**
0048  * A set of flags that determine the type of the file
0049  * in question
0050  */
0051 #define PRTE_FILEM_TYPE_FILE    0
0052 #define PRTE_FILEM_TYPE_DIR     1
0053 #define PRTE_FILEM_TYPE_UNKNOWN 2
0054 #define PRTE_FILEM_TYPE_TAR     3
0055 #define PRTE_FILEM_TYPE_BZIP    4
0056 #define PRTE_FILEM_TYPE_GZIP    5
0057 #define PRTE_FILEM_TYPE_EXE     6
0058
0059 /**
0060  * Type of movement
0061  */
0062 #define PRTE_FILEM_MOVE_TYPE_PUT     0
0063 #define PRTE_FILEM_MOVE_TYPE_GET     1
0064 #define PRTE_FILEM_MOVE_TYPE_RM      2
0065 #define PRTE_FILEM_MOVE_TYPE_UNKNOWN 3
0066
0067 /**
0068  * Hints that describe the local or remote file target for
0069  * optimization purposes.
0070  */
0071 #define PRTE_FILEM_HINT_NONE   0
0072 #define PRTE_FILEM_HINT_SHARED 1
0073
0074 /**
0075  * Define a Process Set
0076  *
0077  * Source: A single source of the operation.
0078  * Sink: Desitination of the operation.
0079  */
0080 struct prte_filem_base_process_set_1_0_0_t {
0081     /** This is an object, so must have a super */
0082     pmix_list_item_t super;
0083
0084     /** Source Process */
0085     pmix_proc_t source;
0086
0087     /** Sink Process */
0088     pmix_proc_t sink;
0089 };
0090 typedef struct prte_filem_base_process_set_1_0_0_t prte_filem_base_process_set_1_0_0_t;
0091 typedef struct prte_filem_base_process_set_1_0_0_t prte_filem_base_process_set_t;
0092
0093 PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_filem_base_process_set_t);
0094
0095 /**
0096  * Define a File Pair
0097  *
0098  * Local: Local file reference
0099  * Remove: Remote file reference
0100  *
0101  * Note: If multiple process sinks are used it is assumed that the
0102  * file reference is the same for each of the sinks. If this is not
0103  * true then more than one filem request needs to be created.
0104  */
0105 struct prte_filem_base_file_set_1_0_0_t {
0106     /** This is an object, so must have a super */
0107     pmix_list_item_t super;
0108
0109     /* the app_index this pertains to, if applicable */
0110     prte_app_idx_t app_idx;
0111
0112     /* Local file reference */
0113     char *local_target;
0114
0115     /* Local file reference hints */
0116     int local_hint;
0117
0118     /* Remove file reference */
0119     char *remote_target;
0120
0121     /* Remote file reference hints */
0122     int remote_hint;
0123
0124     /* Type of file to move */
0125     int target_flag;
0126 };
0127 typedef struct prte_filem_base_file_set_1_0_0_t prte_filem_base_file_set_1_0_0_t;
0128 typedef struct prte_filem_base_file_set_1_0_0_t prte_filem_base_file_set_t;
0129
0130 PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_filem_base_file_set_t);
0131
0132 /**
0133  * Definition of a file movement request
0134  * This will allow:
0135  *  - The movement of one or more files
0136  *  - to/from one or more processes
0137  * in a single call of the API function. Allowing the implementation
0138  * to optimize the sending/receiving of data.
0139  * Used for the following:
0140  *
0141  */
0142 struct prte_filem_base_request_1_0_0_t {
0143     /** This is an object, so must have a super */
0144     pmix_list_item_t super;
0145
0146     /*
0147      * A list of process sets - use WILDCARD to
0148      * indicate all procs of a given vpid/jobid,
0149      * INVALID to indicate not-applicable. For
0150      * example, if you need to move a file at time
0151      * of job start to each node that has a proc
0152      * on it, then the process set would have a
0153      * source proc with vpid=INVALID and a sink proc
0154      * with vpid=WILDCARD, and a remote hint of "shared"
0155      * in the file sets so we don't copy them over
0156      * multiple times
0157      */
0158     pmix_list_t process_sets;
0159
0160     /*
0161      * A list of file pairings
0162      */
0163     pmix_list_t file_sets;
0164
0165     /*
0166      * Internal use:
0167      * Number of movements
0168      */
0169     int num_mv;
0170
0171     /*
0172      * Internal use:
0173      * Boolean to indianate if transfer is complete
0174      */
0175     bool *is_done;
0176
0177     /*
0178      * Internal use:
0179      * Boolean to indianate if transfer is active
0180      */
0181     bool *is_active;
0182
0183     /*
0184      * Internal use:
0185      * Exit status of the copy command
0186      */
0187     int32_t *exit_status;
0188
0189     /*
0190      * Internal use:
0191      * Movement type
0192      */
0193     int movement_type;
0194 };
0195 typedef struct prte_filem_base_request_1_0_0_t prte_filem_base_request_1_0_0_t;
0196 typedef struct prte_filem_base_request_1_0_0_t prte_filem_base_request_t;
0197
0198 PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_filem_base_request_t);
0199
0200 /**
0201  * Module initialization function.
0202  * Returns PRTE_SUCCESS
0203  */
0204 typedef int (*prte_filem_base_module_init_fn_t)(void);
0205
0206 /**
0207  * Module finalization function.
0208  * Returns PRTE_SUCCESS
0209  */
0210 typedef int (*prte_filem_base_module_finalize_fn_t)(void);
0211
0212 /**
0213  * Put a file or directory on the remote machine
0214  *
0215  * Note: By using a relative path for the remote file/directory, the filem
0216  *       component will negotiate the correct absolute path for that file/directory
0217  *       for the remote machine.
0218  *
0219  * @param request FileM request describing the files/directories to send,
0220  *        the remote files/directories to use, and the processes to see the change.
0221  *
0222  * @return PRTE_SUCCESS on successful file transer
0223  * @return PRTE_ERROR on failed file transfer
0224  */
0225 typedef int (*prte_filem_base_put_fn_t)(prte_filem_base_request_t *request);
0226
0227 /**
0228  * Put a file or directory on the remote machine (Async)
0229  *
0230  * Note: By using a relative path for the remote file/directory, the filem
0231  *       component will negotiate the correct absolute path for that file/directory
0232  *       for the remote machine.
0233  *
0234  * @param request FileM request describing the files/directories to send,
0235  *        the remote files/directories to use, and the processes to see the change.
0236  *
0237  * @return PRTE_SUCCESS on successful file transer
0238  * @return PRTE_ERROR on failed file transfer
0239  */
0240 typedef int (*prte_filem_base_put_nb_fn_t)(prte_filem_base_request_t *request);
0241
0242 /**
0243  * Get a file from the remote machine
0244  *
0245  * Note: By using a relative path for the remote file/directory, the filem
0246  *       component will negotiate the correct absolute path for that file/directory
0247  *       for the remote machine.
0248  *
0249  * @param request FileM request describing the files/directories to receive,
0250  *        the remote files/directories to use, and the processes to see the change.
0251  *
0252  * @return PRTE_SUCCESS on successful file transer
0253  * @return PRTE_ERROR on failed file transfer
0254  */
0255 typedef int (*prte_filem_base_get_fn_t)(prte_filem_base_request_t *request);
0256
0257 /**
0258  * Get a file from the remote machine (Async)
0259  *
0260  * Note: By using a relative path for the remote file/directory, the filem
0261  *       component will negotiate the correct absolute path for that file/directory
0262  *       for the remote machine.
0263  *
0264  * @param request FileM request describing the files/directories to receive,
0265  *        the remote files/directories to use, and the processes to see the change.
0266  *
0267  * @return PRTE_SUCCESS on successful file transer
0268  * @return PRTE_ERROR on failed file transfer
0269  */
0270 typedef int (*prte_filem_base_get_nb_fn_t)(prte_filem_base_request_t *request);
0271
0272 /**
0273  * Remove a file from the remote machine
0274  *
0275  * Note: By using a relative path for the remote file/directory, the filem
0276  *       component will negotiate the correct absolute path for that file/directory
0277  *       for the remote machine.
0278  *
0279  * @param request FileM request describing the remote files/directories to remove,
0280  *        the processes to see the change.
0281  *
0282  * @return PRTE_SUCCESS on success
0283  * @return PRTE_ERROR on fail
0284  */
0285 typedef int (*prte_filem_base_rm_fn_t)(prte_filem_base_request_t *request);
0286
0287 /**
0288  * Remove a file from the remote machine (Async)
0289  *
0290  * Note: By using a relative path for the remote file/directory, the filem
0291  *       component will negotiate the correct absolute path for that file/directory
0292  *       for the remote machine.
0293  *
0294  * @param request FileM request describing the remote files/directories to remove,
0295  *        the processes to see the change.
0296  *
0297  * @return PRTE_SUCCESS on success
0298  * @return PRTE_ERROR on fail
0299  */
0300 typedef int (*prte_filem_base_rm_nb_fn_t)(prte_filem_base_request_t *request);
0301
0302 /**
0303  * Wait for a single file movement request to finish
0304  *
0305  * @param request FileM request describing the remote files/directories.
0306  *
0307  * The request must have been passed through one of the non-blocking functions
0308  * before calling wait or wait_all otherwise PRTE_ERROR will be returned.
0309  *
0310  * @return PRTE_SUCCESS on success
0311  * @return PRTE_ERROR on fail
0312  */
0313 typedef int (*prte_filem_base_wait_fn_t)(prte_filem_base_request_t *request);
0314
0315 /**
0316  * Wait for a multiple file movement requests to finish
0317  *
0318  * @param request_list pmix_list_t of FileM requests describing the remote files/directories.
0319  *
0320  * The request must have been passed through one of the non-blocking functions
0321  * before calling wait or wait_all otherwise PRTE_ERROR will be returned.
0322  *
0323  * @return PRTE_SUCCESS on success
0324  * @return PRTE_ERROR on fail
0325  */
0326 typedef int (*prte_filem_base_wait_all_fn_t)(pmix_list_t *request_list);
0327
0328 typedef void (*prte_filem_completion_cbfunc_t)(int status, void *cbdata);
0329
0330 /* Pre-position files
0331  */
0332 typedef int (*prte_filem_base_preposition_files_fn_t)(prte_job_t *jdata,
0333                                                       prte_filem_completion_cbfunc_t cbfunc,
0334                                                       void *cbdata);
0335
0336 /* link local files */
0337 typedef int (*prte_filem_base_link_local_files_fn_t)(prte_job_t *jdata, prte_app_context_t *app);
0338
0339 /**
0340  * Structure for FILEM components.
0341  */
0342 typedef pmix_mca_base_component_t prte_filem_base_component_t;
0343
0344 /**
0345  * Structure for FILEM  modules
0346  */
0347 struct prte_filem_base_module_1_0_0_t {
0348     /** Initialization Function */
0349     prte_filem_base_module_init_fn_t filem_init;
0350     /** Finalization Function */
0351     prte_filem_base_module_finalize_fn_t filem_finalize;
0352
0353     /** Put a file on the remote machine */
0354     prte_filem_base_put_fn_t put;
0355     prte_filem_base_put_nb_fn_t put_nb;
0356     /** Get a file from the remote machine */
0357     prte_filem_base_get_fn_t get;
0358     prte_filem_base_get_nb_fn_t get_nb;
0359
0360     /** Remove a file on the remote machine */
0361     prte_filem_base_rm_fn_t rm;
0362     prte_filem_base_rm_nb_fn_t rm_nb;
0363
0364     /** Test functions for the non-blocking versions */
0365     prte_filem_base_wait_fn_t wait;
0366     prte_filem_base_wait_all_fn_t wait_all;
0367
0368     /* pre-position files to every node */
0369     prte_filem_base_preposition_files_fn_t preposition_files;
0370     /* create local links for all shared files */
0371     prte_filem_base_link_local_files_fn_t link_local_files;
0372 };
0373 typedef struct prte_filem_base_module_1_0_0_t prte_filem_base_module_1_0_0_t;
0374 typedef struct prte_filem_base_module_1_0_0_t prte_filem_base_module_t;
0375
0376 PRTE_EXPORT extern prte_filem_base_module_t prte_filem;
0377
0378 /**
0379  * Macro for use in components that are of type FILEM
0380  */
0381 #define PRTE_FILEM_BASE_VERSION_2_0_0 PRTE_MCA_BASE_VERSION_3_0_0("filem", 2, 0, 0)
0382
0383 END_C_DECLS
0384
0385 #endif /* PRTE_FILEM_H */