src/server/pmix_server_ops.h

0001 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
0002 /*
0003  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
0004  * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
0005  *                         All rights reserved.
0006  * Copyright (c) 2015      Mellanox Technologies, Inc.
0007  *                         All rights reserved.
0008  * Copyright (c) 2016-2020 IBM Corporation.  All rights reserved.
0009  * Copyright (c) 2016-2018 Research Organization for Information Science
0010  *                         and Technology (RIST).  All rights reserved.
0011  * Copyright (c) 2021-2024 Nanook Consulting  All rights reserved.
0012  * $COPYRIGHT$
0013  */
0014
0015 #ifndef PMIX_SERVER_OPS_H
0016 #define PMIX_SERVER_OPS_H
0017
0018 #include <unistd.h>
0019 #ifdef HAVE_SYS_TYPES_H
0020 #    include <sys/types.h>
0021 #endif
0022
0023 #include "src/include/pmix_config.h"
0024 #include "pmix_common.h"
0025 #include "pmix_server.h"
0026
0027 #include "src/class/pmix_hotel.h"
0028 #include "src/include/pmix_globals.h"
0029 #include "src/include/pmix_types.h"
0030 #include "src/threads/pmix_threads.h"
0031 #include "src/util/pmix_hash.h"
0032
0033 #define PMIX_IOF_HOTEL_SIZE 256
0034 #define PMIX_IOF_MAX_STAY   300000000
0035
0036 typedef struct {
0037     pmix_object_t super;
0038     pmix_event_t ev;
0039     pmix_server_trkr_t *trk;
0040 } pmix_trkr_caddy_t;
0041 PMIX_CLASS_DECLARATION(pmix_trkr_caddy_t);
0042
0043 typedef struct {
0044     pmix_object_t super;
0045     pmix_event_t ev;
0046     pmix_lock_t lock;
0047     pmix_peer_t *peer;
0048     char *nspace;
0049     pmix_status_t status;
0050     pmix_status_t *codes;
0051     size_t ncodes;
0052     pmix_proc_t proc;
0053     pmix_proc_t *procs;
0054     size_t nprocs;
0055     uid_t uid;
0056     gid_t gid;
0057     void *server_object;
0058     int nlocalprocs;
0059     pmix_info_t *info;
0060     size_t ninfo;
0061     bool copied;
0062     char **keys;
0063     pmix_app_t *apps;
0064     size_t napps;
0065     pmix_iof_channel_t channels;
0066     pmix_iof_flags_t flags;
0067     pmix_byte_object_t *bo;
0068     size_t nbo;
0069     /* timestamp receipt of the notification so we
0070      * can evict the oldest one if we get overwhelmed */
0071     time_t ts;
0072     /* what room of the hotel they are in */
0073     int room;
0074     pmix_op_cbfunc_t opcbfunc;
0075     pmix_dmodex_response_fn_t cbfunc;
0076     pmix_setup_application_cbfunc_t setupcbfunc;
0077     pmix_lookup_cbfunc_t lkcbfunc;
0078     pmix_spawn_cbfunc_t spcbfunc;
0079     void *cbdata;
0080 } pmix_setup_caddy_t;
0081 PMIX_CLASS_DECLARATION(pmix_setup_caddy_t);
0082
0083 /* define a callback function returning inventory */
0084 typedef void (*pmix_inventory_cbfunc_t)(pmix_status_t status, pmix_list_t *inventory, void *cbdata);
0085
0086 /* define an object for rolling up the inventory*/
0087 typedef struct {
0088     pmix_object_t super;
0089     pmix_lock_t lock;
0090     pmix_event_t ev;
0091     pmix_status_t status;
0092     int requests;
0093     int replies;
0094     pmix_list_t payload; // list of pmix_kval_t containing the replies
0095     pmix_info_t *info;
0096     size_t ninfo;
0097     pmix_inventory_cbfunc_t cbfunc;
0098     pmix_info_cbfunc_t infocbfunc;
0099     pmix_op_cbfunc_t opcbfunc;
0100     void *cbdata;
0101 } pmix_inventory_rollup_t;
0102 PMIX_CLASS_DECLARATION(pmix_inventory_rollup_t);
0103
0104 typedef struct {
0105     pmix_list_item_t super;
0106     pmix_setup_caddy_t *cd;
0107 } pmix_dmdx_remote_t;
0108 PMIX_CLASS_DECLARATION(pmix_dmdx_remote_t);
0109
0110 typedef struct {
0111     pmix_list_item_t super;
0112     pmix_proc_t proc;     // id of proc whose data is being requested
0113     pmix_list_t loc_reqs; // list of pmix_dmdx_request_t elem is keeping track of
0114                           // all local ranks that are interested in this namespace-rank
0115     pmix_info_t *info;    // array of info structs for this request
0116     size_t ninfo;         // number of info structs
0117 } pmix_dmdx_local_t;
0118 PMIX_CLASS_DECLARATION(pmix_dmdx_local_t);
0119
0120 typedef struct {
0121     pmix_list_item_t super;
0122     pmix_event_t ev;
0123     bool event_active;
0124     pmix_dmdx_local_t *lcd;
0125     char *key;
0126     pmix_modex_cbfunc_t cbfunc; // cbfunc to be executed when data is available
0127     void *cbdata;
0128 } pmix_dmdx_request_t;
0129 PMIX_CLASS_DECLARATION(pmix_dmdx_request_t);
0130
0131 /* event/error registration book keeping */
0132 typedef struct {
0133     pmix_list_item_t super;
0134     pmix_peer_t *peer;
0135     bool enviro_events;
0136     pmix_proc_t *affected;
0137     size_t naffected;
0138 } pmix_peer_events_info_t;
0139 PMIX_CLASS_DECLARATION(pmix_peer_events_info_t);
0140
0141 typedef struct {
0142     pmix_list_item_t super;
0143     pmix_list_t peers; // list of pmix_peer_events_info_t
0144     int code;
0145 } pmix_regevents_info_t;
0146 PMIX_CLASS_DECLARATION(pmix_regevents_info_t);
0147
0148 typedef struct {
0149     pmix_list_item_t super;
0150     pmix_group_t *grp;
0151     pmix_rank_t rank;
0152     size_t idx;
0153 } pmix_group_caddy_t;
0154 PMIX_CLASS_DECLARATION(pmix_group_caddy_t);
0155
0156 typedef struct {
0157     pmix_list_item_t super;
0158     pmix_proc_t source;
0159     pmix_iof_channel_t channel;
0160     pmix_byte_object_t *bo;
0161     pmix_info_t *info;
0162     size_t ninfo;
0163 } pmix_iof_cache_t;
0164 PMIX_CLASS_DECLARATION(pmix_iof_cache_t);
0165
0166 typedef struct {
0167     pmix_list_item_t super;
0168     char *name;
0169     pmix_proc_t *members;
0170     size_t nmembers;
0171 } pmix_pset_t;
0172 PMIX_CLASS_DECLARATION(pmix_pset_t);
0173
0174 typedef struct {
0175     pmix_list_t nspaces;          // list of pmix_nspace_t for the nspaces we know about
0176     pmix_pointer_array_t clients; // array of pmix_peer_t local clients
0177     pmix_list_t collectives;      // list of active pmix_server_trkr_t
0178     pmix_list_t remote_pnd; // list of pmix_dmdx_remote_t awaiting arrival of data fror servicing
0179                             // remote req's
0180     pmix_list_t local_reqs;     // list of pmix_dmdx_local_t awaiting arrival of data from local neighbours
0181     pmix_list_t gdata;  // cache of data given to me for passing to all clients
0182     char **genvars;     // argv array of envars given to me for passing to all clients
0183     pmix_list_t events; // list of pmix_regevents_info_t registered events
0184     char **failedgrps;    // group IDs that failed to construct
0185     pmix_list_t iof;    // IO to be forwarded to clients
0186     pmix_list_t iof_residuals;  // leftover bytes waiting for newline
0187     pmix_list_t psets;  // list of known psets and memberships
0188     size_t max_iof_cache; // max number of IOF messages to cache
0189     bool tool_connections_allowed;
0190     char *tmpdir;             // temporary directory for this server
0191     char *system_tmpdir;      // system tmpdir
0192     bool fence_localonly_opt; // local-only fence optimization
0193     // verbosity for server get operations
0194     int get_output;
0195     int get_verbose;
0196     // verbosity for server connect operations
0197     int connect_output;
0198     int connect_verbose;
0199     // verbosity for server fence operations
0200     int fence_output;
0201     int fence_verbose;
0202     // verbosity for server pub operations
0203     int pub_output;
0204     int pub_verbose;
0205     // verbosity for server spawn operations
0206     int spawn_output;
0207     int spawn_verbose;
0208     // verbosity for server event operations
0209     int event_output;
0210     int event_verbose;
0211     // verbosity for server iof operations
0212     int iof_output;
0213     int iof_verbose;
0214     // verbosity for basic server functions
0215     int base_output;
0216     int base_verbose;
0217     // verbosity for server group operations
0218     int group_output;
0219     int group_verbose;
0220 } pmix_server_globals_t;
0221
0222 #define PMIX_GDS_CADDY(c, p, t)              \
0223     do {                                     \
0224         (c) = PMIX_NEW(pmix_server_caddy_t); \
0225         (c)->hdr.tag = (t);                  \
0226         PMIX_RETAIN((p));                    \
0227         (c)->peer = (p);                     \
0228     } while (0)
0229
0230 #define PMIX_SETUP_COLLECTIVE(c, t)        \
0231     do {                                   \
0232         (c) = PMIX_NEW(pmix_trkr_caddy_t); \
0233         (c)->trk = (t);                    \
0234     } while (0)
0235
0236 #define PMIX_EXECUTE_COLLECTIVE(c, t, f)                                            \
0237     do {                                                                            \
0238         PMIX_SETUP_COLLECTIVE(c, t);                                                \
0239         pmix_event_assign(&((c)->ev), pmix_globals.evbase, -1, EV_WRITE, (f), (c)); \
0240         pmix_event_active(&((c)->ev), EV_WRITE, 1);                                 \
0241     } while (0)
0242
0243 PMIX_EXPORT bool pmix_server_trk_update(pmix_server_trkr_t *trk);
0244
0245 PMIX_EXPORT void pmix_pending_nspace_requests(pmix_namespace_t *nptr);
0246 PMIX_EXPORT pmix_status_t pmix_pending_resolve(pmix_namespace_t *nptr, pmix_rank_t rank,
0247                                                pmix_status_t status, pmix_scope_t scope,
0248                                                pmix_dmdx_local_t *lcd);
0249
0250 PMIX_EXPORT pmix_status_t pmix_server_abort(pmix_peer_t *peer, pmix_buffer_t *buf,
0251                                             pmix_op_cbfunc_t cbfunc, void *cbdata);
0252
0253 PMIX_EXPORT pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf);
0254
0255 PMIX_EXPORT pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, pmix_buffer_t *buf,
0256                                             pmix_modex_cbfunc_t modexcbfunc,
0257                                             pmix_op_cbfunc_t opcbfunc);
0258
0259 PMIX_EXPORT pmix_status_t pmix_server_get(pmix_buffer_t *buf, pmix_modex_cbfunc_t cbfunc,
0260                                           void *cbdata);
0261
0262 PMIX_EXPORT pmix_status_t pmix_server_publish(pmix_peer_t *peer, pmix_buffer_t *buf,
0263                                               pmix_op_cbfunc_t cbfunc, void *cbdata);
0264
0265 PMIX_EXPORT pmix_status_t pmix_server_lookup(pmix_peer_t *peer, pmix_buffer_t *buf,
0266                                              pmix_lookup_cbfunc_t cbfunc, void *cbdata);
0267
0268 PMIX_EXPORT pmix_status_t pmix_server_unpublish(pmix_peer_t *peer, pmix_buffer_t *buf,
0269                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
0270
0271 PMIX_EXPORT pmix_status_t pmix_server_spawn(pmix_peer_t *peer, pmix_buffer_t *buf,
0272                                             pmix_spawn_cbfunc_t cbfunc, void *cbdata);
0273 PMIX_EXPORT void pmix_server_spawn_parser(pmix_peer_t *peer,
0274                                           pmix_iof_channel_t *channels,
0275                                           pmix_iof_flags_t *flags,
0276                                           pmix_info_t *info,
0277                                           size_t ninfo);
0278 PMIX_EXPORT pmix_status_t pmix_server_process_iof(pmix_setup_caddy_t *cd,
0279                                                   char nspace[]);
0280
0281 PMIX_EXPORT void pmix_server_spcbfunc(pmix_status_t status, char nspace[], void *cbdata);
0282
0283 PMIX_EXPORT pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, pmix_buffer_t *buf,
0284                                               pmix_op_cbfunc_t cbfunc);
0285
0286 PMIX_EXPORT pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, pmix_buffer_t *buf,
0287                                                  pmix_op_cbfunc_t cbfunc);
0288
0289 PMIX_EXPORT pmix_status_t pmix_server_notify_error(pmix_status_t status, pmix_proc_t procs[],
0290                                                    size_t nprocs, pmix_proc_t error_procs[],
0291                                                    size_t error_nprocs, pmix_info_t info[],
0292                                                    size_t ninfo, pmix_op_cbfunc_t cbfunc,
0293                                                    void *cbdata);
0294
0295 PMIX_EXPORT pmix_status_t pmix_server_register_events(pmix_peer_t *peer, pmix_buffer_t *buf,
0296                                                       pmix_op_cbfunc_t cbfunc, void *cbdata);
0297
0298 PMIX_EXPORT void pmix_server_deregister_events(pmix_peer_t *peer, pmix_buffer_t *buf);
0299
0300 PMIX_EXPORT pmix_status_t pmix_server_query(pmix_peer_t *peer, pmix_buffer_t *buf,
0301                                             pmix_info_cbfunc_t cbfunc, void *cbdata);
0302
0303 PMIX_EXPORT pmix_status_t pmix_server_log(pmix_peer_t *peer, pmix_buffer_t *buf,
0304                                           pmix_op_cbfunc_t cbfunc, void *cbdata);
0305
0306 PMIX_EXPORT pmix_status_t pmix_server_alloc(pmix_peer_t *peer, pmix_buffer_t *buf,
0307                                             pmix_info_cbfunc_t cbfunc, void *cbdata);
0308
0309 PMIX_EXPORT pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, pmix_buffer_t *buf,
0310                                                pmix_info_cbfunc_t cbfunc, void *cbdata);
0311
0312 PMIX_EXPORT pmix_status_t pmix_server_monitor(pmix_peer_t *peer, pmix_buffer_t *buf,
0313                                               pmix_info_cbfunc_t cbfunc, void *cbdata);
0314
0315 PMIX_EXPORT pmix_status_t pmix_server_get_credential(pmix_peer_t *peer, pmix_buffer_t *buf,
0316                                                      pmix_credential_cbfunc_t cbfunc, void *cbdata);
0317
0318 PMIX_EXPORT pmix_status_t pmix_server_validate_credential(pmix_peer_t *peer, pmix_buffer_t *buf,
0319                                                           pmix_validation_cbfunc_t cbfunc,
0320                                                           void *cbdata);
0321
0322 PMIX_EXPORT pmix_status_t pmix_server_iofreg(pmix_peer_t *peer, pmix_buffer_t *buf,
0323                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
0324
0325 PMIX_EXPORT pmix_status_t pmix_server_iofstdin(pmix_peer_t *peer, pmix_buffer_t *buf,
0326                                                pmix_op_cbfunc_t cbfunc, void *cbdata);
0327
0328 PMIX_EXPORT pmix_status_t pmix_server_iofdereg(pmix_peer_t *peer, pmix_buffer_t *buf,
0329                                                pmix_op_cbfunc_t cbfunc, void *cbdata);
0330
0331 PMIX_EXPORT pmix_status_t pmix_server_grpconstruct(pmix_server_caddy_t *cd, pmix_buffer_t *buf);
0332
0333 PMIX_EXPORT pmix_status_t pmix_server_grpdestruct(pmix_server_caddy_t *cd, pmix_buffer_t *buf);
0334
0335 PMIX_EXPORT pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, pmix_buffer_t *buf,
0336                                                               pmix_op_cbfunc_t cbfunc,
0337                                                               void *cbdata);
0338 PMIX_EXPORT void pmix_server_execute_collective(int sd, short args, void *cbdata);
0339
0340 PMIX_EXPORT pmix_status_t pmix_server_initialize(void);
0341
0342 PMIX_EXPORT void pmix_server_message_handler(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr,
0343                                              pmix_buffer_t *buf, void *cbdata);
0344
0345 PMIX_EXPORT void pmix_server_purge_events(pmix_peer_t *peer, pmix_proc_t *proc);
0346
0347 PMIX_EXPORT pmix_status_t pmix_server_fabric_register(pmix_server_caddy_t *cd, pmix_buffer_t *buf,
0348                                                       pmix_info_cbfunc_t cbfunc);
0349
0350 PMIX_EXPORT pmix_status_t pmix_server_fabric_update(pmix_server_caddy_t *cd, pmix_buffer_t *buf,
0351                                                     pmix_info_cbfunc_t cbfunc);
0352
0353 PMIX_EXPORT pmix_status_t pmix_server_fabric_get_vertex_info(pmix_server_caddy_t *cd,
0354                                                              pmix_buffer_t *buf,
0355                                                              pmix_info_cbfunc_t cbfunc);
0356
0357 PMIX_EXPORT pmix_status_t pmix_server_fabric_get_device_index(pmix_server_caddy_t *cd,
0358                                                               pmix_buffer_t *buf,
0359                                                               pmix_info_cbfunc_t cbfunc);
0360
0361 PMIX_EXPORT pmix_status_t pmix_server_device_dists(pmix_server_caddy_t *cd,
0362                                                    pmix_buffer_t *buf,
0363                                                    pmix_device_dist_cbfunc_t cbfunc);
0364
0365 PMIX_EXPORT pmix_status_t pmix_server_refresh_cache(pmix_server_caddy_t *cd,
0366                                                     pmix_buffer_t *buf,
0367                                                     pmix_op_cbfunc_t cbfunc);
0368
0369 PMIX_EXPORT void pmix_server_query_cbfunc(pmix_status_t status,
0370                                           pmix_info_t *info, size_t ninfo, void *cbdata,
0371                                           pmix_release_cbfunc_t release_fn, void *release_cbdata);
0372
0373 PMIX_EXPORT extern pmix_server_module_t pmix_host_server;
0374 PMIX_EXPORT extern pmix_server_globals_t pmix_server_globals;
0375
0376 static inline pmix_peer_t* pmix_get_peer_object(const pmix_proc_t *proc)
0377 {
0378     pmix_peer_t *peer;
0379     int n;
0380
0381     for (n=0; n < pmix_server_globals.clients.size; n++) {
0382         peer = (pmix_peer_t *) pmix_pointer_array_get_item(&pmix_server_globals.clients, n);
0383         if (NULL == peer) {
0384             continue;
0385         }
0386         if (PMIX_CHECK_NSPACE(proc->nspace, peer->info->pname.nspace) &&
0387             proc->rank == peer->info->pname.rank) {
0388             return peer;
0389         }
0390     }
0391     return NULL;
0392 }
0393
0394
0395 #endif // PMIX_SERVER_OPS_H