include/hwloc/openfabrics-verbs.h

0001 /*
0002  * SPDX-License-Identifier: BSD-3-Clause
0003  * Copyright © 2009 CNRS
0004  * Copyright © 2009-2023 Inria.  All rights reserved.
0005  * Copyright © 2009-2010 Université Bordeaux
0006  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
0007  * See COPYING in top-level directory.
0008  */
0009
0010 /** \file
0011  * \brief Macros to help interaction between hwloc and OpenFabrics
0012  * verbs.
0013  *
0014  * Applications that use both hwloc and OpenFabrics verbs may want to
0015  * include this file so as to get topology information for OpenFabrics
0016  * hardware (InfiniBand, etc).
0017  *
0018  */
0019
0020 #ifndef HWLOC_OPENFABRICS_VERBS_H
0021 #define HWLOC_OPENFABRICS_VERBS_H
0022
0023 #include "hwloc.h"
0024 #include "hwloc/autogen/config.h"
0025 #ifdef HWLOC_LINUX_SYS
0026 #include "hwloc/linux.h"
0027 #endif
0028
0029 #include <infiniband/verbs.h>
0030
0031
0032 #ifdef __cplusplus
0033 extern "C" {
0034 #endif
0035
0036
0037 /** \defgroup hwlocality_openfabrics Interoperability with OpenFabrics
0038  *
0039  * This interface offers ways to retrieve topology information about
0040  * OpenFabrics devices (InfiniBand, Omni-Path, usNIC, etc).
0041  *
0042  * @{
0043  */
0044
0045 /** \brief Get the CPU set of processors that are physically
0046  * close to device \p ibdev.
0047  *
0048  * Store in \p set the CPU-set describing the locality of the OpenFabrics
0049  * device \p ibdev (InfiniBand, etc).
0050  *
0051  * Topology \p topology and device \p ibdev must match the local machine.
0052  * I/O devices detection is not needed in the topology.
0053  *
0054  * The function only returns the locality of the device.
0055  * If more information about the device is needed, OS objects should
0056  * be used instead, see hwloc_ibv_get_device_osdev()
0057  * and hwloc_ibv_get_device_osdev_by_name().
0058  *
0059  * This function is currently only implemented in a meaningful way for
0060  * Linux; other systems will simply get a full cpuset.
0061  *
0062  * \return 0 on success.
0063  * \return -1 on error, for instance if device information could not be found.
0064  */
0065 static __hwloc_inline int
0066 hwloc_ibv_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused,
0067                 struct ibv_device *ibdev, hwloc_cpuset_t set)
0068 {
0069 #ifdef HWLOC_LINUX_SYS
0070   /* If we're on Linux, use the verbs-provided sysfs mechanism to
0071      get the local cpus */
0072 #define HWLOC_OPENFABRICS_VERBS_SYSFS_PATH_MAX 128
0073   char path[HWLOC_OPENFABRICS_VERBS_SYSFS_PATH_MAX];
0074
0075   if (!hwloc_topology_is_thissystem(topology)) {
0076     errno = EINVAL;
0077     return -1;
0078   }
0079
0080   sprintf(path, "/sys/class/infiniband/%s/device/local_cpus",
0081       ibv_get_device_name(ibdev));
0082   if (hwloc_linux_read_path_as_cpumask(path, set) < 0
0083       || hwloc_bitmap_iszero(set))
0084     hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
0085 #else
0086   /* Non-Linux systems simply get a full cpuset */
0087   hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
0088 #endif
0089   return 0;
0090 }
0091
0092 /** \brief Get the hwloc OS device object corresponding to the OpenFabrics
0093  * device named \p ibname.
0094  *
0095  * \return The hwloc OS device object describing the OpenFabrics device
0096  * (InfiniBand, Omni-Path, usNIC, etc) whose name is \p ibname
0097  * (mlx5_0, hfi1_0, usnic_0, qib0, etc).
0098  * \return \c NULL if none could be found.
0099  *
0100  * The name \p ibname is usually obtained from ibv_get_device_name().
0101  *
0102  * The topology \p topology does not necessarily have to match the current
0103  * machine. For instance the topology may be an XML import of a remote host.
0104  * I/O devices detection must be enabled in the topology.
0105  *
0106  * \note The corresponding PCI device object can be obtained by looking
0107  * at the OS device parent object.
0108  */
0109 static __hwloc_inline hwloc_obj_t
0110 hwloc_ibv_get_device_osdev_by_name(hwloc_topology_t topology,
0111                    const char *ibname)
0112 {
0113     hwloc_obj_t osdev = NULL;
0114     while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
0115         if (HWLOC_OBJ_OSDEV_OPENFABRICS == osdev->attr->osdev.type
0116             && osdev->name && !strcmp(ibname, osdev->name))
0117             return osdev;
0118     }
0119     return NULL;
0120 }
0121
0122 /** \brief Get the hwloc OS device object corresponding to the OpenFabrics
0123  * device \p ibdev.
0124  *
0125  * \return The hwloc OS device object describing the OpenFabrics
0126  * device \p ibdev (InfiniBand, etc).
0127  * \return \c NULL if none could be found.
0128  *
0129  * Topology \p topology and device \p ibdev must match the local machine.
0130  * I/O devices detection must be enabled in the topology.
0131  * If not, the locality of the object may still be found using
0132  * hwloc_ibv_get_device_cpuset().
0133  *
0134  * \note The corresponding PCI device object can be obtained by looking
0135  * at the OS device parent object.
0136  */
0137 static __hwloc_inline hwloc_obj_t
0138 hwloc_ibv_get_device_osdev(hwloc_topology_t topology,
0139                struct ibv_device *ibdev)
0140 {
0141     if (!hwloc_topology_is_thissystem(topology)) {
0142         errno = EINVAL;
0143         return NULL;
0144     }
0145     return hwloc_ibv_get_device_osdev_by_name(topology, ibv_get_device_name(ibdev));
0146 }
0147
0148 /** @} */
0149
0150
0151 #ifdef __cplusplus
0152 } /* extern "C" */
0153 #endif
0154
0155
0156 #endif /* HWLOC_OPENFABRICS_VERBS_H */