include/hwloc/cuda.h

0001 /*
0002  * SPDX-License-Identifier: BSD-3-Clause
0003  * Copyright © 2010-2023 Inria.  All rights reserved.
0004  * Copyright © 2010-2011 Université Bordeaux
0005  * Copyright © 2011 Cisco Systems, Inc.  All rights reserved.
0006  * See COPYING in top-level directory.
0007  */
0008
0009 /** \file
0010  * \brief Macros to help interaction between hwloc and the CUDA Driver API.
0011  *
0012  * Applications that use both hwloc and the CUDA Driver API may want to
0013  * include this file so as to get topology information for CUDA devices.
0014  *
0015  */
0016
0017 #ifndef HWLOC_CUDA_H
0018 #define HWLOC_CUDA_H
0019
0020 #include "hwloc.h"
0021 #include "hwloc/autogen/config.h"
0022 #include "hwloc/helper.h"
0023 #ifdef HWLOC_LINUX_SYS
0024 #include "hwloc/linux.h"
0025 #endif
0026
0027 #include <cuda.h>
0028
0029
0030 #ifdef __cplusplus
0031 extern "C" {
0032 #endif
0033
0034
0035 /** \defgroup hwlocality_cuda Interoperability with the CUDA Driver API
0036  *
0037  * This interface offers ways to retrieve topology information about
0038  * CUDA devices when using the CUDA Driver API.
0039  *
0040  * @{
0041  */
0042
0043 /** \brief Return the domain, bus and device IDs of the CUDA device \p cudevice.
0044  *
0045  * Device \p cudevice must match the local machine.
0046  *
0047  * \return 0 on success.
0048  * \return -1 on error, for instance if device information could not be found.
0049  */
0050 static __hwloc_inline int
0051 hwloc_cuda_get_device_pci_ids(hwloc_topology_t topology __hwloc_attribute_unused,
0052                   CUdevice cudevice, int *domain, int *bus, int *dev)
0053 {
0054   CUresult cres;
0055
0056 #if CUDA_VERSION >= 4000
0057   cres = cuDeviceGetAttribute(domain, CU_DEVICE_ATTRIBUTE_PCI_DOMAIN_ID, cudevice);
0058   if (cres != CUDA_SUCCESS) {
0059     errno = ENOSYS;
0060     return -1;
0061   }
0062 #else
0063   *domain = 0;
0064 #endif
0065   cres = cuDeviceGetAttribute(bus, CU_DEVICE_ATTRIBUTE_PCI_BUS_ID, cudevice);
0066   if (cres != CUDA_SUCCESS) {
0067     errno = ENOSYS;
0068     return -1;
0069   }
0070   cres = cuDeviceGetAttribute(dev, CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID, cudevice);
0071   if (cres != CUDA_SUCCESS) {
0072     errno = ENOSYS;
0073     return -1;
0074   }
0075
0076   return 0;
0077 }
0078
0079 /** \brief Get the CPU set of processors that are physically
0080  * close to device \p cudevice.
0081  *
0082  * Store in \p set the CPU-set describing the locality of the CUDA device \p cudevice.
0083  *
0084  * Topology \p topology and device \p cudevice must match the local machine.
0085  * I/O devices detection and the CUDA component are not needed in the topology.
0086  *
0087  * The function only returns the locality of the device.
0088  * If more information about the device is needed, OS objects should
0089  * be used instead, see hwloc_cuda_get_device_osdev()
0090  * and hwloc_cuda_get_device_osdev_by_index().
0091  *
0092  * This function is currently only implemented in a meaningful way for
0093  * Linux; other systems will simply get a full cpuset.
0094  *
0095  * \return 0 on success.
0096  * \return -1 on error, for instance if device information could not be found.
0097  */
0098 static __hwloc_inline int
0099 hwloc_cuda_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused,
0100                  CUdevice cudevice, hwloc_cpuset_t set)
0101 {
0102 #ifdef HWLOC_LINUX_SYS
0103   /* If we're on Linux, use the sysfs mechanism to get the local cpus */
0104 #define HWLOC_CUDA_DEVICE_SYSFS_PATH_MAX 128
0105   char path[HWLOC_CUDA_DEVICE_SYSFS_PATH_MAX];
0106   int domainid, busid, deviceid;
0107
0108   if (hwloc_cuda_get_device_pci_ids(topology, cudevice, &domainid, &busid, &deviceid))
0109     return -1;
0110
0111   if (!hwloc_topology_is_thissystem(topology)) {
0112     errno = EINVAL;
0113     return -1;
0114   }
0115
0116   sprintf(path, "/sys/bus/pci/devices/%04x:%02x:%02x.0/local_cpus", domainid, busid, deviceid);
0117   if (hwloc_linux_read_path_as_cpumask(path, set) < 0
0118       || hwloc_bitmap_iszero(set))
0119     hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
0120 #else
0121   /* Non-Linux systems simply get a full cpuset */
0122   hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
0123 #endif
0124   return 0;
0125 }
0126
0127 /** \brief Get the hwloc PCI device object corresponding to the
0128  * CUDA device \p cudevice.
0129  *
0130  * \return The hwloc PCI device object describing the CUDA device \p cudevice.
0131  * \return \c NULL if none could be found.
0132  *
0133  * Topology \p topology and device \p cudevice must match the local machine.
0134  * I/O devices detection must be enabled in topology \p topology.
0135  * The CUDA component is not needed in the topology.
0136  */
0137 static __hwloc_inline hwloc_obj_t
0138 hwloc_cuda_get_device_pcidev(hwloc_topology_t topology, CUdevice cudevice)
0139 {
0140   int domain, bus, dev;
0141
0142   if (hwloc_cuda_get_device_pci_ids(topology, cudevice, &domain, &bus, &dev))
0143     return NULL;
0144
0145   return hwloc_get_pcidev_by_busid(topology, domain, bus, dev, 0);
0146 }
0147
0148 /** \brief Get the hwloc OS device object corresponding to CUDA device \p cudevice.
0149  *
0150  * \return The hwloc OS device object that describes the given CUDA device \p cudevice.
0151  * \return \c NULL if none could be found.
0152  *
0153  * Topology \p topology and device \p cudevice must match the local machine.
0154  * I/O devices detection and the CUDA component must be enabled in the topology.
0155  * If not, the locality of the object may still be found using
0156  * hwloc_cuda_get_device_cpuset().
0157  *
0158  * \note This function cannot work if PCI devices are filtered out.
0159  *
0160  * \note The corresponding hwloc PCI device may be found by looking
0161  * at the result parent pointer (unless PCI devices are filtered out).
0162  */
0163 static __hwloc_inline hwloc_obj_t
0164 hwloc_cuda_get_device_osdev(hwloc_topology_t topology, CUdevice cudevice)
0165 {
0166     hwloc_obj_t osdev = NULL;
0167     int domain, bus, dev;
0168
0169     if (hwloc_cuda_get_device_pci_ids(topology, cudevice, &domain, &bus, &dev))
0170         return NULL;
0171
0172     osdev = NULL;
0173     while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
0174         hwloc_obj_t pcidev = osdev->parent;
0175         if (strncmp(osdev->name, "cuda", 4))
0176             continue;
0177         if (pcidev
0178             && pcidev->type == HWLOC_OBJ_PCI_DEVICE
0179             && (int) pcidev->attr->pcidev.domain == domain
0180             && (int) pcidev->attr->pcidev.bus == bus
0181             && (int) pcidev->attr->pcidev.dev == dev
0182             && pcidev->attr->pcidev.func == 0)
0183             return osdev;
0184         /* if PCI are filtered out, we need a info attr to match on */
0185     }
0186
0187     return NULL;
0188 }
0189
0190 /** \brief Get the hwloc OS device object corresponding to the
0191  * CUDA device whose index is \p idx.
0192  *
0193  * \return The hwloc OS device object describing the CUDA device whose index is \p idx.
0194  * \return \c NULL if none could be found.
0195  *
0196  * The topology \p topology does not necessarily have to match the current
0197  * machine. For instance the topology may be an XML import of a remote host.
0198  * I/O devices detection and the CUDA component must be enabled in the topology.
0199  *
0200  * \note The corresponding PCI device object can be obtained by looking
0201  * at the OS device parent object (unless PCI devices are filtered out).
0202  *
0203  * \note This function is identical to hwloc_cudart_get_device_osdev_by_index().
0204  */
0205 static __hwloc_inline hwloc_obj_t
0206 hwloc_cuda_get_device_osdev_by_index(hwloc_topology_t topology, unsigned idx)
0207 {
0208     hwloc_obj_t osdev = NULL;
0209     while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
0210         if (HWLOC_OBJ_OSDEV_COPROC == osdev->attr->osdev.type
0211             && osdev->name
0212             && !strncmp("cuda", osdev->name, 4)
0213             && atoi(osdev->name + 4) == (int) idx)
0214             return osdev;
0215     }
0216     return NULL;
0217 }
0218
0219 /** @} */
0220
0221
0222 #ifdef __cplusplus
0223 } /* extern "C" */
0224 #endif
0225
0226
0227 #endif /* HWLOC_CUDA_H */