include/hwloc/rsmi.h

0001 /*
0002  * Copyright © 2012-2023 Inria.  All rights reserved.
0003  * Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
0004  * Written by Advanced Micro Devices,
0005  * See COPYING in top-level directory.
0006  */
0007
0008 /** \file
0009  * \brief Macros to help interaction between hwloc and the ROCm SMI Management Library.
0010  *
0011  * Applications that use both hwloc and the ROCm SMI Management Library may want to
0012  * include this file so as to get topology information for AMD GPU devices.
0013  */
0014
0015 #ifndef HWLOC_RSMI_H
0016 #define HWLOC_RSMI_H
0017
0018 #include "hwloc.h"
0019 #include "hwloc/autogen/config.h"
0020 #include "hwloc/helper.h"
0021 #ifdef HWLOC_LINUX_SYS
0022 #include "hwloc/linux.h"
0023 #endif
0024
0025 #include <rocm_smi/rocm_smi.h>
0026
0027
0028 #ifdef __cplusplus
0029 extern "C" {
0030 #endif
0031
0032
0033 /** \defgroup hwlocality_rsmi Interoperability with the ROCm SMI Management Library
0034  *
0035  * This interface offers ways to retrieve topology information about
0036  * devices managed by the ROCm SMI Management Library.
0037  *
0038  * @{
0039  */
0040
0041 /** \brief Get the CPU set of logical processors that are physically
0042  * close to AMD GPU device whose index is \p dv_ind.
0043  *
0044  * Store in \p set the CPU-set describing the locality of the AMD GPU device
0045  * whose index is \p dv_ind.
0046  *
0047  * Topology \p topology and device \p dv_ind must match the local machine.
0048  * I/O devices detection and the ROCm SMI component are not needed in the
0049  * topology.
0050  *
0051  * The function only returns the locality of the device.
0052  * If more information about the device is needed, OS objects should
0053  * be used instead, see hwloc_rsmi_get_device_osdev()
0054  * and hwloc_rsmi_get_device_osdev_by_index().
0055  *
0056  * This function is currently only implemented in a meaningful way for
0057  * Linux; other systems will simply get a full cpuset.
0058  *
0059  * \return 0 on success.
0060  * \return -1 on error, for instance if device information could not be found.
0061  */
0062 static __hwloc_inline int
0063 hwloc_rsmi_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused,
0064                              uint32_t dv_ind, hwloc_cpuset_t set)
0065 {
0066 #ifdef HWLOC_LINUX_SYS
0067   /* If we're on Linux, use the sysfs mechanism to get the local cpus */
0068 #define HWLOC_RSMI_DEVICE_SYSFS_PATH_MAX 128
0069   char path[HWLOC_RSMI_DEVICE_SYSFS_PATH_MAX];
0070   rsmi_status_t ret;
0071   uint64_t bdfid = 0;
0072   unsigned domain, device, bus;
0073
0074   if (!hwloc_topology_is_thissystem(topology)) {
0075     errno = EINVAL;
0076     return -1;
0077   }
0078
0079   ret = rsmi_dev_pci_id_get(dv_ind, &bdfid);
0080   if (RSMI_STATUS_SUCCESS != ret) {
0081     errno = EINVAL;
0082     return -1;
0083   }
0084   domain = (bdfid>>32) & 0xffffffff;
0085   bus = ((bdfid & 0xffff)>>8) & 0xff;
0086   device = ((bdfid & 0xff)>>3) & 0x1f;
0087
0088   sprintf(path, "/sys/bus/pci/devices/%04x:%02x:%02x.0/local_cpus", domain, bus, device);
0089   if (hwloc_linux_read_path_as_cpumask(path, set) < 0
0090       || hwloc_bitmap_iszero(set))
0091     hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
0092 #else
0093   /* Non-Linux systems simply get a full cpuset */
0094   hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
0095 #endif
0096   return 0;
0097 }
0098
0099 /** \brief Get the hwloc OS device object corresponding to the
0100  * AMD GPU device whose index is \p dv_ind.
0101  *
0102  * \return The hwloc OS device object describing the AMD GPU device whose
0103  * index is \p dv_ind.
0104  * \return \c NULL if none could be found.
0105  *
0106  * The topology \p topology does not necessarily have to match the current
0107  * machine. For instance the topology may be an XML import of a remote host.
0108  * I/O devices detection and the ROCm SMI component must be enabled in the
0109  * topology.
0110  *
0111  * \note The corresponding PCI device object can be obtained by looking
0112  * at the OS device parent object (unless PCI devices are filtered out).
0113  */
0114 static __hwloc_inline hwloc_obj_t
0115 hwloc_rsmi_get_device_osdev_by_index(hwloc_topology_t topology, uint32_t dv_ind)
0116 {
0117   hwloc_obj_t osdev = NULL;
0118   while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
0119     if (HWLOC_OBJ_OSDEV_GPU == osdev->attr->osdev.type
0120       && osdev->name
0121       && !strncmp("rsmi", osdev->name, 4)
0122       && atoi(osdev->name + 4) == (int) dv_ind)
0123       return osdev;
0124   }
0125   return NULL;
0126 }
0127
0128 /** \brief Get the hwloc OS device object corresponding to AMD GPU device,
0129  * whose index is \p dv_ind.
0130  *
0131  * \return The hwloc OS device object that describes the given
0132  * AMD GPU, whose index is \p dv_ind.
0133  * \return \c NULL if none could be found.
0134  *
0135  * Topology \p topology and device \p dv_ind must match the local machine.
0136  * I/O devices detection and the ROCm SMI component must be enabled in the
0137  * topology. If not, the locality of the object may still be found using
0138  * hwloc_rsmi_get_device_cpuset().
0139  *
0140  * \note The corresponding hwloc PCI device may be found by looking
0141  * at the result parent pointer (unless PCI devices are filtered out).
0142  */
0143 static __hwloc_inline hwloc_obj_t
0144 hwloc_rsmi_get_device_osdev(hwloc_topology_t topology, uint32_t dv_ind)
0145 {
0146   hwloc_obj_t osdev;
0147   rsmi_status_t ret;
0148   uint64_t bdfid = 0;
0149   unsigned domain, device, bus, func;
0150   uint64_t id;
0151   char uuid[64];
0152
0153   if (!hwloc_topology_is_thissystem(topology)) {
0154     errno = EINVAL;
0155     return NULL;
0156   }
0157
0158   ret = rsmi_dev_pci_id_get(dv_ind, &bdfid);
0159   if (RSMI_STATUS_SUCCESS != ret) {
0160     errno = EINVAL;
0161     return NULL;
0162   }
0163   domain = (bdfid>>32) & 0xffffffff;
0164   bus = ((bdfid & 0xffff)>>8) & 0xff;
0165   device = ((bdfid & 0xff)>>3) & 0x1f;
0166   func = bdfid & 0x7;
0167
0168   ret = rsmi_dev_unique_id_get(dv_ind, &id);
0169   if (RSMI_STATUS_SUCCESS != ret)
0170     uuid[0] = '\0';
0171   else
0172     sprintf(uuid, "%lx", id);
0173
0174   osdev = NULL;
0175   while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
0176     hwloc_obj_t pcidev = osdev->parent;
0177     const char *info;
0178
0179     if (strncmp(osdev->name, "rsmi", 4))
0180       continue;
0181
0182     if (pcidev
0183       && pcidev->type == HWLOC_OBJ_PCI_DEVICE
0184       && pcidev->attr->pcidev.domain == domain
0185       && pcidev->attr->pcidev.bus == bus
0186       && pcidev->attr->pcidev.dev == device
0187       && pcidev->attr->pcidev.func == func)
0188       return osdev;
0189
0190     info = hwloc_obj_get_info_by_name(osdev, "AMDUUID");
0191     if (info && !strcmp(info, uuid))
0192       return osdev;
0193   }
0194
0195   return NULL;
0196 }
0197
0198 /** @} */
0199
0200
0201 #ifdef __cplusplus
0202 } /* extern "C" */
0203 #endif
0204
0205
0206 #endif /* HWLOC_RSMI_H */