JANA/Utils/JProcessorMapping.cc

0001
0002 // Copyright 2020, Jefferson Science Associates, LLC.
0003 // Subject to the terms in the LICENSE file found in the top-level directory.
0004
0005 #include "JProcessorMapping.h"
0006
0007 #include <JANA/Utils/JTablePrinter.h>
0008 #include <iomanip>
0009 #include <unistd.h>
0010 #include <sys/wait.h>
0011 #include <algorithm>
0012
0013 void JProcessorMapping::initialize(AffinityStrategy affinity, LocalityStrategy locality) {
0014
0015     m_affinity_strategy = affinity;
0016     m_locality_strategy = locality;
0017
0018     if (affinity == AffinityStrategy::None && locality == LocalityStrategy::Global) {
0019         // User doesn't care about NUMA awareness, so we can skip building the processor map completely
0020         m_error_msg = ""; // Denotes "no error" as used by stringifier
0021         return;
0022     }
0023
0024     // Capture lscpu info
0025
0026     int pipe_fd[2]; // We want to pipe lscpu's stdout straight to us
0027     if (pipe(pipe_fd) == -1) {
0028         m_error_msg = "Unable to open pipe";
0029         return;
0030     }
0031     pid_t pid = fork();
0032     if (pid == -1) { // Failure
0033         m_error_msg = "Unable to fork";
0034         close(pipe_fd[0]);
0035         close(pipe_fd[1]);
0036         return;
0037     }
0038     else if (pid == 0) { // We are the child process
0039         dup2(pipe_fd[1], 1);  // Redirect stdout to pipe
0040         close(pipe_fd[0]);
0041         close(pipe_fd[1]);
0042         execlp("lscpu", "lscpu", "-b", "-pcpu,core,node,socket", (char*) nullptr);
0043         // Unreachable
0044         exit(-1);
0045     }
0046     else { // We are the parent process
0047         close(pipe_fd[1]);
0048     }
0049
0050     // In case initialize() is called multiple times, we don't want old data to interfere
0051     m_loc_count = 1;
0052     m_mapping.clear();
0053
0054     // Parse into table
0055     FILE* infile = fdopen(pipe_fd[0], "r");
0056
0057     const size_t buffer_size = 300;
0058     char buffer[buffer_size];
0059
0060     while (fgets(buffer, buffer_size, infile) != nullptr) {
0061         if (buffer[0] == '#') continue;
0062         Row row;
0063         int count = sscanf(buffer, "%zu,%zu,%zu,%zu", &row.cpu_id, &row.core_id, &row.numa_domain_id, &row.socket_id);
0064         if (count == 4) {
0065             switch (m_locality_strategy) {
0066                 case LocalityStrategy::CpuLocal:        row.location_id = row.cpu_id; break;
0067                 case LocalityStrategy::CoreLocal:       row.location_id = row.core_id; break;
0068                 case LocalityStrategy::NumaDomainLocal: row.location_id = row.numa_domain_id; break;
0069                 case LocalityStrategy::SocketLocal:     row.location_id = row.socket_id; break;
0070                 case LocalityStrategy::Global:
0071                 default:                                row.location_id = 0; break;
0072             }
0073             if (row.location_id >= m_loc_count) {
0074                 // Assume all of these ids are zero-indexed and contiguous
0075                 m_loc_count = row.location_id + 1;
0076             }
0077             m_mapping.push_back(row);
0078         }
0079         else {
0080             // On machines with no NUMA domains, lscpu returns "" instead of "0"
0081             int count = sscanf(buffer, "%zu,%zu,,%zu", &row.cpu_id, &row.core_id, &row.socket_id);
0082             row.numa_domain_id = row.socket_id;
0083             if (count == 3) {
0084                 switch (m_locality_strategy) {
0085                     case LocalityStrategy::CpuLocal:        row.location_id = row.cpu_id; break;
0086                     case LocalityStrategy::CoreLocal:       row.location_id = row.core_id; break;
0087                     case LocalityStrategy::NumaDomainLocal: row.location_id = row.numa_domain_id; break;
0088                     case LocalityStrategy::SocketLocal:     row.location_id = row.socket_id; break;
0089                     case LocalityStrategy::Global:
0090                     default:                                row.location_id = 0; break;
0091                 }
0092                 if (row.location_id >= m_loc_count) {
0093                     // Assume all of these ids are zero-indexed and contiguous
0094                     m_loc_count = row.location_id + 1;
0095                 }
0096                 m_mapping.push_back(row);
0097             }
0098
0099         }
0100     }
0101     fclose(infile);
0102     int status = 0;
0103     waitpid(pid, &status, 0);  // Wait for child to exit and acknowledge. This prevents child from becoming a zombie.
0104
0105     if (!WIFEXITED(status)) {
0106         m_error_msg = "lscpu child process returned abnormally";
0107         return;
0108     }
0109     else if (WEXITSTATUS(status) != 0) {
0110         m_error_msg = "lscpu child process returned with an exit code of " + std::to_string(WEXITSTATUS(status));
0111         return;
0112     }
0113     else if (m_mapping.empty()){
0114         m_error_msg = "Unable to parse lscpu output";
0115         return;
0116     }
0117
0118     // Apply affinity strategy by sorting over sets of columns
0119     switch (m_affinity_strategy) {
0120
0121         case AffinityStrategy::ComputeBound:
0122
0123             std::stable_sort(m_mapping.begin(), m_mapping.end(),
0124                              [](const Row& lhs, const Row& rhs) -> bool { return lhs.cpu_id < rhs.cpu_id; });
0125             break;
0126
0127         case AffinityStrategy::MemoryBound:
0128
0129             std::stable_sort(m_mapping.begin(), m_mapping.end(),
0130                              [](const Row& lhs, const Row& rhs) -> bool { return lhs.cpu_id < rhs.cpu_id; });
0131
0132             std::stable_sort(m_mapping.begin(), m_mapping.end(),
0133                              [](const Row& lhs, const Row& rhs) -> bool { return lhs.numa_domain_id < rhs.numa_domain_id; });
0134             break;
0135
0136         default:
0137             break;
0138     }
0139
0140     // Apparently we were successful
0141     m_initialized = true;
0142 }
0143
0144 std::ostream& operator<<(std::ostream& os, const JProcessorMapping::AffinityStrategy& s) {
0145     switch (s) {
0146         case JProcessorMapping::AffinityStrategy::ComputeBound: os << "compute-bound (favor fewer hyperthreads)"; break;
0147         case JProcessorMapping::AffinityStrategy::MemoryBound: os << "memory-bound (favor fewer NUMA domains)"; break;
0148         case JProcessorMapping::AffinityStrategy::None: os << "none"; break;
0149     }
0150     return os;
0151 }
0152
0153
0154 std::ostream& operator<<(std::ostream& os, const JProcessorMapping::LocalityStrategy& s) {
0155     switch (s) {
0156         case JProcessorMapping::LocalityStrategy::CpuLocal: os << "cpu-local"; break;
0157         case JProcessorMapping::LocalityStrategy::CoreLocal: os << "core-local"; break;
0158         case JProcessorMapping::LocalityStrategy::NumaDomainLocal: os << "numa-domain-local"; break;
0159         case JProcessorMapping::LocalityStrategy::SocketLocal: os << "socket-local"; break;
0160         case JProcessorMapping::LocalityStrategy::Global: os << "global"; break;
0161     }
0162     return os;
0163 }
0164
0165
0166 std::ostream& operator<<(std::ostream& os, const JProcessorMapping& m) {
0167
0168     os << "NUMA Configuration: " << "affinity=" << m.m_affinity_strategy << ", locality=" << m.m_locality_strategy;
0169     if (m.m_locality_strategy != JProcessorMapping::LocalityStrategy::Global) {
0170         os << " (" << m.m_loc_count << " locations)";
0171     }
0172
0173     if (m.m_initialized) {
0174         os << std::endl;
0175         JTablePrinter table;
0176         table.AddColumn("worker", JTablePrinter::Justify::Right);
0177         table.AddColumn("location", JTablePrinter::Justify::Right);
0178         table.AddColumn("cpu", JTablePrinter::Justify::Right);
0179         table.AddColumn("core", JTablePrinter::Justify::Right);
0180         table.AddColumn("numa node", JTablePrinter::Justify::Right);
0181         table.AddColumn("socket", JTablePrinter::Justify::Right);
0182
0183         size_t worker_id = 0;
0184         for (const JProcessorMapping::Row& row : m.m_mapping) {
0185             table | worker_id++ | row.location_id | row.cpu_id | row.core_id | row.numa_domain_id | row.socket_id;
0186         }
0187         table.Render(os);
0188     }
0189     else if (!m.m_error_msg.empty()) {
0190         os << std::endl << "  Error: " << m.m_error_msg << std::endl;
0191     }
0192     return os;
0193 }