Cuda/Seeding2/main.cpp

0001 // This file is part of the ACTS project.
0002 //
0003 // Copyright (C) 2016 CERN for the benefit of the ACTS project
0004 //
0005 // This Source Code Form is subject to the terms of the Mozilla Public
0006 // License, v. 2.0. If a copy of the MPL was not distributed with this
0007 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
0008
0009 // Local include(s).
0010 #include "CommandLineArguments.hpp"
0011 #include "ReadSeedFile.hpp"
0012 #include "TestDeviceCuts.hpp"
0013 #include "TestHostCuts.hpp"
0014 #include "TestSpacePoint.hpp"
0015
0016 // CUDA plugin include(s).
0017 #include "Acts/Plugins/Cuda/Seeding2/SeedFinder.hpp"
0018 #include "Acts/Plugins/Cuda/Utilities/Info.hpp"
0019 #include "Acts/Plugins/Cuda/Utilities/MemoryManager.hpp"
0020
0021 // Acts include(s).
0022 #include "Acts/EventData/SpacePointData.hpp"
0023 #include "Acts/Seeding/BinnedGroup.hpp"
0024 #include "Acts/Seeding/SeedFilterConfig.hpp"
0025 #include "Acts/Seeding/SeedFinder.hpp"
0026 #include "Acts/Seeding/SeedFinderConfig.hpp"
0027 #include "Acts/Utilities/GridBinFinder.hpp"
0028
0029 // System include(s).
0030 #include <cassert>
0031 #include <chrono>
0032 #include <iomanip>
0033 #include <iostream>
0034 #include <iterator>
0035 #include <memory>
0036
0037 using namespace Acts::UnitLiterals;
0038
0039 int main(int argc, char* argv[]) {
0040   // Interpret the command line arguments passed to the executable.
0041   CommandLineArguments cmdl;
0042   cmdl.interpret(argc, argv);
0043
0044   // Read in the seeds from the input text file.
0045   auto spacepoints = readSeedFile(cmdl.spFile, cmdl.filterDuplicates);
0046   std::cout << "Read " << spacepoints.size()
0047             << " spacepoints from file: " << cmdl.spFile << std::endl;
0048
0049   // Create a "view vector" on top of them. This is necessary to be able to pass
0050   // the objects to the Acts code. While the return type of readSeedFile(...) is
0051   // useful for simplified memory management...
0052   std::vector<const TestSpacePoint*> spView;
0053   spView.reserve(spacepoints.size());
0054   for (const auto& sp : spacepoints) {
0055     spView.push_back(sp.get());
0056   }
0057
0058   int numPhiNeighbors = 1;
0059
0060   std::vector<std::pair<int, int>> zBinNeighborsTop;
0061   std::vector<std::pair<int, int>> zBinNeighborsBottom;
0062
0063   // Create binned groups of these spacepoints.
0064   auto bottomBinFinder = std::make_unique<Acts::GridBinFinder<2ul>>(
0065       numPhiNeighbors, zBinNeighborsBottom);
0066   auto topBinFinder = std::make_unique<Acts::GridBinFinder<2ul>>(
0067       numPhiNeighbors, zBinNeighborsTop);
0068
0069   // Set up the seedFinder configuration.
0070   Acts::SeedFinderConfig<TestSpacePoint> sfConfig;
0071   // silicon detector max
0072   sfConfig.rMax = 160._mm;
0073   sfConfig.deltaRMin = 5._mm;
0074   sfConfig.deltaRMax = 160._mm;
0075   sfConfig.collisionRegionMin = -250._mm;
0076   sfConfig.collisionRegionMax = 250._mm;
0077   sfConfig.zMin = -2800._mm;
0078   sfConfig.zMax = 2800._mm;
0079   sfConfig.maxSeedsPerSpM = 5;
0080   // 2.7 eta
0081   sfConfig.cotThetaMax = 7.40627;
0082   sfConfig.sigmaScattering = 1.00000;
0083   sfConfig.minPt = 500._MeV;
0084   sfConfig.impactMax = 10._mm;
0085   Acts::SeedFinderOptions sfOptions;
0086   sfOptions.bFieldInZ = 2_T;
0087   sfOptions.beamPos = {-.5_mm, -.5_mm};
0088
0089   // Use a size slightly smaller than what modern GPUs are capable of. This is
0090   // because for debugging we can't use all available threads in a block, and
0091   // because early testing shows that using this sort of block size results in
0092   // better performance than using the maximal one. (It probably works better
0093   // with the kind of branching that is present in the CUDA code.)
0094   sfConfig.maxBlockSize = 256;
0095
0096   sfConfig = sfConfig.toInternalUnits().calculateDerivedQuantities();
0097
0098   // Set up the spacepoint grid configuration.
0099   Acts::CylindricalSpacePointGridConfig gridConfig;
0100   gridConfig.minPt = sfConfig.minPt;
0101   gridConfig.rMax = sfConfig.rMax;
0102   gridConfig.zMax = sfConfig.zMax;
0103   gridConfig.zMin = sfConfig.zMin;
0104   gridConfig.deltaRMax = sfConfig.deltaRMax;
0105   gridConfig.cotThetaMax = sfConfig.cotThetaMax;
0106   gridConfig = gridConfig.toInternalUnits();
0107   // Set up the spacepoint grid options
0108   Acts::CylindricalSpacePointGridOptions gridOpts;
0109   gridOpts.bFieldInZ = sfOptions.bFieldInZ;
0110
0111   // Covariance tool, sets covariances per spacepoint as required.
0112   auto ct = [=](const TestSpacePoint& sp, float, float, float)
0113       -> std::tuple<Acts::Vector3, Acts::Vector2, std::optional<float>> {
0114     Acts::Vector3 position(sp.x(), sp.y(), sp.z());
0115     Acts::Vector2 covariance(sp.m_varianceR, sp.m_varianceZ);
0116     return {position, covariance, std::nullopt};
0117   };
0118
0119   // extent used to store r range for middle spacepoint
0120   Acts::Extent rRangeSPExtent;
0121
0122   const Acts::Range1D<float> rMiddleSPRange;
0123
0124   // Create a grid with bin sizes according to the configured geometry, and
0125   // split the spacepoints into groups according to that grid.
0126   auto grid =
0127       Acts::CylindricalSpacePointGridCreator::createGrid<TestSpacePoint>(
0128           gridConfig, gridOpts);
0129   Acts::CylindricalSpacePointGridCreator::fillGrid(sfConfig, sfOptions, grid,
0130                                                    spView.begin(), spView.end(),
0131                                                    ct, rRangeSPExtent);
0132
0133   auto spGroup = Acts::CylindricalBinnedGroup<TestSpacePoint>(
0134       std::move(grid), *bottomBinFinder, *topBinFinder);
0135   // Make a convenient iterator that will be used multiple times later on.
0136   auto spGroup_end = spGroup.end();
0137
0138   // Allocate memory on the selected CUDA device.
0139   if (Acts::Cuda::Info::instance().devices().size() <=
0140       static_cast<std::size_t>(cmdl.cudaDevice)) {
0141     std::cerr << "Invalid CUDA device (" << cmdl.cudaDevice << ") requested"
0142               << std::endl;
0143     return 1;
0144   }
0145   static constexpr std::size_t MEGABYTES = 1024l * 1024l;
0146   std::size_t deviceMemoryAllocation = cmdl.cudaDeviceMemory * MEGABYTES;
0147   if (deviceMemoryAllocation == 0) {
0148     deviceMemoryAllocation =
0149         Acts::Cuda::Info::instance().devices()[cmdl.cudaDevice].totalMemory *
0150         0.8;
0151   }
0152   std::cout << "Allocating " << deviceMemoryAllocation / MEGABYTES
0153             << " MB memory on device:\n"
0154             << Acts::Cuda::Info::instance().devices()[cmdl.cudaDevice]
0155             << std::endl;
0156   Acts::Cuda::MemoryManager::instance().setMemorySize(deviceMemoryAllocation,
0157                                                       cmdl.cudaDevice);
0158
0159   // Set up the seedFinder configuration objects.
0160   TestHostCuts hostCuts;
0161   Acts::SeedFilterConfig filterConfig;
0162   filterConfig = filterConfig.toInternalUnits();
0163   sfConfig.seedFilter = std::make_unique<Acts::SeedFilter<TestSpacePoint>>(
0164       filterConfig, &hostCuts);
0165   auto deviceCuts = testDeviceCuts();
0166
0167   // Set up the seedFinder objects.
0168   Acts::SeedFinder<TestSpacePoint,
0169                    Acts::CylindricalSpacePointGrid<TestSpacePoint>>
0170       seedFinder_host(sfConfig);
0171   Acts::Cuda::SeedFinder<TestSpacePoint> seedFinder_device(
0172       sfConfig, sfOptions, filterConfig, deviceCuts, cmdl.cudaDevice);
0173
0174   //
0175   // Perform the seed finding on the host.
0176   //
0177
0178   // Record the start time.
0179   auto start_host = std::chrono::system_clock::now();
0180   // Create the result object.
0181   std::vector<std::vector<Acts::Seed<TestSpacePoint>>> seeds_host;
0182
0183   std::array<std::vector<std::size_t>, 2ul> navigation;
0184   navigation[0ul].resize(spGroup.grid().numLocalBins()[0ul]);
0185   navigation[1ul].resize(spGroup.grid().numLocalBins()[1ul]);
0186   std::iota(navigation[0ul].begin(), navigation[0ul].end(), 1ul);
0187   std::iota(navigation[1ul].begin(), navigation[1ul].end(), 1ul);
0188
0189   // Perform the seed finding.
0190   if (!cmdl.onlyGPU) {
0191     decltype(seedFinder_host)::SeedingState state;
0192     for (std::size_t i = 0; i < cmdl.groupsToIterate; ++i) {
0193       std::array<std::size_t, 2ul> localPosition =
0194           spGroup.grid().localBinsFromGlobalBin(i);
0195       auto spGroup_itr = Acts::CylindricalBinnedGroupIterator<TestSpacePoint>(
0196           spGroup, localPosition, navigation);
0197       if (spGroup_itr == spGroup.end()) {
0198         break;
0199       }
0200       auto& group = seeds_host.emplace_back();
0201       auto [bottom, middle, top] = *spGroup_itr;
0202
0203       seedFinder_host.createSeedsForGroup(sfOptions, state, spGroup.grid(),
0204                                           std::back_inserter(group), bottom,
0205                                           middle, top, rMiddleSPRange);
0206     }
0207   }
0208
0209   // Record the finish time.
0210   auto end_host = std::chrono::system_clock::now();
0211   double time_host = std::chrono::duration_cast<std::chrono::milliseconds>(
0212                          end_host - start_host)
0213                          .count() *
0214                      0.001;
0215   if (!cmdl.onlyGPU) {
0216     std::cout << "Done with the seedfinding on the host" << std::endl;
0217   }
0218
0219   //
0220   // Perform the seed finding on the accelerator.
0221   //
0222
0223   // Record the start time.
0224   auto start_device = std::chrono::system_clock::now();
0225   // Create the result object.
0226   std::vector<std::vector<Acts::Seed<TestSpacePoint>>> seeds_device;
0227   Acts::SpacePointData spacePointData;
0228   spacePointData.resize(spView.size());
0229
0230   // Perform the seed finding.
0231   for (std::size_t i = 0; i < cmdl.groupsToIterate; ++i) {
0232     std::array<std::size_t, 2ul> localPosition =
0233         spGroup.grid().localBinsFromGlobalBin(i);
0234     auto spGroup_itr = Acts::CylindricalBinnedGroupIterator<TestSpacePoint>(
0235         spGroup, localPosition, navigation);
0236     if (spGroup_itr == spGroup_end) {
0237       break;
0238     }
0239     auto [bottom, middle, top] = *spGroup_itr;
0240     seeds_device.push_back(seedFinder_device.createSeedsForGroup(
0241         spacePointData, spGroup.grid(), bottom, middle, top));
0242   }
0243
0244   // Record the finish time.
0245   auto end_device = std::chrono::system_clock::now();
0246   double time_device = std::chrono::duration_cast<std::chrono::milliseconds>(
0247                            end_device - start_device)
0248                            .count() *
0249                        0.001;
0250   std::cout << "Done with the seedfinding on the device" << std::endl;
0251
0252   //
0253   // Print some summary about the seed finding.
0254   //
0255
0256   // Count the total number of reconstructed seeds.
0257   std::size_t nSeeds_host = 0, nSeeds_device = 0;
0258   for (const auto& seeds : seeds_host) {
0259     nSeeds_host += seeds.size();
0260   }
0261   for (const auto& seeds : seeds_device) {
0262     nSeeds_device += seeds.size();
0263   }
0264
0265   // Count how many seeds, reconstructed on the host, can be matched with seeds
0266   // reconstructed on the accelerator.
0267   std::size_t nMatch = 0;
0268   double matchPercentage = 0.0;
0269   if (!cmdl.onlyGPU) {
0270     assert(seeds_host.size() == seeds_device.size());
0271     for (std::size_t i = 0; i < seeds_host.size(); i++) {
0272       // Access the seeds for this region.
0273       const auto& seeds_in_host_region = seeds_host[i];
0274       const auto& seeds_in_device_region = seeds_device[i];
0275       // Loop over all seeds found on the host.
0276       for (const auto& host_seed : seeds_in_host_region) {
0277         assert(host_seed.sp().size() == 3);
0278         // Try to find a matching seed that was found on the accelerator.
0279         for (const auto& device_seed : seeds_in_device_region) {
0280           assert(device_seed.sp().size() == 3);
0281           if ((*(host_seed.sp()[0]) == *(device_seed.sp()[0])) &&
0282               (*(host_seed.sp()[1]) == *(device_seed.sp()[1])) &&
0283               (*(host_seed.sp()[2]) == *(device_seed.sp()[2]))) {
0284             ++nMatch;
0285             break;
0286           }
0287         }
0288       }
0289     }
0290     matchPercentage = (100.0 * nMatch) / nSeeds_host;
0291   }
0292
0293   // Print the summary results.
0294   std::cout << std::endl;
0295   std::cout << "-------------------------- Results ---------------------------"
0296             << std::endl;
0297   std::cout << "|          |     Host     |    Device    | Speedup/agreement |"
0298             << std::endl;
0299   std::cout << "--------------------------------------------------------------"
0300             << std::endl;
0301   std::cout << "| Time [s] |  " << std::setw(10)
0302             << (cmdl.onlyGPU ? "N/A   " : std::to_string(time_host)) << "  |  "
0303             << std::setw(10) << time_device << "  |     " << std::setw(10)
0304             << (cmdl.onlyGPU ? "N/A    "
0305                              : std::to_string(time_host / time_device))
0306             << "    |" << std::endl;
0307   std::cout << "|   Seeds  |  " << std::setw(10)
0308             << (cmdl.onlyGPU ? "N/A   " : std::to_string(nSeeds_host))
0309             << "  |  " << std::setw(10) << nSeeds_device << "  |     "
0310             << std::setw(10)
0311             << (cmdl.onlyGPU ? "N/A    " : std::to_string(matchPercentage))
0312             << "    |" << std::endl;
0313   std::cout << "--------------------------------------------------------------"
0314             << std::endl;
0315   std::cout << std::endl;
0316
0317   // Return gracefully.
0318   return 0;
0319 }