cuda/apps/truth_finding_example_cuda.cpp

0001 /** TRACCC library, part of the ACTS project (R&D line)
0002  *
0003  * (c) 2023-2026 CERN for the benefit of the ACTS project
0004  *
0005  * Mozilla Public License Version 2.0
0006  */
0007
0008 // Project include(s).
0009 #include "traccc/cuda/finding/combinatorial_kalman_filter_algorithm.hpp"
0010 #include "traccc/cuda/fitting/kalman_fitting_algorithm.hpp"
0011 #include "traccc/cuda/utils/make_magnetic_field.hpp"
0012 #include "traccc/cuda/utils/stream_wrapper.hpp"
0013 #include "traccc/definitions/common.hpp"
0014 #include "traccc/definitions/primitives.hpp"
0015 #include "traccc/device/container_d2h_copy_alg.hpp"
0016 #include "traccc/device/container_h2d_copy_alg.hpp"
0017 #include "traccc/efficiency/finding_performance_writer.hpp"
0018 #include "traccc/examples/make_magnetic_field.hpp"
0019 #include "traccc/finding/combinatorial_kalman_filter_algorithm.hpp"
0020 #include "traccc/fitting/kalman_fitting_algorithm.hpp"
0021 #include "traccc/geometry/detector.hpp"
0022 #include "traccc/geometry/host_detector.hpp"
0023 #include "traccc/io/read_detector.hpp"
0024 #include "traccc/io/read_detector_description.hpp"
0025 #include "traccc/io/read_measurements.hpp"
0026 #include "traccc/io/utils.hpp"
0027 #include "traccc/options/accelerator.hpp"
0028 #include "traccc/options/detector.hpp"
0029 #include "traccc/options/input_data.hpp"
0030 #include "traccc/options/magnetic_field.hpp"
0031 #include "traccc/options/performance.hpp"
0032 #include "traccc/options/program_options.hpp"
0033 #include "traccc/options/track_finding.hpp"
0034 #include "traccc/options/track_fitting.hpp"
0035 #include "traccc/options/track_matching.hpp"
0036 #include "traccc/options/track_propagation.hpp"
0037 #include "traccc/options/truth_finding.hpp"
0038 #include "traccc/performance/collection_comparator.hpp"
0039 #include "traccc/performance/container_comparator.hpp"
0040 #include "traccc/performance/soa_comparator.hpp"
0041 #include "traccc/performance/timer.hpp"
0042 #include "traccc/resolution/fitting_performance_writer.hpp"
0043 #include "traccc/utils/propagation.hpp"
0044 #include "traccc/utils/seed_generator.hpp"
0045
0046 // VecMem include(s).
0047 #include <vecmem/memory/cuda/device_memory_resource.hpp>
0048 #include <vecmem/memory/cuda/host_memory_resource.hpp>
0049 #include <vecmem/memory/cuda/managed_memory_resource.hpp>
0050 #include <vecmem/memory/host_memory_resource.hpp>
0051 #include <vecmem/utils/cuda/async_copy.hpp>
0052 #include <vecmem/utils/cuda/stream_wrapper.hpp>
0053
0054 // System include(s).
0055 #include <exception>
0056 #include <iomanip>
0057 #include <iostream>
0058
0059 using namespace traccc;
0060
0061 int seq_run(const traccc::opts::track_finding& finding_opts,
0062             const traccc::opts::track_propagation& propagation_opts,
0063             const traccc::opts::track_fitting& fitting_opts,
0064             const traccc::opts::input_data& input_opts,
0065             const traccc::opts::detector& detector_opts,
0066             const traccc::opts::magnetic_field& bfield_opts,
0067             const traccc::opts::performance& performance_opts,
0068             const traccc::opts::accelerator& accelerator_opts,
0069             const traccc::opts::truth_finding& truth_finding_opts,
0070             const traccc::opts::track_matching& track_matching_opts,
0071             std::unique_ptr<const traccc::Logger> ilogger) {
0072   TRACCC_LOCAL_LOGGER(std::move(ilogger));
0073
0074   // Memory resources used by the application.
0075   vecmem::host_memory_resource host_mr;
0076   vecmem::cuda::host_memory_resource cuda_host_mr;
0077   vecmem::cuda::managed_memory_resource mng_mr;
0078   vecmem::cuda::device_memory_resource device_mr;
0079   traccc::memory_resource mr{device_mr, &cuda_host_mr};
0080
0081   // Performance writer
0082   traccc::finding_performance_writer find_performance_writer(
0083       traccc::finding_performance_writer::config{
0084           .truth_config = truth_finding_opts,
0085           .track_truth_config = track_matching_opts},
0086       logger().clone("FindingPerformanceWriter"));
0087   traccc::fitting_performance_writer fit_performance_writer(
0088       traccc::fitting_performance_writer::config{},
0089       logger().clone("FittingPerformanceWriter"));
0090
0091   // Output Stats
0092   uint64_t n_found_tracks = 0;
0093   uint64_t n_found_tracks_cuda = 0;
0094   uint64_t n_fitted_tracks = 0;
0095   uint64_t n_fitted_tracks_cuda = 0;
0096
0097   /*****************************
0098    * Build a geometry
0099    *****************************/
0100
0101   // B field value
0102   const auto host_field = traccc::details::make_magnetic_field(bfield_opts);
0103   const auto device_field = traccc::cuda::make_magnetic_field(host_field);
0104
0105   // Construct a Detray detector object, if supported by the configuration.
0106   traccc::host_detector polymorphic_detector;
0107   traccc::io::read_detector(
0108       polymorphic_detector, mng_mr, detector_opts.detector_file,
0109       detector_opts.material_file, detector_opts.grid_file);
0110
0111   /*****************************
0112    * Do the reconstruction
0113    *****************************/
0114
0115   // Stream object
0116   vecmem::cuda::stream_wrapper vecmem_stream;
0117   traccc::cuda::stream_wrapper stream{vecmem_stream.stream()};
0118
0119   // Copy object
0120   vecmem::copy host_copy;
0121   vecmem::cuda::async_copy async_copy{stream.cudaStream()};
0122
0123   const traccc::detector_buffer detector_buffer =
0124       traccc::buffer_from_host_detector(polymorphic_detector, device_mr,
0125                                         async_copy);
0126
0127   // Standard deviations for seed track parameters
0128   static constexpr std::array<double, traccc::e_bound_size> stddevs = {
0129       1e-4 * traccc::unit<double>::mm,
0130       1e-4 * traccc::unit<double>::mm,
0131       1e-3,
0132       1e-3,
0133       1e-4 / traccc::unit<double>::GeV,
0134       1e-4 * traccc::unit<double>::ns};
0135
0136   // Propagation configuration
0137   detray::propagation::config propagation_config(propagation_opts);
0138
0139   // Finding algorithm configuration
0140   traccc::finding_config cfg(finding_opts);
0141   cfg.propagation = propagation_config;
0142
0143   // Finding algorithm object
0144   traccc::host::combinatorial_kalman_filter_algorithm host_finding(
0145       cfg, host_mr, logger().clone("HostFindingAlg"));
0146   traccc::cuda::combinatorial_kalman_filter_algorithm device_finding(
0147       cfg, mr, async_copy, stream, logger().clone("CudaFindingAlg"));
0148
0149   // Fitting algorithm object
0150   traccc::fitting_config fit_cfg(fitting_opts);
0151   fit_cfg.propagation = propagation_config;
0152
0153   traccc::host::kalman_fitting_algorithm host_fitting(
0154       fit_cfg, host_mr, host_copy, logger().clone("HostFittingAlg"));
0155   traccc::cuda::kalman_fitting_algorithm device_fitting(
0156       fit_cfg, mr, async_copy, stream, logger().clone("CudaFittingAlg"));
0157
0158   traccc::performance::timing_info elapsedTimes;
0159
0160   // Iterate over events
0161   for (std::size_t event = input_opts.skip;
0162        event < input_opts.events + input_opts.skip; ++event) {
0163     // Truth Track Candidates
0164     traccc::event_data evt_data(
0165         input_opts.directory, event, host_mr, input_opts.use_acts_geom_source,
0166         &polymorphic_detector, input_opts.format, false);
0167
0168     traccc::edm::measurement_collection::host truth_measurements{host_mr};
0169     traccc::edm::track_container<traccc::default_algebra>::host
0170         truth_track_candidates{host_mr};
0171
0172     host_detector_visitor<detector_type_list>(
0173         polymorphic_detector, [&]<typename detector_traits_t>(
0174                                   const typename detector_traits_t::host& det) {
0175           typename traccc::seed_generator<
0176               typename detector_traits_t::host>::config seed_cfg{};
0177           seed_cfg.initial_sigmas = stddevs;
0178           // Seed generator
0179           traccc::seed_generator<typename detector_traits_t::host> sg(det,
0180                                                                       seed_cfg);
0181           evt_data.generate_truth_candidates(truth_track_candidates,
0182                                              truth_measurements, sg, host_mr,
0183                                              truth_finding_opts.m_pT_min);
0184         });
0185     truth_track_candidates.measurements = vecmem::get_data(truth_measurements);
0186
0187     // Prepare truth seeds
0188     traccc::bound_track_parameters_collection_types::host seeds(mr.host);
0189     const std::size_t n_tracks = truth_track_candidates.tracks.size();
0190     for (std::size_t i_trk = 0; i_trk < n_tracks; i_trk++) {
0191       seeds.push_back(truth_track_candidates.tracks.at(i_trk).params());
0192     }
0193
0194     std::cout << "Number of seeds: " << seeds.size() << std::endl;
0195
0196     traccc::bound_track_parameters_collection_types::buffer seeds_buffer{
0197         static_cast<unsigned int>(seeds.size()), mr.main};
0198     async_copy.setup(seeds_buffer)->wait();
0199     async_copy(vecmem::get_data(seeds), seeds_buffer,
0200                vecmem::copy::type::host_to_device)
0201         ->wait();
0202
0203     // Read measurements
0204     traccc::edm::measurement_collection::host measurements_per_event{host_mr};
0205     traccc::io::read_measurements(
0206         measurements_per_event, event, input_opts.directory,
0207         (input_opts.use_acts_geom_source ? &polymorphic_detector : nullptr),
0208         nullptr, nullptr, input_opts.format);
0209
0210     traccc::edm::measurement_collection::buffer measurements_cuda_buffer(
0211         static_cast<unsigned int>(measurements_per_event.size()), mr.main);
0212     async_copy.setup(measurements_cuda_buffer)->wait();
0213     async_copy(vecmem::get_data(measurements_per_event),
0214                measurements_cuda_buffer)
0215         ->wait();
0216
0217     // Instantiate output cuda containers/collections
0218     traccc::cuda::combinatorial_kalman_filter_algorithm::output_type
0219         track_candidates_cuda_buffer;
0220
0221     {
0222       traccc::performance::timer t("Track finding  (cuda)", elapsedTimes);
0223
0224       // Run finding
0225       track_candidates_cuda_buffer =
0226           device_finding(detector_buffer, device_field,
0227                          measurements_cuda_buffer, seeds_buffer);
0228     }
0229
0230     traccc::edm::track_container<traccc::default_algebra>::host
0231         track_candidates_cuda{host_mr,
0232                               vecmem::get_data(measurements_per_event)};
0233     async_copy(track_candidates_cuda_buffer.tracks,
0234                track_candidates_cuda.tracks, vecmem::copy::type::device_to_host)
0235         ->wait();
0236
0237     // Instantiate cuda containers/collections
0238     traccc::edm::track_container<traccc::default_algebra>::buffer
0239         track_states_cuda_buffer;
0240
0241     {
0242       traccc::performance::timer t("Track fitting  (cuda)", elapsedTimes);
0243
0244       // Run fitting
0245       track_states_cuda_buffer = device_fitting(detector_buffer, device_field,
0246                                                 track_candidates_cuda_buffer);
0247     }
0248     traccc::edm::track_container<traccc::default_algebra>::host
0249         track_states_cuda{host_mr};
0250     async_copy(track_states_cuda_buffer.tracks, track_states_cuda.tracks,
0251                vecmem::copy::type::device_to_host)
0252         ->wait();
0253     async_copy(track_states_cuda_buffer.states, track_states_cuda.states,
0254                vecmem::copy::type::device_to_host)
0255         ->wait();
0256
0257     // CPU containers
0258     traccc::host::combinatorial_kalman_filter_algorithm::output_type
0259         track_candidates{host_mr};
0260     traccc::host::kalman_fitting_algorithm::output_type track_states{host_mr};
0261
0262     if (accelerator_opts.compare_with_cpu) {
0263       {
0264         traccc::performance::timer t("Track finding  (cpu)", elapsedTimes);
0265
0266         // Run finding
0267         track_candidates = host_finding(
0268             polymorphic_detector, host_field,
0269             vecmem::get_data(measurements_per_event), vecmem::get_data(seeds));
0270       }
0271
0272       {
0273         traccc::performance::timer t("Track fitting  (cpu)", elapsedTimes);
0274
0275         // Run fitting
0276         track_states = host_fitting(
0277             polymorphic_detector, host_field,
0278             traccc::edm::track_container<traccc::default_algebra>::const_data(
0279                 track_candidates));
0280       }
0281     }
0282
0283     if (accelerator_opts.compare_with_cpu) {
0284       // Show which event we are currently presenting the results for.
0285       TRACCC_INFO("===>>> Event " << event << " <<<===");
0286
0287       // Compare the track parameters made on the host and on the device.
0288       traccc::soa_comparator<
0289           traccc::edm::track_collection<traccc::default_algebra>>
0290           compare_track_candidates{
0291               "track candidates",
0292               traccc::details::comparator_factory<traccc::edm::track_collection<
0293                   traccc::default_algebra>::const_device::const_proxy_type>{
0294                   vecmem::get_data(measurements_per_event),
0295                   vecmem::get_data(measurements_per_event),
0296                   {},
0297                   {}}};
0298       compare_track_candidates(vecmem::get_data(track_candidates.tracks),
0299                                vecmem::get_data(track_candidates_cuda.tracks));
0300     }
0301
0302     /// Statistics
0303     n_found_tracks += track_candidates.tracks.size();
0304     n_fitted_tracks += track_states.tracks.size();
0305     n_found_tracks_cuda += track_candidates_cuda.tracks.size();
0306     n_fitted_tracks_cuda += track_states_cuda.tracks.size();
0307
0308     if (performance_opts.run) {
0309       find_performance_writer.write(
0310           traccc::edm::track_container<traccc::default_algebra>::const_data(
0311               track_candidates_cuda),
0312
0313           evt_data);
0314
0315       for (unsigned int i = 0; i < track_states_cuda.tracks.size(); i++) {
0316         host_detector_visitor<detector_type_list>(
0317             polymorphic_detector,
0318             [&]<typename detector_traits_t>(
0319                 const typename detector_traits_t::host& det) {
0320               fit_performance_writer.write(
0321                   track_states_cuda.tracks.at(i), track_states_cuda.states,
0322                   measurements_per_event, det, evt_data);
0323             });
0324       }
0325     }
0326   }
0327
0328   if (performance_opts.run) {
0329     find_performance_writer.finalize();
0330     fit_performance_writer.finalize();
0331   }
0332
0333   TRACCC_INFO("==> Statistics ... ");
0334   TRACCC_INFO("- created (cuda) " << n_found_tracks_cuda << " found tracks");
0335   TRACCC_INFO("- created (cuda) " << n_fitted_tracks_cuda << " fitted tracks");
0336   TRACCC_INFO("- created  (cpu) " << n_found_tracks << " found tracks");
0337   TRACCC_INFO("- created  (cpu) " << n_fitted_tracks << " fitted tracks");
0338   TRACCC_INFO("==>Elapsed times... " << elapsedTimes);
0339
0340   return 1;
0341 }
0342
0343 // The main routine
0344 //
0345 int main(int argc, char* argv[]) {
0346   std::unique_ptr<const traccc::Logger> logger = traccc::getDefaultLogger(
0347       "TracccExampleTruthFindingCuda", traccc::Logging::Level::INFO);
0348
0349   // Program options.
0350   traccc::opts::detector detector_opts;
0351   traccc::opts::magnetic_field bfield_opts;
0352   traccc::opts::input_data input_opts;
0353   traccc::opts::track_finding finding_opts;
0354   traccc::opts::track_propagation propagation_opts;
0355   traccc::opts::track_fitting fitting_opts;
0356   traccc::opts::performance performance_opts;
0357   traccc::opts::accelerator accelerator_opts;
0358   traccc::opts::truth_finding truth_finding_config;
0359   traccc::opts::track_matching track_matching_opts;
0360   traccc::opts::program_options program_opts{
0361       "Truth Track Finding Using CUDA",
0362       {detector_opts, bfield_opts, input_opts, finding_opts, propagation_opts,
0363        fitting_opts, performance_opts, accelerator_opts, truth_finding_config,
0364        track_matching_opts},
0365       argc,
0366       argv,
0367       logger->cloneWithSuffix("Options")};
0368
0369   // Run the application.
0370   return seq_run(finding_opts, propagation_opts, fitting_opts, input_opts,
0371                  detector_opts, bfield_opts, performance_opts, accelerator_opts,
0372                  truth_finding_config, track_matching_opts, logger->clone());
0373 }