sycl/apps/seq_example_sycl.cpp

0001 /* TRACCC library, part of the ACTS project (R&D line)
0002  *
0003  * (c) 2021-2026 CERN for the benefit of the ACTS project
0004  *
0005  * Mozilla Public License Version 2.0
0006  */
0007
0008 // core
0009 #include "traccc/geometry/detector.hpp"
0010
0011 // io
0012 #include "traccc/io/read_cells.hpp"
0013 #include "traccc/io/read_detector.hpp"
0014 #include "traccc/io/read_detector_description.hpp"
0015 #include "traccc/io/utils.hpp"
0016
0017 // algorithms
0018 #include "traccc/clusterization/clusterization_algorithm.hpp"
0019 #include "traccc/device/container_d2h_copy_alg.hpp"
0020 #include "traccc/finding/combinatorial_kalman_filter_algorithm.hpp"
0021 #include "traccc/gbts_seeding/gbts_seeding_config.hpp"
0022 #include "traccc/seeding/seeding_algorithm.hpp"
0023 #include "traccc/seeding/silicon_pixel_spacepoint_formation_algorithm.hpp"
0024 #include "traccc/seeding/track_params_estimation.hpp"
0025 #include "traccc/sycl/clusterization/clusterization_algorithm.hpp"
0026 #include "traccc/sycl/clusterization/measurement_sorting_algorithm.hpp"
0027 #include "traccc/sycl/finding/combinatorial_kalman_filter_algorithm.hpp"
0028 #include "traccc/sycl/gbts_seeding/gbts_seeding_algorithm.hpp"
0029 #include "traccc/sycl/seeding/seed_parameter_estimation_algorithm.hpp"
0030 #include "traccc/sycl/seeding/silicon_pixel_spacepoint_formation_algorithm.hpp"
0031 #include "traccc/sycl/seeding/triplet_seeding_algorithm.hpp"
0032 #include "traccc/sycl/utils/make_magnetic_field.hpp"
0033
0034 // performance
0035 #include "traccc/efficiency/seeding_performance_writer.hpp"
0036 #include "traccc/performance/collection_comparator.hpp"
0037 #include "traccc/performance/soa_comparator.hpp"
0038 #include "traccc/performance/timer.hpp"
0039
0040 // options
0041 #include "traccc/options/accelerator.hpp"
0042 #include "traccc/options/clusterization.hpp"
0043 #include "traccc/options/detector.hpp"
0044 #include "traccc/options/input_data.hpp"
0045 #include "traccc/options/magnetic_field.hpp"
0046 #include "traccc/options/performance.hpp"
0047 #include "traccc/options/program_options.hpp"
0048 #include "traccc/options/track_finding.hpp"
0049 #include "traccc/options/track_fitting.hpp"
0050 #include "traccc/options/track_gbts_seeding.hpp"
0051 #include "traccc/options/track_propagation.hpp"
0052 #include "traccc/options/track_seeding.hpp"
0053
0054 // examples
0055 #include "traccc/examples/make_magnetic_field.hpp"
0056
0057 // Vecmem include(s)
0058 #include <vecmem/memory/host_memory_resource.hpp>
0059 #include <vecmem/memory/sycl/device_memory_resource.hpp>
0060 #include <vecmem/memory/sycl/host_memory_resource.hpp>
0061 #include <vecmem/memory/sycl/shared_memory_resource.hpp>
0062 #include <vecmem/utils/sycl/async_copy.hpp>
0063
0064 // Project include(s).
0065 #include "traccc/utils/memory_resource.hpp"
0066
0067 // System include(s).
0068 #include <exception>
0069 #include <iomanip>
0070 #include <iostream>
0071 #include <memory>
0072
0073 int seq_run(const traccc::opts::detector& detector_opts,
0074             const traccc::opts::magnetic_field& bfield_opts,
0075             const traccc::opts::input_data& input_opts,
0076             const traccc::opts::clusterization& clusterization_opts,
0077             const traccc::opts::track_seeding& seeding_opts,
0078             const traccc::opts::track_gbts_seeding& seeding_gbts_opts,
0079             const traccc::opts::track_finding& finding_opts,
0080             const traccc::opts::track_propagation& propagation_opts,
0081             const traccc::opts::track_fitting& /*fitting_opts*/,
0082             const traccc::opts::performance& performance_opts,
0083             const traccc::opts::accelerator& accelerator_opts,
0084             std::unique_ptr<const traccc::Logger> ilogger, bool usingGBTS) {
0085   TRACCC_LOCAL_LOGGER(std::move(ilogger));
0086
0087   // Creating SYCL queue object
0088   vecmem::sycl::queue_wrapper vecmem_queue;
0089   traccc::sycl::queue_wrapper traccc_queue{vecmem_queue.queue()};
0090   TRACCC_INFO("Running on device: " << vecmem_queue.device_name());
0091
0092   // Memory resources used by the application.
0093   vecmem::host_memory_resource host_mr;
0094   vecmem::sycl::host_memory_resource sycl_host_mr{vecmem_queue};
0095   vecmem::sycl::device_memory_resource device_mr{vecmem_queue};
0096   traccc::memory_resource mr{device_mr, &sycl_host_mr};
0097
0098   // Copy object for asynchronous data transfers.
0099   vecmem::sycl::async_copy copy{vecmem_queue};
0100
0101   // Construct the detector description object.
0102   traccc::detector_design_description::host host_det_descr{host_mr};
0103   traccc::detector_conditions_description::host host_det_cond{host_mr};
0104   traccc::io::read_detector_description(
0105       host_det_descr, host_det_cond, detector_opts.detector_file,
0106       detector_opts.digitization_file, detector_opts.conditions_file,
0107       traccc::data_format::json);
0108   traccc::detector_design_description::data host_det_descr_data{
0109       vecmem::get_data(host_det_descr)};
0110   traccc::detector_conditions_description::data host_det_cond_data{
0111       vecmem::get_data(host_det_cond)};
0112   traccc::detector_design_description::buffer device_det_descr{
0113       [&]() {
0114         // number of elements in the detector design description
0115         std::vector<unsigned int> sizes(host_det_descr.size());
0116         for (std::size_t i = 0; i < host_det_descr.size(); ++i) {
0117           auto this_design = host_det_descr.at(i);
0118           // now for each element, set the size to the largest size of
0119           // that element across all modules
0120           sizes[i] = std::max(
0121               static_cast<unsigned int>(((this_design.bin_edges_x()).size())),
0122               static_cast<unsigned int>(((this_design.bin_edges_y()).size())));
0123         }
0124         return sizes;
0125       }(),
0126       device_mr, &host_mr, vecmem::data::buffer_type::resizable};
0127   copy.setup(device_det_descr)->wait();
0128   copy(host_det_descr_data, device_det_descr)->wait();
0129
0130   traccc::detector_conditions_description::buffer device_det_cond{
0131       static_cast<traccc::detector_conditions_description::buffer::size_type>(
0132           host_det_cond.size()),
0133       device_mr};
0134   copy.setup(device_det_cond)->wait();
0135   copy(host_det_cond_data, device_det_cond)->wait();
0136
0137   // Construct a Detray detector object, if supported by the configuration.
0138   traccc::host_detector host_det;
0139   traccc::io::read_detector(host_det, host_mr, detector_opts.detector_file,
0140                             detector_opts.material_file,
0141                             detector_opts.grid_file);
0142
0143   const traccc::detector_buffer detector_buffer =
0144       traccc::buffer_from_host_detector(host_det, device_mr, copy);
0145
0146   // Output stats
0147   uint64_t n_cells = 0;
0148   // uint64_t n_clusters = 0;
0149   uint64_t n_measurements = 0;
0150   uint64_t n_spacepoints = 0;
0151   uint64_t n_spacepoints_sycl = 0;
0152   uint64_t n_seeds = 0;
0153   uint64_t n_seeds_sycl = 0;
0154   uint64_t n_found_tracks = 0;
0155   uint64_t n_found_tracks_sycl = 0;
0156
0157   // Constant B field for the track finding and fitting
0158   const traccc::vector3 field_vec(seeding_opts);
0159   const auto host_field = traccc::details::make_magnetic_field(bfield_opts);
0160   const auto device_field =
0161       traccc::sycl::make_magnetic_field(host_field, traccc_queue);
0162
0163   // Algorithm configuration(s).
0164   const traccc::seedfinder_config seedfinder_config(seeding_opts);
0165   const traccc::seedfilter_config seedfilter_config(seeding_opts);
0166   const traccc::spacepoint_grid_config spacepoint_grid_config(seeding_opts);
0167
0168   // GBTS seeding configuration
0169   const traccc::gbts_seedfinder_config gbts_config(seeding_gbts_opts);
0170
0171   detray::propagation::config propagation_config(propagation_opts);
0172
0173   traccc::finding_config finding_cfg(finding_opts);
0174   finding_cfg.propagation = propagation_config;
0175
0176   // Algorithms.
0177   traccc::host::clusterization_algorithm ca(
0178       host_mr, logger().clone("HostClusteringAlg"));
0179   traccc::host::silicon_pixel_spacepoint_formation_algorithm sf(
0180       host_mr, logger().clone("HostSpFormationAlg"));
0181   traccc::host::seeding_algorithm sa(seedfinder_config, spacepoint_grid_config,
0182                                      seedfilter_config, host_mr,
0183                                      logger().clone("HostSeedingAlg"));
0184   traccc::track_params_estimation_config track_params_estimation_config;
0185   traccc::host::track_params_estimation tp(
0186       track_params_estimation_config, host_mr,
0187       logger().clone("HostTrackParEstAlg"));
0188   traccc::host::combinatorial_kalman_filter_algorithm finding_alg{
0189       finding_cfg, host_mr, logger().clone("HostFindingAlg")};
0190
0191   traccc::sycl::clusterization_algorithm ca_sycl(
0192       mr, copy, traccc_queue, clusterization_opts,
0193       logger().clone("SyclClusteringAlg"));
0194   traccc::sycl::measurement_sorting_algorithm ms_sycl(
0195       mr, copy, traccc_queue, logger().clone("SyclMeasSortingAlg"));
0196   traccc::sycl::silicon_pixel_spacepoint_formation_algorithm sf_sycl(
0197       mr, copy, traccc_queue, logger().clone("SyclSpFormationAlg"));
0198   traccc::sycl::triplet_seeding_algorithm sa_sycl(
0199       seedfinder_config, spacepoint_grid_config, seedfilter_config, mr, copy,
0200       traccc_queue, logger().clone("SyclSeedingAlg"));
0201   traccc::sycl::gbts_seeding_algorithm gbts_sa_sycl(
0202       gbts_config, mr, copy, traccc_queue,
0203       logger().clone("SyclGbtsSeedingAlg"));
0204   traccc::sycl::seed_parameter_estimation_algorithm tp_sycl(
0205       track_params_estimation_config, mr, copy, traccc_queue,
0206       logger().clone("SyclTrackParEstAlg"));
0207   traccc::sycl::combinatorial_kalman_filter_algorithm finding_alg_sycl{
0208       finding_cfg, mr, copy, traccc_queue, logger().clone("SyclFindingAlg")};
0209
0210   // performance writer
0211   traccc::seeding_performance_writer sd_performance_writer(
0212       traccc::seeding_performance_writer::config{},
0213       logger().clone("SeedingPerformanceWriter"));
0214
0215   traccc::performance::timing_info elapsedTimes;
0216
0217   // Loop over events
0218   for (std::size_t event = input_opts.skip;
0219        event < input_opts.events + input_opts.skip; ++event) {
0220     // Instantiate host containers/collections
0221     traccc::host::clusterization_algorithm::output_type measurements_per_event{
0222         host_mr};
0223     traccc::host::silicon_pixel_spacepoint_formation_algorithm::output_type
0224         spacepoints_per_event{host_mr};
0225     traccc::host::seeding_algorithm::output_type seeds{host_mr};
0226     traccc::host::track_params_estimation::output_type params{&host_mr};
0227     traccc::host::combinatorial_kalman_filter_algorithm::output_type
0228         track_candidates{host_mr};
0229
0230     // Instantiate SYCL containers/collections
0231     traccc::edm::measurement_collection::buffer measurements_sycl_buffer;
0232     traccc::sycl::silicon_pixel_spacepoint_formation_algorithm::output_type
0233         spacepoints_sycl_buffer;
0234     traccc::sycl::triplet_seeding_algorithm::output_type seeds_sycl_buffer;
0235     traccc::sycl::seed_parameter_estimation_algorithm::output_type
0236         params_sycl_buffer(0, *mr.host);
0237     traccc::sycl::combinatorial_kalman_filter_algorithm::output_type
0238         track_candidates_sycl_buffer;
0239
0240     {
0241       traccc::performance::timer wall_t("Wall time", elapsedTimes);
0242
0243       traccc::edm::silicon_cell_collection::host cells_per_event{host_mr};
0244
0245       {
0246         traccc::performance::timer t("File reading  (cpu)", elapsedTimes);
0247         // Read the cells from the relevant event file into host memory.
0248         static constexpr bool DEDUPLICATE = true;
0249         traccc::io::read_cells(cells_per_event, event, input_opts.directory,
0250                                logger().clone(), &host_det_cond,
0251                                input_opts.format, DEDUPLICATE,
0252                                input_opts.use_acts_geom_source);
0253       }  // stop measuring file reading timer
0254
0255       n_cells += cells_per_event.size();
0256
0257       // Create device copy of input collections
0258       traccc::edm::silicon_cell_collection::buffer cells_buffer(
0259           static_cast<unsigned int>(cells_per_event.size()), mr.main);
0260       copy(vecmem::get_data(cells_per_event), cells_buffer)->wait();
0261
0262       // SYCL
0263       {
0264         traccc::performance::timer t("Clusterization (sycl)", elapsedTimes);
0265         // Reconstruct it into spacepoints on the device.
0266         auto unsorted_measurements =
0267             ca_sycl(cells_buffer, device_det_descr, device_det_cond);
0268         measurements_sycl_buffer = ms_sycl(unsorted_measurements);
0269         vecmem_queue.synchronize();
0270       }  // stop measuring clusterization sycl timer
0271
0272       // CPU
0273       if (accelerator_opts.compare_with_cpu) {
0274         traccc::performance::timer t("Clusterization  (cpu)", elapsedTimes);
0275         measurements_per_event = ca(vecmem::get_data(cells_per_event),
0276                                     host_det_descr_data, host_det_cond_data);
0277       }
0278
0279       // Perform seeding, track finding and fitting only when using a
0280       // Detray geometry.
0281       // SYCL
0282       {
0283         traccc::performance::timer t("Spacepoint formation (sycl)",
0284                                      elapsedTimes);
0285         // Reconstruct it into spacepoints on the device.
0286         spacepoints_sycl_buffer =
0287             sf_sycl(detector_buffer, measurements_sycl_buffer);
0288         vecmem_queue.synchronize();
0289       }  // stop measuring clusterization sycl timer
0290
0291       // CPU
0292       if (accelerator_opts.compare_with_cpu) {
0293         traccc::performance::timer t("Spacepoint formation  (cpu)",
0294                                      elapsedTimes);
0295         spacepoints_per_event =
0296             sf(host_det, vecmem::get_data(measurements_per_event));
0297       }
0298
0299       // SYCL
0300       {
0301         traccc::performance::timer t("Seeding (sycl)", elapsedTimes);
0302         if (usingGBTS) {
0303           seeds_sycl_buffer =
0304               gbts_sa_sycl(spacepoints_sycl_buffer, measurements_sycl_buffer);
0305         } else {
0306           seeds_sycl_buffer = sa_sycl(spacepoints_sycl_buffer);
0307         }
0308         vecmem_queue.synchronize();
0309       }  // stop measuring seeding sycl timer
0310
0311       // CPU
0312       if (accelerator_opts.compare_with_cpu) {
0313         traccc::performance::timer t("Seeding  (cpu)", elapsedTimes);
0314         seeds = sa(vecmem::get_data(spacepoints_per_event));
0315       }  // stop measuring seeding cpu timer
0316
0317       // SYCL
0318       {
0319         traccc::performance::timer t("Track params (sycl)", elapsedTimes);
0320         params_sycl_buffer =
0321             tp_sycl(device_field, measurements_sycl_buffer,
0322                     spacepoints_sycl_buffer, seeds_sycl_buffer);
0323         vecmem_queue.synchronize();
0324       }  // stop measuring track params timer
0325
0326       // CPU
0327       if (accelerator_opts.compare_with_cpu) {
0328         traccc::performance::timer t("Track params  (cpu)", elapsedTimes);
0329         params = tp(vecmem::get_data(measurements_per_event),
0330                     vecmem::get_data(spacepoints_per_event),
0331                     vecmem::get_data(seeds), field_vec);
0332       }  // stop measuring track params cpu timer
0333
0334       // SYCL
0335       {
0336         traccc::performance::timer timer{"Track finding (sycl)", elapsedTimes};
0337         track_candidates_sycl_buffer =
0338             finding_alg_sycl(detector_buffer, device_field,
0339                              measurements_sycl_buffer, params_sycl_buffer);
0340         vecmem_queue.synchronize();
0341       }
0342
0343       // CPU
0344       if (accelerator_opts.compare_with_cpu) {
0345         traccc::performance::timer timer{"Track finding  (cpu)", elapsedTimes};
0346         track_candidates = finding_alg(host_det, host_field,
0347                                        vecmem::get_data(measurements_per_event),
0348                                        vecmem::get_data(params));
0349       }
0350     }  // stop measuring wall time
0351
0352     /*----------------------------------
0353       compare cpu and sycl result
0354       ----------------------------------*/
0355
0356     traccc::edm::measurement_collection::host measurements_per_event_sycl{
0357         host_mr};
0358     traccc::edm::spacepoint_collection::host spacepoints_per_event_sycl{
0359         host_mr};
0360     traccc::edm::seed_collection::host seeds_sycl{host_mr};
0361     traccc::bound_track_parameters_collection_types::host params_sycl{&host_mr};
0362     traccc::edm::track_collection<traccc::default_algebra>::host
0363         track_candidates_sycl{host_mr};
0364
0365     copy(measurements_sycl_buffer, measurements_per_event_sycl)->wait();
0366     copy(spacepoints_sycl_buffer, spacepoints_per_event_sycl)->wait();
0367     copy(seeds_sycl_buffer, seeds_sycl)->wait();
0368     copy(params_sycl_buffer, params_sycl)->wait();
0369     copy(track_candidates_sycl_buffer.tracks, track_candidates_sycl,
0370          vecmem::copy::type::device_to_host)
0371         ->wait();
0372
0373     if (accelerator_opts.compare_with_cpu) {
0374       // Show which event we are currently presenting the results for.
0375       TRACCC_INFO("===>>> Event " << event << " <<<===");
0376
0377       // Compare the measurements made on the host and on the device.
0378       traccc::soa_comparator<traccc::edm::measurement_collection>
0379           compare_measurements{"measurements"};
0380       compare_measurements(vecmem::get_data(measurements_per_event),
0381                            vecmem::get_data(measurements_per_event_sycl));
0382
0383       // Compare the spacepoints made on the host and on the device.
0384       traccc::soa_comparator<traccc::edm::spacepoint_collection>
0385           compare_spacepoints{"spacepoints"};
0386       compare_spacepoints(vecmem::get_data(spacepoints_per_event),
0387                           vecmem::get_data(spacepoints_per_event_sycl));
0388
0389       // Compare the seeds made on the host and on the device
0390       traccc::soa_comparator<traccc::edm::seed_collection> compare_seeds{
0391           "seeds",
0392           traccc::details::comparator_factory<
0393               traccc::edm::seed_collection::const_device::const_proxy_type>{
0394               vecmem::get_data(spacepoints_per_event),
0395               vecmem::get_data(spacepoints_per_event_sycl)}};
0396       compare_seeds(vecmem::get_data(seeds), vecmem::get_data(seeds_sycl));
0397
0398       // Compare the track parameters made on the host and on the device.
0399       traccc::collection_comparator<traccc::bound_track_parameters<>>
0400           compare_track_parameters{"track parameters"};
0401       compare_track_parameters(vecmem::get_data(params),
0402                                vecmem::get_data(params_sycl));
0403
0404       // Compare tracks found on the host and on the device.
0405       traccc::soa_comparator<
0406           traccc::edm::track_collection<traccc::default_algebra>>
0407           compare_track_candidates{
0408               "track candidates",
0409               traccc::details::comparator_factory<traccc::edm::track_collection<
0410                   traccc::default_algebra>::const_device::const_proxy_type>{
0411                   vecmem::get_data(measurements_per_event),
0412                   vecmem::get_data(measurements_per_event_sycl),
0413                   {},
0414                   {}}};
0415       compare_track_candidates(vecmem::get_data(track_candidates.tracks),
0416                                vecmem::get_data(track_candidates_sycl));
0417     }
0418
0419     /// Statistics
0420     n_measurements += measurements_per_event.size();
0421     n_spacepoints += spacepoints_per_event.size();
0422     n_spacepoints_sycl += spacepoints_per_event_sycl.size();
0423     n_seeds_sycl += seeds_sycl.size();
0424     n_seeds += seeds.size();
0425     n_found_tracks += track_candidates.tracks.size();
0426     n_found_tracks_sycl += track_candidates_sycl.size();
0427
0428     if (performance_opts.run) {
0429       traccc::event_data evt_data(input_opts.directory, event, host_mr,
0430                                   input_opts.use_acts_geom_source, &host_det,
0431                                   input_opts.format, true);
0432
0433       sd_performance_writer.write(
0434           vecmem::get_data(seeds_sycl), vecmem::get_data(spacepoints_per_event),
0435           vecmem::get_data(measurements_per_event), evt_data);
0436     }
0437   }
0438
0439   if (performance_opts.run) {
0440     sd_performance_writer.finalize();
0441   }
0442
0443   TRACCC_INFO("==> Statistics ... ");
0444   TRACCC_INFO("- read    " << n_cells << " cells");
0445   TRACCC_INFO("- created (cpu)  " << n_measurements << " measurements");
0446   TRACCC_INFO("- created (cpu)  " << n_spacepoints << " spacepoints");
0447   TRACCC_INFO("- created (sycl) " << n_spacepoints_sycl << " spacepoints     ");
0448
0449   TRACCC_INFO("- created  (cpu) " << n_seeds << " seeds");
0450   TRACCC_INFO("- created (sycl) " << n_seeds_sycl << " seeds");
0451   TRACCC_INFO("- found (cpu)    " << n_found_tracks << " tracks");
0452   TRACCC_INFO("- found (sycl)   " << n_found_tracks_sycl << " tracks");
0453   TRACCC_INFO("==>Elapsed times...\n" << elapsedTimes);
0454
0455   return 0;
0456 }
0457 //
0458 // The main routine
0459 //
0460 int main(int argc, char* argv[]) {
0461   std::unique_ptr<const traccc::Logger> logger = traccc::getDefaultLogger(
0462       "TracccExampleSeqSycl", traccc::Logging::Level::INFO);
0463
0464   // Program options.
0465   traccc::opts::detector detector_opts;
0466   traccc::opts::magnetic_field bfield_opts;
0467   traccc::opts::input_data input_opts;
0468   traccc::opts::clusterization clusterization_opts;
0469   traccc::opts::track_seeding seeding_opts;
0470   traccc::opts::track_gbts_seeding seeding_gbts_opts;
0471   traccc::opts::track_finding finding_opts;
0472   traccc::opts::track_propagation propagation_opts;
0473   traccc::opts::track_fitting fitting_opts;
0474   traccc::opts::performance performance_opts;
0475   traccc::opts::accelerator accelerator_opts;
0476   traccc::opts::program_options program_opts{
0477       "Full Tracking Chain Using SYCL",
0478       {detector_opts, bfield_opts, input_opts, clusterization_opts,
0479        seeding_opts, seeding_gbts_opts, finding_opts, propagation_opts,
0480        fitting_opts, performance_opts, accelerator_opts},
0481       argc,
0482       argv,
0483       logger->cloneWithSuffix("Options")};
0484
0485   // Run the application.
0486   return seq_run(detector_opts, bfield_opts, input_opts, clusterization_opts,
0487                  seeding_opts, seeding_gbts_opts, finding_opts,
0488                  propagation_opts, fitting_opts, performance_opts,
0489                  accelerator_opts, logger->clone(), seeding_gbts_opts.useGBTS);
0490 }