File indexing completed on 2026-07-26 08:22:04
0001
0002
0003
0004
0005
0006
0007
0008
0009 #include "traccc/alpaka/gbts_seeding/gbts_seeding_algorithm.hpp"
0010
0011 #include "../utils/barrier.hpp"
0012 #include "../utils/get_queue.hpp"
0013 #include "../utils/parallel_algorithms.hpp"
0014 #include "../utils/thread_id.hpp"
0015 #include "../utils/utils.hpp"
0016
0017
0018 #include "traccc/gbts_seeding/device/gbts_add_terminus_to_path_store.hpp"
0019 #include "traccc/gbts_seeding/device/gbts_bid_seeds_for_hits.hpp"
0020 #include "traccc/gbts_seeding/device/gbts_bin_spacepoints.hpp"
0021 #include "traccc/gbts_seeding/device/gbts_compress_graph.hpp"
0022 #include "traccc/gbts_seeding/device/gbts_convert_seeds.hpp"
0023 #include "traccc/gbts_seeding/device/gbts_count_eta_phi_bins.hpp"
0024 #include "traccc/gbts_seeding/device/gbts_count_spacepoints_by_layer.hpp"
0025 #include "traccc/gbts_seeding/device/gbts_count_terminus_edges.hpp"
0026 #include "traccc/gbts_seeding/device/gbts_fill_path_store.hpp"
0027 #include "traccc/gbts_seeding/device/gbts_find_minmax_radius.hpp"
0028 #include "traccc/gbts_seeding/device/gbts_fit_segments.hpp"
0029 #include "traccc/gbts_seeding/device/gbts_link_graph_edges.hpp"
0030 #include "traccc/gbts_seeding/device/gbts_make_graph_edges.hpp"
0031 #include "traccc/gbts_seeding/device/gbts_match_graph_edges.hpp"
0032 #include "traccc/gbts_seeding/device/gbts_prefix_sum_eta_phi_bins.hpp"
0033 #include "traccc/gbts_seeding/device/gbts_rebid_seeds_for_edges.hpp"
0034 #include "traccc/gbts_seeding/device/gbts_reindex_edges.hpp"
0035 #include "traccc/gbts_seeding/device/gbts_reset_edge_bids.hpp"
0036 #include "traccc/gbts_seeding/device/gbts_run_cca_iteration.hpp"
0037 #include "traccc/gbts_seeding/device/gbts_sort_nodes.hpp"
0038 #include "traccc/gbts_seeding/gbts_types.hpp"
0039
0040
0041 #include <vecmem/containers/data/vector_view.hpp>
0042 #include <vecmem/containers/device_vector.hpp>
0043
0044
0045 #include <algorithm>
0046
0047 namespace traccc::alpaka {
0048
0049 namespace kernels {
0050
0051
0052
0053
0054
0055
0056 struct gbts_count_spacepoints_by_layer {
0057 template <typename TAcc>
0058 ALPAKA_FN_ACC void operator()(
0059 TAcc const& acc,
0060 const device::gbts_count_spacepoints_by_layer_payload payload) const {
0061 device::gbts_count_spacepoints_by_layer(details::thread_id1{acc}, payload);
0062 }
0063 };
0064
0065
0066 struct gbts_bin_spacepoints {
0067 template <typename TAcc>
0068 ALPAKA_FN_ACC void operator()(
0069 TAcc const& acc,
0070 const device::gbts_bin_spacepoints_payload payload) const {
0071 device::gbts_bin_spacepoints(details::thread_id1{acc}, payload);
0072 }
0073 };
0074
0075
0076 struct gbts_count_eta_phi_bins {
0077 template <typename TAcc>
0078 ALPAKA_FN_ACC void operator()(
0079 TAcc const& acc,
0080 const device::gbts_count_eta_phi_bins_payload payload) const {
0081 device::gbts_count_eta_phi_bins(details::thread_id1{acc}, payload);
0082 }
0083 };
0084
0085
0086 struct gbts_prefix_sum_eta_phi_bins {
0087 template <typename TAcc>
0088 ALPAKA_FN_ACC void operator()(
0089 TAcc const& acc,
0090 const device::gbts_prefix_sum_eta_phi_bins_payload payload) const {
0091 device::gbts_prefix_sum_eta_phi_bins(details::thread_id1{acc}, payload);
0092 }
0093 };
0094
0095
0096 struct gbts_sort_nodes {
0097 template <typename TAcc>
0098 ALPAKA_FN_ACC void operator()(
0099 TAcc const& acc, const device::gbts_sort_nodes_payload payload) const {
0100 device::gbts_sort_nodes(details::thread_id1{acc}, payload);
0101 }
0102 };
0103
0104
0105 struct gbts_find_minmax_radius {
0106 template <typename TAcc>
0107 ALPAKA_FN_ACC void operator()(
0108 TAcc const& acc,
0109 const device::gbts_find_minmax_radius_payload payload) const {
0110 device::gbts_find_minmax_radius(details::thread_id1{acc}, payload);
0111 }
0112 };
0113
0114
0115
0116
0117
0118
0119 struct gbts_make_graph_edges {
0120 template <typename TAcc>
0121 ALPAKA_FN_ACC void operator()(
0122 TAcc const& acc,
0123 const device::gbts_make_graph_edges_payload payload) const {
0124 auto& phi = ::alpaka::declareSharedVar<
0125 float[traccc::device::gbts_consts::node_buffer_length], __COUNTER__>(
0126 acc);
0127 auto& node_pack = ::alpaka::declareSharedVar<
0128 traccc::float4[traccc::device::gbts_consts::node_buffer_length],
0129 __COUNTER__>(acc);
0130 const alpaka::barrier<TAcc> barrier(&acc);
0131
0132 device::gbts_make_graph_edges(
0133 details::thread_id1{acc}, barrier, payload,
0134 {vecmem::data::vector_view<float>(
0135 traccc::device::gbts_consts::node_buffer_length, &phi[0]),
0136 vecmem::data::vector_view<traccc::float4>(
0137 traccc::device::gbts_consts::node_buffer_length, &node_pack[0])});
0138 }
0139 };
0140
0141
0142 struct gbts_link_graph_edges {
0143 template <typename TAcc>
0144 ALPAKA_FN_ACC void operator()(
0145 TAcc const& acc,
0146 const device::gbts_link_graph_edges_payload payload) const {
0147 device::gbts_link_graph_edges(details::thread_id1{acc}, payload);
0148 }
0149 };
0150
0151
0152 struct gbts_match_graph_edges {
0153 template <typename TAcc>
0154 ALPAKA_FN_ACC void operator()(
0155 TAcc const& acc,
0156 const device::gbts_match_graph_edges_payload payload) const {
0157 device::gbts_match_graph_edges(details::thread_id1{acc}, payload);
0158 }
0159 };
0160
0161
0162 struct gbts_reindex_edges {
0163 template <typename TAcc>
0164 ALPAKA_FN_ACC void operator()(
0165 TAcc const& acc, const device::gbts_reindex_edges_payload payload) const {
0166 device::gbts_reindex_edges(details::thread_id1{acc}, payload);
0167 }
0168 };
0169
0170
0171 struct gbts_compress_graph {
0172 template <typename TAcc>
0173 ALPAKA_FN_ACC void operator()(
0174 TAcc const& acc,
0175 const device::gbts_compress_graph_payload payload) const {
0176 device::gbts_compress_graph(details::thread_id1{acc}, payload);
0177 }
0178 };
0179
0180
0181
0182
0183
0184
0185 struct gbts_run_cca_iteration {
0186 template <typename TAcc>
0187 ALPAKA_FN_ACC void operator()(
0188 TAcc const& acc,
0189 const device::gbts_run_cca_iteration_payload payload) const {
0190 device::gbts_run_cca_iteration(details::thread_id1{acc}, payload);
0191 }
0192 };
0193
0194
0195 struct gbts_count_terminus_edges {
0196 template <typename TAcc>
0197 ALPAKA_FN_ACC void operator()(
0198 TAcc const& acc,
0199 const device::gbts_count_terminus_edges_payload payload) const {
0200 device::gbts_count_terminus_edges(details::thread_id1{acc}, payload);
0201 }
0202 };
0203
0204
0205 struct gbts_add_terminus_to_path_store {
0206 template <typename TAcc>
0207 ALPAKA_FN_ACC void operator()(
0208 TAcc const& acc,
0209 const device::gbts_add_terminus_to_path_store_payload payload) const {
0210 device::gbts_add_terminus_to_path_store(details::thread_id1{acc}, payload);
0211 }
0212 };
0213
0214
0215 struct gbts_fill_path_store {
0216 template <typename TAcc>
0217 ALPAKA_FN_ACC void operator()(
0218 TAcc const& acc,
0219 const device::gbts_fill_path_store_payload payload) const {
0220 auto& live_paths = ::alpaka::declareSharedVar<
0221 traccc::uint2[traccc::device::gbts_consts::live_path_buffer],
0222 __COUNTER__>(acc);
0223 auto& n_live_paths = ::alpaka::declareSharedVar<int, __COUNTER__>(acc);
0224 const alpaka::barrier<TAcc> barrier(&acc);
0225
0226 device::gbts_fill_path_store(
0227 details::thread_id1{acc}, barrier, payload,
0228 {vecmem::data::vector_view<traccc::uint2>(
0229 traccc::device::gbts_consts::live_path_buffer, &live_paths[0]),
0230 n_live_paths});
0231 }
0232 };
0233
0234
0235 struct gbts_fit_segments {
0236 template <typename TAcc>
0237 ALPAKA_FN_ACC void operator()(
0238 TAcc const& acc, const device::gbts_fit_segments_payload payload) const {
0239 device::gbts_fit_segments(details::thread_id1{acc}, payload);
0240 }
0241 };
0242
0243
0244 struct gbts_reset_edge_bids {
0245 template <typename TAcc>
0246 ALPAKA_FN_ACC void operator()(
0247 TAcc const& acc,
0248 const device::gbts_reset_edge_bids_payload payload) const {
0249 device::gbts_reset_edge_bids(details::thread_id1{acc}, payload);
0250 }
0251 };
0252
0253
0254 struct gbts_rebid_seeds_for_edges {
0255 template <typename TAcc>
0256 ALPAKA_FN_ACC void operator()(
0257 TAcc const& acc,
0258 const device::gbts_rebid_seeds_for_edges_payload payload) const {
0259 device::gbts_rebid_seeds_for_edges(details::thread_id1{acc}, payload);
0260 }
0261 };
0262
0263
0264 struct gbts_bid_seeds_for_hits {
0265 template <typename TAcc>
0266 ALPAKA_FN_ACC void operator()(
0267 TAcc const& acc,
0268 const device::gbts_bid_seeds_for_hits_payload payload) const {
0269 device::gbts_bid_seeds_for_hits(details::thread_id1{acc}, payload);
0270 }
0271 };
0272
0273
0274 struct gbts_convert_seeds {
0275 template <typename TAcc>
0276 ALPAKA_FN_ACC void operator()(
0277 TAcc const& acc, const device::gbts_convert_seeds_payload payload) const {
0278 device::gbts_convert_seeds(details::thread_id1{acc}, payload);
0279 }
0280 };
0281
0282 }
0283
0284
0285
0286
0287
0288 gbts_seeding_algorithm::gbts_seeding_algorithm(
0289 const gbts_seedfinder_config& cfg, const memory_resource& mr,
0290 const vecmem::copy& copy, alpaka::queue& q,
0291 std::unique_ptr<const Logger> logger)
0292 : device::gbts_seeding_algorithm(cfg, mr, copy, std::move(logger)),
0293 alpaka::algorithm_base{q} {}
0294
0295 void gbts_seeding_algorithm::gbts_count_spacepoints_by_layer_kernel(
0296 const device::gbts_count_spacepoints_by_layer_payload& payload) const {
0297 const unsigned int n_threads = 128;
0298 const unsigned int n_blocks = 1 + (payload.nSp - 1) / n_threads;
0299 ::alpaka::exec<Acc>(details::get_queue(queue()),
0300 makeWorkDiv<Acc>(n_blocks, n_threads),
0301 kernels::gbts_count_spacepoints_by_layer{}, payload);
0302 vecmem::device_vector<unsigned int> d_layer_sums(payload.layerCounts);
0303 details::inclusive_scan(details::get_queue(queue()), mr(),
0304 d_layer_sums.begin(), d_layer_sums.end(),
0305 d_layer_sums.begin());
0306 }
0307
0308 void gbts_seeding_algorithm::gbts_bin_spacepoints_kernel(
0309 const device::gbts_bin_spacepoints_payload& payload) const {
0310 const unsigned int n_threads = 128;
0311 const unsigned int n_blocks = 1 + (payload.nSp - 1) / n_threads;
0312 ::alpaka::exec<Acc>(details::get_queue(queue()),
0313 makeWorkDiv<Acc>(n_blocks, n_threads),
0314 kernels::gbts_bin_spacepoints{}, payload);
0315 }
0316
0317 void gbts_seeding_algorithm::gbts_count_eta_phi_bins_kernel(
0318 const device::gbts_count_eta_phi_bins_payload& payload) const {
0319 const unsigned int n_threads = 128;
0320 const unsigned int n_blocks = 1 + (payload.nEtaBins - 1) / n_threads;
0321 ::alpaka::exec<Acc>(details::get_queue(queue()),
0322 makeWorkDiv<Acc>(n_blocks, n_threads),
0323 kernels::gbts_count_eta_phi_bins{}, payload);
0324 vecmem::device_vector<unsigned int> d_eta_sums(payload.eta_node_counter);
0325 details::inclusive_scan(details::get_queue(queue()), mr(), d_eta_sums.begin(),
0326 d_eta_sums.end(), d_eta_sums.begin());
0327 }
0328
0329 void gbts_seeding_algorithm::gbts_prefix_sum_eta_phi_bins_kernel(
0330 const device::gbts_prefix_sum_eta_phi_bins_payload& payload) const {
0331 const unsigned int n_threads = 128;
0332 const unsigned int n_blocks = 1 + (payload.nEtaBins - 1) / n_threads;
0333 ::alpaka::exec<Acc>(details::get_queue(queue()),
0334 makeWorkDiv<Acc>(n_blocks, n_threads),
0335 kernels::gbts_prefix_sum_eta_phi_bins{}, payload);
0336 }
0337
0338 void gbts_seeding_algorithm::gbts_sort_nodes_kernel(
0339 const device::gbts_sort_nodes_payload& payload) const {
0340 const unsigned int n_threads = 256;
0341 const unsigned int n_blocks = 1 + (payload.nNodes - 1) / n_threads;
0342 ::alpaka::exec<Acc>(details::get_queue(queue()),
0343 makeWorkDiv<Acc>(n_blocks, n_threads),
0344 kernels::gbts_sort_nodes{}, payload);
0345 }
0346
0347 void gbts_seeding_algorithm::gbts_find_minmax_radius_kernel(
0348 const device::gbts_find_minmax_radius_payload& payload) const {
0349 const unsigned int n_threads = 128;
0350 const unsigned int n_blocks = 1 + (payload.nEtaBins - 1) / n_threads;
0351 ::alpaka::exec<Acc>(details::get_queue(queue()),
0352 makeWorkDiv<Acc>(n_blocks, n_threads),
0353 kernels::gbts_find_minmax_radius{}, payload);
0354 }
0355
0356 void gbts_seeding_algorithm::gbts_make_graph_edges_kernel(
0357 const device::gbts_make_graph_edges_payload& payload) const {
0358 const unsigned int n_threads = 128;
0359 const unsigned int n_blocks = payload.nUsedBinPairs;
0360 ::alpaka::exec<Acc>(details::get_queue(queue()),
0361 makeWorkDiv<Acc>(n_blocks, n_threads),
0362 kernels::gbts_make_graph_edges{}, payload);
0363 vecmem::device_vector<unsigned int> d_num_outgoing_edges(
0364 payload.num_outgoing_edges);
0365 details::inclusive_scan(
0366 details::get_queue(queue()), mr(), d_num_outgoing_edges.begin(),
0367 d_num_outgoing_edges.end(), d_num_outgoing_edges.begin());
0368 }
0369
0370 void gbts_seeding_algorithm::gbts_link_graph_edges_kernel(
0371 const device::gbts_link_graph_edges_payload& payload) const {
0372 const unsigned int n_threads = 256;
0373 const unsigned int n_blocks = 1 + (payload.nEdges - 1) / n_threads;
0374 ::alpaka::exec<Acc>(details::get_queue(queue()),
0375 makeWorkDiv<Acc>(n_blocks, n_threads),
0376 kernels::gbts_link_graph_edges{}, payload);
0377 }
0378
0379 void gbts_seeding_algorithm::gbts_match_graph_edges_kernel(
0380 const device::gbts_match_graph_edges_payload& payload) const {
0381 const unsigned int n_threads = 256;
0382 const unsigned int n_blocks = 1 + (payload.nEdges - 1) / n_threads;
0383 ::alpaka::exec<Acc>(details::get_queue(queue()),
0384 makeWorkDiv<Acc>(n_blocks, n_threads),
0385 kernels::gbts_match_graph_edges{}, payload);
0386 }
0387
0388 void gbts_seeding_algorithm::gbts_reindex_edges_kernel(
0389 const device::gbts_reindex_edges_payload& payload) const {
0390 const unsigned int n_threads = 256;
0391 const unsigned int n_blocks = 1 + (payload.nEdges - 1) / n_threads;
0392 ::alpaka::exec<Acc>(details::get_queue(queue()),
0393 makeWorkDiv<Acc>(n_blocks, n_threads),
0394 kernels::gbts_reindex_edges{}, payload);
0395 }
0396
0397 void gbts_seeding_algorithm::gbts_compress_graph_kernel(
0398 const device::gbts_compress_graph_payload& payload) const {
0399 const unsigned int n_threads = 256;
0400 const unsigned int n_blocks = 1 + (payload.nEdges - 1) / n_threads;
0401 ::alpaka::exec<Acc>(details::get_queue(queue()),
0402 makeWorkDiv<Acc>(n_blocks, n_threads),
0403 kernels::gbts_compress_graph{}, payload);
0404 }
0405
0406 void gbts_seeding_algorithm::gbts_run_cca_iteration_kernel(
0407 const device::gbts_run_cca_iteration_payload& payload) const {
0408 const unsigned int n_threads = 128;
0409 const unsigned int n_blocks = 1 + (payload.nConnectedEdges - 1) / n_threads;
0410 ::alpaka::exec<Acc>(details::get_queue(queue()),
0411 makeWorkDiv<Acc>(n_blocks, n_threads),
0412 kernels::gbts_run_cca_iteration{}, payload);
0413 }
0414
0415 void gbts_seeding_algorithm::gbts_count_terminus_edges_kernel(
0416 const device::gbts_count_terminus_edges_payload& payload) const {
0417 const unsigned int n_threads = 128;
0418 const unsigned int n_blocks = 1 + (payload.nConnectedEdges - 1) / n_threads;
0419 ::alpaka::exec<Acc>(details::get_queue(queue()),
0420 makeWorkDiv<Acc>(n_blocks, n_threads),
0421 kernels::gbts_count_terminus_edges{}, payload);
0422 }
0423
0424 void gbts_seeding_algorithm::gbts_add_terminus_to_path_store_kernel(
0425 const device::gbts_add_terminus_to_path_store_payload& payload) const {
0426 const unsigned int n_threads = 128;
0427 const unsigned int n_blocks = 1 + (payload.nConnectedEdges - 1) / n_threads;
0428 ::alpaka::exec<Acc>(details::get_queue(queue()),
0429 makeWorkDiv<Acc>(n_blocks, n_threads),
0430 kernels::gbts_add_terminus_to_path_store{}, payload);
0431 }
0432
0433 void gbts_seeding_algorithm::gbts_fill_path_store_kernel(
0434 const device::gbts_fill_path_store_payload& payload) const {
0435 const unsigned int n_threads = 128;
0436 const unsigned int pathsPerTerminus =
0437 1 + (payload.nPaths - 1) / payload.nTerminusEdges;
0438 const unsigned int terminusPerBlock = std::min(
0439 n_threads, 1 + (traccc::device::gbts_consts::live_path_buffer - 1) /
0440 pathsPerTerminus);
0441 const unsigned int n_blocks =
0442 1 + (payload.nTerminusEdges - 1) / terminusPerBlock;
0443 ::alpaka::exec<Acc>(details::get_queue(queue()),
0444 makeWorkDiv<Acc>(n_blocks, n_threads),
0445 kernels::gbts_fill_path_store{}, payload);
0446 }
0447
0448 void gbts_seeding_algorithm::gbts_fit_segments_kernel(
0449 const device::gbts_fit_segments_payload& payload) const {
0450 const unsigned int n_threads = 128;
0451 const unsigned int n_blocks = 1 + (payload.nPaths - 1) / n_threads;
0452 ::alpaka::exec<Acc>(details::get_queue(queue()),
0453 makeWorkDiv<Acc>(n_blocks, n_threads),
0454 kernels::gbts_fit_segments{}, payload);
0455 }
0456
0457 void gbts_seeding_algorithm::gbts_reset_edge_bids_kernel(
0458 const device::gbts_reset_edge_bids_payload& payload) const {
0459 const unsigned int n_threads = 128;
0460 const unsigned int n_blocks = 1 + (payload.nProps - 1) / n_threads;
0461 ::alpaka::exec<Acc>(details::get_queue(queue()),
0462 makeWorkDiv<Acc>(n_blocks, n_threads),
0463 kernels::gbts_reset_edge_bids{}, payload);
0464 }
0465
0466 void gbts_seeding_algorithm::gbts_rebid_seeds_for_edges_kernel(
0467 const device::gbts_rebid_seeds_for_edges_payload& payload) const {
0468 const unsigned int n_threads = 128;
0469 const unsigned int n_blocks = 1 + (payload.nProps - 1) / n_threads;
0470 ::alpaka::exec<Acc>(details::get_queue(queue()),
0471 makeWorkDiv<Acc>(n_blocks, n_threads),
0472 kernels::gbts_rebid_seeds_for_edges{}, payload);
0473 }
0474
0475 void gbts_seeding_algorithm::gbts_bid_seeds_for_hits_kernel(
0476 const device::gbts_bid_seeds_for_hits_payload& payload) const {
0477 const unsigned int n_threads = 128;
0478 const unsigned int n_blocks = 1 + (payload.nProps - 1) / n_threads;
0479 ::alpaka::exec<Acc>(details::get_queue(queue()),
0480 makeWorkDiv<Acc>(n_blocks, n_threads),
0481 kernels::gbts_bid_seeds_for_hits{}, payload);
0482 }
0483
0484 void gbts_seeding_algorithm::gbts_convert_seeds_kernel(
0485 const device::gbts_convert_seeds_payload& payload) const {
0486 const unsigned int n_threads = 128;
0487 const unsigned int n_blocks = 1 + (payload.nProps - 1) / n_threads;
0488 ::alpaka::exec<Acc>(details::get_queue(queue()),
0489 makeWorkDiv<Acc>(n_blocks, n_threads),
0490 kernels::gbts_convert_seeds{}, payload);
0491 }
0492
0493 }