__pstl/cpu_algos/find_if.h

0001 //===----------------------------------------------------------------------===//
0002 //
0003 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0004 // See https://llvm.org/LICENSE.txt for license information.
0005 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0006 //
0007 //===----------------------------------------------------------------------===//
0008
0009 #ifndef _LIBCPP___CXX03___PSTL_CPU_ALGOS_FIND_IF_H
0010 #define _LIBCPP___CXX03___PSTL_CPU_ALGOS_FIND_IF_H
0011
0012 #include <__cxx03/__algorithm/find_if.h>
0013 #include <__cxx03/__assert>
0014 #include <__cxx03/__atomic/atomic.h>
0015 #include <__cxx03/__config>
0016 #include <__cxx03/__functional/operations.h>
0017 #include <__cxx03/__iterator/concepts.h>
0018 #include <__cxx03/__iterator/iterator_traits.h>
0019 #include <__cxx03/__pstl/backend_fwd.h>
0020 #include <__cxx03/__pstl/cpu_algos/cpu_traits.h>
0021 #include <__cxx03/__type_traits/is_execution_policy.h>
0022 #include <__cxx03/__utility/move.h>
0023 #include <__cxx03/__utility/pair.h>
0024 #include <__cxx03/cstddef>
0025 #include <__cxx03/optional>
0026
0027 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
0028 #  pragma GCC system_header
0029 #endif
0030
0031 _LIBCPP_PUSH_MACROS
0032 #include <__cxx03/__undef_macros>
0033
0034 _LIBCPP_BEGIN_NAMESPACE_STD
0035 namespace __pstl {
0036
0037 template <class _Backend, class _Index, class _Brick, class _Compare>
0038 _LIBCPP_HIDE_FROM_ABI optional<_Index>
0039 __parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool __b_first) {
0040   typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
0041   const _DifferenceType __n      = __last - __first;
0042   _DifferenceType __initial_dist = __b_first ? __n : -1;
0043   std::atomic<_DifferenceType> __extremum(__initial_dist);
0044   // TODO: find out what is better here: parallel_for or parallel_reduce
0045   auto __res =
0046       __cpu_traits<_Backend>::__for_each(__first, __last, [__comp, __f, __first, &__extremum](_Index __i, _Index __j) {
0047         // See "Reducing Contention Through Priority Updates", PPoPP '13, for discussion of
0048         // why using a shared variable scales fairly well in this situation.
0049         if (__comp(__i - __first, __extremum)) {
0050           _Index __result = __f(__i, __j);
0051           // If not '__last' returned then we found what we want so put this to extremum
0052           if (__result != __j) {
0053             const _DifferenceType __k = __result - __first;
0054             for (_DifferenceType __old = __extremum; __comp(__k, __old); __old = __extremum) {
0055               __extremum.compare_exchange_weak(__old, __k);
0056             }
0057           }
0058         }
0059       });
0060   if (!__res)
0061     return nullopt;
0062   return __extremum.load() != __initial_dist ? __first + __extremum.load() : __last;
0063 }
0064
0065 template <class _Backend, class _Index, class _DifferenceType, class _Compare>
0066 _LIBCPP_HIDE_FROM_ABI _Index
0067 __simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp) noexcept {
0068   // Experiments show good block sizes like this
0069   const _DifferenceType __block_size                                                = 8;
0070   alignas(__cpu_traits<_Backend>::__lane_size) _DifferenceType __lane[__block_size] = {0};
0071   while (__end - __begin >= __block_size) {
0072     _DifferenceType __found = 0;
0073     _PSTL_PRAGMA_SIMD_REDUCTION(| : __found) for (_DifferenceType __i = __begin; __i < __begin + __block_size; ++__i) {
0074       const _DifferenceType __t = __comp(__first, __i);
0075       __lane[__i - __begin]     = __t;
0076       __found |= __t;
0077     }
0078     if (__found) {
0079       _DifferenceType __i;
0080       // This will vectorize
0081       for (__i = 0; __i < __block_size; ++__i) {
0082         if (__lane[__i]) {
0083           break;
0084         }
0085       }
0086       return __first + __begin + __i;
0087     }
0088     __begin += __block_size;
0089   }
0090
0091   // Keep remainder scalar
0092   while (__begin != __end) {
0093     if (__comp(__first, __begin)) {
0094       return __first + __begin;
0095     }
0096     ++__begin;
0097   }
0098   return __first + __end;
0099 }
0100
0101 template <class _Backend, class _RawExecutionPolicy>
0102 struct __cpu_parallel_find_if {
0103   template <class _Policy, class _ForwardIterator, class _Predicate>
0104   _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator>
0105   operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) const noexcept {
0106     if constexpr (__is_parallel_execution_policy_v<_RawExecutionPolicy> &&
0107                   __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
0108       return __pstl::__parallel_find<_Backend>(
0109           __first,
0110           __last,
0111           [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
0112             using _FindIfUnseq = __pstl::__find_if<_Backend, __remove_parallel_policy_t<_RawExecutionPolicy>>;
0113             auto __res = _FindIfUnseq()(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred);
0114             _LIBCPP_ASSERT_INTERNAL(__res, "unseq/seq should never try to allocate!");
0115             return *std::move(__res);
0116           },
0117           less<>{},
0118           true);
0119     } else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
0120                          __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
0121       using __diff_t = __iter_diff_t<_ForwardIterator>;
0122       return __pstl::__simd_first<_Backend>(
0123           __first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) {
0124             return __pred(__iter[__i]);
0125           });
0126     } else {
0127       return std::find_if(__first, __last, __pred);
0128     }
0129   }
0130 };
0131
0132 } // namespace __pstl
0133 _LIBCPP_END_NAMESPACE_STD
0134
0135 _LIBCPP_POP_MACROS
0136
0137 #endif // _LIBCPP___CXX03___PSTL_CPU_ALGOS_FIND_IF_H