tbb/detail/_task.h

0001 /*
0002     Copyright (c) 2020-2023 Intel Corporation
0003
0004     Licensed under the Apache License, Version 2.0 (the "License");
0005     you may not use this file except in compliance with the License.
0006     You may obtain a copy of the License at
0007
0008         http://www.apache.org/licenses/LICENSE-2.0
0009
0010     Unless required by applicable law or agreed to in writing, software
0011     distributed under the License is distributed on an "AS IS" BASIS,
0012     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0013     See the License for the specific language governing permissions and
0014     limitations under the License.
0015 */
0016
0017 #ifndef __TBB__task_H
0018 #define __TBB__task_H
0019
0020 #include "_config.h"
0021 #include "_assert.h"
0022 #include "_template_helpers.h"
0023 #include "_small_object_pool.h"
0024
0025 #include "../profiling.h"
0026
0027 #include <cstddef>
0028 #include <cstdint>
0029 #include <climits>
0030 #include <utility>
0031 #include <atomic>
0032 #include <mutex>
0033
0034 namespace tbb {
0035 namespace detail {
0036
0037 namespace d1 {
0038 using slot_id = unsigned short;
0039 constexpr slot_id no_slot = slot_id(~0);
0040 constexpr slot_id any_slot = slot_id(~1);
0041
0042 class task;
0043 class wait_context;
0044 class task_group_context;
0045 struct execution_data;
0046 }
0047
0048 namespace r1 {
0049 //! Task spawn/wait entry points
0050 TBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx);
0051 TBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx, d1::slot_id id);
0052 TBB_EXPORT void __TBB_EXPORTED_FUNC execute_and_wait(d1::task& t, d1::task_group_context& t_ctx, d1::wait_context&, d1::task_group_context& w_ctx);
0053 TBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::wait_context&, d1::task_group_context& ctx);
0054 TBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::execution_data*);
0055 TBB_EXPORT d1::task_group_context* __TBB_EXPORTED_FUNC current_context();
0056
0057 // Do not place under __TBB_RESUMABLE_TASKS. It is a stub for unsupported platforms.
0058 struct suspend_point_type;
0059 using suspend_callback_type = void(*)(void*, suspend_point_type*);
0060 //! The resumable tasks entry points
0061 TBB_EXPORT void __TBB_EXPORTED_FUNC suspend(suspend_callback_type suspend_callback, void* user_callback);
0062 TBB_EXPORT void __TBB_EXPORTED_FUNC resume(suspend_point_type* tag);
0063 TBB_EXPORT suspend_point_type* __TBB_EXPORTED_FUNC current_suspend_point();
0064 TBB_EXPORT void __TBB_EXPORTED_FUNC notify_waiters(std::uintptr_t wait_ctx_addr);
0065
0066 class thread_data;
0067 class task_dispatcher;
0068 class external_waiter;
0069 struct task_accessor;
0070 struct task_arena_impl;
0071 } // namespace r1
0072
0073 namespace d1 {
0074
0075 class task_arena;
0076 using suspend_point = r1::suspend_point_type*;
0077
0078 #if __TBB_RESUMABLE_TASKS
0079 template <typename F>
0080 static void suspend_callback(void* user_callback, suspend_point sp) {
0081     // Copy user function to a new stack after the context switch to avoid a race when the previous
0082     // suspend point is resumed while the user_callback is being called.
0083     F user_callback_copy = *static_cast<F*>(user_callback);
0084     user_callback_copy(sp);
0085 }
0086
0087 template <typename F>
0088 void suspend(F f) {
0089     r1::suspend(&suspend_callback<F>, &f);
0090 }
0091
0092 inline void resume(suspend_point tag) {
0093     r1::resume(tag);
0094 }
0095 #endif /* __TBB_RESUMABLE_TASKS */
0096
0097 // TODO align wait_context on cache lane
0098 class wait_context {
0099     static constexpr std::uint64_t overflow_mask = ~((1LLU << 32) - 1);
0100
0101     std::uint64_t m_version_and_traits{1};
0102     std::atomic<std::uint64_t> m_ref_count{};
0103
0104     void add_reference(std::int64_t delta) {
0105         call_itt_task_notify(releasing, this);
0106         std::uint64_t r = m_ref_count.fetch_add(static_cast<std::uint64_t>(delta)) + static_cast<std::uint64_t>(delta);
0107
0108         __TBB_ASSERT_EX((r & overflow_mask) == 0, "Overflow is detected");
0109
0110         if (!r) {
0111             // Some external waiters or coroutine waiters sleep in wait list
0112             // Should to notify them that work is done
0113             std::uintptr_t wait_ctx_addr = std::uintptr_t(this);
0114             r1::notify_waiters(wait_ctx_addr);
0115         }
0116     }
0117
0118     bool continue_execution() const {
0119         std::uint64_t r = m_ref_count.load(std::memory_order_acquire);
0120         __TBB_ASSERT_EX((r & overflow_mask) == 0, "Overflow is detected");
0121         return r > 0;
0122     }
0123
0124     friend class r1::thread_data;
0125     friend class r1::task_dispatcher;
0126     friend class r1::external_waiter;
0127     friend class task_group;
0128     friend class task_group_base;
0129     friend struct r1::task_arena_impl;
0130     friend struct r1::suspend_point_type;
0131 public:
0132     // Despite the internal reference count is uin64_t we limit the user interface with uint32_t
0133     // to preserve a part of the internal reference count for special needs.
0134     wait_context(std::uint32_t ref_count) : m_ref_count{ref_count} { suppress_unused_warning(m_version_and_traits); }
0135     wait_context(const wait_context&) = delete;
0136
0137     ~wait_context() {
0138         __TBB_ASSERT(!continue_execution(), nullptr);
0139     }
0140
0141     void reserve(std::uint32_t delta = 1) {
0142         add_reference(delta);
0143     }
0144
0145     void release(std::uint32_t delta = 1) {
0146         add_reference(-std::int64_t(delta));
0147     }
0148 };
0149
0150 struct execution_data {
0151     task_group_context* context{};
0152     slot_id original_slot{};
0153     slot_id affinity_slot{};
0154 };
0155
0156 inline task_group_context* context(const execution_data& ed) {
0157     return ed.context;
0158 }
0159
0160 inline slot_id original_slot(const execution_data& ed) {
0161     return ed.original_slot;
0162 }
0163
0164 inline slot_id affinity_slot(const execution_data& ed) {
0165     return ed.affinity_slot;
0166 }
0167
0168 inline slot_id execution_slot(const execution_data& ed) {
0169     return r1::execution_slot(&ed);
0170 }
0171
0172 inline bool is_same_affinity(const execution_data& ed) {
0173     return affinity_slot(ed) == no_slot || affinity_slot(ed) == execution_slot(ed);
0174 }
0175
0176 inline bool is_stolen(const execution_data& ed) {
0177     return original_slot(ed) != execution_slot(ed);
0178 }
0179
0180 inline void spawn(task& t, task_group_context& ctx) {
0181     call_itt_task_notify(releasing, &t);
0182     r1::spawn(t, ctx);
0183 }
0184
0185 inline void spawn(task& t, task_group_context& ctx, slot_id id) {
0186     call_itt_task_notify(releasing, &t);
0187     r1::spawn(t, ctx, id);
0188 }
0189
0190 inline void execute_and_wait(task& t, task_group_context& t_ctx, wait_context& wait_ctx, task_group_context& w_ctx) {
0191     r1::execute_and_wait(t, t_ctx, wait_ctx, w_ctx);
0192     call_itt_task_notify(acquired, &wait_ctx);
0193     call_itt_task_notify(destroy, &wait_ctx);
0194 }
0195
0196 inline void wait(wait_context& wait_ctx, task_group_context& ctx) {
0197     r1::wait(wait_ctx, ctx);
0198     call_itt_task_notify(acquired, &wait_ctx);
0199     call_itt_task_notify(destroy, &wait_ctx);
0200 }
0201
0202 using r1::current_context;
0203
0204 class task_traits {
0205     std::uint64_t m_version_and_traits{};
0206     friend struct r1::task_accessor;
0207 };
0208
0209 //! Alignment for a task object
0210 static constexpr std::size_t task_alignment = 64;
0211
0212 //! Base class for user-defined tasks.
0213 /** @ingroup task_scheduling */
0214 class alignas(task_alignment) task : public task_traits {
0215 protected:
0216     virtual ~task() = default;
0217
0218 public:
0219     virtual task* execute(execution_data&) = 0;
0220     virtual task* cancel(execution_data&) = 0;
0221
0222 private:
0223     std::uint64_t m_reserved[6]{};
0224     friend struct r1::task_accessor;
0225 };
0226 static_assert(sizeof(task) == task_alignment, "task size is broken");
0227
0228 } // namespace d1
0229 } // namespace detail
0230 } // namespace tbb
0231
0232 #endif /* __TBB__task_H */