boost/context/fiber_ucontext.hpp

0001
0002 //          Copyright Oliver Kowalke 2017.
0003 // Distributed under the Boost Software License, Version 1.0.
0004 //    (See accompanying file LICENSE_1_0.txt or copy at
0005 //          http://www.boost.org/LICENSE_1_0.txt)
0006
0007 #ifndef BOOST_CONTEXT_FIBER_H
0008 #define BOOST_CONTEXT_FIBER_H
0009
0010 #include <boost/predef/os.h>
0011 #if BOOST_OS_MACOS
0012 #define _XOPEN_SOURCE 600
0013 #endif
0014
0015 extern "C" {
0016 #include <ucontext.h>
0017 }
0018
0019 #include <boost/predef.h>
0020 #include <boost/context/detail/config.hpp>
0021
0022 #include <algorithm>
0023 #include <cstddef>
0024 #include <cstdint>
0025 #include <cstdlib>
0026 #include <cstring>
0027 #include <functional>
0028 #include <memory>
0029 #include <ostream>
0030 #include <system_error>
0031 #include <tuple>
0032 #include <utility>
0033
0034 #include <boost/assert.hpp>
0035 #include <boost/config.hpp>
0036 #include <boost/predef.h>
0037
0038 #include <boost/context/detail/disable_overload.hpp>
0039 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0040 #include <boost/context/detail/exchange.hpp>
0041 #endif
0042 #include <boost/context/detail/externc.hpp>
0043 #if defined(BOOST_NO_CXX17_STD_INVOKE)
0044 #include <boost/context/detail/invoke.hpp>
0045 #endif
0046 #include <boost/context/fixedsize_stack.hpp>
0047 #include <boost/context/flags.hpp>
0048 #include <boost/context/preallocated.hpp>
0049 #if defined(BOOST_USE_SEGMENTED_STACKS)
0050 #include <boost/context/segmented_stack.hpp>
0051 #endif
0052 #include <boost/context/stack_context.hpp>
0053
0054 #ifdef BOOST_HAS_ABI_HEADERS
0055 # include BOOST_ABI_PREFIX
0056 #endif
0057
0058 #ifdef BOOST_USE_TSAN
0059 #include <sanitizer/tsan_interface.h>
0060 #endif
0061
0062 namespace boost {
0063 namespace context {
0064 namespace detail {
0065
0066 // tampoline function
0067 // entered if the execution context
0068 // is resumed for the first time
0069 template <typename Record>
0070 #if BOOST_OS_MACOS
0071 static void fiber_entry_func(std::uint32_t data_high,
0072                              std::uint32_t data_low) noexcept {
0073   auto data =
0074       reinterpret_cast<void *>(std::uint64_t(data_high) << 32 | data_low);
0075 #else
0076 static void fiber_entry_func(void *data) noexcept {
0077 #endif
0078   Record *record = static_cast<Record *>(data);
0079   BOOST_ASSERT(nullptr != record);
0080   // start execution of toplevel context-function
0081   record->run();
0082 }
0083
0084 struct BOOST_CONTEXT_DECL fiber_activation_record {
0085     ucontext_t                                                  uctx{};
0086     stack_context                                               sctx{};
0087     bool                                                        main_ctx{ true };
0088     fiber_activation_record                                       * from{ nullptr };
0089     std::function< fiber_activation_record*(fiber_activation_record*&) >    ontop{};
0090     bool                                                        terminated{ false };
0091     bool                                                        force_unwind{ false };
0092 #if defined(BOOST_USE_ASAN)
0093     void                                                    *   fake_stack{ nullptr };
0094     void                                                    *   stack_bottom{ nullptr };
0095     std::size_t                                                 stack_size{ 0 };
0096 #endif
0097
0098 #if defined(BOOST_USE_TSAN)
0099     void * tsan_fiber{ nullptr };
0100     bool destroy_tsan_fiber{ true };
0101 #endif
0102
0103     static fiber_activation_record *& current() noexcept;
0104
0105     // used for toplevel-context
0106     // (e.g. main context, thread-entry context)
0107     fiber_activation_record() {
0108         if ( BOOST_UNLIKELY( 0 != ::getcontext( & uctx) ) ) {
0109             throw std::system_error(
0110                     std::error_code( errno, std::system_category() ),
0111                     "getcontext() failed");
0112         }
0113
0114 #if defined(BOOST_USE_TSAN)
0115         tsan_fiber = __tsan_get_current_fiber();
0116         destroy_tsan_fiber = false;
0117 #endif
0118     }
0119
0120     fiber_activation_record( stack_context sctx_) noexcept :
0121         sctx( sctx_ ),
0122         main_ctx( false ) {
0123     }
0124
0125     virtual ~fiber_activation_record() {
0126 #if defined(BOOST_USE_TSAN)
0127         if (destroy_tsan_fiber)
0128             __tsan_destroy_fiber(tsan_fiber);
0129 #endif
0130     }
0131
0132     fiber_activation_record( fiber_activation_record const&) = delete;
0133     fiber_activation_record & operator=( fiber_activation_record const&) = delete;
0134
0135     bool is_main_context() const noexcept {
0136         return main_ctx;
0137     }
0138
0139     fiber_activation_record * resume() {
0140         from = current();
0141         // store `this` in static, thread local pointer
0142         // `this` will become the active (running) context
0143         current() = this;
0144 #if defined(BOOST_USE_SEGMENTED_STACKS)
0145         // adjust segmented stack properties
0146         __splitstack_getcontext( from->sctx.segments_ctx);
0147         __splitstack_setcontext( sctx.segments_ctx);
0148 #endif
0149 #if defined(BOOST_USE_ASAN)
0150         if ( terminated) {
0151             __sanitizer_start_switch_fiber( nullptr, stack_bottom, stack_size);
0152         } else {
0153             __sanitizer_start_switch_fiber( & from->fake_stack, stack_bottom, stack_size);
0154         }
0155 #endif
0156 #if defined (BOOST_USE_TSAN)
0157         __tsan_switch_to_fiber(tsan_fiber, 0);
0158 #endif
0159         // context switch from parent context to `this`-context
0160         ::swapcontext( & from->uctx, & uctx);
0161 #if defined(BOOST_USE_ASAN)
0162         __sanitizer_finish_switch_fiber( current()->fake_stack,
0163                                          (const void **) & current()->from->stack_bottom,
0164                                          & current()->from->stack_size);
0165 #endif
0166 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0167         return exchange( current()->from, nullptr);
0168 #else
0169         return std::exchange( current()->from, nullptr);
0170 #endif
0171     }
0172
0173     template< typename Ctx, typename Fn >
0174     fiber_activation_record * resume_with( Fn && fn) {
0175         from = current();
0176         // store `this` in static, thread local pointer
0177         // `this` will become the active (running) context
0178         // returned by fiber::current()
0179         current() = this;
0180 #if defined(BOOST_NO_CXX14_GENERIC_LAMBDAS)
0181         current()->ontop = std::bind(
0182                 [](typename std::decay< Fn >::type & fn, fiber_activation_record *& ptr){
0183                     Ctx c{ ptr };
0184                     c = fn( std::move( c) );
0185                     if ( ! c) {
0186                         ptr = nullptr;
0187                     }
0188 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0189                     return exchange( c.ptr_, nullptr);
0190 #else
0191                     return std::exchange( c.ptr_, nullptr);
0192 #endif
0193                 },
0194                 std::forward< Fn >( fn),
0195                 std::placeholders::_1);
0196 #else
0197         current()->ontop = [fn=std::forward<Fn>(fn)](fiber_activation_record *& ptr){
0198             Ctx c{ ptr };
0199             c = fn( std::move( c) );
0200             if ( ! c) {
0201                 ptr = nullptr;
0202             }
0203 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0204             return exchange( c.ptr_, nullptr);
0205 #else
0206             return std::exchange( c.ptr_, nullptr);
0207 #endif
0208         };
0209 #endif
0210 #if defined(BOOST_USE_SEGMENTED_STACKS)
0211         // adjust segmented stack properties
0212         __splitstack_getcontext( from->sctx.segments_ctx);
0213         __splitstack_setcontext( sctx.segments_ctx);
0214 #endif
0215 #if defined(BOOST_USE_ASAN)
0216         __sanitizer_start_switch_fiber( & from->fake_stack, stack_bottom, stack_size);
0217 #endif
0218 #if defined (BOOST_USE_TSAN)
0219         __tsan_switch_to_fiber(tsan_fiber, 0);
0220 #endif
0221         // context switch from parent context to `this`-context
0222         ::swapcontext( & from->uctx, & uctx);
0223 #if defined(BOOST_USE_ASAN)
0224         __sanitizer_finish_switch_fiber( current()->fake_stack,
0225                                          (const void **) & current()->from->stack_bottom,
0226                                          & current()->from->stack_size);
0227 #endif
0228 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0229         return exchange( current()->from, nullptr);
0230 #else
0231         return std::exchange( current()->from, nullptr);
0232 #endif
0233     }
0234
0235     virtual void deallocate() noexcept {
0236     }
0237 };
0238
0239 struct BOOST_CONTEXT_DECL fiber_activation_record_initializer {
0240     fiber_activation_record_initializer() noexcept;
0241     ~fiber_activation_record_initializer();
0242 };
0243
0244 struct forced_unwind {
0245     fiber_activation_record  *  from{ nullptr };
0246
0247     forced_unwind( fiber_activation_record * from_) noexcept :
0248         from{ from_ } {
0249     }
0250 };
0251
0252 template< typename Ctx, typename StackAlloc, typename Fn >
0253 class fiber_capture_record : public fiber_activation_record {
0254 private:
0255     typename std::decay< StackAlloc >::type             salloc_;
0256     typename std::decay< Fn >::type                     fn_;
0257
0258     static void destroy( fiber_capture_record * p) noexcept {
0259         typename std::decay< StackAlloc >::type salloc = std::move( p->salloc_);
0260         stack_context sctx = p->sctx;
0261         // deallocate activation record
0262         p->~fiber_capture_record();
0263         // destroy stack with stack allocator
0264         salloc.deallocate( sctx);
0265     }
0266
0267 public:
0268     fiber_capture_record( stack_context sctx, StackAlloc && salloc, Fn && fn) noexcept :
0269         fiber_activation_record{ sctx },
0270         salloc_{ std::forward< StackAlloc >( salloc) },
0271         fn_( std::forward< Fn >( fn) ) {
0272     }
0273
0274     void deallocate() noexcept override final {
0275         BOOST_ASSERT( main_ctx || ( ! main_ctx && terminated) );
0276         destroy( this);
0277     }
0278
0279     void run() {
0280 #if defined(BOOST_USE_ASAN)
0281         __sanitizer_finish_switch_fiber( fake_stack,
0282                                          (const void **) & from->stack_bottom,
0283                                          & from->stack_size);
0284 #endif
0285         Ctx c{ from };
0286         try {
0287             // invoke context-function
0288 #if defined(BOOST_NO_CXX17_STD_INVOKE)
0289             c = boost::context::detail::invoke( fn_, std::move( c) );
0290 #else
0291             c = std::invoke( fn_, std::move( c) );
0292 #endif
0293         } catch ( forced_unwind const& ex) {
0294             c = Ctx{ ex.from };
0295         }
0296         // this context has finished its task
0297         from = nullptr;
0298         ontop = nullptr;
0299         terminated = true;
0300         force_unwind = false;
0301         std::move( c).resume();
0302         BOOST_ASSERT_MSG( false, "fiber already terminated");
0303     }
0304 };
0305
0306 template< typename Ctx, typename StackAlloc, typename Fn >
0307 static fiber_activation_record * create_fiber1( StackAlloc && salloc, Fn && fn) {
0308     typedef fiber_capture_record< Ctx, StackAlloc, Fn >  capture_t;
0309
0310     auto sctx = salloc.allocate();
0311     // reserve space for control structure
0312     void * storage = reinterpret_cast< void * >(
0313             ( reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sizeof( capture_t) ) )
0314             & ~ static_cast< uintptr_t >( 0xff) );
0315     // placment new for control structure on context stack
0316     capture_t * record = new ( storage) capture_t{
0317             sctx, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) };
0318     // stack bottom
0319     void * stack_bottom = reinterpret_cast< void * >(
0320             reinterpret_cast< uintptr_t >( sctx.sp) - static_cast< uintptr_t >( sctx.size) );
0321     // create user-context
0322     if ( BOOST_UNLIKELY( 0 != ::getcontext( & record->uctx) ) ) {
0323         record->~capture_t();
0324         salloc.deallocate( sctx);
0325         throw std::system_error(
0326                 std::error_code( errno, std::system_category() ),
0327                 "getcontext() failed");
0328     }
0329 #if BOOST_OS_BSD_FREE
0330     // because FreeBSD defines stack_t::ss_sp as char *
0331     record->uctx.uc_stack.ss_sp = static_cast< char * >( stack_bottom);
0332 #else
0333     record->uctx.uc_stack.ss_sp = stack_bottom;
0334 #endif
0335     // 64byte gap between control structure and stack top
0336     record->uctx.uc_stack.ss_size = reinterpret_cast< uintptr_t >( storage) -
0337             reinterpret_cast< uintptr_t >( stack_bottom) - static_cast< uintptr_t >( 64);
0338     record->uctx.uc_link = nullptr;
0339 #if BOOST_OS_MACOS
0340     const auto integer = std::uint64_t(record);
0341     ::makecontext(&record->uctx, (void (*)()) & fiber_entry_func<capture_t>, 2,
0342                   std::uint32_t((integer >> 32) & 0xFFFFFFFF),
0343                   std::uint32_t(integer));
0344 #else
0345     ::makecontext(&record->uctx, (void (*)()) & fiber_entry_func<capture_t>, 1,
0346                   record);
0347 #endif
0348 #if defined(BOOST_USE_ASAN)
0349     record->stack_bottom = record->uctx.uc_stack.ss_sp;
0350     record->stack_size = record->uctx.uc_stack.ss_size;
0351 #endif
0352 #if defined (BOOST_USE_TSAN)
0353     record->tsan_fiber = __tsan_create_fiber(0);
0354 #endif
0355     return record;
0356 }
0357
0358 template< typename Ctx, typename StackAlloc, typename Fn >
0359 static fiber_activation_record * create_fiber2( preallocated palloc, StackAlloc && salloc, Fn && fn) {
0360     typedef fiber_capture_record< Ctx, StackAlloc, Fn >  capture_t;
0361
0362     // reserve space for control structure
0363     void * storage = reinterpret_cast< void * >(
0364             ( reinterpret_cast< uintptr_t >( palloc.sp) - static_cast< uintptr_t >( sizeof( capture_t) ) )
0365             & ~ static_cast< uintptr_t >( 0xff) );
0366     // placment new for control structure on context stack
0367     capture_t * record = new ( storage) capture_t{
0368             palloc.sctx, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) };
0369     // stack bottom
0370     void * stack_bottom = reinterpret_cast< void * >(
0371             reinterpret_cast< uintptr_t >( palloc.sctx.sp) - static_cast< uintptr_t >( palloc.sctx.size) );
0372     // create user-context
0373     if ( BOOST_UNLIKELY( 0 != ::getcontext( & record->uctx) ) ) {
0374         record->~capture_t();
0375         salloc.deallocate( palloc.sctx);
0376         throw std::system_error(
0377                 std::error_code( errno, std::system_category() ),
0378                 "getcontext() failed");
0379     }
0380 #if BOOST_OS_BSD_FREE
0381     // because FreeBSD defines stack_t::ss_sp as char *
0382     record->uctx.uc_stack.ss_sp = static_cast< char * >( stack_bottom);
0383 #else
0384     record->uctx.uc_stack.ss_sp = stack_bottom;
0385 #endif
0386     // 64byte gap between control structure and stack top
0387     record->uctx.uc_stack.ss_size = reinterpret_cast< uintptr_t >( storage) -
0388             reinterpret_cast< uintptr_t >( stack_bottom) - static_cast< uintptr_t >( 64);
0389     record->uctx.uc_link = nullptr;
0390 #if BOOST_OS_MACOS
0391     const auto integer = std::uint64_t(record);
0392     ::makecontext(&record->uctx, (void (*)()) & fiber_entry_func<capture_t>, 2,
0393                   std::uint32_t((integer >> 32) & 0xFFFFFFFF),
0394                   std::uint32_t(integer));
0395 #else
0396     ::makecontext(&record->uctx, (void (*)()) & fiber_entry_func<capture_t>, 1,
0397                   record);
0398 #endif
0399 #if defined(BOOST_USE_ASAN)
0400     record->stack_bottom = record->uctx.uc_stack.ss_sp;
0401     record->stack_size = record->uctx.uc_stack.ss_size;
0402 #endif
0403 #if defined (BOOST_USE_TSAN)
0404     record->tsan_fiber = __tsan_create_fiber(0);
0405 #endif
0406     return record;
0407 }
0408
0409 }
0410
0411 class BOOST_CONTEXT_DECL fiber {
0412 private:
0413     friend struct detail::fiber_activation_record;
0414
0415     template< typename Ctx, typename StackAlloc, typename Fn >
0416     friend class detail::fiber_capture_record;
0417
0418     template< typename Ctx, typename StackAlloc, typename Fn >
0419     friend detail::fiber_activation_record * detail::create_fiber1( StackAlloc &&, Fn &&);
0420
0421     template< typename Ctx, typename StackAlloc, typename Fn >
0422     friend detail::fiber_activation_record * detail::create_fiber2( preallocated, StackAlloc &&, Fn &&);
0423
0424     detail::fiber_activation_record   *   ptr_{ nullptr };
0425
0426     fiber( detail::fiber_activation_record * ptr) noexcept :
0427         ptr_{ ptr } {
0428     }
0429
0430 public:
0431     fiber() = default;
0432
0433     template< typename Fn, typename = detail::disable_overload< fiber, Fn > >
0434     fiber( Fn && fn) :
0435         fiber{
0436                 std::allocator_arg,
0437 #if defined(BOOST_USE_SEGMENTED_STACKS)
0438                 segmented_stack(),
0439 #else
0440                 fixedsize_stack(),
0441 #endif
0442                 std::forward< Fn >( fn) } {
0443     }
0444
0445     template< typename StackAlloc, typename Fn >
0446     fiber( std::allocator_arg_t, StackAlloc && salloc, Fn && fn) :
0447         ptr_{ detail::create_fiber1< fiber >(
0448                 std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) ) } {
0449     }
0450
0451     template< typename StackAlloc, typename Fn >
0452     fiber( std::allocator_arg_t, preallocated palloc, StackAlloc && salloc, Fn && fn) :
0453         ptr_{ detail::create_fiber2< fiber >(
0454                 palloc, std::forward< StackAlloc >( salloc), std::forward< Fn >( fn) ) } {
0455     }
0456
0457     ~fiber() {
0458         if ( BOOST_UNLIKELY( nullptr != ptr_) && ! ptr_->main_ctx) {
0459             if ( BOOST_LIKELY( ! ptr_->terminated) ) {
0460                 ptr_->force_unwind = true;
0461                 ptr_->resume();
0462                 BOOST_ASSERT( ptr_->terminated);
0463             }
0464             ptr_->deallocate();
0465         }
0466     }
0467
0468     fiber( fiber const&) = delete;
0469     fiber & operator=( fiber const&) = delete;
0470
0471     fiber( fiber && other) noexcept {
0472         swap( other);
0473     }
0474
0475     fiber & operator=( fiber && other) noexcept {
0476         if ( BOOST_LIKELY( this != & other) ) {
0477             fiber tmp = std::move( other);
0478             swap( tmp);
0479         }
0480         return * this;
0481     }
0482
0483     fiber resume() && {
0484         BOOST_ASSERT( nullptr != ptr_);
0485 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0486         detail::fiber_activation_record * ptr = detail::exchange( ptr_, nullptr)->resume();
0487 #else
0488         detail::fiber_activation_record * ptr = std::exchange( ptr_, nullptr)->resume();
0489 #endif
0490         if ( BOOST_UNLIKELY( detail::fiber_activation_record::current()->force_unwind) ) {
0491             throw detail::forced_unwind{ ptr};
0492         } else if ( BOOST_UNLIKELY( nullptr != detail::fiber_activation_record::current()->ontop) ) {
0493             ptr = detail::fiber_activation_record::current()->ontop( ptr);
0494             detail::fiber_activation_record::current()->ontop = nullptr;
0495         }
0496         return { ptr };
0497     }
0498
0499     template< typename Fn >
0500     fiber resume_with( Fn && fn) && {
0501         BOOST_ASSERT( nullptr != ptr_);
0502 #if defined(BOOST_NO_CXX14_STD_EXCHANGE)
0503         detail::fiber_activation_record * ptr =
0504             detail::exchange( ptr_, nullptr)->resume_with< fiber >( std::forward< Fn >( fn) );
0505 #else
0506         detail::fiber_activation_record * ptr =
0507             std::exchange( ptr_, nullptr)->resume_with< fiber >( std::forward< Fn >( fn) );
0508 #endif
0509         if ( BOOST_UNLIKELY( detail::fiber_activation_record::current()->force_unwind) ) {
0510             throw detail::forced_unwind{ ptr};
0511         } else if ( BOOST_UNLIKELY( nullptr != detail::fiber_activation_record::current()->ontop) ) {
0512             ptr = detail::fiber_activation_record::current()->ontop( ptr);
0513             detail::fiber_activation_record::current()->ontop = nullptr;
0514         }
0515         return { ptr };
0516     }
0517
0518     explicit operator bool() const noexcept {
0519         return nullptr != ptr_ && ! ptr_->terminated;
0520     }
0521
0522     bool operator!() const noexcept {
0523         return nullptr == ptr_ || ptr_->terminated;
0524     }
0525
0526     bool operator<( fiber const& other) const noexcept {
0527         return ptr_ < other.ptr_;
0528     }
0529
0530     #if !defined(BOOST_EMBTC)
0531
0532     template< typename charT, class traitsT >
0533     friend std::basic_ostream< charT, traitsT > &
0534     operator<<( std::basic_ostream< charT, traitsT > & os, fiber const& other) {
0535         if ( nullptr != other.ptr_) {
0536             return os << other.ptr_;
0537         } else {
0538             return os << "{not-a-context}";
0539         }
0540     }
0541
0542     #else
0543
0544     template< typename charT, class traitsT >
0545     friend std::basic_ostream< charT, traitsT > &
0546     operator<<( std::basic_ostream< charT, traitsT > & os, fiber const& other);
0547
0548     #endif
0549
0550     void swap( fiber & other) noexcept {
0551         std::swap( ptr_, other.ptr_);
0552     }
0553 };
0554
0555 #if defined(BOOST_EMBTC)
0556
0557     template< typename charT, class traitsT >
0558     inline std::basic_ostream< charT, traitsT > &
0559     operator<<( std::basic_ostream< charT, traitsT > & os, fiber const& other) {
0560         if ( nullptr != other.ptr_) {
0561             return os << other.ptr_;
0562         } else {
0563             return os << "{not-a-context}";
0564         }
0565     }
0566
0567 #endif
0568
0569 inline
0570 void swap( fiber & l, fiber & r) noexcept {
0571     l.swap( r);
0572 }
0573
0574 typedef fiber fiber_context;
0575
0576 }}
0577
0578 #ifdef BOOST_HAS_ABI_HEADERS
0579 # include BOOST_ABI_SUFFIX
0580 #endif
0581
0582 #endif // BOOST_CONTEXT_FIBER_H