absl/base/call_once.h

0001 // Copyright 2017 The Abseil Authors.
0002 //
0003 // Licensed under the Apache License, Version 2.0 (the "License");
0004 // you may not use this file except in compliance with the License.
0005 // You may obtain a copy of the License at
0006 //
0007 //      https://www.apache.org/licenses/LICENSE-2.0
0008 //
0009 // Unless required by applicable law or agreed to in writing, software
0010 // distributed under the License is distributed on an "AS IS" BASIS,
0011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0012 // See the License for the specific language governing permissions and
0013 // limitations under the License.
0014 //
0015 // -----------------------------------------------------------------------------
0016 // File: call_once.h
0017 // -----------------------------------------------------------------------------
0018 //
0019 // This header file provides an Abseil version of `std::call_once` for invoking
0020 // a given function at most once, across all threads. This Abseil version is
0021 // faster than the C++11 version and incorporates the C++17 argument-passing
0022 // fix, so that (for example) non-const references may be passed to the invoked
0023 // function.
0024
0025 #ifndef ABSL_BASE_CALL_ONCE_H_
0026 #define ABSL_BASE_CALL_ONCE_H_
0027
0028 #include <algorithm>
0029 #include <atomic>
0030 #include <cstdint>
0031 #include <type_traits>
0032 #include <utility>
0033
0034 #include "absl/base/internal/invoke.h"
0035 #include "absl/base/internal/low_level_scheduling.h"
0036 #include "absl/base/internal/raw_logging.h"
0037 #include "absl/base/internal/scheduling_mode.h"
0038 #include "absl/base/internal/spinlock_wait.h"
0039 #include "absl/base/macros.h"
0040 #include "absl/base/nullability.h"
0041 #include "absl/base/optimization.h"
0042 #include "absl/base/port.h"
0043
0044 namespace absl {
0045 ABSL_NAMESPACE_BEGIN
0046
0047 class once_flag;
0048
0049 namespace base_internal {
0050 absl::Nonnull<std::atomic<uint32_t>*> ControlWord(
0051     absl::Nonnull<absl::once_flag*> flag);
0052 }  // namespace base_internal
0053
0054 // call_once()
0055 //
0056 // For all invocations using a given `once_flag`, invokes a given `fn` exactly
0057 // once across all threads. The first call to `call_once()` with a particular
0058 // `once_flag` argument (that does not throw an exception) will run the
0059 // specified function with the provided `args`; other calls with the same
0060 // `once_flag` argument will not run the function, but will wait
0061 // for the provided function to finish running (if it is still running).
0062 //
0063 // This mechanism provides a safe, simple, and fast mechanism for one-time
0064 // initialization in a multi-threaded process.
0065 //
0066 // Example:
0067 //
0068 // class MyInitClass {
0069 //  public:
0070 //  ...
0071 //  mutable absl::once_flag once_;
0072 //
0073 //  MyInitClass* init() const {
0074 //    absl::call_once(once_, &MyInitClass::Init, this);
0075 //    return ptr_;
0076 //  }
0077 //
0078 template <typename Callable, typename... Args>
0079 void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args);
0080
0081 // once_flag
0082 //
0083 // Objects of this type are used to distinguish calls to `call_once()` and
0084 // ensure the provided function is only invoked once across all threads. This
0085 // type is not copyable or movable. However, it has a `constexpr`
0086 // constructor, and is safe to use as a namespace-scoped global variable.
0087 class once_flag {
0088  public:
0089   constexpr once_flag() : control_(0) {}
0090   once_flag(const once_flag&) = delete;
0091   once_flag& operator=(const once_flag&) = delete;
0092
0093  private:
0094   friend absl::Nonnull<std::atomic<uint32_t>*> base_internal::ControlWord(
0095       absl::Nonnull<once_flag*> flag);
0096   std::atomic<uint32_t> control_;
0097 };
0098
0099 //------------------------------------------------------------------------------
0100 // End of public interfaces.
0101 // Implementation details follow.
0102 //------------------------------------------------------------------------------
0103
0104 namespace base_internal {
0105
0106 // Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to
0107 // initialize entities used by the scheduler implementation.
0108 template <typename Callable, typename... Args>
0109 void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn,
0110                       Args&&... args);
0111
0112 // Disables scheduling while on stack when scheduling mode is non-cooperative.
0113 // No effect for cooperative scheduling modes.
0114 class SchedulingHelper {
0115  public:
0116   explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) {
0117     if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
0118       guard_result_ = base_internal::SchedulingGuard::DisableRescheduling();
0119     }
0120   }
0121
0122   ~SchedulingHelper() {
0123     if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
0124       base_internal::SchedulingGuard::EnableRescheduling(guard_result_);
0125     }
0126   }
0127
0128  private:
0129   base_internal::SchedulingMode mode_;
0130   bool guard_result_ = false;
0131 };
0132
0133 // Bit patterns for call_once state machine values.  Internal implementation
0134 // detail, not for use by clients.
0135 //
0136 // The bit patterns are arbitrarily chosen from unlikely values, to aid in
0137 // debugging.  However, kOnceInit must be 0, so that a zero-initialized
0138 // once_flag will be valid for immediate use.
0139 enum {
0140   kOnceInit = 0,
0141   kOnceRunning = 0x65C2937B,
0142   kOnceWaiter = 0x05A308D2,
0143   // A very small constant is chosen for kOnceDone so that it fit in a single
0144   // compare with immediate instruction for most common ISAs.  This is verified
0145   // for x86, POWER and ARM.
0146   kOnceDone = 221,    // Random Number
0147 };
0148
0149 template <typename Callable, typename... Args>
0150 ABSL_ATTRIBUTE_NOINLINE void CallOnceImpl(
0151     absl::Nonnull<std::atomic<uint32_t>*> control,
0152     base_internal::SchedulingMode scheduling_mode, Callable&& fn,
0153     Args&&... args) {
0154 #ifndef NDEBUG
0155   {
0156     uint32_t old_control = control->load(std::memory_order_relaxed);
0157     if (old_control != kOnceInit &&
0158         old_control != kOnceRunning &&
0159         old_control != kOnceWaiter &&
0160         old_control != kOnceDone) {
0161       ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx",
0162                    static_cast<unsigned long>(old_control));  // NOLINT
0163     }
0164   }
0165 #endif  // NDEBUG
0166   static const base_internal::SpinLockWaitTransition trans[] = {
0167       {kOnceInit, kOnceRunning, true},
0168       {kOnceRunning, kOnceWaiter, false},
0169       {kOnceDone, kOnceDone, true}};
0170
0171   // Must do this before potentially modifying control word's state.
0172   base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode);
0173   // Short circuit the simplest case to avoid procedure call overhead.
0174   // The base_internal::SpinLockWait() call returns either kOnceInit or
0175   // kOnceDone. If it returns kOnceDone, it must have loaded the control word
0176   // with std::memory_order_acquire and seen a value of kOnceDone.
0177   uint32_t old_control = kOnceInit;
0178   if (control->compare_exchange_strong(old_control, kOnceRunning,
0179                                        std::memory_order_relaxed) ||
0180       base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans,
0181                                   scheduling_mode) == kOnceInit) {
0182     base_internal::invoke(std::forward<Callable>(fn),
0183                           std::forward<Args>(args)...);
0184     old_control =
0185         control->exchange(base_internal::kOnceDone, std::memory_order_release);
0186     if (old_control == base_internal::kOnceWaiter) {
0187       base_internal::SpinLockWake(control, true);
0188     }
0189   }  // else *control is already kOnceDone
0190 }
0191
0192 inline absl::Nonnull<std::atomic<uint32_t>*> ControlWord(
0193     absl::Nonnull<once_flag*> flag) {
0194   return &flag->control_;
0195 }
0196
0197 template <typename Callable, typename... Args>
0198 void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn,
0199                       Args&&... args) {
0200   std::atomic<uint32_t>* once = base_internal::ControlWord(flag);
0201   uint32_t s = once->load(std::memory_order_acquire);
0202   if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
0203     base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY,
0204                                 std::forward<Callable>(fn),
0205                                 std::forward<Args>(args)...);
0206   }
0207 }
0208
0209 }  // namespace base_internal
0210
0211 template <typename Callable, typename... Args>
0212 void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) {
0213   std::atomic<uint32_t>* once = base_internal::ControlWord(&flag);
0214   uint32_t s = once->load(std::memory_order_acquire);
0215   if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
0216     base_internal::CallOnceImpl(
0217         once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL,
0218         std::forward<Callable>(fn), std::forward<Args>(args)...);
0219   }
0220 }
0221
0222 ABSL_NAMESPACE_END
0223 }  // namespace absl
0224
0225 #endif  // ABSL_BASE_CALL_ONCE_H_