oneapi/tbb/spin_rw_mutex.h

0001 /*
0002     Copyright (c) 2005-2021 Intel Corporation
0003
0004     Licensed under the Apache License, Version 2.0 (the "License");
0005     you may not use this file except in compliance with the License.
0006     You may obtain a copy of the License at
0007
0008         http://www.apache.org/licenses/LICENSE-2.0
0009
0010     Unless required by applicable law or agreed to in writing, software
0011     distributed under the License is distributed on an "AS IS" BASIS,
0012     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0013     See the License for the specific language governing permissions and
0014     limitations under the License.
0015 */
0016
0017 #ifndef __TBB_spin_rw_mutex_H
0018 #define __TBB_spin_rw_mutex_H
0019
0020 #include "detail/_namespace_injection.h"
0021 #include "detail/_mutex_common.h"
0022
0023 #include "profiling.h"
0024
0025 #include "detail/_assert.h"
0026 #include "detail/_utils.h"
0027 #include "detail/_scoped_lock.h"
0028
0029 #include <atomic>
0030
0031 namespace tbb {
0032 namespace detail {
0033 namespace d1 {
0034
0035 #if __TBB_TSX_INTRINSICS_PRESENT
0036 class rtm_rw_mutex;
0037 #endif
0038
0039 //! Fast, unfair, spinning reader-writer lock with backoff and writer-preference
0040 /** @ingroup synchronization */
0041 class spin_rw_mutex {
0042 public:
0043     //! Constructors
0044     spin_rw_mutex() noexcept : m_state(0) {
0045        create_itt_sync(this, "tbb::spin_rw_mutex", "");
0046     }
0047
0048     //! Destructor
0049     ~spin_rw_mutex() {
0050         __TBB_ASSERT(!m_state, "destruction of an acquired mutex");
0051     }
0052
0053     //! No Copy
0054     spin_rw_mutex(const spin_rw_mutex&) = delete;
0055     spin_rw_mutex& operator=(const spin_rw_mutex&) = delete;
0056
0057     using scoped_lock = rw_scoped_lock<spin_rw_mutex>;
0058
0059     //! Mutex traits
0060     static constexpr bool is_rw_mutex = true;
0061     static constexpr bool is_recursive_mutex = false;
0062     static constexpr bool is_fair_mutex = false;
0063
0064     //! Acquire lock
0065     void lock() {
0066         call_itt_notify(prepare, this);
0067         for (atomic_backoff backoff; ; backoff.pause()) {
0068             state_type s = m_state.load(std::memory_order_relaxed);
0069             if (!(s & BUSY)) { // no readers, no writers
0070                 if (m_state.compare_exchange_strong(s, WRITER))
0071                     break; // successfully stored writer flag
0072                 backoff.reset(); // we could be very close to complete op.
0073             } else if (!(s & WRITER_PENDING)) { // no pending writers
0074                 m_state |= WRITER_PENDING;
0075             }
0076         }
0077         call_itt_notify(acquired, this);
0078     }
0079
0080     //! Try acquiring lock (non-blocking)
0081     /** Return true if lock acquired; false otherwise. */
0082     bool try_lock() {
0083         // for a writer: only possible to acquire if no active readers or writers
0084         state_type s = m_state.load(std::memory_order_relaxed);
0085         if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101
0086             if (m_state.compare_exchange_strong(s, WRITER)) {
0087                 call_itt_notify(acquired, this);
0088                 return true; // successfully stored writer flag
0089             }
0090         }
0091         return false;
0092     }
0093
0094     //! Release lock
0095     void unlock() {
0096         call_itt_notify(releasing, this);
0097         m_state &= READERS;
0098     }
0099
0100     //! Lock shared ownership mutex
0101     void lock_shared() {
0102         call_itt_notify(prepare, this);
0103         for (atomic_backoff b; ; b.pause()) {
0104             state_type s = m_state.load(std::memory_order_relaxed);
0105             if (!(s & (WRITER | WRITER_PENDING))) { // no writer or write requests
0106                 state_type prev_state = m_state.fetch_add(ONE_READER);
0107                 if (!(prev_state & WRITER)) {
0108                     break; // successfully stored increased number of readers
0109                 }
0110                 // writer got there first, undo the increment
0111                 m_state -= ONE_READER;
0112             }
0113         }
0114         call_itt_notify(acquired, this);
0115         __TBB_ASSERT(m_state & READERS, "invalid state of a read lock: no readers");
0116     }
0117
0118     //! Try lock shared ownership mutex
0119     bool try_lock_shared() {
0120         // for a reader: acquire if no active or waiting writers
0121         state_type s = m_state.load(std::memory_order_relaxed);
0122         if (!(s & (WRITER | WRITER_PENDING))) { // no writers
0123             state_type prev_state = m_state.fetch_add(ONE_READER);
0124             if (!(prev_state & WRITER)) {  // got the lock
0125                 call_itt_notify(acquired, this);
0126                 return true; // successfully stored increased number of readers
0127             }
0128             // writer got there first, undo the increment
0129             m_state -= ONE_READER;
0130         }
0131         return false;
0132     }
0133
0134     //! Unlock shared ownership mutex
0135     void unlock_shared() {
0136         __TBB_ASSERT(m_state & READERS, "invalid state of a read lock: no readers");
0137         call_itt_notify(releasing, this);
0138         m_state -= ONE_READER;
0139     }
0140
0141 protected:
0142     /** Internal non ISO C++ standard API **/
0143     //! This API is used through the scoped_lock class
0144
0145     //! Upgrade reader to become a writer.
0146     /** Returns whether the upgrade happened without releasing and re-acquiring the lock */
0147     bool upgrade() {
0148         state_type s = m_state.load(std::memory_order_relaxed);
0149         __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers ");
0150         // Check and set writer-pending flag.
0151         // Required conditions: either no pending writers, or we are the only reader
0152         // (with multiple readers and pending writer, another upgrade could have been requested)
0153         while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) {
0154             if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) {
0155                 atomic_backoff backoff;
0156                 while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) backoff.pause();
0157                 __TBB_ASSERT((m_state & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), "invalid state when upgrading to writer");
0158                 // Both new readers and writers are blocked at this time
0159                 m_state -= (ONE_READER + WRITER_PENDING);
0160                 return true; // successfully upgraded
0161             }
0162         }
0163         // Slow reacquire
0164         unlock_shared();
0165         lock();
0166         return false;
0167     }
0168
0169     //! Downgrade writer to a reader
0170     void downgrade() {
0171         call_itt_notify(releasing, this);
0172         m_state += (ONE_READER - WRITER);
0173         __TBB_ASSERT(m_state & READERS, "invalid state after downgrade: no readers");
0174     }
0175
0176     using state_type = std::intptr_t;
0177     static constexpr state_type WRITER = 1;
0178     static constexpr state_type WRITER_PENDING = 2;
0179     static constexpr state_type READERS = ~(WRITER | WRITER_PENDING);
0180     static constexpr state_type ONE_READER = 4;
0181     static constexpr state_type BUSY = WRITER | READERS;
0182     friend scoped_lock;
0183     //! State of lock
0184     /** Bit 0 = writer is holding lock
0185         Bit 1 = request by a writer to acquire lock (hint to readers to wait)
0186         Bit 2..N = number of readers holding lock */
0187     std::atomic<state_type> m_state;
0188 }; // class spin_rw_mutex
0189
0190 #if TBB_USE_PROFILING_TOOLS
0191 inline void set_name(spin_rw_mutex& obj, const char* name) {
0192     itt_set_sync_name(&obj, name);
0193 }
0194 #if (_WIN32||_WIN64)
0195 inline void set_name(spin_rw_mutex& obj, const wchar_t* name) {
0196     itt_set_sync_name(&obj, name);
0197 }
0198 #endif // WIN
0199 #else
0200 inline void set_name(spin_rw_mutex&, const char*) {}
0201 #if (_WIN32||_WIN64)
0202 inline void set_name(spin_rw_mutex&, const wchar_t*) {}
0203 #endif // WIN
0204 #endif
0205 } // namespace d1
0206 } // namespace detail
0207
0208 inline namespace v1 {
0209 using detail::d1::spin_rw_mutex;
0210 } // namespace v1
0211 namespace profiling {
0212     using detail::d1::set_name;
0213 }
0214 } // namespace tbb
0215
0216 #include "detail/_rtm_rw_mutex.h"
0217
0218 namespace tbb {
0219 inline namespace v1 {
0220 #if __TBB_TSX_INTRINSICS_PRESENT
0221     using speculative_spin_rw_mutex = detail::d1::rtm_rw_mutex;
0222 #else
0223     using speculative_spin_rw_mutex = detail::d1::spin_rw_mutex;
0224 #endif
0225 }
0226 }
0227
0228 #endif /* __TBB_spin_rw_mutex_H */
0229