oneapi/tbb/rw_mutex.h

0001 /*
0002     Copyright (c) 2021 Intel Corporation
0003
0004     Licensed under the Apache License, Version 2.0 (the "License");
0005     you may not use this file except in compliance with the License.
0006     You may obtain a copy of the License at
0007
0008         http://www.apache.org/licenses/LICENSE-2.0
0009
0010     Unless required by applicable law or agreed to in writing, software
0011     distributed under the License is distributed on an "AS IS" BASIS,
0012     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0013     See the License for the specific language governing permissions and
0014     limitations under the License.
0015 */
0016
0017 #ifndef __TBB_rw_mutex_H
0018 #define __TBB_rw_mutex_H
0019
0020 #include "detail/_namespace_injection.h"
0021 #include "detail/_utils.h"
0022 #include "detail/_waitable_atomic.h"
0023 #include "detail/_scoped_lock.h"
0024 #include "detail/_mutex_common.h"
0025 #include "profiling.h"
0026
0027 namespace tbb {
0028 namespace detail {
0029 namespace d1 {
0030
0031 class rw_mutex {
0032 public:
0033     //! Constructors
0034     rw_mutex() noexcept : m_state(0) {
0035        create_itt_sync(this, "tbb::rw_mutex", "");
0036     }
0037
0038     //! Destructor
0039     ~rw_mutex() {
0040         __TBB_ASSERT(!m_state.load(std::memory_order_relaxed), "destruction of an acquired mutex");
0041     }
0042
0043     //! No Copy
0044     rw_mutex(const rw_mutex&) = delete;
0045     rw_mutex& operator=(const rw_mutex&) = delete;
0046
0047     using scoped_lock = rw_scoped_lock<rw_mutex>;
0048
0049     //! Mutex traits
0050     static constexpr bool is_rw_mutex = true;
0051     static constexpr bool is_recursive_mutex = false;
0052     static constexpr bool is_fair_mutex = false;
0053
0054     //! Acquire lock
0055     void lock() {
0056         call_itt_notify(prepare, this);
0057         while (!try_lock()) {
0058             if (!(m_state.load(std::memory_order_relaxed) & WRITER_PENDING)) { // no pending writers
0059                 m_state |= WRITER_PENDING;
0060             }
0061
0062             auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & BUSY); };
0063             adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT);
0064         }
0065
0066         call_itt_notify(acquired, this);
0067     }
0068
0069     //! Try acquiring lock (non-blocking)
0070     /** Return true if lock acquired; false otherwise. */
0071     bool try_lock() {
0072         // for a writer: only possible to acquire if no active readers or writers
0073         // Use relaxed memory fence is OK here because
0074         // Acquire memory fence guaranteed by compare_exchange_strong()
0075         state_type s = m_state.load(std::memory_order_relaxed);
0076         if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101
0077             if (m_state.compare_exchange_strong(s, WRITER)) {
0078                 call_itt_notify(acquired, this);
0079                 return true; // successfully stored writer flag
0080             }
0081         }
0082         return false;
0083     }
0084
0085     //! Release lock
0086     void unlock() {
0087         call_itt_notify(releasing, this);
0088         state_type curr_state = (m_state &= READERS | WRITER_PENDING); // Returns current state
0089
0090         if (curr_state & WRITER_PENDING) {
0091             r1::notify_by_address(this, WRITER_CONTEXT);
0092         } else {
0093             // It's possible that WRITER sleeps without WRITER_PENDING,
0094             // because other thread might clear this bit at upgrade()
0095             r1::notify_by_address_all(this);
0096         }
0097     }
0098
0099     //! Lock shared ownership mutex
0100     void lock_shared() {
0101         call_itt_notify(prepare, this);
0102         while (!try_lock_shared()) {
0103             state_type has_writer = WRITER | WRITER_PENDING;
0104             auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & has_writer); };
0105             adaptive_wait_on_address(this, wakeup_condition, READER_CONTEXT);
0106         }
0107         __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers");
0108     }
0109
0110     //! Try lock shared ownership mutex
0111     bool try_lock_shared() {
0112         // for a reader: acquire if no active or waiting writers
0113         // Use relaxed memory fence is OK here because
0114         // Acquire memory fence guaranteed by fetch_add()
0115         state_type has_writer = WRITER | WRITER_PENDING;
0116         if (!(m_state.load(std::memory_order_relaxed) & has_writer)) {
0117             if (m_state.fetch_add(ONE_READER) & has_writer) {
0118                 m_state -= ONE_READER;
0119                 r1::notify_by_address(this, WRITER_CONTEXT);
0120             } else {
0121                 call_itt_notify(acquired, this);
0122                 return true; // successfully stored increased number of readers
0123             }
0124         }
0125         return false;
0126     }
0127
0128     //! Unlock shared ownership mutex
0129     void unlock_shared() {
0130         __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers");
0131         call_itt_notify(releasing, this);
0132
0133         state_type curr_state = (m_state -= ONE_READER); // Returns current state
0134
0135         if (curr_state & (WRITER_PENDING)) {
0136             r1::notify_by_address(this, WRITER_CONTEXT);
0137         } else {
0138             // It's possible that WRITER sleeps without WRITER_PENDING,
0139             // because other thread might clear this bit at upgrade()
0140             r1::notify_by_address_all(this);
0141         }
0142     }
0143
0144 private:
0145     /** Internal non ISO C++ standard API **/
0146     //! This API is used through the scoped_lock class
0147
0148     //! Upgrade reader to become a writer.
0149     /** Returns whether the upgrade happened without releasing and re-acquiring the lock */
0150     bool upgrade() {
0151         state_type s = m_state.load(std::memory_order_relaxed);
0152         __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers ");
0153         // Check and set writer-pending flag.
0154         // Required conditions: either no pending writers, or we are the only reader
0155         // (with multiple readers and pending writer, another upgrade could have been requested)
0156         while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) {
0157             if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) {
0158                 auto wakeup_condition = [&] { return (m_state.load(std::memory_order_relaxed) & READERS) == ONE_READER; };
0159                 while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) {
0160                     adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT);
0161                 }
0162
0163                 __TBB_ASSERT((m_state.load(std::memory_order_relaxed) & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER),
0164                              "invalid state when upgrading to writer");
0165                 // Both new readers and writers are blocked at this time
0166                 m_state -= (ONE_READER + WRITER_PENDING);
0167                 return true; // successfully upgraded
0168             }
0169         }
0170         // Slow reacquire
0171         unlock_shared();
0172         lock();
0173         return false;
0174     }
0175
0176     //! Downgrade writer to a reader
0177     void downgrade() {
0178         __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & WRITER, nullptr),
0179         call_itt_notify(releasing, this);
0180         m_state += (ONE_READER - WRITER);
0181
0182         if (!(m_state & WRITER_PENDING)) {
0183             r1::notify_by_address(this, READER_CONTEXT);
0184         }
0185
0186         __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state after downgrade: no readers");
0187     }
0188
0189     using state_type = std::intptr_t;
0190     static constexpr state_type WRITER = 1;
0191     static constexpr state_type WRITER_PENDING = 2;
0192     static constexpr state_type READERS = ~(WRITER | WRITER_PENDING);
0193     static constexpr state_type ONE_READER = 4;
0194     static constexpr state_type BUSY = WRITER | READERS;
0195
0196     using context_type = std::uintptr_t;
0197     static constexpr context_type WRITER_CONTEXT = 0;
0198     static constexpr context_type READER_CONTEXT = 1;
0199     friend scoped_lock;
0200     //! State of lock
0201     /** Bit 0 = writer is holding lock
0202         Bit 1 = request by a writer to acquire lock (hint to readers to wait)
0203         Bit 2..N = number of readers holding lock */
0204     std::atomic<state_type> m_state;
0205 }; // class rw_mutex
0206
0207 } // namespace d1
0208 } // namespace detail
0209
0210 inline namespace v1 {
0211 using detail::d1::rw_mutex;
0212 } // namespace v1
0213
0214 } // namespace tbb
0215
0216 #endif // __TBB_rw_mutex_H