cppgc/internal/member-storage.h

0001 // Copyright 2022 the V8 project authors. All rights reserved.
0002 // Use of this source code is governed by a BSD-style license that can be
0003 // found in the LICENSE file.
0004
0005 #ifndef INCLUDE_CPPGC_INTERNAL_MEMBER_STORAGE_H_
0006 #define INCLUDE_CPPGC_INTERNAL_MEMBER_STORAGE_H_
0007
0008 #include <atomic>
0009 #include <cstddef>
0010 #include <type_traits>
0011
0012 #include "cppgc/internal/api-constants.h"
0013 #include "cppgc/internal/caged-heap.h"
0014 #include "cppgc/internal/logging.h"
0015 #include "cppgc/sentinel-pointer.h"
0016 #include "v8config.h"  // NOLINT(build/include_directory)
0017
0018 namespace cppgc {
0019 namespace internal {
0020
0021 enum class WriteBarrierSlotType {
0022   kCompressed,
0023   kUncompressed,
0024 };
0025
0026 #if defined(CPPGC_POINTER_COMPRESSION)
0027
0028 #if defined(__clang__)
0029 // Attribute const allows the compiler to assume that CageBaseGlobal::g_base_
0030 // doesn't change (e.g. across calls) and thereby avoid redundant loads.
0031 #define CPPGC_CONST __attribute__((const))
0032 #define CPPGC_REQUIRE_CONSTANT_INIT \
0033   __attribute__((require_constant_initialization))
0034 #else  // defined(__clang__)
0035 #define CPPGC_CONST
0036 #define CPPGC_REQUIRE_CONSTANT_INIT
0037 #endif  // defined(__clang__)
0038
0039 class V8_EXPORT CageBaseGlobal final {
0040  public:
0041   V8_INLINE CPPGC_CONST static uintptr_t Get() {
0042     CPPGC_DCHECK(IsBaseConsistent());
0043     return g_base_.base;
0044   }
0045
0046   V8_INLINE CPPGC_CONST static bool IsSet() {
0047     CPPGC_DCHECK(IsBaseConsistent());
0048     return (g_base_.base & ~kLowerHalfWordMask) != 0;
0049   }
0050
0051  private:
0052   // We keep the lower halfword as ones to speed up decompression.
0053   static constexpr uintptr_t kLowerHalfWordMask =
0054       (api_constants::kCagedHeapReservationAlignment - 1);
0055
0056   static union alignas(api_constants::kCachelineSize) Base {
0057     uintptr_t base;
0058     char cache_line[api_constants::kCachelineSize];
0059   } g_base_ CPPGC_REQUIRE_CONSTANT_INIT;
0060
0061   CageBaseGlobal() = delete;
0062
0063   V8_INLINE static bool IsBaseConsistent() {
0064     return kLowerHalfWordMask == (g_base_.base & kLowerHalfWordMask);
0065   }
0066
0067   friend class CageBaseGlobalUpdater;
0068 };
0069
0070 #undef CPPGC_REQUIRE_CONSTANT_INIT
0071 #undef CPPGC_CONST
0072
0073 class V8_TRIVIAL_ABI CompressedPointer final {
0074  public:
0075   struct AtomicInitializerTag {};
0076
0077   using IntegralType = uint32_t;
0078   static constexpr auto kWriteBarrierSlotType =
0079       WriteBarrierSlotType::kCompressed;
0080
0081   V8_INLINE CompressedPointer() : value_(0u) {}
0082   V8_INLINE explicit CompressedPointer(const void* value,
0083                                        AtomicInitializerTag) {
0084     StoreAtomic(value);
0085   }
0086   V8_INLINE explicit CompressedPointer(const void* ptr)
0087       : value_(Compress(ptr)) {}
0088   V8_INLINE explicit CompressedPointer(std::nullptr_t) : value_(0u) {}
0089   V8_INLINE explicit CompressedPointer(SentinelPointer)
0090       : value_(kCompressedSentinel) {}
0091
0092   V8_INLINE const void* Load() const { return Decompress(value_); }
0093   V8_INLINE const void* LoadAtomic() const {
0094     return Decompress(
0095         reinterpret_cast<const std::atomic<IntegralType>&>(value_).load(
0096             std::memory_order_relaxed));
0097   }
0098
0099   V8_INLINE void Store(const void* ptr) { value_ = Compress(ptr); }
0100   V8_INLINE void StoreAtomic(const void* value) {
0101     reinterpret_cast<std::atomic<IntegralType>&>(value_).store(
0102         Compress(value), std::memory_order_relaxed);
0103   }
0104
0105   V8_INLINE void Clear() { value_ = 0u; }
0106   V8_INLINE bool IsCleared() const { return !value_; }
0107
0108   V8_INLINE bool IsSentinel() const { return value_ == kCompressedSentinel; }
0109
0110   V8_INLINE uint32_t GetAsInteger() const { return value_; }
0111
0112   V8_INLINE friend bool operator==(CompressedPointer a, CompressedPointer b) {
0113     return a.value_ == b.value_;
0114   }
0115   V8_INLINE friend bool operator!=(CompressedPointer a, CompressedPointer b) {
0116     return a.value_ != b.value_;
0117   }
0118   V8_INLINE friend bool operator<(CompressedPointer a, CompressedPointer b) {
0119     return a.value_ < b.value_;
0120   }
0121   V8_INLINE friend bool operator<=(CompressedPointer a, CompressedPointer b) {
0122     return a.value_ <= b.value_;
0123   }
0124   V8_INLINE friend bool operator>(CompressedPointer a, CompressedPointer b) {
0125     return a.value_ > b.value_;
0126   }
0127   V8_INLINE friend bool operator>=(CompressedPointer a, CompressedPointer b) {
0128     return a.value_ >= b.value_;
0129   }
0130
0131   static V8_INLINE IntegralType Compress(const void* ptr) {
0132     static_assert(SentinelPointer::kSentinelValue ==
0133                       1 << api_constants::kPointerCompressionShift,
0134                   "The compression scheme relies on the sentinel encoded as 1 "
0135                   "<< kPointerCompressionShift");
0136     static constexpr size_t kGigaCageMask =
0137         ~(api_constants::kCagedHeapReservationAlignment - 1);
0138     static constexpr size_t kPointerCompressionShiftMask =
0139         (1 << api_constants::kPointerCompressionShift) - 1;
0140
0141     CPPGC_DCHECK(CageBaseGlobal::IsSet());
0142     const uintptr_t base = CageBaseGlobal::Get();
0143     CPPGC_DCHECK(!ptr || ptr == kSentinelPointer ||
0144                  (base & kGigaCageMask) ==
0145                      (reinterpret_cast<uintptr_t>(ptr) & kGigaCageMask));
0146     CPPGC_DCHECK(
0147         (reinterpret_cast<uintptr_t>(ptr) & kPointerCompressionShiftMask) == 0);
0148
0149     const auto uptr = reinterpret_cast<uintptr_t>(ptr);
0150     // Shift the pointer and truncate.
0151     auto compressed = static_cast<IntegralType>(
0152         uptr >> api_constants::kPointerCompressionShift);
0153     // Normal compressed pointers must have the MSB set. This is guaranteed by
0154     // the cage alignment.
0155     CPPGC_DCHECK((!compressed || compressed == kCompressedSentinel) ||
0156                  (compressed & (1 << 31)));
0157     return compressed;
0158   }
0159
0160   static V8_INLINE void* Decompress(IntegralType ptr) {
0161     CPPGC_DCHECK(CageBaseGlobal::IsSet());
0162     const uintptr_t base = CageBaseGlobal::Get();
0163     return Decompress(ptr, base);
0164   }
0165
0166   static V8_INLINE void* Decompress(IntegralType ptr, uintptr_t base) {
0167     CPPGC_DCHECK(CageBaseGlobal::IsSet());
0168     CPPGC_DCHECK(base == CageBaseGlobal::Get());
0169     // Sign-extend compressed pointer to full width. This ensure that normal
0170     // pointers have only 1s in the base part of the address. It's also
0171     // important to shift the unsigned value, as otherwise it would result in
0172     // undefined behavior.
0173     const uint64_t mask = static_cast<uint64_t>(static_cast<int32_t>(ptr))
0174                           << api_constants::kPointerCompressionShift;
0175     // Set the base part of the address for normal compressed pointers. Note
0176     // that nullptr and the sentinel value do not have 1s in the base part and
0177     // remain as-is in this operation.
0178     return reinterpret_cast<void*>(mask & base);
0179   }
0180
0181   // For a given memory `address`, this method iterates all possible pointers
0182   // that can be reasonably recovered with the current compression scheme and
0183   // passes them to `callback`.
0184   template <typename Callback>
0185   static V8_INLINE void VisitPossiblePointers(const void* address,
0186                                               Callback callback);
0187
0188  private:
0189   static constexpr IntegralType kCompressedSentinel =
0190       SentinelPointer::kSentinelValue >>
0191       api_constants::kPointerCompressionShift;
0192   // All constructors initialize `value_`. Do not add a default value here as it
0193   // results in a non-atomic write on some builds, even when the atomic version
0194   // of the constructor is used.
0195   IntegralType value_;
0196 };
0197
0198 template <typename Callback>
0199 // static
0200 void CompressedPointer::VisitPossiblePointers(const void* address,
0201                                               Callback callback) {
0202   const uintptr_t base = CageBaseGlobal::Get();
0203   CPPGC_DCHECK(base);
0204   // We may have random compressed pointers on stack (e.g. due to inlined
0205   // collections). These could be present in both halfwords.
0206   const uint32_t compressed_low =
0207       static_cast<uint32_t>(reinterpret_cast<uintptr_t>(address));
0208   callback(CompressedPointer::Decompress(compressed_low, base));
0209   const uint32_t compressed_high = static_cast<uint32_t>(
0210       reinterpret_cast<uintptr_t>(address) >> (sizeof(uint32_t) * CHAR_BIT));
0211   callback(CompressedPointer::Decompress(compressed_high, base));
0212   // Iterate possible intermediate values, see `Decompress()`. The intermediate
0213   // value of decompressing is a 64-bit value where 35 bits are the offset. We
0214   // don't assume sign extension is stored and recover that part.
0215   //
0216   // Note that this case conveniently also recovers the full pointer.
0217   static constexpr uintptr_t kBitForIntermediateValue =
0218       (sizeof(uint32_t) * CHAR_BIT) + api_constants::kPointerCompressionShift;
0219   static constexpr uintptr_t kSignExtensionMask =
0220       ~((uintptr_t{1} << kBitForIntermediateValue) - 1);
0221   const uintptr_t intermediate_sign_extended =
0222       reinterpret_cast<uintptr_t>(address) | kSignExtensionMask;
0223   callback(reinterpret_cast<void*>(intermediate_sign_extended & base));
0224 }
0225
0226 #endif  // defined(CPPGC_POINTER_COMPRESSION)
0227
0228 class V8_TRIVIAL_ABI RawPointer final {
0229  public:
0230   struct AtomicInitializerTag {};
0231
0232   using IntegralType = uintptr_t;
0233   static constexpr auto kWriteBarrierSlotType =
0234       WriteBarrierSlotType::kUncompressed;
0235
0236   V8_INLINE RawPointer() : ptr_(nullptr) {}
0237   V8_INLINE explicit RawPointer(const void* ptr, AtomicInitializerTag) {
0238     StoreAtomic(ptr);
0239   }
0240   V8_INLINE explicit RawPointer(const void* ptr) : ptr_(ptr) {}
0241
0242   V8_INLINE const void* Load() const { return ptr_; }
0243   V8_INLINE const void* LoadAtomic() const {
0244     return reinterpret_cast<const std::atomic<const void*>&>(ptr_).load(
0245         std::memory_order_relaxed);
0246   }
0247
0248   V8_INLINE void Store(const void* ptr) { ptr_ = ptr; }
0249   V8_INLINE void StoreAtomic(const void* ptr) {
0250     reinterpret_cast<std::atomic<const void*>&>(ptr_).store(
0251         ptr, std::memory_order_relaxed);
0252   }
0253
0254   V8_INLINE void Clear() { ptr_ = nullptr; }
0255   V8_INLINE bool IsCleared() const { return !ptr_; }
0256
0257   V8_INLINE bool IsSentinel() const { return ptr_ == kSentinelPointer; }
0258
0259   V8_INLINE uintptr_t GetAsInteger() const {
0260     return reinterpret_cast<uintptr_t>(ptr_);
0261   }
0262
0263   V8_INLINE friend bool operator==(RawPointer a, RawPointer b) {
0264     return a.ptr_ == b.ptr_;
0265   }
0266   V8_INLINE friend bool operator!=(RawPointer a, RawPointer b) {
0267     return a.ptr_ != b.ptr_;
0268   }
0269   V8_INLINE friend bool operator<(RawPointer a, RawPointer b) {
0270     return a.ptr_ < b.ptr_;
0271   }
0272   V8_INLINE friend bool operator<=(RawPointer a, RawPointer b) {
0273     return a.ptr_ <= b.ptr_;
0274   }
0275   V8_INLINE friend bool operator>(RawPointer a, RawPointer b) {
0276     return a.ptr_ > b.ptr_;
0277   }
0278   V8_INLINE friend bool operator>=(RawPointer a, RawPointer b) {
0279     return a.ptr_ >= b.ptr_;
0280   }
0281
0282   template <typename Callback>
0283   static V8_INLINE void VisitPossiblePointers(const void* address,
0284                                               Callback callback) {
0285     // Pass along the full pointer.
0286     return callback(const_cast<void*>(address));
0287   }
0288
0289  private:
0290   // All constructors initialize `ptr_`. Do not add a default value here as it
0291   // results in a non-atomic write on some builds, even when the atomic version
0292   // of the constructor is used.
0293   const void* ptr_;
0294 };
0295
0296 #if defined(CPPGC_POINTER_COMPRESSION)
0297 using DefaultMemberStorage = CompressedPointer;
0298 #else   // !defined(CPPGC_POINTER_COMPRESSION)
0299 using DefaultMemberStorage = RawPointer;
0300 #endif  // !defined(CPPGC_POINTER_COMPRESSION)
0301
0302 }  // namespace internal
0303 }  // namespace cppgc
0304
0305 #endif  // INCLUDE_CPPGC_INTERNAL_MEMBER_STORAGE_H_