xsimd/memory/xsimd_aligned_allocator.hpp

0001 /***************************************************************************
0002  * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
0003  * Martin Renou                                                             *
0004  * Copyright (c) QuantStack                                                 *
0005  * Copyright (c) Serge Guelton                                              *
0006  *                                                                          *
0007  * Distributed under the terms of the BSD 3-Clause License.                 *
0008  *                                                                          *
0009  * The full license is in the file LICENSE, distributed with this software. *
0010  ****************************************************************************/
0011
0012 #ifndef XSIMD_ALIGNED_ALLOCATOR_HPP
0013 #define XSIMD_ALIGNED_ALLOCATOR_HPP
0014
0015 #include <algorithm>
0016 #include <cstddef>
0017 #include <utility>
0018 #ifdef _WIN32
0019 #include <malloc.h>
0020 #else
0021 #include <cstdlib>
0022 #endif
0023
0024 #include <cassert>
0025 #include <memory>
0026
0027 #include "../config/xsimd_arch.hpp"
0028
0029 namespace xsimd
0030 {
0031
0032     /**
0033      * @class aligned_allocator
0034      * @brief Allocator for aligned memory
0035      *
0036      * The aligned_allocator class template is an allocator that
0037      * performs memory allocation aligned by the specified value.
0038      *
0039      * @tparam T type of objects to allocate.
0040      * @tparam Align alignment in bytes.
0041      */
0042     template <class T, size_t Align>
0043     class aligned_allocator
0044     {
0045     public:
0046         using value_type = T;
0047         using pointer = T*;
0048         using const_pointer = const T*;
0049         using reference = T&;
0050         using const_reference = const T&;
0051         using size_type = size_t;
0052         using difference_type = ptrdiff_t;
0053
0054         static constexpr size_t alignment = Align;
0055
0056         template <class U>
0057         struct rebind
0058         {
0059             using other = aligned_allocator<U, Align>;
0060         };
0061
0062         XSIMD_INLINE aligned_allocator() noexcept;
0063         XSIMD_INLINE aligned_allocator(const aligned_allocator& rhs) noexcept;
0064
0065         template <class U>
0066         XSIMD_INLINE aligned_allocator(const aligned_allocator<U, Align>& rhs) noexcept;
0067
0068         XSIMD_INLINE ~aligned_allocator();
0069
0070         XSIMD_INLINE pointer address(reference) noexcept;
0071         XSIMD_INLINE const_pointer address(const_reference) const noexcept;
0072
0073         XSIMD_INLINE pointer allocate(size_type n, const void* hint = 0);
0074         XSIMD_INLINE void deallocate(pointer p, size_type n);
0075
0076         XSIMD_INLINE size_type max_size() const noexcept;
0077         XSIMD_INLINE size_type size_max() const noexcept;
0078
0079         template <class U, class... Args>
0080         XSIMD_INLINE void construct(U* p, Args&&... args);
0081
0082         template <class U>
0083         XSIMD_INLINE void destroy(U* p);
0084     };
0085
0086     template <class T1, size_t Align1, class T2, size_t Align2>
0087     XSIMD_INLINE bool operator==(const aligned_allocator<T1, Align1>& lhs,
0088                                  const aligned_allocator<T2, Align2>& rhs) noexcept;
0089
0090     template <class T1, size_t Align1, class T2, size_t Align2>
0091     XSIMD_INLINE bool operator!=(const aligned_allocator<T1, Align1>& lhs,
0092                                  const aligned_allocator<T2, Align2>& rhs) noexcept;
0093
0094     XSIMD_INLINE void* aligned_malloc(size_t size, size_t alignment);
0095     XSIMD_INLINE void aligned_free(void* ptr);
0096
0097     template <class T>
0098     XSIMD_INLINE size_t get_alignment_offset(const T* p, size_t size, size_t block_size);
0099
0100     /************************************
0101      * aligned_allocator implementation *
0102      ************************************/
0103
0104     /**
0105      * Default constructor.
0106      */
0107     template <class T, size_t A>
0108     XSIMD_INLINE aligned_allocator<T, A>::aligned_allocator() noexcept
0109     {
0110     }
0111
0112     /**
0113      * Copy constructor.
0114      */
0115     template <class T, size_t A>
0116     XSIMD_INLINE aligned_allocator<T, A>::aligned_allocator(const aligned_allocator&) noexcept
0117     {
0118     }
0119
0120     /**
0121      * Extended copy constructor.
0122      */
0123     template <class T, size_t A>
0124     template <class U>
0125     XSIMD_INLINE aligned_allocator<T, A>::aligned_allocator(const aligned_allocator<U, A>&) noexcept
0126     {
0127     }
0128
0129     /**
0130      * Destructor.
0131      */
0132     template <class T, size_t A>
0133     XSIMD_INLINE aligned_allocator<T, A>::~aligned_allocator()
0134     {
0135     }
0136
0137     /**
0138      * Returns the actual address of \c r even in presence of overloaded \c operator&.
0139      * @param r the object to acquire address of.
0140      * @return the actual address of \c r.
0141      */
0142     template <class T, size_t A>
0143     XSIMD_INLINE auto
0144     aligned_allocator<T, A>::address(reference r) noexcept -> pointer
0145     {
0146         return &r;
0147     }
0148
0149     /**
0150      * Returns the actual address of \c r even in presence of overloaded \c operator&.
0151      * @param r the object to acquire address of.
0152      * @return the actual address of \c r.
0153      */
0154     template <class T, size_t A>
0155     XSIMD_INLINE auto
0156     aligned_allocator<T, A>::address(const_reference r) const noexcept -> const_pointer
0157     {
0158         return &r;
0159     }
0160
0161     /**
0162      * Allocates <tt>n * sizeof(T)</tt> bytes of uninitialized memory, aligned by \c A.
0163      * The alignment may require some extra memory allocation.
0164      * @param n the number of objects to allocate storage for.
0165      * @param hint unused parameter provided for standard compliance.
0166      * @return a pointer to the first byte of a memory block suitably aligned and sufficient to
0167      * hold an array of \c n objects of type \c T.
0168      */
0169     template <class T, size_t A>
0170     XSIMD_INLINE auto
0171     aligned_allocator<T, A>::allocate(size_type n, const void*) -> pointer
0172     {
0173         pointer res = reinterpret_cast<pointer>(aligned_malloc(sizeof(T) * n, A));
0174 #if defined(_CPPUNWIND) || defined(__cpp_exceptions)
0175         if (res == nullptr)
0176             throw std::bad_alloc();
0177 #endif
0178         return res;
0179     }
0180
0181     /**
0182      * Deallocates the storage referenced by the pointer p, which must be a pointer obtained by
0183      * an earlier call to allocate(). The argument \c n must be equal to the first argument of the call
0184      * to allocate() that originally produced \c p; otherwise, the behavior is undefined.
0185      * @param p pointer obtained from allocate().
0186      * @param n number of objects earlier passed to allocate().
0187      */
0188     template <class T, size_t A>
0189     XSIMD_INLINE void aligned_allocator<T, A>::deallocate(pointer p, size_type)
0190     {
0191         aligned_free(p);
0192     }
0193
0194     /**
0195      * Returns the maximum theoretically possible value of \c n, for which the
0196      * call allocate(n, 0) could succeed.
0197      * @return the maximum supported allocated size.
0198      */
0199     template <class T, size_t A>
0200     XSIMD_INLINE auto
0201     aligned_allocator<T, A>::max_size() const noexcept -> size_type
0202     {
0203         return size_type(-1) / sizeof(T);
0204     }
0205
0206     /**
0207      * This method is deprecated, use max_size() instead
0208      */
0209     template <class T, size_t A>
0210     XSIMD_INLINE auto
0211     aligned_allocator<T, A>::size_max() const noexcept -> size_type
0212     {
0213         return size_type(-1) / sizeof(T);
0214     }
0215
0216     /**
0217      * Constructs an object of type \c T in allocated uninitialized memory
0218      * pointed to by \c p, using placement-new.
0219      * @param p pointer to allocated uninitialized memory.
0220      * @param args the constructor arguments to use.
0221      */
0222     template <class T, size_t A>
0223     template <class U, class... Args>
0224     XSIMD_INLINE void aligned_allocator<T, A>::construct(U* p, Args&&... args)
0225     {
0226         new ((void*)p) U(std::forward<Args>(args)...);
0227     }
0228
0229     /**
0230      * Calls the destructor of the object pointed to by \c p.
0231      * @param p pointer to the object that is going to be destroyed.
0232      */
0233     template <class T, size_t A>
0234     template <class U>
0235     XSIMD_INLINE void aligned_allocator<T, A>::destroy(U* p)
0236     {
0237         p->~U();
0238     }
0239
0240     /**
0241      * @defgroup allocator_comparison Comparison operators
0242      */
0243
0244     /**
0245      * @ingroup allocator_comparison
0246      * Compares two aligned memory allocator for equality. Since allocators
0247      * are stateless, return \c true iff <tt>A1 == A2</tt>.
0248      * @param lhs aligned_allocator to compare.
0249      * @param rhs aligned_allocator to compare.
0250      * @return true if the allocators have the same alignment.
0251      */
0252     template <class T1, size_t A1, class T2, size_t A2>
0253     XSIMD_INLINE bool operator==(const aligned_allocator<T1, A1>& lhs,
0254                                  const aligned_allocator<T2, A2>& rhs) noexcept
0255     {
0256         return lhs.alignment == rhs.alignment;
0257     }
0258
0259     /**
0260      * @ingroup allocator_comparison
0261      * Compares two aligned memory allocator for inequality. Since allocators
0262      * are stateless, return \c true iff <tt>A1 != A2</tt>.
0263      * @param lhs aligned_allocator to compare.
0264      * @param rhs aligned_allocator to compare.
0265      * @return true if the allocators have different alignments.
0266      */
0267     template <class T1, size_t A1, class T2, size_t A2>
0268     XSIMD_INLINE bool operator!=(const aligned_allocator<T1, A1>& lhs,
0269                                  const aligned_allocator<T2, A2>& rhs) noexcept
0270     {
0271         return !(lhs == rhs);
0272     }
0273
0274     /****************************************
0275      * aligned malloc / free implementation *
0276      ****************************************/
0277
0278     namespace detail
0279     {
0280         XSIMD_INLINE void* xaligned_malloc(size_t size, size_t alignment)
0281         {
0282             assert(((alignment & (alignment - 1)) == 0) && "alignment must be a power of two");
0283             assert((alignment >= sizeof(void*)) && "alignment must be at least the size of a pointer");
0284             void* res = nullptr;
0285 #ifdef _WIN32
0286             res = _aligned_malloc(size, alignment);
0287 #else
0288             if (posix_memalign(&res, alignment, size) != 0)
0289             {
0290                 res = nullptr;
0291             }
0292 #endif
0293             return res;
0294         }
0295
0296         XSIMD_INLINE void xaligned_free(void* ptr)
0297         {
0298 #ifdef _WIN32
0299             _aligned_free(ptr);
0300 #else
0301             free(ptr);
0302 #endif
0303         }
0304     }
0305
0306     XSIMD_INLINE void* aligned_malloc(size_t size, size_t alignment)
0307     {
0308         return detail::xaligned_malloc(size, alignment);
0309     }
0310
0311     XSIMD_INLINE void aligned_free(void* ptr)
0312     {
0313         detail::xaligned_free(ptr);
0314     }
0315
0316     template <class T>
0317     XSIMD_INLINE size_t get_alignment_offset(const T* p, size_t size, size_t block_size)
0318     {
0319         // size_t block_size = simd_traits<T>::size;
0320         if (block_size == 1)
0321         {
0322             // The simd_block consists of exactly one scalar so that all
0323             // elements of the array
0324             // are "well" aligned.
0325             return 0;
0326         }
0327         else if (size_t(p) & (sizeof(T) - 1))
0328         {
0329             // The array is not aligned to the size of a single element, so that
0330             // no element
0331             // of the array is well aligned
0332             return size;
0333         }
0334         else
0335         {
0336             size_t block_mask = block_size - 1;
0337             return std::min<size_t>(
0338                 (block_size - ((size_t(p) / sizeof(T)) & block_mask)) & block_mask,
0339                 size);
0340         }
0341     }
0342
0343     template <class T, class A = default_arch>
0344     using default_allocator = typename std::conditional<A::requires_alignment(),
0345                                                         aligned_allocator<T, A::alignment()>,
0346                                                         std::allocator<T>>::type;
0347 }
0348
0349 #endif