Vc/common/storage.h

0001 /*  This file is part of the Vc library. {{{
0002 Copyright © 2010-2015 Matthias Kretz <kretz@kde.org>
0003
0004 Redistribution and use in source and binary forms, with or without
0005 modification, are permitted provided that the following conditions are met:
0006     * Redistributions of source code must retain the above copyright
0007       notice, this list of conditions and the following disclaimer.
0008     * Redistributions in binary form must reproduce the above copyright
0009       notice, this list of conditions and the following disclaimer in the
0010       documentation and/or other materials provided with the distribution.
0011     * Neither the names of contributing organizations nor the
0012       names of its contributors may be used to endorse or promote products
0013       derived from this software without specific prior written permission.
0014
0015 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
0016 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
0017 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
0018 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
0019 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
0020 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
0021 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
0022 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
0023 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
0024 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0025
0026 }}}*/
0027
0028 #ifndef VC_COMMON_STORAGE_H_
0029 #define VC_COMMON_STORAGE_H_
0030
0031 #include "aliasingentryhelper.h"
0032 #include "types.h"
0033 #include "maskbool.h"
0034 #ifdef Vc_IMPL_AVX
0035 #include "../avx/intrinsics.h"
0036 #endif
0037 #include "macros.h"
0038
0039 namespace Vc_VERSIONED_NAMESPACE
0040 {
0041 namespace Detail
0042 {
0043 template <typename V> inline V zero();
0044 }  // namespace Detail
0045 namespace Common
0046 {
0047 namespace Detail
0048 {
0049 #ifdef Vc_IMPL_AVX
0050 template <typename ValueType, size_t Size> struct IntrinsicType {
0051     using type = typename std::conditional<
0052         std::is_integral<ValueType>::value,
0053         typename std::conditional<sizeof(ValueType) * Size == 16, __m128i, __m256i>::type,
0054         typename std::conditional<
0055             std::is_same<ValueType, double>::value,
0056             typename std::conditional<sizeof(ValueType) * Size == 16, __m128d,
0057                                       __m256d>::type,
0058             typename std::conditional<sizeof(ValueType) * Size == 16, __m128,
0059                                       __m256>::type>::type>::type;
0060 };
0061 #elif defined Vc_IMPL_SSE
0062 template <typename ValueType, size_t Size> struct IntrinsicType {
0063     using type = typename std::conditional<
0064         std::is_integral<ValueType>::value, __m128i,
0065         typename std::conditional<std::is_same<ValueType, double>::value, __m128d,
0066                                   __m128>::type>::type;
0067 };
0068 #else
0069 template <typename ValueType, size_t Size> struct IntrinsicType {
0070     static_assert(Size == 1,
0071                   "IntrinsicType without SIMD target support may only have Size = 1");
0072     using type = ValueType;
0073 };
0074 #endif
0075 template <typename ValueType, size_t Size, size_t Bytes = sizeof(ValueType) * Size>
0076 struct BuiltinType;
0077 #ifdef Vc_USE_BUILTIN_VECTOR_TYPES
0078 #define Vc_VECBUILTIN __attribute__((__vector_size__(16)))
0079 template <size_t Size> struct BuiltinType<         double   , Size, 16> { typedef          double    type Vc_VECBUILTIN; };
0080 template <size_t Size> struct BuiltinType<         float    , Size, 16> { typedef          float     type Vc_VECBUILTIN; };
0081 template <size_t Size> struct BuiltinType<         long long, Size, 16> { typedef          long long type Vc_VECBUILTIN; };
0082 template <size_t Size> struct BuiltinType<unsigned long long, Size, 16> { typedef unsigned long long type Vc_VECBUILTIN; };
0083 template <size_t Size> struct BuiltinType<         long     , Size, 16> { typedef          long      type Vc_VECBUILTIN; };
0084 template <size_t Size> struct BuiltinType<unsigned long     , Size, 16> { typedef unsigned long      type Vc_VECBUILTIN; };
0085 template <size_t Size> struct BuiltinType<         int      , Size, 16> { typedef          int       type Vc_VECBUILTIN; };
0086 template <size_t Size> struct BuiltinType<unsigned int      , Size, 16> { typedef unsigned int       type Vc_VECBUILTIN; };
0087 template <size_t Size> struct BuiltinType<         short    , Size, 16> { typedef          short     type Vc_VECBUILTIN; };
0088 template <size_t Size> struct BuiltinType<unsigned short    , Size, 16> { typedef unsigned short     type Vc_VECBUILTIN; };
0089 template <size_t Size> struct BuiltinType<         char     , Size, 16> { typedef          char      type Vc_VECBUILTIN; };
0090 template <size_t Size> struct BuiltinType<unsigned char     , Size, 16> { typedef unsigned char      type Vc_VECBUILTIN; };
0091 template <size_t Size> struct BuiltinType<  signed char     , Size, 16> { typedef   signed char      type Vc_VECBUILTIN; };
0092 template <size_t Size> struct BuiltinType<         bool     , Size, 16> { typedef unsigned char      type Vc_VECBUILTIN; };
0093 #undef Vc_VECBUILTIN
0094 #define Vc_VECBUILTIN __attribute__((__vector_size__(32)))
0095 template <size_t Size> struct BuiltinType<         double   , Size, 32> { typedef          double    type Vc_VECBUILTIN; };
0096 template <size_t Size> struct BuiltinType<         float    , Size, 32> { typedef          float     type Vc_VECBUILTIN; };
0097 template <size_t Size> struct BuiltinType<         long long, Size, 32> { typedef          long long type Vc_VECBUILTIN; };
0098 template <size_t Size> struct BuiltinType<unsigned long long, Size, 32> { typedef unsigned long long type Vc_VECBUILTIN; };
0099 template <size_t Size> struct BuiltinType<         long     , Size, 32> { typedef          long      type Vc_VECBUILTIN; };
0100 template <size_t Size> struct BuiltinType<unsigned long     , Size, 32> { typedef unsigned long      type Vc_VECBUILTIN; };
0101 template <size_t Size> struct BuiltinType<         int      , Size, 32> { typedef          int       type Vc_VECBUILTIN; };
0102 template <size_t Size> struct BuiltinType<unsigned int      , Size, 32> { typedef unsigned int       type Vc_VECBUILTIN; };
0103 template <size_t Size> struct BuiltinType<         short    , Size, 32> { typedef          short     type Vc_VECBUILTIN; };
0104 template <size_t Size> struct BuiltinType<unsigned short    , Size, 32> { typedef unsigned short     type Vc_VECBUILTIN; };
0105 template <size_t Size> struct BuiltinType<         char     , Size, 32> { typedef          char      type Vc_VECBUILTIN; };
0106 template <size_t Size> struct BuiltinType<unsigned char     , Size, 32> { typedef unsigned char      type Vc_VECBUILTIN; };
0107 template <size_t Size> struct BuiltinType<  signed char     , Size, 32> { typedef   signed char      type Vc_VECBUILTIN; };
0108 template <size_t Size> struct BuiltinType<         bool     , Size, 32> { typedef unsigned char      type Vc_VECBUILTIN; };
0109 #undef Vc_VECBUILTIN
0110 #endif
0111 }  // namespace Detail
0112
0113 template <typename ValueType, size_t Size>
0114 using IntrinsicType = typename Detail::IntrinsicType<ValueType, Size>::type;
0115
0116 template <typename ValueType, size_t Size>
0117 using BuiltinType = typename Detail::BuiltinType<ValueType, Size>::type;
0118
0119 namespace AliasStrategy
0120 {
0121 struct Union {};
0122 struct MayAlias {};
0123 struct VectorBuiltin {};
0124 struct UnionMembers {};
0125 }  // namespace AliasStrategy
0126
0127 using DefaultStrategy =
0128 #if defined Vc_USE_BUILTIN_VECTOR_TYPES
0129     AliasStrategy::VectorBuiltin;
0130 #elif defined Vc_MSVC
0131     AliasStrategy::UnionMembers;
0132 #elif defined Vc_ICC
0133     AliasStrategy::Union;
0134 #elif defined __GNUC__
0135     AliasStrategy::MayAlias;
0136 #else
0137     AliasStrategy::Union;
0138 #endif
0139
0140 template <typename ValueType, size_t Size, typename Strategy = DefaultStrategy>
0141 class Storage;
0142
0143 // GCC 6 forbids `EntryType m[]` altogether
0144 template <typename ValueType, size_t Size>
0145 class Storage<ValueType, Size, AliasStrategy::Union>
0146 {
0147     static_assert(std::is_fundamental<ValueType>::value &&
0148                       std::is_arithmetic<ValueType>::value,
0149                   "Only works for fundamental arithmetic types.");
0150
0151 public:
0152     using VectorType = IntrinsicType<ValueType, Size>;
0153     using EntryType = ValueType;
0154
0155     union Alias {
0156         Vc_INTRINSIC Alias(VectorType vv) : v(vv) {}
0157         VectorType v;
0158         EntryType m[Size];
0159     };
0160
0161     Vc_INTRINSIC Storage() : data(Vc::Detail::zero<VectorType>()) {}
0162     Vc_INTRINSIC Storage(const VectorType &x) : data(x) { assertCorrectAlignment(&data); }
0163     template <typename U>
0164     Vc_INTRINSIC explicit Storage(const U &x,
0165                                   enable_if<sizeof(U) == sizeof(VectorType)> = nullarg)
0166         : data(reinterpret_cast<VectorType>(x))
0167     {
0168         assertCorrectAlignment(&data);
0169     }
0170
0171     Vc_INTRINSIC Storage(const Storage &) = default;
0172     Vc_INTRINSIC Storage &operator=(const Storage &) = default;
0173
0174     Vc_INTRINSIC operator const VectorType &() const { return data; }
0175     Vc_INTRINSIC Vc_PURE VectorType &v() { return data; }
0176     Vc_INTRINSIC Vc_PURE const VectorType &v() const { return data; }
0177     Vc_INTRINSIC Vc_PURE EntryType m(size_t i) const { return Alias(data).m[i]; }
0178     Vc_INTRINSIC void set(size_t i, EntryType x)
0179     {
0180         Alias a(data);
0181         a.m[i] = x;
0182         data = a.v;
0183     }
0184
0185 private:
0186     VectorType data;
0187 };
0188
0189 template <typename ValueType, size_t Size>
0190 class Storage<ValueType, Size, AliasStrategy::MayAlias>
0191 {
0192     static_assert(std::is_fundamental<ValueType>::value &&
0193                       std::is_arithmetic<ValueType>::value,
0194                   "Only works for fundamental arithmetic types.");
0195
0196 public:
0197     using VectorType = IntrinsicType<ValueType, Size>;
0198     using EntryType = ValueType;
0199
0200     Vc_INTRINSIC Storage() : data() { assertCorrectAlignment(&data); }
0201     Vc_INTRINSIC Storage(const VectorType &x) : data(x)
0202     {
0203         assertCorrectAlignment(&data);
0204     }
0205     template <typename U>
0206     Vc_INTRINSIC explicit Storage(const U &x,
0207                                   enable_if<sizeof(U) == sizeof(VectorType)> = nullarg)
0208         : data(reinterpret_cast<const VectorType &>(x))
0209     {
0210         assertCorrectAlignment(&data);
0211     }
0212     Vc_INTRINSIC Storage &operator=(const VectorType &x)
0213     {
0214         data = x;
0215         return *this;
0216     }
0217
0218     Vc_INTRINSIC Storage(const Storage &) = default;
0219     Vc_INTRINSIC Storage &operator=(const Storage &) = default;
0220
0221     Vc_INTRINSIC operator const VectorType &() const { return v(); }
0222     Vc_INTRINSIC Vc_PURE VectorType &v() { return data; }
0223     Vc_INTRINSIC Vc_PURE const VectorType &v() const { return data; }
0224
0225     Vc_INTRINSIC Vc_PURE EntryType m(size_t i) const
0226     {
0227         return aliasing_cast<EntryType>(&data)[i];
0228     }
0229     Vc_INTRINSIC void set(size_t i, EntryType x)
0230     {
0231         aliasing_cast<EntryType>(&data)[i] = x;
0232     }
0233
0234 private:
0235     VectorType data;
0236 };
0237
0238 template <typename ValueType, size_t Size>
0239 class Storage<ValueType, Size, AliasStrategy::VectorBuiltin>
0240 {
0241     static_assert(std::is_fundamental<ValueType>::value &&
0242                       std::is_arithmetic<ValueType>::value,
0243                   "Only works for fundamental arithmetic types.");
0244
0245     using Builtin = BuiltinType<ValueType, Size>;
0246
0247 public:
0248     using VectorType =
0249 #ifdef Vc_TEMPLATES_DROP_ATTRIBUTES
0250         MayAlias<IntrinsicType<ValueType, Size>>;
0251 #else
0252         IntrinsicType<ValueType, Size>;
0253 #endif
0254     using EntryType = ValueType;
0255
0256     Vc_INTRINSIC Storage() : data() { assertCorrectAlignment(&data); }
0257     Vc_INTRINSIC Storage(const Storage &) = default;
0258     Vc_INTRINSIC Storage &operator=(const Storage &) = default;
0259
0260     Vc_INTRINSIC Storage(const VectorType &x)
0261         : data(aliasing_cast<Builtin>(x))
0262     {
0263         assertCorrectAlignment(&data);
0264     }
0265     template <typename U>
0266     Vc_INTRINSIC explicit Storage(const U &x,
0267                                   enable_if<sizeof(U) == sizeof(VectorType)> = nullarg)
0268         : data(aliasing_cast<Builtin>(x))
0269     {
0270         assertCorrectAlignment(&data);
0271     }
0272     Vc_INTRINSIC Storage &operator=(const VectorType &x)
0273     {
0274         data = aliasing_cast<Builtin>(x);
0275         return *this;
0276     }
0277
0278     Vc_INTRINSIC operator const VectorType &() const { return v(); }
0279     Vc_INTRINSIC Vc_PURE VectorType &v() { return reinterpret_cast<VectorType &>(data); }
0280     Vc_INTRINSIC Vc_PURE const VectorType &v() const { return reinterpret_cast<const VectorType &>(data); }
0281
0282     Vc_INTRINSIC Vc_PURE EntryType m(size_t i) const { return data[i]; }
0283     Vc_INTRINSIC void set(size_t i, EntryType x) { data[i] = x; }
0284
0285     Vc_INTRINSIC Builtin &builtin() { return data; }
0286     Vc_INTRINSIC const Builtin &builtin() const { return data; }
0287
0288 private:
0289     Builtin data;
0290 };
0291
0292 template <typename ValueType, size_t Size>
0293 class Storage<ValueType, Size, AliasStrategy::UnionMembers>
0294 {
0295     static_assert(std::is_fundamental<ValueType>::value &&
0296                       std::is_arithmetic<ValueType>::value,
0297                   "Only works for fundamental arithmetic types.");
0298
0299 public:
0300     using VectorType = IntrinsicType<ValueType, Size>;
0301     using EntryType = ValueType;
0302
0303     Vc_INTRINSIC Storage() : data() { assertCorrectAlignment(&data); }
0304     Vc_INTRINSIC Storage(const VectorType &x) : data(x)
0305     {
0306         assertCorrectAlignment(&data);
0307     }
0308     template <typename U>
0309     Vc_INTRINSIC explicit Storage(const U &x,
0310                                   enable_if<sizeof(U) == sizeof(VectorType)> = nullarg)
0311         : data(reinterpret_cast<const VectorType &>(x))
0312     {
0313         assertCorrectAlignment(&data);
0314     }
0315     Vc_INTRINSIC Storage &operator=(const VectorType &x)
0316     {
0317         data = x;
0318         return *this;
0319     }
0320
0321     Vc_INTRINSIC Storage(const Storage &) = default;
0322     Vc_INTRINSIC Storage &operator=(const Storage &) = default;
0323
0324     Vc_INTRINSIC Vc_PURE VectorType &v() { return data; }
0325     Vc_INTRINSIC Vc_PURE const VectorType &v() const { return data; }
0326
0327     Vc_INTRINSIC_L Vc_PURE_L EntryType m(size_t i) const Vc_INTRINSIC_R Vc_PURE_R;
0328     Vc_INTRINSIC void set(size_t i, EntryType x) { ref(i) = x; }
0329
0330 private:
0331     Vc_INTRINSIC_L Vc_PURE_L EntryType &ref(size_t i) Vc_INTRINSIC_R Vc_PURE_R;
0332     VectorType data;
0333 };
0334
0335 #ifdef Vc_MSVC
0336 template <> Vc_INTRINSIC Vc_PURE          double Storage<         double, 2, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128d_f64[i]; }
0337 template <> Vc_INTRINSIC Vc_PURE          float  Storage<         float , 4, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128_f32[i]; }
0338 template <> Vc_INTRINSIC Vc_PURE   signed int    Storage<  signed int   , 4, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128i_i32[i]; }
0339 template <> Vc_INTRINSIC Vc_PURE   signed short  Storage<  signed short , 8, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128i_i16[i]; }
0340 template <> Vc_INTRINSIC Vc_PURE   signed char   Storage<  signed char  ,16, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128i_i8[i]; }
0341 template <> Vc_INTRINSIC Vc_PURE unsigned int    Storage<unsigned int   , 4, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128i_u32[i]; }
0342 template <> Vc_INTRINSIC Vc_PURE unsigned short  Storage<unsigned short , 8, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128i_u16[i]; }
0343 template <> Vc_INTRINSIC Vc_PURE unsigned char   Storage<unsigned char  ,16, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m128i_u8[i]; }
0344
0345 template <> Vc_INTRINSIC Vc_PURE          double &Storage<         double, 2, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128d_f64[i]; }
0346 template <> Vc_INTRINSIC Vc_PURE          float  &Storage<         float , 4, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128_f32[i]; }
0347 template <> Vc_INTRINSIC Vc_PURE   signed int    &Storage<  signed int   , 4, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128i_i32[i]; }
0348 template <> Vc_INTRINSIC Vc_PURE   signed short  &Storage<  signed short , 8, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128i_i16[i]; }
0349 template <> Vc_INTRINSIC Vc_PURE   signed char   &Storage<  signed char  ,16, AliasStrategy::UnionMembers>::ref(size_t i) { return reinterpret_cast<signed char &>(data.m128i_i8[i]); }
0350 template <> Vc_INTRINSIC Vc_PURE unsigned int    &Storage<unsigned int   , 4, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128i_u32[i]; }
0351 template <> Vc_INTRINSIC Vc_PURE unsigned short  &Storage<unsigned short , 8, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128i_u16[i]; }
0352 template <> Vc_INTRINSIC Vc_PURE unsigned char   &Storage<unsigned char  ,16, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m128i_u8[i]; }
0353
0354 #ifdef Vc_IMPL_AVX
0355 template <> Vc_INTRINSIC Vc_PURE          double Storage<         double, 4, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256d_f64[i]; }
0356 template <> Vc_INTRINSIC Vc_PURE          float  Storage<         float , 8, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256_f32[i]; }
0357 template <> Vc_INTRINSIC Vc_PURE   signed int    Storage<  signed int   , 8, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256i_i32[i]; }
0358 template <> Vc_INTRINSIC Vc_PURE   signed short  Storage<  signed short ,16, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256i_i16[i]; }
0359 template <> Vc_INTRINSIC Vc_PURE   signed char   Storage<  signed char  ,32, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256i_i8[i]; }
0360 template <> Vc_INTRINSIC Vc_PURE unsigned int    Storage<unsigned int   , 8, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256i_u32[i]; }
0361 template <> Vc_INTRINSIC Vc_PURE unsigned short  Storage<unsigned short ,16, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256i_u16[i]; }
0362 template <> Vc_INTRINSIC Vc_PURE unsigned char   Storage<unsigned char  ,32, AliasStrategy::UnionMembers>::m(size_t i) const { return data.m256i_u8[i]; }
0363
0364 template <> Vc_INTRINSIC Vc_PURE          double &Storage<         double, 4, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256d_f64[i]; }
0365 template <> Vc_INTRINSIC Vc_PURE          float  &Storage<         float , 8, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256_f32[i]; }
0366 template <> Vc_INTRINSIC Vc_PURE   signed int    &Storage<  signed int   , 8, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256i_i32[i]; }
0367 template <> Vc_INTRINSIC Vc_PURE   signed short  &Storage<  signed short ,16, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256i_i16[i]; }
0368 template <> Vc_INTRINSIC Vc_PURE   signed char   &Storage<  signed char  ,32, AliasStrategy::UnionMembers>::ref(size_t i) { return reinterpret_cast<signed char &>(data.m256i_i8[i]); }
0369 template <> Vc_INTRINSIC Vc_PURE unsigned int    &Storage<unsigned int   , 8, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256i_u32[i]; }
0370 template <> Vc_INTRINSIC Vc_PURE unsigned short  &Storage<unsigned short ,16, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256i_u16[i]; }
0371 template <> Vc_INTRINSIC Vc_PURE unsigned char   &Storage<unsigned char  ,32, AliasStrategy::UnionMembers>::ref(size_t i) { return data.m256i_u8[i]; }
0372 #endif
0373 #endif  // Vc_MSVC
0374
0375 template <typename VectorType, typename EntryType>
0376 using VectorMemoryUnion = Storage<EntryType, sizeof(VectorType) / sizeof(EntryType)>;
0377
0378 }  // namespace Common
0379 }  // namespace Vc
0380
0381 #endif // VC_COMMON_STORAGE_H_