Back to home page

EIC code displayed by LXR
 
 

    

File indexing completed on 2025-01-19 09:51:47 

0001 // This file is part of Eigen, a lightweight C++ template library
0002 // for linear algebra.
0003 //
0004 // Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org>
0005 //
0006 // This Source Code Form is subject to the terms of the Mozilla
0007 // Public License v. 2.0. If a copy of the MPL was not distributed
0008 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
0009 
0010 #ifndef EIGEN_PACKET_MATH_ZVECTOR_H
0011 #define EIGEN_PACKET_MATH_ZVECTOR_H
0012 
0013 namespace Eigen {
0014 
0015 namespace internal {
0016 
0017 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
0018 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 16
0019 #endif
0020 
0021 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
0022 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
0023 #endif
0024 
0025 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
0026 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS  32
0027 #endif
0028 
0029 typedef __vector int                 Packet4i;
0030 typedef __vector unsigned int        Packet4ui;
0031 typedef __vector __bool int          Packet4bi;
0032 typedef __vector short int           Packet8i;
0033 typedef __vector unsigned char       Packet16uc;
0034 typedef __vector double              Packet2d;
0035 typedef __vector unsigned long long  Packet2ul;
0036 typedef __vector long long           Packet2l;
0037 
0038 // Z14 has builtin support for float vectors
0039 #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
0040 typedef __vector float               Packet4f;
0041 #else
0042 typedef struct {
0043     Packet2d  v4f[2];
0044 } Packet4f;
0045 #endif
0046 
0047 typedef union {
0048   numext::int32_t   i[4];
0049   numext::uint32_t ui[4];
0050   numext::int64_t   l[2];
0051   numext::uint64_t ul[2];
0052   double    d[2];
0053   float     f[4];
0054   Packet4i  v4i;
0055   Packet4ui v4ui;
0056   Packet2l  v2l;
0057   Packet2ul v2ul;
0058   Packet2d  v2d;
0059 #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
0060   Packet4f  v4f;
0061 #endif
0062 } Packet;
0063 
0064 // We don't want to write the same code all the time, but we need to reuse the constants
0065 // and it doesn't really work to declare them global, so we define macros instead
0066 
0067 #define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \
0068   Packet4i p4i_##NAME = reinterpret_cast<Packet4i>(vec_splat_s32(X))
0069 
0070 #define _EIGEN_DECLARE_CONST_FAST_Packet2d(NAME,X) \
0071   Packet2d p2d_##NAME = reinterpret_cast<Packet2d>(vec_splat_s64(X))
0072 
0073 #define _EIGEN_DECLARE_CONST_FAST_Packet2l(NAME,X) \
0074   Packet2l p2l_##NAME = reinterpret_cast<Packet2l>(vec_splat_s64(X))
0075 
0076 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
0077   Packet4i p4i_##NAME = pset1<Packet4i>(X)
0078 
0079 #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \
0080   Packet2d p2d_##NAME = pset1<Packet2d>(X)
0081 
0082 #define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \
0083   Packet2l p2l_##NAME = pset1<Packet2l>(X)
0084 
0085 // These constants are endian-agnostic
0086 static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,}
0087 static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1}
0088 
0089 static _EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0);
0090 static _EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0);
0091 static _EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1);
0092 
0093 static Packet2d p2d_ONE = { 1.0, 1.0 };
0094 static Packet2d p2d_ZERO_ = { numext::bit_cast<double>0x8000000000000000ull),
0095                               numext::bit_cast<double>0x8000000000000000ull) };
0096 
0097 #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
0098 #define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \
0099   Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X))
0100 
0101 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
0102   Packet4f p4f_##NAME = pset1<Packet4f>(X)
0103 
0104 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
0105   const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
0106 
0107 static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0}
0108 static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1}
0109 static Packet4f p4f_MZERO = { 0x80000000, 0x80000000, 0x80000000, 0x80000000};
0110 #endif
0111 
0112 static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
0113 static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
0114 static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet16uc>(p2d_ZERO), reinterpret_cast<Packet16uc>(p2d_ONE), 8));
0115 
0116 static Packet16uc p16uc_PSET64_HI = { 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
0117 static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 };
0118 
0119 // Mask alignment
0120 #define _EIGEN_MASK_ALIGNMENT   0xfffffffffffffff0
0121 
0122 #define _EIGEN_ALIGNED_PTR(x)   ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT)
0123 
0124 // Handle endianness properly while loading constants
0125 // Define global static constants:
0126 
0127 static Packet16uc p16uc_FORWARD =   { 0,1,2,3, 4,5,6,7, 8,9,10,11, 12,13,14,15 };
0128 static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 };
0129 static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
0130 
0131 static Packet16uc p16uc_PSET32_WODD   = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
0132 static Packet16uc p16uc_PSET32_WEVEN  = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
0133 /*static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8);      //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
0134 
0135 static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);     //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };*/
0136 static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);     //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
0137 /*static Packet16uc p16uc_TRANSPOSE64_HI = vec_add(p16uc_PSET64_HI, p16uc_HALF64_0_16);                                         //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
0138 static Packet16uc p16uc_TRANSPOSE64_LO = vec_add(p16uc_PSET64_LO, p16uc_HALF64_0_16);                                         //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};*/
0139 static Packet16uc p16uc_TRANSPOSE64_HI = { 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
0140 static Packet16uc p16uc_TRANSPOSE64_LO = { 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};
0141 
0142 static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8);                                         //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
0143 
0144 static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8);                                            //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
0145 
0146 
0147 #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
0148   #define EIGEN_ZVECTOR_PREFETCH(ADDR) __builtin_prefetch(ADDR);
0149 #else
0150   #define EIGEN_ZVECTOR_PREFETCH(ADDR) asm( "   pfd [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" );
0151 #endif
0152 
0153 template<> struct packet_traits<int>    : default_packet_traits
0154 {
0155   typedef Packet4i type;
0156   typedef Packet4i half;
0157   enum {
0158     Vectorizable = 1,
0159     AlignedOnScalar = 1,
0160     size = 4,
0161     HasHalfPacket = 0,
0162 
0163     HasAdd  = 1,
0164     HasSub  = 1,
0165     HasMul  = 1,
0166     HasDiv  = 1,
0167     HasBlend = 1
0168   };
0169 };
0170 
0171 template <>
0172 struct packet_traits<float> : default_packet_traits {
0173   typedef Packet4f type;
0174   typedef Packet4f half;
0175   enum {
0176     Vectorizable = 1,
0177     AlignedOnScalar = 1,
0178     size = 4,
0179     HasHalfPacket = 0,
0180 
0181     HasAdd = 1,
0182     HasSub = 1,
0183     HasMul = 1,
0184     HasDiv = 1,
0185     HasMin = 1,
0186     HasMax = 1,
0187     HasAbs = 1,
0188     HasSin = 0,
0189     HasCos = 0,
0190     HasLog = 0,
0191     HasExp = 1,
0192     HasSqrt = 1,
0193     HasRsqrt = 1,
0194     HasTanh = 1,
0195     HasErf = 1,
0196     HasRound = 1,
0197     HasFloor = 1,
0198     HasCeil = 1,
0199     HasNegate = 1,
0200     HasBlend = 1
0201   };
0202 };
0203 
0204 template<> struct packet_traits<double> : default_packet_traits
0205 {
0206   typedef Packet2d type;
0207   typedef Packet2d half;
0208   enum {
0209     Vectorizable = 1,
0210     AlignedOnScalar = 1,
0211     size=2,
0212     HasHalfPacket = 1,
0213 
0214     HasAdd  = 1,
0215     HasSub  = 1,
0216     HasMul  = 1,
0217     HasDiv  = 1,
0218     HasMin  = 1,
0219     HasMax  = 1,
0220     HasAbs  = 1,
0221     HasSin  = 0,
0222     HasCos  = 0,
0223     HasLog  = 0,
0224     HasExp  = 1,
0225     HasSqrt = 1,
0226     HasRsqrt = 1,
0227     HasRound = 1,
0228     HasFloor = 1,
0229     HasCeil = 1,
0230     HasNegate = 1,
0231     HasBlend = 1
0232   };
0233 };
0234 
0235 template<> struct unpacket_traits<Packet4i> { typedef int    type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4i half; };
0236 template<> struct unpacket_traits<Packet4f> { typedef float  type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4f half; };
0237 template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet2d half; };
0238 
0239 /* Forward declaration */
0240 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f,4>& kernel);
0241  
0242 inline std::ostream & operator <<(std::ostream & s, const Packet4i & v)
0243 {
0244   Packet vt;
0245   vt.v4i = v;
0246   s << vt.i[0] << ", " << vt.i[1] << ", " << vt.i[2] << ", " << vt.i[3];
0247   return s;
0248 }
0249 
0250 inline std::ostream & operator <<(std::ostream & s, const Packet4ui & v)
0251 {
0252   Packet vt;
0253   vt.v4ui = v;
0254   s << vt.ui[0] << ", " << vt.ui[1] << ", " << vt.ui[2] << ", " << vt.ui[3];
0255   return s;
0256 }
0257 
0258 inline std::ostream & operator <<(std::ostream & s, const Packet2l & v)
0259 {
0260   Packet vt;
0261   vt.v2l = v;
0262   s << vt.l[0] << ", " << vt.l[1];
0263   return s;
0264 }
0265 
0266 inline std::ostream & operator <<(std::ostream & s, const Packet2ul & v)
0267 {
0268   Packet vt;
0269   vt.v2ul = v;
0270   s << vt.ul[0] << ", " << vt.ul[1] ;
0271   return s;
0272 }
0273 
0274 inline std::ostream & operator <<(std::ostream & s, const Packet2d & v)
0275 {
0276   Packet vt;
0277   vt.v2d = v;
0278   s << vt.d[0] << ", " << vt.d[1];
0279   return s;
0280 }
0281 
0282 #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
0283 inline std::ostream & operator <<(std::ostream & s, const Packet4f & v)
0284 {
0285   Packet vt;
0286   vt.v4f = v;
0287   s << vt.f[0] << ", " << vt.f[1] << ", " << vt.f[2] << ", " << vt.f[3];
0288   return s;
0289 }
0290 #endif
0291 
0292 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int*     from)
0293 {
0294   // FIXME: No intrinsic yet
0295   EIGEN_DEBUG_ALIGNED_LOAD
0296   Packet *vfrom;
0297   vfrom = (Packet *) from;
0298   return vfrom->v4i;
0299 }
0300 
0301 template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from)
0302 {
0303   // FIXME: No intrinsic yet
0304   EIGEN_DEBUG_ALIGNED_LOAD
0305   Packet *vfrom;
0306   vfrom = (Packet *) from;
0307   return vfrom->v2d;
0308 }
0309 
0310 template<> EIGEN_STRONG_INLINE void pstore<int>(int*       to, const Packet4i& from)
0311 {
0312   // FIXME: No intrinsic yet
0313   EIGEN_DEBUG_ALIGNED_STORE
0314   Packet *vto;
0315   vto = (Packet *) to;
0316   vto->v4i = from;
0317 }
0318 
0319 template<> EIGEN_STRONG_INLINE void pstore<double>(double*   to, const Packet2d& from)
0320 {
0321   // FIXME: No intrinsic yet
0322   EIGEN_DEBUG_ALIGNED_STORE
0323   Packet *vto;
0324   vto = (Packet *) to;
0325   vto->v2d = from;
0326 }
0327 
0328 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int&    from)
0329 {
0330   return vec_splats(from);
0331 }
0332 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
0333   return vec_splats(from);
0334 }
0335 
0336 template<> EIGEN_STRONG_INLINE void
0337 pbroadcast4<Packet4i>(const int *a,
0338                       Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3)
0339 {
0340   a3 = pload<Packet4i>(a);
0341   a0 = vec_splat(a3, 0);
0342   a1 = vec_splat(a3, 1);
0343   a2 = vec_splat(a3, 2);
0344   a3 = vec_splat(a3, 3);
0345 }
0346 
0347 template<> EIGEN_STRONG_INLINE void
0348 pbroadcast4<Packet2d>(const double *a,
0349                       Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
0350 {
0351   a1 = pload<Packet2d>(a);
0352   a0 = vec_splat(a1, 0);
0353   a1 = vec_splat(a1, 1);
0354   a3 = pload<Packet2d>(a+2);
0355   a2 = vec_splat(a3, 0);
0356   a3 = vec_splat(a3, 1);
0357 }
0358 
0359 template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride)
0360 {
0361   int EIGEN_ALIGN16 ai[4];
0362   ai[0] = from[0*stride];
0363   ai[1] = from[1*stride];
0364   ai[2] = from[2*stride];
0365   ai[3] = from[3*stride];
0366  return pload<Packet4i>(ai);
0367 }
0368 
0369 template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride)
0370 {
0371   double EIGEN_ALIGN16 af[2];
0372   af[0] = from[0*stride];
0373   af[1] = from[1*stride];
0374  return pload<Packet2d>(af);
0375 }
0376 
0377 template<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride)
0378 {
0379   int EIGEN_ALIGN16 ai[4];
0380   pstore<int>((int *)ai, from);
0381   to[0*stride] = ai[0];
0382   to[1*stride] = ai[1];
0383   to[2*stride] = ai[2];
0384   to[3*stride] = ai[3];
0385 }
0386 
0387 template<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride)
0388 {
0389   double EIGEN_ALIGN16 af[2];
0390   pstore<double>(af, from);
0391   to[0*stride] = af[0];
0392   to[1*stride] = af[1];
0393 }
0394 
0395 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a + b); }
0396 template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a + b); }
0397 
0398 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a - b); }
0399 template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a - b); }
0400 
0401 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a * b); }
0402 template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a * b); }
0403 
0404 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a / b); }
0405 template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a / b); }
0406 
0407 template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return (-a); }
0408 template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return (-a); }
0409 
0410 template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
0411 template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
0412 
0413 template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd<Packet4i>(pmul<Packet4i>(a, b), c); }
0414 template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); }
0415 
0416 template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a)    { return padd<Packet4i>(pset1<Packet4i>(a), p4i_COUNTDOWN); }
0417 template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return padd<Packet2d>(pset1<Packet2d>(a), p2d_COUNTDOWN); }
0418 
0419 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_min(a, b); }
0420 template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_min(a, b); }
0421 
0422 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_max(a, b); }
0423 template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_max(a, b); }
0424 
0425 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_and(a, b); }
0426 template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, b); }
0427 
0428 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_or(a, b); }
0429 template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_or(a, b); }
0430 
0431 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_xor(a, b); }
0432 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_xor(a, b); }
0433 
0434 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return pand<Packet4i>(a, vec_nor(b, b)); }
0435 template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); }
0436 
0437 template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return vec_round(a); }
0438 template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const  Packet2d& a) { return vec_ceil(a); }
0439 template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return vec_floor(a); }
0440 
0441 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int*       from) { return pload<Packet4i>(from); }
0442 template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double*    from) { return pload<Packet2d>(from); }
0443 
0444 
0445 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int*     from)
0446 {
0447   Packet4i p = pload<Packet4i>(from);
0448   return vec_perm(p, p, p16uc_DUPLICATE32_HI);
0449 }
0450 
0451 template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double*   from)
0452 {
0453   Packet2d p = pload<Packet2d>(from);
0454   return vec_perm(p, p, p16uc_PSET64_HI);
0455 }
0456 
0457 template<> EIGEN_STRONG_INLINE void pstoreu<int>(int*        to, const Packet4i& from) { pstore<int>(to, from); }
0458 template<> EIGEN_STRONG_INLINE void pstoreu<double>(double*  to, const Packet2d& from) { pstore<double>(to, from); }
0459 
0460 template<> EIGEN_STRONG_INLINE void prefetch<int>(const int*       addr) { EIGEN_ZVECTOR_PREFETCH(addr); }
0461 template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_ZVECTOR_PREFETCH(addr); }
0462 
0463 template<> EIGEN_STRONG_INLINE int    pfirst<Packet4i>(const Packet4i& a) { int    EIGEN_ALIGN16 x[4]; pstore(x, a); return x[0]; }
0464 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double EIGEN_ALIGN16 x[2]; pstore(x, a); return x[0]; }
0465 
0466 template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
0467 {
0468   return reinterpret_cast<Packet4i>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
0469 }
0470 
0471 template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
0472 {
0473   return reinterpret_cast<Packet2d>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64));
0474 }
0475 
0476 template<> EIGEN_STRONG_INLINE Packet4i pabs<Packet4i>(const Packet4i& a) { return vec_abs(a); }
0477 template<> EIGEN_STRONG_INLINE Packet2d pabs<Packet2d>(const Packet2d& a) { return vec_abs(a); }
0478 
0479 template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
0480 {
0481   Packet4i b, sum;
0482   b   = vec_sld(a, a, 8);
0483   sum = padd<Packet4i>(a, b);
0484   b   = vec_sld(sum, sum, 4);
0485   sum = padd<Packet4i>(sum, b);
0486   return pfirst(sum);
0487 }
0488 
0489 template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
0490 {
0491   Packet2d b, sum;
0492   b   = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8));
0493   sum = padd<Packet2d>(a, b);
0494   return pfirst(sum);
0495 }
0496 
0497 // Other reduction functions:
0498 // mul
0499 template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
0500 {
0501   EIGEN_ALIGN16 int aux[4];
0502   pstore(aux, a);
0503   return aux[0] * aux[1] * aux[2] * aux[3];
0504 }
0505 
0506 template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
0507 {
0508   return pfirst(pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
0509 }
0510 
0511 // min
0512 template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
0513 {
0514   Packet4i b, res;
0515   b   = pmin<Packet4i>(a, vec_sld(a, a, 8));
0516   res = pmin<Packet4i>(b, vec_sld(b, b, 4));
0517   return pfirst(res);
0518 }
0519 
0520 template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
0521 {
0522   return pfirst(pmin<Packet2d>(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
0523 }
0524 
0525 // max
0526 template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
0527 {
0528   Packet4i b, res;
0529   b = pmax<Packet4i>(a, vec_sld(a, a, 8));
0530   res = pmax<Packet4i>(b, vec_sld(b, b, 4));
0531   return pfirst(res);
0532 }
0533 
0534 // max
0535 template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
0536 {
0537   return pfirst(pmax<Packet2d>(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
0538 }
0539 
0540 EIGEN_DEVICE_FUNC inline void
0541 ptranspose(PacketBlock<Packet4i,4>& kernel) {
0542   Packet4i t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
0543   Packet4i t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
0544   Packet4i t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
0545   Packet4i t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
0546   kernel.packet[0] = vec_mergeh(t0, t2);
0547   kernel.packet[1] = vec_mergel(t0, t2);
0548   kernel.packet[2] = vec_mergeh(t1, t3);
0549   kernel.packet[3] = vec_mergel(t1, t3);
0550 }
0551 
0552 EIGEN_DEVICE_FUNC inline void
0553 ptranspose(PacketBlock<Packet2d,2>& kernel) {
0554   Packet2d t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
0555   Packet2d t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
0556   kernel.packet[0] = t0;
0557   kernel.packet[1] = t1;
0558 }
0559 
0560 template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) {
0561   Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
0562   Packet4ui mask = vec_cmpeq(select, reinterpret_cast<Packet4ui>(p4i_ONE));
0563   return vec_sel(elsePacket, thenPacket, mask);
0564 }
0565 
0566 
0567 template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) {
0568   Packet2ul select = { ifPacket.select[0], ifPacket.select[1] };
0569   Packet2ul mask = vec_cmpeq(select, reinterpret_cast<Packet2ul>(p2l_ONE));
0570   return vec_sel(elsePacket, thenPacket, mask);
0571 }
0572 
0573 /* z13 has no vector float support so we emulate that with double
0574    z14 has proper vector float support.
0575 */
0576 #if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12)
0577 /* Helper function to simulate a vec_splat_packet4f
0578  */
0579 template<int element> EIGEN_STRONG_INLINE Packet4f vec_splat_packet4f(const Packet4f&   from)
0580 {
0581   Packet4f splat;
0582   switch (element) {
0583   case 0:
0584     splat.v4f[0] = vec_splat(from.v4f[0], 0);
0585     splat.v4f[1] = splat.v4f[0];
0586     break;
0587   case 1:
0588     splat.v4f[0] = vec_splat(from.v4f[0], 1);
0589     splat.v4f[1] = splat.v4f[0];
0590     break;
0591   case 2:
0592     splat.v4f[0] = vec_splat(from.v4f[1], 0);
0593     splat.v4f[1] = splat.v4f[0];
0594     break;
0595   case 3:
0596     splat.v4f[0] = vec_splat(from.v4f[1], 1);
0597     splat.v4f[1] = splat.v4f[0];
0598     break;
0599   }
0600   return splat;
0601 }
0602 
0603 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float*   from)
0604 {
0605   // FIXME: No intrinsic yet
0606   EIGEN_DEBUG_ALIGNED_LOAD
0607   Packet4f vfrom;
0608   vfrom.v4f[0] = vec_ld2f(&from[0]);
0609   vfrom.v4f[1] = vec_ld2f(&from[2]);
0610   return vfrom;
0611 }
0612 
0613 template<> EIGEN_STRONG_INLINE void pstore<float>(float*   to, const Packet4f& from)
0614 {
0615   // FIXME: No intrinsic yet
0616   EIGEN_DEBUG_ALIGNED_STORE
0617   vec_st2f(from.v4f[0], &to[0]);
0618   vec_st2f(from.v4f[1], &to[2]);
0619 }
0620 
0621 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float&    from)
0622 {
0623   Packet4f to;
0624   to.v4f[0] = pset1<Packet2d>(static_cast<const double&>(from));
0625   to.v4f[1] = to.v4f[0];
0626   return to;
0627 }
0628 
0629 template<> EIGEN_STRONG_INLINE void
0630 pbroadcast4<Packet4f>(const float *a,
0631                       Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
0632 {
0633   a3 = pload<Packet4f>(a);
0634   a0 = vec_splat_packet4f<0>(a3);
0635   a1 = vec_splat_packet4f<1>(a3);
0636   a2 = vec_splat_packet4f<2>(a3);
0637   a3 = vec_splat_packet4f<3>(a3);
0638 }
0639 
0640 template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride)
0641 {
0642   float EIGEN_ALIGN16 ai[4];
0643   ai[0] = from[0*stride];
0644   ai[1] = from[1*stride];
0645   ai[2] = from[2*stride];
0646   ai[3] = from[3*stride];
0647  return pload<Packet4f>(ai);
0648 }
0649 
0650 template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
0651 {
0652   float EIGEN_ALIGN16 ai[4];
0653   pstore<float>((float *)ai, from);
0654   to[0*stride] = ai[0];
0655   to[1*stride] = ai[1];
0656   to[2*stride] = ai[2];
0657   to[3*stride] = ai[3];
0658 }
0659 
0660 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b)
0661 {
0662   Packet4f c;
0663   c.v4f[0] = a.v4f[0] + b.v4f[0];
0664   c.v4f[1] = a.v4f[1] + b.v4f[1];
0665   return c;
0666 }
0667 
0668 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b)
0669 {
0670   Packet4f c;
0671   c.v4f[0] = a.v4f[0] - b.v4f[0];
0672   c.v4f[1] = a.v4f[1] - b.v4f[1];
0673   return c;
0674 }
0675 
0676 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b)
0677 {
0678   Packet4f c;
0679   c.v4f[0] = a.v4f[0] * b.v4f[0];
0680   c.v4f[1] = a.v4f[1] * b.v4f[1];
0681   return c;
0682 }
0683 
0684 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
0685 {
0686   Packet4f c;
0687   c.v4f[0] = a.v4f[0] / b.v4f[0];
0688   c.v4f[1] = a.v4f[1] / b.v4f[1];
0689   return c;
0690 }
0691 
0692 template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
0693 {
0694   Packet4f c;
0695   c.v4f[0] = -a.v4f[0];
0696   c.v4f[1] = -a.v4f[1];
0697   return c;
0698 }
0699 
0700 template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
0701 {
0702   Packet4f res;
0703   res.v4f[0] = vec_madd(a.v4f[0], b.v4f[0], c.v4f[0]);
0704   res.v4f[1] = vec_madd(a.v4f[1], b.v4f[1], c.v4f[1]);
0705   return res;
0706 }
0707 
0708 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b)
0709 {
0710   Packet4f res;
0711   res.v4f[0] = pmin(a.v4f[0], b.v4f[0]);
0712   res.v4f[1] = pmin(a.v4f[1], b.v4f[1]);
0713   return res;
0714 }
0715 
0716 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b)
0717 {
0718   Packet4f res;
0719   res.v4f[0] = pmax(a.v4f[0], b.v4f[0]);
0720   res.v4f[1] = pmax(a.v4f[1], b.v4f[1]);
0721   return res;
0722 }
0723 
0724 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b)
0725 {
0726   Packet4f res;
0727   res.v4f[0] = pand(a.v4f[0], b.v4f[0]);
0728   res.v4f[1] = pand(a.v4f[1], b.v4f[1]);
0729   return res;
0730 }
0731 
0732 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b)
0733 {
0734   Packet4f res;
0735   res.v4f[0] = por(a.v4f[0], b.v4f[0]);
0736   res.v4f[1] = por(a.v4f[1], b.v4f[1]);
0737   return res;
0738 }
0739 
0740 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b)
0741 {
0742   Packet4f res;
0743   res.v4f[0] = pxor(a.v4f[0], b.v4f[0]);
0744   res.v4f[1] = pxor(a.v4f[1], b.v4f[1]);
0745   return res;
0746 }
0747 
0748 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b)
0749 {
0750   Packet4f res;
0751   res.v4f[0] = pandnot(a.v4f[0], b.v4f[0]);
0752   res.v4f[1] = pandnot(a.v4f[1], b.v4f[1]);
0753   return res;
0754 }
0755 
0756 template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a)
0757 {
0758   Packet4f res;
0759   res.v4f[0] = vec_round(a.v4f[0]);
0760   res.v4f[1] = vec_round(a.v4f[1]);
0761   return res;
0762 }
0763 
0764 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const  Packet4f& a)
0765 {
0766   Packet4f res;
0767   res.v4f[0] = vec_ceil(a.v4f[0]);
0768   res.v4f[1] = vec_ceil(a.v4f[1]);
0769   return res;
0770 }
0771 
0772 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
0773 {
0774   Packet4f res;
0775   res.v4f[0] = vec_floor(a.v4f[0]);
0776   res.v4f[1] = vec_floor(a.v4f[1]);
0777   return res;
0778 }
0779 
0780 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float*    from)
0781 {
0782   Packet4f p = pload<Packet4f>(from);
0783   p.v4f[1] = vec_splat(p.v4f[0], 1);
0784   p.v4f[0] = vec_splat(p.v4f[0], 0);
0785   return p;
0786 }
0787 
0788 template<> EIGEN_STRONG_INLINE float  pfirst<Packet4f>(const Packet4f& a) { float  EIGEN_ALIGN16 x[2]; vec_st2f(a.v4f[0], &x[0]); return x[0]; }
0789 
0790 template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
0791 {
0792   Packet4f rev;
0793   rev.v4f[0] = preverse<Packet2d>(a.v4f[1]);
0794   rev.v4f[1] = preverse<Packet2d>(a.v4f[0]);
0795   return rev;
0796 }
0797 
0798 template<> EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(const Packet4f& a)
0799 {
0800   Packet4f res;
0801   res.v4f[0] = pabs(a.v4f[0]);
0802   res.v4f[1] = pabs(a.v4f[1]);
0803   return res;
0804 }
0805 
0806 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
0807 {
0808   Packet2d sum;
0809   sum = padd<Packet2d>(a.v4f[0], a.v4f[1]);
0810   double first = predux<Packet2d>(sum);
0811   return static_cast<float>(first);
0812 }
0813 
0814 template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
0815 {
0816   // Return predux_mul<Packet2d> of the subvectors product
0817   return static_cast<float>(pfirst(predux_mul(pmul(a.v4f[0], a.v4f[1]))));
0818 }
0819 
0820 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
0821 {
0822   Packet2d b, res;
0823   b   = pmin<Packet2d>(a.v4f[0], a.v4f[1]);
0824   res = pmin<Packet2d>(b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8)));
0825   return static_cast<float>(pfirst(res));
0826 }
0827 
0828 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
0829 {
0830   Packet2d b, res;
0831   b   = pmax<Packet2d>(a.v4f[0], a.v4f[1]);
0832   res = pmax<Packet2d>(b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8)));
0833   return static_cast<float>(pfirst(res));
0834 }
0835 
0836 /* Split the Packet4f PacketBlock into 4 Packet2d PacketBlocks and transpose each one
0837  */
0838 EIGEN_DEVICE_FUNC inline void
0839 ptranspose(PacketBlock<Packet4f,4>& kernel) {
0840   PacketBlock<Packet2d,2> t0,t1,t2,t3;
0841   // copy top-left 2x2 Packet2d block
0842   t0.packet[0] = kernel.packet[0].v4f[0];
0843   t0.packet[1] = kernel.packet[1].v4f[0];
0844 
0845   // copy top-right 2x2 Packet2d block
0846   t1.packet[0] = kernel.packet[0].v4f[1];
0847   t1.packet[1] = kernel.packet[1].v4f[1];
0848 
0849   // copy bottom-left 2x2 Packet2d block
0850   t2.packet[0] = kernel.packet[2].v4f[0];
0851   t2.packet[1] = kernel.packet[3].v4f[0];
0852 
0853   // copy bottom-right 2x2 Packet2d block
0854   t3.packet[0] = kernel.packet[2].v4f[1];
0855   t3.packet[1] = kernel.packet[3].v4f[1];
0856 
0857   // Transpose all 2x2 blocks
0858   ptranspose(t0);
0859   ptranspose(t1);
0860   ptranspose(t2);
0861   ptranspose(t3);
0862 
0863   // Copy back transposed blocks, but exchange t1 and t2 due to transposition
0864   kernel.packet[0].v4f[0] = t0.packet[0];
0865   kernel.packet[0].v4f[1] = t2.packet[0];
0866   kernel.packet[1].v4f[0] = t0.packet[1];
0867   kernel.packet[1].v4f[1] = t2.packet[1];
0868   kernel.packet[2].v4f[0] = t1.packet[0];
0869   kernel.packet[2].v4f[1] = t3.packet[0];
0870   kernel.packet[3].v4f[0] = t1.packet[1];
0871   kernel.packet[3].v4f[1] = t3.packet[1];
0872 }
0873 
0874 template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) {
0875   Packet2ul select_hi = { ifPacket.select[0], ifPacket.select[1] };
0876   Packet2ul select_lo = { ifPacket.select[2], ifPacket.select[3] };
0877   Packet2ul mask_hi = vec_cmpeq(select_hi, reinterpret_cast<Packet2ul>(p2l_ONE));
0878   Packet2ul mask_lo = vec_cmpeq(select_lo, reinterpret_cast<Packet2ul>(p2l_ONE));
0879   Packet4f result;
0880   result.v4f[0] = vec_sel(elsePacket.v4f[0], thenPacket.v4f[0], mask_hi);
0881   result.v4f[1] = vec_sel(elsePacket.v4f[1], thenPacket.v4f[1], mask_lo);
0882   return result;
0883 }
0884 
0885 template<> Packet4f EIGEN_STRONG_INLINE pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b)
0886 {
0887   Packet4f res;
0888   res.v4f[0] = pcmp_le(a.v4f[0], b.v4f[0]);
0889   res.v4f[1] = pcmp_le(a.v4f[1], b.v4f[1]);
0890   return res;
0891 }
0892 
0893 template<> Packet4f EIGEN_STRONG_INLINE pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b)
0894 {
0895   Packet4f res;
0896   res.v4f[0] = pcmp_lt(a.v4f[0], b.v4f[0]);
0897   res.v4f[1] = pcmp_lt(a.v4f[1], b.v4f[1]);
0898   return res;
0899 }
0900 
0901 template<> Packet4f EIGEN_STRONG_INLINE pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b)
0902 {
0903   Packet4f res;
0904   res.v4f[0] = pcmp_eq(a.v4f[0], b.v4f[0]);
0905   res.v4f[1] = pcmp_eq(a.v4f[1], b.v4f[1]);
0906   return res;
0907 }
0908 
0909 #else
0910 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from)
0911 {
0912   // FIXME: No intrinsic yet
0913   EIGEN_DEBUG_ALIGNED_LOAD
0914   Packet *vfrom;
0915   vfrom = (Packet *) from;
0916   return vfrom->v4f;
0917 }
0918 
0919 template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from)
0920 {
0921   // FIXME: No intrinsic yet
0922   EIGEN_DEBUG_ALIGNED_STORE
0923   Packet *vto;
0924   vto = (Packet *) to;
0925   vto->v4f = from;
0926 }
0927 
0928 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from)
0929 {
0930   return vec_splats(from);
0931 }
0932 
0933 template<> EIGEN_STRONG_INLINE void
0934 pbroadcast4<Packet4f>(const float *a,
0935                       Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
0936 {
0937   a3 = pload<Packet4f>(a);
0938   a0 = vec_splat(a3, 0);
0939   a1 = vec_splat(a3, 1);
0940   a2 = vec_splat(a3, 2);
0941   a3 = vec_splat(a3, 3);
0942 }
0943 
0944 template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride)
0945 {
0946   float EIGEN_ALIGN16 af[4];
0947   af[0] = from[0*stride];
0948   af[1] = from[1*stride];
0949   af[2] = from[2*stride];
0950   af[3] = from[3*stride];
0951  return pload<Packet4f>(af);
0952 }
0953 
0954 template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
0955 {
0956   float EIGEN_ALIGN16 af[4];
0957   pstore<float>((float*)af, from);
0958   to[0*stride] = af[0];
0959   to[1*stride] = af[1];
0960   to[2*stride] = af[2];
0961   to[3*stride] = af[3];
0962 }
0963 
0964 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a + b); }
0965 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a - b); }
0966 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a * b); }
0967 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return (a / b); }
0968 template<> EIGEN_STRONG_INLINE Packet4f pnegate<Packet4f>(const Packet4f& a) { return (-a); }
0969 template<> EIGEN_STRONG_INLINE Packet4f pconj<Packet4f>  (const Packet4f& a) { return a; }
0970 template<> EIGEN_STRONG_INLINE Packet4f pmadd<Packet4f>  (const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a, b, c); }
0971 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>   (const Packet4f& a, const Packet4f& b) { return vec_min(a, b); }
0972 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>   (const Packet4f& a, const Packet4f& b) { return vec_max(a, b); }
0973 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>   (const Packet4f& a, const Packet4f& b) { return vec_and(a, b); }
0974 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>    (const Packet4f& a, const Packet4f& b) { return vec_or(a, b); }
0975 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>   (const Packet4f& a, const Packet4f& b) { return vec_xor(a, b); }
0976 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_and(a, vec_nor(b, b)); }
0977 template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f> (const Packet4f& a) { return vec_round(a); }
0978 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>  (const Packet4f& a) { return vec_ceil(a); }
0979 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f> (const Packet4f& a) { return vec_floor(a); }
0980 template<> EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>   (const Packet4f& a) { return vec_abs(a); }
0981 template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; pstore(x, a); return x[0]; }
0982 
0983 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
0984 {
0985   Packet4f p = pload<Packet4f>(from);
0986   return vec_perm(p, p, p16uc_DUPLICATE32_HI);
0987 }
0988 
0989 template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
0990 {
0991   return reinterpret_cast<Packet4f>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
0992 }
0993 
0994 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
0995 {
0996   Packet4f b, sum;
0997   b   = vec_sld(a, a, 8);
0998   sum = padd<Packet4f>(a, b);
0999   b   = vec_sld(sum, sum, 4);
1000   sum = padd<Packet4f>(sum, b);
1001   return pfirst(sum);
1002 }
1003 
1004 // Other reduction functions:
1005 // mul
1006 template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
1007 {
1008   Packet4f prod;
1009   prod = pmul(a, vec_sld(a, a, 8));
1010   return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
1011 }
1012 
1013 // min
1014 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
1015 {
1016   Packet4f b, res;
1017   b   = pmin<Packet4f>(a, vec_sld(a, a, 8));
1018   res = pmin<Packet4f>(b, vec_sld(b, b, 4));
1019   return pfirst(res);
1020 }
1021 
1022 // max
1023 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
1024 {
1025   Packet4f b, res;
1026   b = pmax<Packet4f>(a, vec_sld(a, a, 8));
1027   res = pmax<Packet4f>(b, vec_sld(b, b, 4));
1028   return pfirst(res);
1029 }
1030 
1031 EIGEN_DEVICE_FUNC inline void
1032 ptranspose(PacketBlock<Packet4f,4>& kernel) {
1033   Packet4f t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
1034   Packet4f t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
1035   Packet4f t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
1036   Packet4f t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
1037   kernel.packet[0] = vec_mergeh(t0, t2);
1038   kernel.packet[1] = vec_mergel(t0, t2);
1039   kernel.packet[2] = vec_mergeh(t1, t3);
1040   kernel.packet[3] = vec_mergel(t1, t3);
1041 }
1042 
1043 template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) {
1044   Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
1045   Packet4ui mask = vec_cmpeq(select, reinterpret_cast<Packet4ui>(p4i_ONE));
1046   return vec_sel(elsePacket, thenPacket, mask);
1047 }
1048 
1049 #endif
1050 
1051 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float*   addr) { EIGEN_ZVECTOR_PREFETCH(addr); }
1052 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f> (const float* from) { return pload<Packet4f>(from); }
1053 template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { pstore<float>(to, from); }
1054 template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>  (const float& a)  { return padd<Packet4f>(pset1<Packet4f>(a), p4f_COUNTDOWN); }
1055 
1056 } // end namespace internal
1057 
1058 } // end namespace Eigen
1059 
1060 #endif // EIGEN_PACKET_MATH_ZVECTOR_H
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.3.7 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!