arch/AVX512/Complex.h

0001 // This file is part of Eigen, a lightweight C++ template library
0002 // for linear algebra.
0003 //
0004 // Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr>
0005 //
0006 // This Source Code Form is subject to the terms of the Mozilla
0007 // Public License v. 2.0. If a copy of the MPL was not distributed
0008 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
0009
0010 #ifndef EIGEN_COMPLEX_AVX512_H
0011 #define EIGEN_COMPLEX_AVX512_H
0012
0013 namespace Eigen {
0014
0015 namespace internal {
0016
0017 //---------- float ----------
0018 struct Packet8cf
0019 {
0020   EIGEN_STRONG_INLINE Packet8cf() {}
0021   EIGEN_STRONG_INLINE explicit Packet8cf(const __m512& a) : v(a) {}
0022   __m512  v;
0023 };
0024
0025 template<> struct packet_traits<std::complex<float> >  : default_packet_traits
0026 {
0027   typedef Packet8cf type;
0028   typedef Packet4cf half;
0029   enum {
0030     Vectorizable = 1,
0031     AlignedOnScalar = 1,
0032     size = 8,
0033     HasHalfPacket = 1,
0034
0035     HasAdd    = 1,
0036     HasSub    = 1,
0037     HasMul    = 1,
0038     HasDiv    = 1,
0039     HasNegate = 1,
0040     HasSqrt   = 1,
0041     HasAbs    = 0,
0042     HasAbs2   = 0,
0043     HasMin    = 0,
0044     HasMax    = 0,
0045     HasSetLinear = 0
0046   };
0047 };
0048
0049 template<> struct unpacket_traits<Packet8cf> {
0050   typedef std::complex<float> type;
0051   typedef Packet4cf half;
0052   typedef Packet16f as_real;
0053   enum {
0054     size = 8,
0055     alignment=unpacket_traits<Packet16f>::alignment,
0056     vectorizable=true,
0057     masked_load_available=false,
0058     masked_store_available=false
0059   };
0060 };
0061
0062 template<> EIGEN_STRONG_INLINE Packet8cf ptrue<Packet8cf>(const Packet8cf& a) { return Packet8cf(ptrue(Packet16f(a.v))); }
0063 template<> EIGEN_STRONG_INLINE Packet8cf padd<Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(_mm512_add_ps(a.v,b.v)); }
0064 template<> EIGEN_STRONG_INLINE Packet8cf psub<Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(_mm512_sub_ps(a.v,b.v)); }
0065 template<> EIGEN_STRONG_INLINE Packet8cf pnegate(const Packet8cf& a)
0066 {
0067   return Packet8cf(pnegate(a.v));
0068 }
0069 template<> EIGEN_STRONG_INLINE Packet8cf pconj(const Packet8cf& a)
0070 {
0071   const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32(
0072     0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,
0073     0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000));
0074   return Packet8cf(pxor(a.v,mask));
0075 }
0076
0077 template<> EIGEN_STRONG_INLINE Packet8cf pmul<Packet8cf>(const Packet8cf& a, const Packet8cf& b)
0078 {
0079   __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1)));
0080   return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2));
0081 }
0082
0083 template<> EIGEN_STRONG_INLINE Packet8cf pand   <Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pand(a.v,b.v)); }
0084 template<> EIGEN_STRONG_INLINE Packet8cf por    <Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(por(a.v,b.v)); }
0085 template<> EIGEN_STRONG_INLINE Packet8cf pxor   <Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pxor(a.v,b.v)); }
0086 template<> EIGEN_STRONG_INLINE Packet8cf pandnot<Packet8cf>(const Packet8cf& a, const Packet8cf& b) { return Packet8cf(pandnot(a.v,b.v)); }
0087
0088 template <>
0089 EIGEN_STRONG_INLINE Packet8cf pcmp_eq(const Packet8cf& a, const Packet8cf& b) {
0090   __m512 eq = pcmp_eq<Packet16f>(a.v, b.v);
0091   return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1)));
0092 }
0093
0094 template<> EIGEN_STRONG_INLINE Packet8cf pload <Packet8cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet8cf(pload<Packet16f>(&numext::real_ref(*from))); }
0095 template<> EIGEN_STRONG_INLINE Packet8cf ploadu<Packet8cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet8cf(ploadu<Packet16f>(&numext::real_ref(*from))); }
0096
0097
0098 template<> EIGEN_STRONG_INLINE Packet8cf pset1<Packet8cf>(const std::complex<float>& from)
0099 {
0100   return Packet8cf(_mm512_castpd_ps(pload1<Packet8d>((const double*)(const void*)&from)));
0101 }
0102
0103 template<> EIGEN_STRONG_INLINE Packet8cf ploaddup<Packet8cf>(const std::complex<float>* from)
0104 {
0105   return Packet8cf( _mm512_castpd_ps( ploaddup<Packet8d>((const double*)(const void*)from )) );
0106 }
0107 template<> EIGEN_STRONG_INLINE Packet8cf ploadquad<Packet8cf>(const std::complex<float>* from)
0108 {
0109   return Packet8cf( _mm512_castpd_ps( ploadquad<Packet8d>((const double*)(const void*)from )) );
0110 }
0111
0112 template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float>* to, const Packet8cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); }
0113 template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet8cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); }
0114
0115 template<> EIGEN_DEVICE_FUNC inline Packet8cf pgather<std::complex<float>, Packet8cf>(const std::complex<float>* from, Index stride)
0116 {
0117   return Packet8cf(_mm512_castpd_ps(pgather<double,Packet8d>((const double*)(const void*)from, stride)));
0118 }
0119
0120 template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet8cf>(std::complex<float>* to, const Packet8cf& from, Index stride)
0121 {
0122   pscatter((double*)(void*)to, _mm512_castps_pd(from.v), stride);
0123 }
0124
0125 template<> EIGEN_STRONG_INLINE std::complex<float>  pfirst<Packet8cf>(const Packet8cf& a)
0126 {
0127   return pfirst(Packet2cf(_mm512_castps512_ps128(a.v)));
0128 }
0129
0130 template<> EIGEN_STRONG_INLINE Packet8cf preverse(const Packet8cf& a) {
0131   return Packet8cf(_mm512_castsi512_ps(
0132             _mm512_permutexvar_epi64( _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7),
0133                                       _mm512_castps_si512(a.v))));
0134 }
0135
0136 template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet8cf>(const Packet8cf& a)
0137 {
0138   return predux(padd(Packet4cf(extract256<0>(a.v)),
0139                      Packet4cf(extract256<1>(a.v))));
0140 }
0141
0142 template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet8cf>(const Packet8cf& a)
0143 {
0144   return predux_mul(pmul(Packet4cf(extract256<0>(a.v)),
0145                          Packet4cf(extract256<1>(a.v))));
0146 }
0147
0148 template <>
0149 EIGEN_STRONG_INLINE Packet4cf predux_half_dowto4<Packet8cf>(const Packet8cf& a) {
0150   __m256 lane0 = extract256<0>(a.v);
0151   __m256 lane1 = extract256<1>(a.v);
0152   __m256 res = _mm256_add_ps(lane0, lane1);
0153   return Packet4cf(res);
0154 }
0155
0156 EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf,Packet16f)
0157
0158 template<> EIGEN_STRONG_INLINE Packet8cf pdiv<Packet8cf>(const Packet8cf& a, const Packet8cf& b)
0159 {
0160   Packet8cf num = pmul(a, pconj(b));
0161   __m512 tmp = _mm512_mul_ps(b.v, b.v);
0162   __m512 tmp2    = _mm512_shuffle_ps(tmp,tmp,0xB1);
0163   __m512 denom = _mm512_add_ps(tmp, tmp2);
0164   return Packet8cf(_mm512_div_ps(num.v, denom));
0165 }
0166
0167 template<> EIGEN_STRONG_INLINE Packet8cf pcplxflip<Packet8cf>(const Packet8cf& x)
0168 {
0169   return Packet8cf(_mm512_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1)));
0170 }
0171
0172 //---------- double ----------
0173 struct Packet4cd
0174 {
0175   EIGEN_STRONG_INLINE Packet4cd() {}
0176   EIGEN_STRONG_INLINE explicit Packet4cd(const __m512d& a) : v(a) {}
0177   __m512d  v;
0178 };
0179
0180 template<> struct packet_traits<std::complex<double> >  : default_packet_traits
0181 {
0182   typedef Packet4cd type;
0183   typedef Packet2cd half;
0184   enum {
0185     Vectorizable = 1,
0186     AlignedOnScalar = 0,
0187     size = 4,
0188     HasHalfPacket = 1,
0189
0190     HasAdd    = 1,
0191     HasSub    = 1,
0192     HasMul    = 1,
0193     HasDiv    = 1,
0194     HasNegate = 1,
0195     HasSqrt   = 1,
0196     HasAbs    = 0,
0197     HasAbs2   = 0,
0198     HasMin    = 0,
0199     HasMax    = 0,
0200     HasSetLinear = 0
0201   };
0202 };
0203
0204 template<> struct unpacket_traits<Packet4cd> {
0205   typedef std::complex<double> type;
0206   typedef Packet2cd half;
0207   typedef Packet8d as_real;
0208   enum {
0209     size = 4,
0210     alignment = unpacket_traits<Packet8d>::alignment,
0211     vectorizable=true,
0212     masked_load_available=false,
0213     masked_store_available=false
0214   };
0215 };
0216
0217 template<> EIGEN_STRONG_INLINE Packet4cd padd<Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(_mm512_add_pd(a.v,b.v)); }
0218 template<> EIGEN_STRONG_INLINE Packet4cd psub<Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(_mm512_sub_pd(a.v,b.v)); }
0219 template<> EIGEN_STRONG_INLINE Packet4cd pnegate(const Packet4cd& a) { return Packet4cd(pnegate(a.v)); }
0220 template<> EIGEN_STRONG_INLINE Packet4cd pconj(const Packet4cd& a)
0221 {
0222   const __m512d mask = _mm512_castsi512_pd(
0223           _mm512_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0,
0224                            0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0));
0225   return Packet4cd(pxor(a.v,mask));
0226 }
0227
0228 template<> EIGEN_STRONG_INLINE Packet4cd pmul<Packet4cd>(const Packet4cd& a, const Packet4cd& b)
0229 {
0230   __m512d tmp1 = _mm512_shuffle_pd(a.v,a.v,0x0);
0231   __m512d tmp2 = _mm512_shuffle_pd(a.v,a.v,0xFF);
0232   __m512d tmp3 = _mm512_shuffle_pd(b.v,b.v,0x55);
0233   __m512d odd  = _mm512_mul_pd(tmp2, tmp3);
0234   return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd));
0235 }
0236
0237 template<> EIGEN_STRONG_INLINE Packet4cd ptrue<Packet4cd>(const Packet4cd& a) { return Packet4cd(ptrue(Packet8d(a.v))); }
0238 template<> EIGEN_STRONG_INLINE Packet4cd pand   <Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pand(a.v,b.v)); }
0239 template<> EIGEN_STRONG_INLINE Packet4cd por    <Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(por(a.v,b.v)); }
0240 template<> EIGEN_STRONG_INLINE Packet4cd pxor   <Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pxor(a.v,b.v)); }
0241 template<> EIGEN_STRONG_INLINE Packet4cd pandnot<Packet4cd>(const Packet4cd& a, const Packet4cd& b) { return Packet4cd(pandnot(a.v,b.v)); }
0242
0243 template <>
0244 EIGEN_STRONG_INLINE Packet4cd pcmp_eq(const Packet4cd& a, const Packet4cd& b) {
0245   __m512d eq = pcmp_eq<Packet8d>(a.v, b.v);
0246   return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55)));
0247 }
0248
0249 template<> EIGEN_STRONG_INLINE Packet4cd pload <Packet4cd>(const std::complex<double>* from)
0250 { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cd(pload<Packet8d>((const double*)from)); }
0251 template<> EIGEN_STRONG_INLINE Packet4cd ploadu<Packet4cd>(const std::complex<double>* from)
0252 { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cd(ploadu<Packet8d>((const double*)from)); }
0253
0254 template<> EIGEN_STRONG_INLINE Packet4cd pset1<Packet4cd>(const std::complex<double>& from)
0255 {
0256   #ifdef EIGEN_VECTORIZE_AVX512DQ
0257   return Packet4cd(_mm512_broadcast_f64x2(pset1<Packet1cd>(from).v));
0258   #else
0259   return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4( _mm_castpd_ps(pset1<Packet1cd>(from).v))));
0260   #endif
0261 }
0262
0263 template<> EIGEN_STRONG_INLINE Packet4cd ploaddup<Packet4cd>(const std::complex<double>* from) {
0264   return Packet4cd(_mm512_insertf64x4(
0265           _mm512_castpd256_pd512(ploaddup<Packet2cd>(from).v), ploaddup<Packet2cd>(from+1).v, 1));
0266 }
0267
0268 template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> *   to, const Packet4cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); }
0269 template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> *   to, const Packet4cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); }
0270
0271 template<> EIGEN_DEVICE_FUNC inline Packet4cd pgather<std::complex<double>, Packet4cd>(const std::complex<double>* from, Index stride)
0272 {
0273   return Packet4cd(_mm512_insertf64x4(_mm512_castpd256_pd512(
0274             _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from+0*stride).v), ploadu<Packet1cd>(from+1*stride).v,1)),
0275             _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from+2*stride).v), ploadu<Packet1cd>(from+3*stride).v,1), 1));
0276 }
0277
0278 template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet4cd>(std::complex<double>* to, const Packet4cd& from, Index stride)
0279 {
0280   __m512i fromi = _mm512_castpd_si512(from.v);
0281   double* tod = (double*)(void*)to;
0282   _mm_storeu_pd(tod+0*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,0)) );
0283   _mm_storeu_pd(tod+2*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,1)) );
0284   _mm_storeu_pd(tod+4*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,2)) );
0285   _mm_storeu_pd(tod+6*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,3)) );
0286 }
0287
0288 template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet4cd>(const Packet4cd& a)
0289 {
0290   __m128d low = extract128<0>(a.v);
0291   EIGEN_ALIGN16 double res[2];
0292   _mm_store_pd(res, low);
0293   return std::complex<double>(res[0],res[1]);
0294 }
0295
0296 template<> EIGEN_STRONG_INLINE Packet4cd preverse(const Packet4cd& a) {
0297   return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, (shuffle_mask<3,2,1,0>::mask)));
0298 }
0299
0300 template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet4cd>(const Packet4cd& a)
0301 {
0302   return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v,0)),
0303                      Packet2cd(_mm512_extractf64x4_pd(a.v,1))));
0304 }
0305
0306 template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet4cd>(const Packet4cd& a)
0307 {
0308   return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v,0)),
0309                          Packet2cd(_mm512_extractf64x4_pd(a.v,1))));
0310 }
0311
0312 template<> struct conj_helper<Packet4cd, Packet4cd, false,true>
0313 {
0314   EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const
0315   { return padd(pmul(x,y),c); }
0316
0317   EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const
0318   {
0319     return internal::pmul(a, pconj(b));
0320   }
0321 };
0322
0323 template<> struct conj_helper<Packet4cd, Packet4cd, true,false>
0324 {
0325   EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const
0326   { return padd(pmul(x,y),c); }
0327
0328   EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const
0329   {
0330     return internal::pmul(pconj(a), b);
0331   }
0332 };
0333
0334 template<> struct conj_helper<Packet4cd, Packet4cd, true,true>
0335 {
0336   EIGEN_STRONG_INLINE Packet4cd pmadd(const Packet4cd& x, const Packet4cd& y, const Packet4cd& c) const
0337   { return padd(pmul(x,y),c); }
0338
0339   EIGEN_STRONG_INLINE Packet4cd pmul(const Packet4cd& a, const Packet4cd& b) const
0340   {
0341     return pconj(internal::pmul(a, b));
0342   }
0343 };
0344
0345 EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cd,Packet8d)
0346
0347 template<> EIGEN_STRONG_INLINE Packet4cd pdiv<Packet4cd>(const Packet4cd& a, const Packet4cd& b)
0348 {
0349   Packet4cd num = pmul(a, pconj(b));
0350   __m512d tmp = _mm512_mul_pd(b.v, b.v);
0351   __m512d denom =  padd(_mm512_permute_pd(tmp,0x55), tmp);
0352   return Packet4cd(_mm512_div_pd(num.v, denom));
0353 }
0354
0355 template<> EIGEN_STRONG_INLINE Packet4cd pcplxflip<Packet4cd>(const Packet4cd& x)
0356 {
0357   return Packet4cd(_mm512_permute_pd(x.v,0x55));
0358 }
0359
0360 EIGEN_DEVICE_FUNC inline void
0361 ptranspose(PacketBlock<Packet8cf,4>& kernel) {
0362   PacketBlock<Packet8d,4> pb;
0363
0364   pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
0365   pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
0366   pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
0367   pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
0368   ptranspose(pb);
0369   kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
0370   kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
0371   kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
0372   kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
0373 }
0374
0375 EIGEN_DEVICE_FUNC inline void
0376 ptranspose(PacketBlock<Packet8cf,8>& kernel) {
0377   PacketBlock<Packet8d,8> pb;
0378
0379   pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
0380   pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
0381   pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
0382   pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
0383   pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v);
0384   pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v);
0385   pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v);
0386   pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v);
0387   ptranspose(pb);
0388   kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
0389   kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
0390   kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
0391   kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
0392   kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]);
0393   kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]);
0394   kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]);
0395   kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]);
0396 }
0397
0398 EIGEN_DEVICE_FUNC inline void
0399 ptranspose(PacketBlock<Packet4cd,4>& kernel) {
0400   __m512d T0 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<0,1,0,1>::mask)); // [a0 a1 b0 b1]
0401   __m512d T1 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<2,3,2,3>::mask)); // [a2 a3 b2 b3]
0402   __m512d T2 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<0,1,0,1>::mask)); // [c0 c1 d0 d1]
0403   __m512d T3 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<2,3,2,3>::mask)); // [c2 c3 d2 d3]
0404
0405   kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<1,3,1,3>::mask))); // [a3 b3 c3 d3]
0406   kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<0,2,0,2>::mask))); // [a2 b2 c2 d2]
0407   kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<1,3,1,3>::mask))); // [a1 b1 c1 d1]
0408   kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<0,2,0,2>::mask))); // [a0 b0 c0 d0]
0409 }
0410
0411 template<> EIGEN_STRONG_INLINE Packet4cd psqrt<Packet4cd>(const Packet4cd& a) {
0412   return psqrt_complex<Packet4cd>(a);
0413 }
0414
0415 template<> EIGEN_STRONG_INLINE Packet8cf psqrt<Packet8cf>(const Packet8cf& a) {
0416   return psqrt_complex<Packet8cf>(a);
0417 }
0418
0419 } // end namespace internal
0420 } // end namespace Eigen
0421
0422 #endif // EIGEN_COMPLEX_AVX512_H