[136] | 1 | // This file is part of Eigen, a lightweight C++ template library
|
---|
| 2 | // for linear algebra.
|
---|
| 3 | //
|
---|
| 4 | // Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
|
---|
| 5 | //
|
---|
| 6 | // This Source Code Form is subject to the terms of the Mozilla
|
---|
| 7 | // Public License v. 2.0. If a copy of the MPL was not distributed
|
---|
| 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
---|
| 9 |
|
---|
| 10 | #ifndef EIGEN_PACKET_MATH_SSE_H
|
---|
| 11 | #define EIGEN_PACKET_MATH_SSE_H
|
---|
| 12 |
|
---|
| 13 | namespace Eigen {
|
---|
| 14 |
|
---|
| 15 | namespace internal {
|
---|
| 16 |
|
---|
| 17 | #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
|
---|
| 18 | #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
|
---|
| 19 | #endif
|
---|
| 20 |
|
---|
| 21 | #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
|
---|
| 22 | #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*))
|
---|
| 23 | #endif
|
---|
| 24 |
|
---|
| 25 | typedef __m128 Packet4f;
|
---|
| 26 | typedef __m128i Packet4i;
|
---|
| 27 | typedef __m128d Packet2d;
|
---|
| 28 |
|
---|
| 29 | template<> struct is_arithmetic<__m128> { enum { value = true }; };
|
---|
| 30 | template<> struct is_arithmetic<__m128i> { enum { value = true }; };
|
---|
| 31 | template<> struct is_arithmetic<__m128d> { enum { value = true }; };
|
---|
| 32 |
|
---|
| 33 | #define vec4f_swizzle1(v,p,q,r,s) \
|
---|
| 34 | (_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), ((s)<<6|(r)<<4|(q)<<2|(p)))))
|
---|
| 35 |
|
---|
| 36 | #define vec4i_swizzle1(v,p,q,r,s) \
|
---|
| 37 | (_mm_shuffle_epi32( v, ((s)<<6|(r)<<4|(q)<<2|(p))))
|
---|
| 38 |
|
---|
| 39 | #define vec2d_swizzle1(v,p,q) \
|
---|
| 40 | (_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), ((q*2+1)<<6|(q*2)<<4|(p*2+1)<<2|(p*2)))))
|
---|
| 41 |
|
---|
| 42 | #define vec4f_swizzle2(a,b,p,q,r,s) \
|
---|
| 43 | (_mm_shuffle_ps( (a), (b), ((s)<<6|(r)<<4|(q)<<2|(p))))
|
---|
| 44 |
|
---|
| 45 | #define vec4i_swizzle2(a,b,p,q,r,s) \
|
---|
| 46 | (_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), ((s)<<6|(r)<<4|(q)<<2|(p))))))
|
---|
| 47 |
|
---|
| 48 | #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
|
---|
| 49 | const Packet4f p4f_##NAME = pset1<Packet4f>(X)
|
---|
| 50 |
|
---|
| 51 | #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \
|
---|
| 52 | const Packet2d p2d_##NAME = pset1<Packet2d>(X)
|
---|
| 53 |
|
---|
| 54 | #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
|
---|
| 55 | const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X))
|
---|
| 56 |
|
---|
| 57 | #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
|
---|
| 58 | const Packet4i p4i_##NAME = pset1<Packet4i>(X)
|
---|
| 59 |
|
---|
| 60 |
|
---|
| 61 | template<> struct packet_traits<float> : default_packet_traits
|
---|
| 62 | {
|
---|
| 63 | typedef Packet4f type;
|
---|
| 64 | enum {
|
---|
| 65 | Vectorizable = 1,
|
---|
| 66 | AlignedOnScalar = 1,
|
---|
| 67 | size=4,
|
---|
| 68 |
|
---|
| 69 | HasDiv = 1,
|
---|
| 70 | HasSin = EIGEN_FAST_MATH,
|
---|
| 71 | HasCos = EIGEN_FAST_MATH,
|
---|
| 72 | HasLog = 1,
|
---|
| 73 | HasExp = 1,
|
---|
| 74 | HasSqrt = 1
|
---|
| 75 | };
|
---|
| 76 | };
|
---|
| 77 | template<> struct packet_traits<double> : default_packet_traits
|
---|
| 78 | {
|
---|
| 79 | typedef Packet2d type;
|
---|
| 80 | enum {
|
---|
| 81 | Vectorizable = 1,
|
---|
| 82 | AlignedOnScalar = 1,
|
---|
| 83 | size=2,
|
---|
| 84 |
|
---|
| 85 | HasDiv = 1,
|
---|
| 86 | HasExp = 1,
|
---|
| 87 | HasSqrt = 1
|
---|
| 88 | };
|
---|
| 89 | };
|
---|
| 90 | template<> struct packet_traits<int> : default_packet_traits
|
---|
| 91 | {
|
---|
| 92 | typedef Packet4i type;
|
---|
| 93 | enum {
|
---|
| 94 | // FIXME check the Has*
|
---|
| 95 | Vectorizable = 1,
|
---|
| 96 | AlignedOnScalar = 1,
|
---|
| 97 | size=4
|
---|
| 98 | };
|
---|
| 99 | };
|
---|
| 100 |
|
---|
| 101 | template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4}; };
|
---|
| 102 | template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2}; };
|
---|
| 103 | template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4}; };
|
---|
| 104 |
|
---|
| 105 | #if defined(_MSC_VER) && (_MSC_VER==1500)
|
---|
| 106 | // Workaround MSVC 9 internal compiler error.
|
---|
| 107 | // TODO: It has been detected with win64 builds (amd64), so let's check whether it also happens in 32bits+SSE mode
|
---|
| 108 | // TODO: let's check whether there does not exist a better fix, like adding a pset0() function. (it crashed on pset1(0)).
|
---|
| 109 | template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set_ps(from,from,from,from); }
|
---|
| 110 | template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set_pd(from,from); }
|
---|
| 111 | template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set_epi32(from,from,from,from); }
|
---|
| 112 | #else
|
---|
| 113 | template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set1_ps(from); }
|
---|
| 114 | template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); }
|
---|
| 115 | template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); }
|
---|
| 116 | #endif
|
---|
| 117 |
|
---|
| 118 | template<> EIGEN_STRONG_INLINE Packet4f plset<float>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); }
|
---|
| 119 | template<> EIGEN_STRONG_INLINE Packet2d plset<double>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); }
|
---|
| 120 | template<> EIGEN_STRONG_INLINE Packet4i plset<int>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); }
|
---|
| 121 |
|
---|
| 122 | template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); }
|
---|
| 123 | template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); }
|
---|
| 124 | template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); }
|
---|
| 125 |
|
---|
| 126 | template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); }
|
---|
| 127 | template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); }
|
---|
| 128 | template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); }
|
---|
| 129 |
|
---|
| 130 | template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
|
---|
| 131 | {
|
---|
| 132 | const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000));
|
---|
| 133 | return _mm_xor_ps(a,mask);
|
---|
| 134 | }
|
---|
| 135 | template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a)
|
---|
| 136 | {
|
---|
| 137 | const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000));
|
---|
| 138 | return _mm_xor_pd(a,mask);
|
---|
| 139 | }
|
---|
| 140 | template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a)
|
---|
| 141 | {
|
---|
| 142 | return psub(_mm_setr_epi32(0,0,0,0), a);
|
---|
| 143 | }
|
---|
| 144 |
|
---|
| 145 | template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
|
---|
| 146 | template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
|
---|
| 147 | template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
|
---|
| 148 |
|
---|
| 149 | template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); }
|
---|
| 150 | template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); }
|
---|
| 151 | template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b)
|
---|
| 152 | {
|
---|
| 153 | #ifdef EIGEN_VECTORIZE_SSE4_1
|
---|
| 154 | return _mm_mullo_epi32(a,b);
|
---|
| 155 | #else
|
---|
| 156 | // this version is slightly faster than 4 scalar products
|
---|
| 157 | return vec4i_swizzle1(
|
---|
| 158 | vec4i_swizzle2(
|
---|
| 159 | _mm_mul_epu32(a,b),
|
---|
| 160 | _mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2),
|
---|
| 161 | vec4i_swizzle1(b,1,0,3,2)),
|
---|
| 162 | 0,2,0,2),
|
---|
| 163 | 0,2,1,3);
|
---|
| 164 | #endif
|
---|
| 165 | }
|
---|
| 166 |
|
---|
| 167 | template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); }
|
---|
| 168 | template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); }
|
---|
| 169 | template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
|
---|
| 170 | { eigen_assert(false && "packet integer division are not supported by SSE");
|
---|
| 171 | return pset1<Packet4i>(0);
|
---|
| 172 | }
|
---|
| 173 |
|
---|
| 174 | // for some weird raisons, it has to be overloaded for packet of integers
|
---|
| 175 | template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); }
|
---|
| 176 |
|
---|
| 177 | template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); }
|
---|
| 178 | template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); }
|
---|
| 179 | template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b)
|
---|
| 180 | {
|
---|
| 181 | #ifdef EIGEN_VECTORIZE_SSE4_1
|
---|
| 182 | return _mm_min_epi32(a,b);
|
---|
| 183 | #else
|
---|
| 184 | // after some bench, this version *is* faster than a scalar implementation
|
---|
| 185 | Packet4i mask = _mm_cmplt_epi32(a,b);
|
---|
| 186 | return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
|
---|
| 187 | #endif
|
---|
| 188 | }
|
---|
| 189 |
|
---|
| 190 | template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); }
|
---|
| 191 | template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); }
|
---|
| 192 | template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b)
|
---|
| 193 | {
|
---|
| 194 | #ifdef EIGEN_VECTORIZE_SSE4_1
|
---|
| 195 | return _mm_max_epi32(a,b);
|
---|
| 196 | #else
|
---|
| 197 | // after some bench, this version *is* faster than a scalar implementation
|
---|
| 198 | Packet4i mask = _mm_cmpgt_epi32(a,b);
|
---|
| 199 | return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
|
---|
| 200 | #endif
|
---|
| 201 | }
|
---|
| 202 |
|
---|
| 203 | template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); }
|
---|
| 204 | template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); }
|
---|
| 205 | template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); }
|
---|
| 206 |
|
---|
| 207 | template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); }
|
---|
| 208 | template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); }
|
---|
| 209 | template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); }
|
---|
| 210 |
|
---|
| 211 | template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); }
|
---|
| 212 | template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); }
|
---|
| 213 | template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); }
|
---|
| 214 |
|
---|
| 215 | template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); }
|
---|
| 216 | template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); }
|
---|
| 217 | template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); }
|
---|
| 218 |
|
---|
| 219 | template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); }
|
---|
| 220 | template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); }
|
---|
| 221 | template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const Packet4i*>(from)); }
|
---|
| 222 |
|
---|
| 223 | #if defined(_MSC_VER)
|
---|
| 224 | template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
|
---|
| 225 | EIGEN_DEBUG_UNALIGNED_LOAD
|
---|
| 226 | #if (_MSC_VER==1600)
|
---|
| 227 | // NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps
|
---|
| 228 | // (i.e., it does not generate an unaligned load!!
|
---|
| 229 | // TODO On most architectures this version should also be faster than a single _mm_loadu_ps
|
---|
| 230 | // so we could also enable it for MSVC08 but first we have to make this later does not generate crap when doing so...
|
---|
| 231 | __m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from));
|
---|
| 232 | res = _mm_loadh_pi(res, (const __m64*)(from+2));
|
---|
| 233 | return res;
|
---|
| 234 | #else
|
---|
| 235 | return _mm_loadu_ps(from);
|
---|
| 236 | #endif
|
---|
| 237 | }
|
---|
| 238 | #else
|
---|
| 239 | // NOTE: with the code below, MSVC's compiler crashes!
|
---|
| 240 |
|
---|
| 241 | template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
|
---|
| 242 | {
|
---|
| 243 | EIGEN_DEBUG_UNALIGNED_LOAD
|
---|
| 244 | return _mm_loadu_ps(from);
|
---|
| 245 | }
|
---|
| 246 | #endif
|
---|
| 247 |
|
---|
| 248 | template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
|
---|
| 249 | {
|
---|
| 250 | EIGEN_DEBUG_UNALIGNED_LOAD
|
---|
| 251 | return _mm_loadu_pd(from);
|
---|
| 252 | }
|
---|
| 253 | template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
|
---|
| 254 | {
|
---|
| 255 | EIGEN_DEBUG_UNALIGNED_LOAD
|
---|
| 256 | return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
|
---|
| 257 | }
|
---|
| 258 |
|
---|
| 259 |
|
---|
| 260 | template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
|
---|
| 261 | {
|
---|
| 262 | return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(from))), 0, 0, 1, 1);
|
---|
| 263 | }
|
---|
| 264 | template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
|
---|
| 265 | { return pset1<Packet2d>(from[0]); }
|
---|
| 266 | template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
|
---|
| 267 | {
|
---|
| 268 | Packet4i tmp;
|
---|
| 269 | tmp = _mm_loadl_epi64(reinterpret_cast<const Packet4i*>(from));
|
---|
| 270 | return vec4i_swizzle1(tmp, 0, 0, 1, 1);
|
---|
| 271 | }
|
---|
| 272 |
|
---|
| 273 | template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); }
|
---|
| 274 | template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); }
|
---|
| 275 | template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<Packet4i*>(to), from); }
|
---|
| 276 |
|
---|
| 277 | template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
|
---|
| 278 | EIGEN_DEBUG_UNALIGNED_STORE
|
---|
| 279 | _mm_storel_pd((to), from);
|
---|
| 280 | _mm_storeh_pd((to+1), from);
|
---|
| 281 | }
|
---|
| 282 | template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(reinterpret_cast<double*>(to), _mm_castps_pd(from)); }
|
---|
| 283 | template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(reinterpret_cast<double*>(to), _mm_castsi128_pd(from)); }
|
---|
| 284 |
|
---|
| 285 | // some compilers might be tempted to perform multiple moves instead of using a vector path.
|
---|
| 286 | template<> EIGEN_STRONG_INLINE void pstore1<Packet4f>(float* to, const float& a)
|
---|
| 287 | {
|
---|
| 288 | Packet4f pa = _mm_set_ss(a);
|
---|
| 289 | pstore(to, vec4f_swizzle1(pa,0,0,0,0));
|
---|
| 290 | }
|
---|
| 291 | // some compilers might be tempted to perform multiple moves instead of using a vector path.
|
---|
| 292 | template<> EIGEN_STRONG_INLINE void pstore1<Packet2d>(double* to, const double& a)
|
---|
| 293 | {
|
---|
| 294 | Packet2d pa = _mm_set_sd(a);
|
---|
| 295 | pstore(to, vec2d_swizzle1(pa,0,0));
|
---|
| 296 | }
|
---|
| 297 |
|
---|
| 298 | template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
---|
| 299 | template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
---|
| 300 | template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
|
---|
| 301 |
|
---|
| 302 | #if defined(_MSC_VER) && defined(_WIN64) && !defined(__INTEL_COMPILER)
|
---|
| 303 | // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
|
---|
| 304 | // Direct of the struct members fixed bug #62.
|
---|
| 305 | template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; }
|
---|
| 306 | template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; }
|
---|
| 307 | template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
|
---|
| 308 | #elif defined(_MSC_VER) && !defined(__INTEL_COMPILER)
|
---|
| 309 | // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
|
---|
| 310 | template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; }
|
---|
| 311 | template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; }
|
---|
| 312 | template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
|
---|
| 313 | #else
|
---|
| 314 | template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); }
|
---|
| 315 | template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); }
|
---|
| 316 | template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); }
|
---|
| 317 | #endif
|
---|
| 318 |
|
---|
| 319 | template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
|
---|
| 320 | { return _mm_shuffle_ps(a,a,0x1B); }
|
---|
| 321 | template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
|
---|
| 322 | { return _mm_shuffle_pd(a,a,0x1); }
|
---|
| 323 | template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
|
---|
| 324 | { return _mm_shuffle_epi32(a,0x1B); }
|
---|
| 325 |
|
---|
| 326 |
|
---|
| 327 | template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a)
|
---|
| 328 | {
|
---|
| 329 | const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF));
|
---|
| 330 | return _mm_and_ps(a,mask);
|
---|
| 331 | }
|
---|
| 332 | template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a)
|
---|
| 333 | {
|
---|
| 334 | const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF));
|
---|
| 335 | return _mm_and_pd(a,mask);
|
---|
| 336 | }
|
---|
| 337 | template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a)
|
---|
| 338 | {
|
---|
| 339 | #ifdef EIGEN_VECTORIZE_SSSE3
|
---|
| 340 | return _mm_abs_epi32(a);
|
---|
| 341 | #else
|
---|
| 342 | Packet4i aux = _mm_srai_epi32(a,31);
|
---|
| 343 | return _mm_sub_epi32(_mm_xor_si128(a,aux),aux);
|
---|
| 344 | #endif
|
---|
| 345 | }
|
---|
| 346 |
|
---|
| 347 | EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs)
|
---|
| 348 | {
|
---|
| 349 | vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
|
---|
| 350 | vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
|
---|
| 351 | vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF));
|
---|
| 352 | vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
|
---|
| 353 | }
|
---|
| 354 |
|
---|
| 355 | #ifdef EIGEN_VECTORIZE_SSE3
|
---|
| 356 | // TODO implement SSE2 versions as well as integer versions
|
---|
| 357 | template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
|
---|
| 358 | {
|
---|
| 359 | return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3]));
|
---|
| 360 | }
|
---|
| 361 | template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
|
---|
| 362 | {
|
---|
| 363 | return _mm_hadd_pd(vecs[0], vecs[1]);
|
---|
| 364 | }
|
---|
| 365 | // SSSE3 version:
|
---|
| 366 | // EIGEN_STRONG_INLINE Packet4i preduxp(const Packet4i* vecs)
|
---|
| 367 | // {
|
---|
| 368 | // return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3]));
|
---|
| 369 | // }
|
---|
| 370 |
|
---|
| 371 | template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
|
---|
| 372 | {
|
---|
| 373 | Packet4f tmp0 = _mm_hadd_ps(a,a);
|
---|
| 374 | return pfirst(_mm_hadd_ps(tmp0, tmp0));
|
---|
| 375 | }
|
---|
| 376 |
|
---|
| 377 | template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return pfirst(_mm_hadd_pd(a, a)); }
|
---|
| 378 |
|
---|
| 379 | // SSSE3 version:
|
---|
| 380 | // EIGEN_STRONG_INLINE float predux(const Packet4i& a)
|
---|
| 381 | // {
|
---|
| 382 | // Packet4i tmp0 = _mm_hadd_epi32(a,a);
|
---|
| 383 | // return pfirst(_mm_hadd_epi32(tmp0, tmp0));
|
---|
| 384 | // }
|
---|
| 385 | #else
|
---|
| 386 | // SSE2 versions
|
---|
| 387 | template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
|
---|
| 388 | {
|
---|
| 389 | Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a));
|
---|
| 390 | return pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
---|
| 391 | }
|
---|
| 392 | template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
|
---|
| 393 | {
|
---|
| 394 | return pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a)));
|
---|
| 395 | }
|
---|
| 396 |
|
---|
| 397 | template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
|
---|
| 398 | {
|
---|
| 399 | Packet4f tmp0, tmp1, tmp2;
|
---|
| 400 | tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]);
|
---|
| 401 | tmp1 = _mm_unpackhi_ps(vecs[0], vecs[1]);
|
---|
| 402 | tmp2 = _mm_unpackhi_ps(vecs[2], vecs[3]);
|
---|
| 403 | tmp0 = _mm_add_ps(tmp0, tmp1);
|
---|
| 404 | tmp1 = _mm_unpacklo_ps(vecs[2], vecs[3]);
|
---|
| 405 | tmp1 = _mm_add_ps(tmp1, tmp2);
|
---|
| 406 | tmp2 = _mm_movehl_ps(tmp1, tmp0);
|
---|
| 407 | tmp0 = _mm_movelh_ps(tmp0, tmp1);
|
---|
| 408 | return _mm_add_ps(tmp0, tmp2);
|
---|
| 409 | }
|
---|
| 410 |
|
---|
| 411 | template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
|
---|
| 412 | {
|
---|
| 413 | return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1]));
|
---|
| 414 | }
|
---|
| 415 | #endif // SSE3
|
---|
| 416 |
|
---|
| 417 | template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
|
---|
| 418 | {
|
---|
| 419 | Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a));
|
---|
| 420 | return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1));
|
---|
| 421 | }
|
---|
| 422 |
|
---|
| 423 | template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
|
---|
| 424 | {
|
---|
| 425 | Packet4i tmp0, tmp1, tmp2;
|
---|
| 426 | tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
|
---|
| 427 | tmp1 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
|
---|
| 428 | tmp2 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
|
---|
| 429 | tmp0 = _mm_add_epi32(tmp0, tmp1);
|
---|
| 430 | tmp1 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
|
---|
| 431 | tmp1 = _mm_add_epi32(tmp1, tmp2);
|
---|
| 432 | tmp2 = _mm_unpacklo_epi64(tmp0, tmp1);
|
---|
| 433 | tmp0 = _mm_unpackhi_epi64(tmp0, tmp1);
|
---|
| 434 | return _mm_add_epi32(tmp0, tmp2);
|
---|
| 435 | }
|
---|
| 436 |
|
---|
| 437 | // Other reduction functions:
|
---|
| 438 |
|
---|
| 439 | // mul
|
---|
| 440 | template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
|
---|
| 441 | {
|
---|
| 442 | Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a));
|
---|
| 443 | return pfirst(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
---|
| 444 | }
|
---|
| 445 | template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
|
---|
| 446 | {
|
---|
| 447 | return pfirst(_mm_mul_sd(a, _mm_unpackhi_pd(a,a)));
|
---|
| 448 | }
|
---|
| 449 | template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
|
---|
| 450 | {
|
---|
| 451 | // after some experiments, it is seems this is the fastest way to implement it
|
---|
| 452 | // for GCC (eg., reusing pmul is very slow !)
|
---|
| 453 | // TODO try to call _mm_mul_epu32 directly
|
---|
| 454 | EIGEN_ALIGN16 int aux[4];
|
---|
| 455 | pstore(aux, a);
|
---|
| 456 | return (aux[0] * aux[1]) * (aux[2] * aux[3]);;
|
---|
| 457 | }
|
---|
| 458 |
|
---|
| 459 | // min
|
---|
| 460 | template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
|
---|
| 461 | {
|
---|
| 462 | Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a));
|
---|
| 463 | return pfirst(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
---|
| 464 | }
|
---|
| 465 | template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
|
---|
| 466 | {
|
---|
| 467 | return pfirst(_mm_min_sd(a, _mm_unpackhi_pd(a,a)));
|
---|
| 468 | }
|
---|
| 469 | template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
|
---|
| 470 | {
|
---|
| 471 | // after some experiments, it is seems this is the fastest way to implement it
|
---|
| 472 | // for GCC (eg., it does not like using std::min after the pstore !!)
|
---|
| 473 | EIGEN_ALIGN16 int aux[4];
|
---|
| 474 | pstore(aux, a);
|
---|
| 475 | int aux0 = aux[0]<aux[1] ? aux[0] : aux[1];
|
---|
| 476 | int aux2 = aux[2]<aux[3] ? aux[2] : aux[3];
|
---|
| 477 | return aux0<aux2 ? aux0 : aux2;
|
---|
| 478 | }
|
---|
| 479 |
|
---|
| 480 | // max
|
---|
| 481 | template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
|
---|
| 482 | {
|
---|
| 483 | Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a));
|
---|
| 484 | return pfirst(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
|
---|
| 485 | }
|
---|
| 486 | template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
|
---|
| 487 | {
|
---|
| 488 | return pfirst(_mm_max_sd(a, _mm_unpackhi_pd(a,a)));
|
---|
| 489 | }
|
---|
| 490 | template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
|
---|
| 491 | {
|
---|
| 492 | // after some experiments, it is seems this is the fastest way to implement it
|
---|
| 493 | // for GCC (eg., it does not like using std::min after the pstore !!)
|
---|
| 494 | EIGEN_ALIGN16 int aux[4];
|
---|
| 495 | pstore(aux, a);
|
---|
| 496 | int aux0 = aux[0]>aux[1] ? aux[0] : aux[1];
|
---|
| 497 | int aux2 = aux[2]>aux[3] ? aux[2] : aux[3];
|
---|
| 498 | return aux0>aux2 ? aux0 : aux2;
|
---|
| 499 | }
|
---|
| 500 |
|
---|
| 501 | #if (defined __GNUC__)
|
---|
| 502 | // template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
|
---|
| 503 | // {
|
---|
| 504 | // Packet4f res = b;
|
---|
| 505 | // asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c));
|
---|
| 506 | // return res;
|
---|
| 507 | // }
|
---|
| 508 | // EIGEN_STRONG_INLINE Packet4i _mm_alignr_epi8(const Packet4i& a, const Packet4i& b, const int i)
|
---|
| 509 | // {
|
---|
| 510 | // Packet4i res = a;
|
---|
| 511 | // asm("palignr %[i], %[a], %[b] " : [b] "+x" (res) : [a] "x" (a), [i] "i" (i));
|
---|
| 512 | // return res;
|
---|
| 513 | // }
|
---|
| 514 | #endif
|
---|
| 515 |
|
---|
| 516 | #ifdef EIGEN_VECTORIZE_SSSE3
|
---|
| 517 | // SSSE3 versions
|
---|
| 518 | template<int Offset>
|
---|
| 519 | struct palign_impl<Offset,Packet4f>
|
---|
| 520 | {
|
---|
| 521 | static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
|
---|
| 522 | {
|
---|
| 523 | if (Offset!=0)
|
---|
| 524 | first = _mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128(second), _mm_castps_si128(first), Offset*4));
|
---|
| 525 | }
|
---|
| 526 | };
|
---|
| 527 |
|
---|
| 528 | template<int Offset>
|
---|
| 529 | struct palign_impl<Offset,Packet4i>
|
---|
| 530 | {
|
---|
| 531 | static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
|
---|
| 532 | {
|
---|
| 533 | if (Offset!=0)
|
---|
| 534 | first = _mm_alignr_epi8(second,first, Offset*4);
|
---|
| 535 | }
|
---|
| 536 | };
|
---|
| 537 |
|
---|
| 538 | template<int Offset>
|
---|
| 539 | struct palign_impl<Offset,Packet2d>
|
---|
| 540 | {
|
---|
| 541 | static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
|
---|
| 542 | {
|
---|
| 543 | if (Offset==1)
|
---|
| 544 | first = _mm_castsi128_pd(_mm_alignr_epi8(_mm_castpd_si128(second), _mm_castpd_si128(first), 8));
|
---|
| 545 | }
|
---|
| 546 | };
|
---|
| 547 | #else
|
---|
| 548 | // SSE2 versions
|
---|
| 549 | template<int Offset>
|
---|
| 550 | struct palign_impl<Offset,Packet4f>
|
---|
| 551 | {
|
---|
| 552 | static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
|
---|
| 553 | {
|
---|
| 554 | if (Offset==1)
|
---|
| 555 | {
|
---|
| 556 | first = _mm_move_ss(first,second);
|
---|
| 557 | first = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(first),0x39));
|
---|
| 558 | }
|
---|
| 559 | else if (Offset==2)
|
---|
| 560 | {
|
---|
| 561 | first = _mm_movehl_ps(first,first);
|
---|
| 562 | first = _mm_movelh_ps(first,second);
|
---|
| 563 | }
|
---|
| 564 | else if (Offset==3)
|
---|
| 565 | {
|
---|
| 566 | first = _mm_move_ss(first,second);
|
---|
| 567 | first = _mm_shuffle_ps(first,second,0x93);
|
---|
| 568 | }
|
---|
| 569 | }
|
---|
| 570 | };
|
---|
| 571 |
|
---|
| 572 | template<int Offset>
|
---|
| 573 | struct palign_impl<Offset,Packet4i>
|
---|
| 574 | {
|
---|
| 575 | static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
|
---|
| 576 | {
|
---|
| 577 | if (Offset==1)
|
---|
| 578 | {
|
---|
| 579 | first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
|
---|
| 580 | first = _mm_shuffle_epi32(first,0x39);
|
---|
| 581 | }
|
---|
| 582 | else if (Offset==2)
|
---|
| 583 | {
|
---|
| 584 | first = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(first)));
|
---|
| 585 | first = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
|
---|
| 586 | }
|
---|
| 587 | else if (Offset==3)
|
---|
| 588 | {
|
---|
| 589 | first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
|
---|
| 590 | first = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second),0x93));
|
---|
| 591 | }
|
---|
| 592 | }
|
---|
| 593 | };
|
---|
| 594 |
|
---|
| 595 | template<int Offset>
|
---|
| 596 | struct palign_impl<Offset,Packet2d>
|
---|
| 597 | {
|
---|
| 598 | static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
|
---|
| 599 | {
|
---|
| 600 | if (Offset==1)
|
---|
| 601 | {
|
---|
| 602 | first = _mm_castps_pd(_mm_movehl_ps(_mm_castpd_ps(first),_mm_castpd_ps(first)));
|
---|
| 603 | first = _mm_castps_pd(_mm_movelh_ps(_mm_castpd_ps(first),_mm_castpd_ps(second)));
|
---|
| 604 | }
|
---|
| 605 | }
|
---|
| 606 | };
|
---|
| 607 | #endif
|
---|
| 608 |
|
---|
| 609 | } // end namespace internal
|
---|
| 610 |
|
---|
| 611 | } // end namespace Eigen
|
---|
| 612 |
|
---|
| 613 | #endif // EIGEN_PACKET_MATH_SSE_H
|
---|