[136] | 1 | // This file is part of Eigen, a lightweight C++ template library
|
---|
| 2 | // for linear algebra.
|
---|
| 3 | //
|
---|
| 4 | // Copyright (C) 2007 Michael Olbrich <michael.olbrich@gmx.net>
|
---|
| 5 | // Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
|
---|
| 6 | // Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
|
---|
| 7 | //
|
---|
| 8 | // This Source Code Form is subject to the terms of the Mozilla
|
---|
| 9 | // Public License v. 2.0. If a copy of the MPL was not distributed
|
---|
| 10 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
---|
| 11 |
|
---|
| 12 | #ifndef EIGEN_ASSIGN_H
|
---|
| 13 | #define EIGEN_ASSIGN_H
|
---|
| 14 |
|
---|
| 15 | namespace Eigen {
|
---|
| 16 |
|
---|
| 17 | namespace internal {
|
---|
| 18 |
|
---|
| 19 | /***************************************************************************
|
---|
| 20 | * Part 1 : the logic deciding a strategy for traversal and unrolling *
|
---|
| 21 | ***************************************************************************/
|
---|
| 22 |
|
---|
| 23 | template <typename Derived, typename OtherDerived>
|
---|
| 24 | struct assign_traits
|
---|
| 25 | {
|
---|
| 26 | public:
|
---|
| 27 | enum {
|
---|
| 28 | DstIsAligned = Derived::Flags & AlignedBit,
|
---|
| 29 | DstHasDirectAccess = Derived::Flags & DirectAccessBit,
|
---|
| 30 | SrcIsAligned = OtherDerived::Flags & AlignedBit,
|
---|
| 31 | JointAlignment = bool(DstIsAligned) && bool(SrcIsAligned) ? Aligned : Unaligned
|
---|
| 32 | };
|
---|
| 33 |
|
---|
| 34 | private:
|
---|
| 35 | enum {
|
---|
| 36 | InnerSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::SizeAtCompileTime)
|
---|
| 37 | : int(Derived::Flags)&RowMajorBit ? int(Derived::ColsAtCompileTime)
|
---|
| 38 | : int(Derived::RowsAtCompileTime),
|
---|
| 39 | InnerMaxSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::MaxSizeAtCompileTime)
|
---|
| 40 | : int(Derived::Flags)&RowMajorBit ? int(Derived::MaxColsAtCompileTime)
|
---|
| 41 | : int(Derived::MaxRowsAtCompileTime),
|
---|
| 42 | MaxSizeAtCompileTime = Derived::SizeAtCompileTime,
|
---|
| 43 | PacketSize = packet_traits<typename Derived::Scalar>::size
|
---|
| 44 | };
|
---|
| 45 |
|
---|
| 46 | enum {
|
---|
| 47 | StorageOrdersAgree = (int(Derived::IsRowMajor) == int(OtherDerived::IsRowMajor)),
|
---|
| 48 | MightVectorize = StorageOrdersAgree
|
---|
| 49 | && (int(Derived::Flags) & int(OtherDerived::Flags) & ActualPacketAccessBit),
|
---|
| 50 | MayInnerVectorize = MightVectorize && int(InnerSize)!=Dynamic && int(InnerSize)%int(PacketSize)==0
|
---|
| 51 | && int(DstIsAligned) && int(SrcIsAligned),
|
---|
| 52 | MayLinearize = StorageOrdersAgree && (int(Derived::Flags) & int(OtherDerived::Flags) & LinearAccessBit),
|
---|
| 53 | MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess
|
---|
| 54 | && (DstIsAligned || MaxSizeAtCompileTime == Dynamic),
|
---|
| 55 | /* If the destination isn't aligned, we have to do runtime checks and we don't unroll,
|
---|
| 56 | so it's only good for large enough sizes. */
|
---|
| 57 | MaySliceVectorize = MightVectorize && DstHasDirectAccess
|
---|
| 58 | && (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=3*PacketSize)
|
---|
| 59 | /* slice vectorization can be slow, so we only want it if the slices are big, which is
|
---|
| 60 | indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block
|
---|
| 61 | in a fixed-size matrix */
|
---|
| 62 | };
|
---|
| 63 |
|
---|
| 64 | public:
|
---|
| 65 | enum {
|
---|
| 66 | Traversal = int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
|
---|
| 67 | : int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
|
---|
| 68 | : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
|
---|
| 69 | : int(MayLinearize) ? int(LinearTraversal)
|
---|
| 70 | : int(DefaultTraversal),
|
---|
| 71 | Vectorized = int(Traversal) == InnerVectorizedTraversal
|
---|
| 72 | || int(Traversal) == LinearVectorizedTraversal
|
---|
| 73 | || int(Traversal) == SliceVectorizedTraversal
|
---|
| 74 | };
|
---|
| 75 |
|
---|
| 76 | private:
|
---|
| 77 | enum {
|
---|
| 78 | UnrollingLimit = EIGEN_UNROLLING_LIMIT * (Vectorized ? int(PacketSize) : 1),
|
---|
| 79 | MayUnrollCompletely = int(Derived::SizeAtCompileTime) != Dynamic
|
---|
| 80 | && int(OtherDerived::CoeffReadCost) != Dynamic
|
---|
| 81 | && int(Derived::SizeAtCompileTime) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit),
|
---|
| 82 | MayUnrollInner = int(InnerSize) != Dynamic
|
---|
| 83 | && int(OtherDerived::CoeffReadCost) != Dynamic
|
---|
| 84 | && int(InnerSize) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit)
|
---|
| 85 | };
|
---|
| 86 |
|
---|
| 87 | public:
|
---|
| 88 | enum {
|
---|
| 89 | Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
|
---|
| 90 | ? (
|
---|
| 91 | int(MayUnrollCompletely) ? int(CompleteUnrolling)
|
---|
| 92 | : int(MayUnrollInner) ? int(InnerUnrolling)
|
---|
| 93 | : int(NoUnrolling)
|
---|
| 94 | )
|
---|
| 95 | : int(Traversal) == int(LinearVectorizedTraversal)
|
---|
| 96 | ? ( bool(MayUnrollCompletely) && bool(DstIsAligned) ? int(CompleteUnrolling) : int(NoUnrolling) )
|
---|
| 97 | : int(Traversal) == int(LinearTraversal)
|
---|
| 98 | ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling) : int(NoUnrolling) )
|
---|
| 99 | : int(NoUnrolling)
|
---|
| 100 | };
|
---|
| 101 |
|
---|
| 102 | #ifdef EIGEN_DEBUG_ASSIGN
|
---|
| 103 | static void debug()
|
---|
| 104 | {
|
---|
| 105 | EIGEN_DEBUG_VAR(DstIsAligned)
|
---|
| 106 | EIGEN_DEBUG_VAR(SrcIsAligned)
|
---|
| 107 | EIGEN_DEBUG_VAR(JointAlignment)
|
---|
| 108 | EIGEN_DEBUG_VAR(InnerSize)
|
---|
| 109 | EIGEN_DEBUG_VAR(InnerMaxSize)
|
---|
| 110 | EIGEN_DEBUG_VAR(PacketSize)
|
---|
| 111 | EIGEN_DEBUG_VAR(StorageOrdersAgree)
|
---|
| 112 | EIGEN_DEBUG_VAR(MightVectorize)
|
---|
| 113 | EIGEN_DEBUG_VAR(MayLinearize)
|
---|
| 114 | EIGEN_DEBUG_VAR(MayInnerVectorize)
|
---|
| 115 | EIGEN_DEBUG_VAR(MayLinearVectorize)
|
---|
| 116 | EIGEN_DEBUG_VAR(MaySliceVectorize)
|
---|
| 117 | EIGEN_DEBUG_VAR(Traversal)
|
---|
| 118 | EIGEN_DEBUG_VAR(UnrollingLimit)
|
---|
| 119 | EIGEN_DEBUG_VAR(MayUnrollCompletely)
|
---|
| 120 | EIGEN_DEBUG_VAR(MayUnrollInner)
|
---|
| 121 | EIGEN_DEBUG_VAR(Unrolling)
|
---|
| 122 | }
|
---|
| 123 | #endif
|
---|
| 124 | };
|
---|
| 125 |
|
---|
| 126 | /***************************************************************************
|
---|
| 127 | * Part 2 : meta-unrollers
|
---|
| 128 | ***************************************************************************/
|
---|
| 129 |
|
---|
| 130 | /************************
|
---|
| 131 | *** Default traversal ***
|
---|
| 132 | ************************/
|
---|
| 133 |
|
---|
| 134 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
| 135 | struct assign_DefaultTraversal_CompleteUnrolling
|
---|
| 136 | {
|
---|
| 137 | enum {
|
---|
| 138 | outer = Index / Derived1::InnerSizeAtCompileTime,
|
---|
| 139 | inner = Index % Derived1::InnerSizeAtCompileTime
|
---|
| 140 | };
|
---|
| 141 |
|
---|
| 142 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 143 | {
|
---|
| 144 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
| 145 | assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, Index+1, Stop>::run(dst, src);
|
---|
| 146 | }
|
---|
| 147 | };
|
---|
| 148 |
|
---|
| 149 | template<typename Derived1, typename Derived2, int Stop>
|
---|
| 150 | struct assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
| 151 | {
|
---|
| 152 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &) {}
|
---|
| 153 | };
|
---|
| 154 |
|
---|
| 155 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
| 156 | struct assign_DefaultTraversal_InnerUnrolling
|
---|
| 157 | {
|
---|
| 158 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src, typename Derived1::Index outer)
|
---|
| 159 | {
|
---|
| 160 | dst.copyCoeffByOuterInner(outer, Index, src);
|
---|
| 161 | assign_DefaultTraversal_InnerUnrolling<Derived1, Derived2, Index+1, Stop>::run(dst, src, outer);
|
---|
| 162 | }
|
---|
| 163 | };
|
---|
| 164 |
|
---|
| 165 | template<typename Derived1, typename Derived2, int Stop>
|
---|
| 166 | struct assign_DefaultTraversal_InnerUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
| 167 | {
|
---|
| 168 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &, typename Derived1::Index) {}
|
---|
| 169 | };
|
---|
| 170 |
|
---|
| 171 | /***********************
|
---|
| 172 | *** Linear traversal ***
|
---|
| 173 | ***********************/
|
---|
| 174 |
|
---|
| 175 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
| 176 | struct assign_LinearTraversal_CompleteUnrolling
|
---|
| 177 | {
|
---|
| 178 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 179 | {
|
---|
| 180 | dst.copyCoeff(Index, src);
|
---|
| 181 | assign_LinearTraversal_CompleteUnrolling<Derived1, Derived2, Index+1, Stop>::run(dst, src);
|
---|
| 182 | }
|
---|
| 183 | };
|
---|
| 184 |
|
---|
| 185 | template<typename Derived1, typename Derived2, int Stop>
|
---|
| 186 | struct assign_LinearTraversal_CompleteUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
| 187 | {
|
---|
| 188 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &) {}
|
---|
| 189 | };
|
---|
| 190 |
|
---|
| 191 | /**************************
|
---|
| 192 | *** Inner vectorization ***
|
---|
| 193 | **************************/
|
---|
| 194 |
|
---|
| 195 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
| 196 | struct assign_innervec_CompleteUnrolling
|
---|
| 197 | {
|
---|
| 198 | enum {
|
---|
| 199 | outer = Index / Derived1::InnerSizeAtCompileTime,
|
---|
| 200 | inner = Index % Derived1::InnerSizeAtCompileTime,
|
---|
| 201 | JointAlignment = assign_traits<Derived1,Derived2>::JointAlignment
|
---|
| 202 | };
|
---|
| 203 |
|
---|
| 204 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 205 | {
|
---|
| 206 | dst.template copyPacketByOuterInner<Derived2, Aligned, JointAlignment>(outer, inner, src);
|
---|
| 207 | assign_innervec_CompleteUnrolling<Derived1, Derived2,
|
---|
| 208 | Index+packet_traits<typename Derived1::Scalar>::size, Stop>::run(dst, src);
|
---|
| 209 | }
|
---|
| 210 | };
|
---|
| 211 |
|
---|
| 212 | template<typename Derived1, typename Derived2, int Stop>
|
---|
| 213 | struct assign_innervec_CompleteUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
| 214 | {
|
---|
| 215 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &) {}
|
---|
| 216 | };
|
---|
| 217 |
|
---|
| 218 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
| 219 | struct assign_innervec_InnerUnrolling
|
---|
| 220 | {
|
---|
| 221 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src, typename Derived1::Index outer)
|
---|
| 222 | {
|
---|
| 223 | dst.template copyPacketByOuterInner<Derived2, Aligned, Aligned>(outer, Index, src);
|
---|
| 224 | assign_innervec_InnerUnrolling<Derived1, Derived2,
|
---|
| 225 | Index+packet_traits<typename Derived1::Scalar>::size, Stop>::run(dst, src, outer);
|
---|
| 226 | }
|
---|
| 227 | };
|
---|
| 228 |
|
---|
| 229 | template<typename Derived1, typename Derived2, int Stop>
|
---|
| 230 | struct assign_innervec_InnerUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
| 231 | {
|
---|
| 232 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &, typename Derived1::Index) {}
|
---|
| 233 | };
|
---|
| 234 |
|
---|
| 235 | /***************************************************************************
|
---|
| 236 | * Part 3 : implementation of all cases
|
---|
| 237 | ***************************************************************************/
|
---|
| 238 |
|
---|
| 239 | template<typename Derived1, typename Derived2,
|
---|
| 240 | int Traversal = assign_traits<Derived1, Derived2>::Traversal,
|
---|
| 241 | int Unrolling = assign_traits<Derived1, Derived2>::Unrolling,
|
---|
| 242 | int Version = Specialized>
|
---|
| 243 | struct assign_impl;
|
---|
| 244 |
|
---|
| 245 | /************************
|
---|
| 246 | *** Default traversal ***
|
---|
| 247 | ************************/
|
---|
| 248 |
|
---|
| 249 | template<typename Derived1, typename Derived2, int Unrolling, int Version>
|
---|
| 250 | struct assign_impl<Derived1, Derived2, InvalidTraversal, Unrolling, Version>
|
---|
| 251 | {
|
---|
| 252 | static inline void run(Derived1 &, const Derived2 &) { }
|
---|
| 253 | };
|
---|
| 254 |
|
---|
| 255 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 256 | struct assign_impl<Derived1, Derived2, DefaultTraversal, NoUnrolling, Version>
|
---|
| 257 | {
|
---|
| 258 | typedef typename Derived1::Index Index;
|
---|
| 259 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
| 260 | {
|
---|
| 261 | const Index innerSize = dst.innerSize();
|
---|
| 262 | const Index outerSize = dst.outerSize();
|
---|
| 263 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
| 264 | for(Index inner = 0; inner < innerSize; ++inner)
|
---|
| 265 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
| 266 | }
|
---|
| 267 | };
|
---|
| 268 |
|
---|
| 269 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 270 | struct assign_impl<Derived1, Derived2, DefaultTraversal, CompleteUnrolling, Version>
|
---|
| 271 | {
|
---|
| 272 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 273 | {
|
---|
| 274 | assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, 0, Derived1::SizeAtCompileTime>
|
---|
| 275 | ::run(dst, src);
|
---|
| 276 | }
|
---|
| 277 | };
|
---|
| 278 |
|
---|
| 279 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 280 | struct assign_impl<Derived1, Derived2, DefaultTraversal, InnerUnrolling, Version>
|
---|
| 281 | {
|
---|
| 282 | typedef typename Derived1::Index Index;
|
---|
| 283 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 284 | {
|
---|
| 285 | const Index outerSize = dst.outerSize();
|
---|
| 286 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
| 287 | assign_DefaultTraversal_InnerUnrolling<Derived1, Derived2, 0, Derived1::InnerSizeAtCompileTime>
|
---|
| 288 | ::run(dst, src, outer);
|
---|
| 289 | }
|
---|
| 290 | };
|
---|
| 291 |
|
---|
| 292 | /***********************
|
---|
| 293 | *** Linear traversal ***
|
---|
| 294 | ***********************/
|
---|
| 295 |
|
---|
| 296 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 297 | struct assign_impl<Derived1, Derived2, LinearTraversal, NoUnrolling, Version>
|
---|
| 298 | {
|
---|
| 299 | typedef typename Derived1::Index Index;
|
---|
| 300 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
| 301 | {
|
---|
| 302 | const Index size = dst.size();
|
---|
| 303 | for(Index i = 0; i < size; ++i)
|
---|
| 304 | dst.copyCoeff(i, src);
|
---|
| 305 | }
|
---|
| 306 | };
|
---|
| 307 |
|
---|
| 308 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 309 | struct assign_impl<Derived1, Derived2, LinearTraversal, CompleteUnrolling, Version>
|
---|
| 310 | {
|
---|
| 311 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 312 | {
|
---|
| 313 | assign_LinearTraversal_CompleteUnrolling<Derived1, Derived2, 0, Derived1::SizeAtCompileTime>
|
---|
| 314 | ::run(dst, src);
|
---|
| 315 | }
|
---|
| 316 | };
|
---|
| 317 |
|
---|
| 318 | /**************************
|
---|
| 319 | *** Inner vectorization ***
|
---|
| 320 | **************************/
|
---|
| 321 |
|
---|
| 322 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 323 | struct assign_impl<Derived1, Derived2, InnerVectorizedTraversal, NoUnrolling, Version>
|
---|
| 324 | {
|
---|
| 325 | typedef typename Derived1::Index Index;
|
---|
| 326 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
| 327 | {
|
---|
| 328 | const Index innerSize = dst.innerSize();
|
---|
| 329 | const Index outerSize = dst.outerSize();
|
---|
| 330 | const Index packetSize = packet_traits<typename Derived1::Scalar>::size;
|
---|
| 331 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
| 332 | for(Index inner = 0; inner < innerSize; inner+=packetSize)
|
---|
| 333 | dst.template copyPacketByOuterInner<Derived2, Aligned, Aligned>(outer, inner, src);
|
---|
| 334 | }
|
---|
| 335 | };
|
---|
| 336 |
|
---|
| 337 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 338 | struct assign_impl<Derived1, Derived2, InnerVectorizedTraversal, CompleteUnrolling, Version>
|
---|
| 339 | {
|
---|
| 340 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 341 | {
|
---|
| 342 | assign_innervec_CompleteUnrolling<Derived1, Derived2, 0, Derived1::SizeAtCompileTime>
|
---|
| 343 | ::run(dst, src);
|
---|
| 344 | }
|
---|
| 345 | };
|
---|
| 346 |
|
---|
| 347 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 348 | struct assign_impl<Derived1, Derived2, InnerVectorizedTraversal, InnerUnrolling, Version>
|
---|
| 349 | {
|
---|
| 350 | typedef typename Derived1::Index Index;
|
---|
| 351 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 352 | {
|
---|
| 353 | const Index outerSize = dst.outerSize();
|
---|
| 354 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
| 355 | assign_innervec_InnerUnrolling<Derived1, Derived2, 0, Derived1::InnerSizeAtCompileTime>
|
---|
| 356 | ::run(dst, src, outer);
|
---|
| 357 | }
|
---|
| 358 | };
|
---|
| 359 |
|
---|
| 360 | /***************************
|
---|
| 361 | *** Linear vectorization ***
|
---|
| 362 | ***************************/
|
---|
| 363 |
|
---|
| 364 | template <bool IsAligned = false>
|
---|
| 365 | struct unaligned_assign_impl
|
---|
| 366 | {
|
---|
| 367 | template <typename Derived, typename OtherDerived>
|
---|
| 368 | static EIGEN_STRONG_INLINE void run(const Derived&, OtherDerived&, typename Derived::Index, typename Derived::Index) {}
|
---|
| 369 | };
|
---|
| 370 |
|
---|
| 371 | template <>
|
---|
| 372 | struct unaligned_assign_impl<false>
|
---|
| 373 | {
|
---|
| 374 | // MSVC must not inline this functions. If it does, it fails to optimize the
|
---|
| 375 | // packet access path.
|
---|
| 376 | #ifdef _MSC_VER
|
---|
| 377 | template <typename Derived, typename OtherDerived>
|
---|
| 378 | static EIGEN_DONT_INLINE void run(const Derived& src, OtherDerived& dst, typename Derived::Index start, typename Derived::Index end)
|
---|
| 379 | #else
|
---|
| 380 | template <typename Derived, typename OtherDerived>
|
---|
| 381 | static EIGEN_STRONG_INLINE void run(const Derived& src, OtherDerived& dst, typename Derived::Index start, typename Derived::Index end)
|
---|
| 382 | #endif
|
---|
| 383 | {
|
---|
| 384 | for (typename Derived::Index index = start; index < end; ++index)
|
---|
| 385 | dst.copyCoeff(index, src);
|
---|
| 386 | }
|
---|
| 387 | };
|
---|
| 388 |
|
---|
| 389 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 390 | struct assign_impl<Derived1, Derived2, LinearVectorizedTraversal, NoUnrolling, Version>
|
---|
| 391 | {
|
---|
| 392 | typedef typename Derived1::Index Index;
|
---|
| 393 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 394 | {
|
---|
| 395 | const Index size = dst.size();
|
---|
| 396 | typedef packet_traits<typename Derived1::Scalar> PacketTraits;
|
---|
| 397 | enum {
|
---|
| 398 | packetSize = PacketTraits::size,
|
---|
| 399 | dstAlignment = PacketTraits::AlignedOnScalar ? Aligned : int(assign_traits<Derived1,Derived2>::DstIsAligned) ,
|
---|
| 400 | srcAlignment = assign_traits<Derived1,Derived2>::JointAlignment
|
---|
| 401 | };
|
---|
| 402 | const Index alignedStart = assign_traits<Derived1,Derived2>::DstIsAligned ? 0
|
---|
| 403 | : internal::first_aligned(&dst.coeffRef(0), size);
|
---|
| 404 | const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
|
---|
| 405 |
|
---|
| 406 | unaligned_assign_impl<assign_traits<Derived1,Derived2>::DstIsAligned!=0>::run(src,dst,0,alignedStart);
|
---|
| 407 |
|
---|
| 408 | for(Index index = alignedStart; index < alignedEnd; index += packetSize)
|
---|
| 409 | {
|
---|
| 410 | dst.template copyPacket<Derived2, dstAlignment, srcAlignment>(index, src);
|
---|
| 411 | }
|
---|
| 412 |
|
---|
| 413 | unaligned_assign_impl<>::run(src,dst,alignedEnd,size);
|
---|
| 414 | }
|
---|
| 415 | };
|
---|
| 416 |
|
---|
| 417 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 418 | struct assign_impl<Derived1, Derived2, LinearVectorizedTraversal, CompleteUnrolling, Version>
|
---|
| 419 | {
|
---|
| 420 | typedef typename Derived1::Index Index;
|
---|
| 421 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
| 422 | {
|
---|
| 423 | enum { size = Derived1::SizeAtCompileTime,
|
---|
| 424 | packetSize = packet_traits<typename Derived1::Scalar>::size,
|
---|
| 425 | alignedSize = (size/packetSize)*packetSize };
|
---|
| 426 |
|
---|
| 427 | assign_innervec_CompleteUnrolling<Derived1, Derived2, 0, alignedSize>::run(dst, src);
|
---|
| 428 | assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, alignedSize, size>::run(dst, src);
|
---|
| 429 | }
|
---|
| 430 | };
|
---|
| 431 |
|
---|
| 432 | /**************************
|
---|
| 433 | *** Slice vectorization ***
|
---|
| 434 | ***************************/
|
---|
| 435 |
|
---|
| 436 | template<typename Derived1, typename Derived2, int Version>
|
---|
| 437 | struct assign_impl<Derived1, Derived2, SliceVectorizedTraversal, NoUnrolling, Version>
|
---|
| 438 | {
|
---|
| 439 | typedef typename Derived1::Index Index;
|
---|
| 440 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
| 441 | {
|
---|
| 442 | typedef typename Derived1::Scalar Scalar;
|
---|
| 443 | typedef packet_traits<Scalar> PacketTraits;
|
---|
| 444 | enum {
|
---|
| 445 | packetSize = PacketTraits::size,
|
---|
| 446 | alignable = PacketTraits::AlignedOnScalar,
|
---|
| 447 | dstIsAligned = assign_traits<Derived1,Derived2>::DstIsAligned,
|
---|
| 448 | dstAlignment = alignable ? Aligned : int(dstIsAligned),
|
---|
| 449 | srcAlignment = assign_traits<Derived1,Derived2>::JointAlignment
|
---|
| 450 | };
|
---|
| 451 | const Scalar *dst_ptr = &dst.coeffRef(0,0);
|
---|
| 452 | if((!bool(dstIsAligned)) && (size_t(dst_ptr) % sizeof(Scalar))>0)
|
---|
| 453 | {
|
---|
| 454 | // the pointer is not aligend-on scalar, so alignment is not possible
|
---|
| 455 | return assign_impl<Derived1,Derived2,DefaultTraversal,NoUnrolling>::run(dst, src);
|
---|
| 456 | }
|
---|
| 457 | const Index packetAlignedMask = packetSize - 1;
|
---|
| 458 | const Index innerSize = dst.innerSize();
|
---|
| 459 | const Index outerSize = dst.outerSize();
|
---|
| 460 | const Index alignedStep = alignable ? (packetSize - dst.outerStride() % packetSize) & packetAlignedMask : 0;
|
---|
| 461 | Index alignedStart = ((!alignable) || bool(dstIsAligned)) ? 0 : internal::first_aligned(dst_ptr, innerSize);
|
---|
| 462 |
|
---|
| 463 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
| 464 | {
|
---|
| 465 | const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
|
---|
| 466 | // do the non-vectorizable part of the assignment
|
---|
| 467 | for(Index inner = 0; inner<alignedStart ; ++inner)
|
---|
| 468 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
| 469 |
|
---|
| 470 | // do the vectorizable part of the assignment
|
---|
| 471 | for(Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
|
---|
| 472 | dst.template copyPacketByOuterInner<Derived2, dstAlignment, Unaligned>(outer, inner, src);
|
---|
| 473 |
|
---|
| 474 | // do the non-vectorizable part of the assignment
|
---|
| 475 | for(Index inner = alignedEnd; inner<innerSize ; ++inner)
|
---|
| 476 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
| 477 |
|
---|
| 478 | alignedStart = std::min<Index>((alignedStart+alignedStep)%packetSize, innerSize);
|
---|
| 479 | }
|
---|
| 480 | }
|
---|
| 481 | };
|
---|
| 482 |
|
---|
| 483 | } // end namespace internal
|
---|
| 484 |
|
---|
| 485 | /***************************************************************************
|
---|
| 486 | * Part 4 : implementation of DenseBase methods
|
---|
| 487 | ***************************************************************************/
|
---|
| 488 |
|
---|
| 489 | template<typename Derived>
|
---|
| 490 | template<typename OtherDerived>
|
---|
| 491 | EIGEN_STRONG_INLINE Derived& DenseBase<Derived>
|
---|
| 492 | ::lazyAssign(const DenseBase<OtherDerived>& other)
|
---|
| 493 | {
|
---|
| 494 | enum{
|
---|
| 495 | SameType = internal::is_same<typename Derived::Scalar,typename OtherDerived::Scalar>::value
|
---|
| 496 | };
|
---|
| 497 |
|
---|
| 498 | EIGEN_STATIC_ASSERT_LVALUE(Derived)
|
---|
| 499 | EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Derived,OtherDerived)
|
---|
| 500 | EIGEN_STATIC_ASSERT(SameType,YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY)
|
---|
| 501 |
|
---|
| 502 | #ifdef EIGEN_DEBUG_ASSIGN
|
---|
| 503 | internal::assign_traits<Derived, OtherDerived>::debug();
|
---|
| 504 | #endif
|
---|
| 505 | eigen_assert(rows() == other.rows() && cols() == other.cols());
|
---|
| 506 | internal::assign_impl<Derived, OtherDerived, int(SameType) ? int(internal::assign_traits<Derived, OtherDerived>::Traversal)
|
---|
| 507 | : int(InvalidTraversal)>::run(derived(),other.derived());
|
---|
| 508 | #ifndef EIGEN_NO_DEBUG
|
---|
| 509 | checkTransposeAliasing(other.derived());
|
---|
| 510 | #endif
|
---|
| 511 | return derived();
|
---|
| 512 | }
|
---|
| 513 |
|
---|
| 514 | namespace internal {
|
---|
| 515 |
|
---|
| 516 | template<typename Derived, typename OtherDerived,
|
---|
| 517 | bool EvalBeforeAssigning = (int(internal::traits<OtherDerived>::Flags) & EvalBeforeAssigningBit) != 0,
|
---|
| 518 | bool NeedToTranspose = ((int(Derived::RowsAtCompileTime) == 1 && int(OtherDerived::ColsAtCompileTime) == 1)
|
---|
| 519 | | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&".
|
---|
| 520 | // revert to || as soon as not needed anymore.
|
---|
| 521 | (int(Derived::ColsAtCompileTime) == 1 && int(OtherDerived::RowsAtCompileTime) == 1))
|
---|
| 522 | && int(Derived::SizeAtCompileTime) != 1>
|
---|
| 523 | struct assign_selector;
|
---|
| 524 |
|
---|
| 525 | template<typename Derived, typename OtherDerived>
|
---|
| 526 | struct assign_selector<Derived,OtherDerived,false,false> {
|
---|
| 527 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.derived()); }
|
---|
| 528 | template<typename ActualDerived, typename ActualOtherDerived>
|
---|
| 529 | static EIGEN_STRONG_INLINE Derived& evalTo(ActualDerived& dst, const ActualOtherDerived& other) { other.evalTo(dst); return dst; }
|
---|
| 530 | };
|
---|
| 531 | template<typename Derived, typename OtherDerived>
|
---|
| 532 | struct assign_selector<Derived,OtherDerived,true,false> {
|
---|
| 533 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.eval()); }
|
---|
| 534 | };
|
---|
| 535 | template<typename Derived, typename OtherDerived>
|
---|
| 536 | struct assign_selector<Derived,OtherDerived,false,true> {
|
---|
| 537 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose()); }
|
---|
| 538 | template<typename ActualDerived, typename ActualOtherDerived>
|
---|
| 539 | static EIGEN_STRONG_INLINE Derived& evalTo(ActualDerived& dst, const ActualOtherDerived& other) { Transpose<ActualDerived> dstTrans(dst); other.evalTo(dstTrans); return dst; }
|
---|
| 540 | };
|
---|
| 541 | template<typename Derived, typename OtherDerived>
|
---|
| 542 | struct assign_selector<Derived,OtherDerived,true,true> {
|
---|
| 543 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose().eval()); }
|
---|
| 544 | };
|
---|
| 545 |
|
---|
| 546 | } // end namespace internal
|
---|
| 547 |
|
---|
| 548 | template<typename Derived>
|
---|
| 549 | template<typename OtherDerived>
|
---|
| 550 | EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase<OtherDerived>& other)
|
---|
| 551 | {
|
---|
| 552 | return internal::assign_selector<Derived,OtherDerived>::run(derived(), other.derived());
|
---|
| 553 | }
|
---|
| 554 |
|
---|
| 555 | template<typename Derived>
|
---|
| 556 | EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase& other)
|
---|
| 557 | {
|
---|
| 558 | return internal::assign_selector<Derived,Derived>::run(derived(), other.derived());
|
---|
| 559 | }
|
---|
| 560 |
|
---|
| 561 | template<typename Derived>
|
---|
| 562 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const MatrixBase& other)
|
---|
| 563 | {
|
---|
| 564 | return internal::assign_selector<Derived,Derived>::run(derived(), other.derived());
|
---|
| 565 | }
|
---|
| 566 |
|
---|
| 567 | template<typename Derived>
|
---|
| 568 | template <typename OtherDerived>
|
---|
| 569 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const DenseBase<OtherDerived>& other)
|
---|
| 570 | {
|
---|
| 571 | return internal::assign_selector<Derived,OtherDerived>::run(derived(), other.derived());
|
---|
| 572 | }
|
---|
| 573 |
|
---|
| 574 | template<typename Derived>
|
---|
| 575 | template <typename OtherDerived>
|
---|
| 576 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const EigenBase<OtherDerived>& other)
|
---|
| 577 | {
|
---|
| 578 | return internal::assign_selector<Derived,OtherDerived,false>::evalTo(derived(), other.derived());
|
---|
| 579 | }
|
---|
| 580 |
|
---|
| 581 | template<typename Derived>
|
---|
| 582 | template<typename OtherDerived>
|
---|
| 583 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other)
|
---|
| 584 | {
|
---|
| 585 | return internal::assign_selector<Derived,OtherDerived,false>::evalTo(derived(), other.derived());
|
---|
| 586 | }
|
---|
| 587 |
|
---|
| 588 | } // end namespace Eigen
|
---|
| 589 |
|
---|
| 590 | #endif // EIGEN_ASSIGN_H
|
---|