1 | // This file is part of Eigen, a lightweight C++ template library
|
---|
2 | // for linear algebra.
|
---|
3 | //
|
---|
4 | // Copyright (C) 2007 Michael Olbrich <michael.olbrich@gmx.net>
|
---|
5 | // Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
|
---|
6 | // Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
|
---|
7 | //
|
---|
8 | // This Source Code Form is subject to the terms of the Mozilla
|
---|
9 | // Public License v. 2.0. If a copy of the MPL was not distributed
|
---|
10 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
---|
11 |
|
---|
12 | #ifndef EIGEN_ASSIGN_H
|
---|
13 | #define EIGEN_ASSIGN_H
|
---|
14 |
|
---|
15 | namespace Eigen {
|
---|
16 |
|
---|
17 | namespace internal {
|
---|
18 |
|
---|
19 | /***************************************************************************
|
---|
20 | * Part 1 : the logic deciding a strategy for traversal and unrolling *
|
---|
21 | ***************************************************************************/
|
---|
22 |
|
---|
23 | template <typename Derived, typename OtherDerived>
|
---|
24 | struct assign_traits
|
---|
25 | {
|
---|
26 | public:
|
---|
27 | enum {
|
---|
28 | DstIsAligned = Derived::Flags & AlignedBit,
|
---|
29 | DstHasDirectAccess = Derived::Flags & DirectAccessBit,
|
---|
30 | SrcIsAligned = OtherDerived::Flags & AlignedBit,
|
---|
31 | JointAlignment = bool(DstIsAligned) && bool(SrcIsAligned) ? Aligned : Unaligned
|
---|
32 | };
|
---|
33 |
|
---|
34 | private:
|
---|
35 | enum {
|
---|
36 | InnerSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::SizeAtCompileTime)
|
---|
37 | : int(Derived::Flags)&RowMajorBit ? int(Derived::ColsAtCompileTime)
|
---|
38 | : int(Derived::RowsAtCompileTime),
|
---|
39 | InnerMaxSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::MaxSizeAtCompileTime)
|
---|
40 | : int(Derived::Flags)&RowMajorBit ? int(Derived::MaxColsAtCompileTime)
|
---|
41 | : int(Derived::MaxRowsAtCompileTime),
|
---|
42 | MaxSizeAtCompileTime = Derived::SizeAtCompileTime,
|
---|
43 | PacketSize = packet_traits<typename Derived::Scalar>::size
|
---|
44 | };
|
---|
45 |
|
---|
46 | enum {
|
---|
47 | StorageOrdersAgree = (int(Derived::IsRowMajor) == int(OtherDerived::IsRowMajor)),
|
---|
48 | MightVectorize = StorageOrdersAgree
|
---|
49 | && (int(Derived::Flags) & int(OtherDerived::Flags) & ActualPacketAccessBit),
|
---|
50 | MayInnerVectorize = MightVectorize && int(InnerSize)!=Dynamic && int(InnerSize)%int(PacketSize)==0
|
---|
51 | && int(DstIsAligned) && int(SrcIsAligned),
|
---|
52 | MayLinearize = StorageOrdersAgree && (int(Derived::Flags) & int(OtherDerived::Flags) & LinearAccessBit),
|
---|
53 | MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess
|
---|
54 | && (DstIsAligned || MaxSizeAtCompileTime == Dynamic),
|
---|
55 | /* If the destination isn't aligned, we have to do runtime checks and we don't unroll,
|
---|
56 | so it's only good for large enough sizes. */
|
---|
57 | MaySliceVectorize = MightVectorize && DstHasDirectAccess
|
---|
58 | && (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=3*PacketSize)
|
---|
59 | /* slice vectorization can be slow, so we only want it if the slices are big, which is
|
---|
60 | indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block
|
---|
61 | in a fixed-size matrix */
|
---|
62 | };
|
---|
63 |
|
---|
64 | public:
|
---|
65 | enum {
|
---|
66 | Traversal = int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
|
---|
67 | : int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
|
---|
68 | : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
|
---|
69 | : int(MayLinearize) ? int(LinearTraversal)
|
---|
70 | : int(DefaultTraversal),
|
---|
71 | Vectorized = int(Traversal) == InnerVectorizedTraversal
|
---|
72 | || int(Traversal) == LinearVectorizedTraversal
|
---|
73 | || int(Traversal) == SliceVectorizedTraversal
|
---|
74 | };
|
---|
75 |
|
---|
76 | private:
|
---|
77 | enum {
|
---|
78 | UnrollingLimit = EIGEN_UNROLLING_LIMIT * (Vectorized ? int(PacketSize) : 1),
|
---|
79 | MayUnrollCompletely = int(Derived::SizeAtCompileTime) != Dynamic
|
---|
80 | && int(OtherDerived::CoeffReadCost) != Dynamic
|
---|
81 | && int(Derived::SizeAtCompileTime) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit),
|
---|
82 | MayUnrollInner = int(InnerSize) != Dynamic
|
---|
83 | && int(OtherDerived::CoeffReadCost) != Dynamic
|
---|
84 | && int(InnerSize) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit)
|
---|
85 | };
|
---|
86 |
|
---|
87 | public:
|
---|
88 | enum {
|
---|
89 | Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
|
---|
90 | ? (
|
---|
91 | int(MayUnrollCompletely) ? int(CompleteUnrolling)
|
---|
92 | : int(MayUnrollInner) ? int(InnerUnrolling)
|
---|
93 | : int(NoUnrolling)
|
---|
94 | )
|
---|
95 | : int(Traversal) == int(LinearVectorizedTraversal)
|
---|
96 | ? ( bool(MayUnrollCompletely) && bool(DstIsAligned) ? int(CompleteUnrolling) : int(NoUnrolling) )
|
---|
97 | : int(Traversal) == int(LinearTraversal)
|
---|
98 | ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling) : int(NoUnrolling) )
|
---|
99 | : int(NoUnrolling)
|
---|
100 | };
|
---|
101 |
|
---|
102 | #ifdef EIGEN_DEBUG_ASSIGN
|
---|
103 | static void debug()
|
---|
104 | {
|
---|
105 | EIGEN_DEBUG_VAR(DstIsAligned)
|
---|
106 | EIGEN_DEBUG_VAR(SrcIsAligned)
|
---|
107 | EIGEN_DEBUG_VAR(JointAlignment)
|
---|
108 | EIGEN_DEBUG_VAR(InnerSize)
|
---|
109 | EIGEN_DEBUG_VAR(InnerMaxSize)
|
---|
110 | EIGEN_DEBUG_VAR(PacketSize)
|
---|
111 | EIGEN_DEBUG_VAR(StorageOrdersAgree)
|
---|
112 | EIGEN_DEBUG_VAR(MightVectorize)
|
---|
113 | EIGEN_DEBUG_VAR(MayLinearize)
|
---|
114 | EIGEN_DEBUG_VAR(MayInnerVectorize)
|
---|
115 | EIGEN_DEBUG_VAR(MayLinearVectorize)
|
---|
116 | EIGEN_DEBUG_VAR(MaySliceVectorize)
|
---|
117 | EIGEN_DEBUG_VAR(Traversal)
|
---|
118 | EIGEN_DEBUG_VAR(UnrollingLimit)
|
---|
119 | EIGEN_DEBUG_VAR(MayUnrollCompletely)
|
---|
120 | EIGEN_DEBUG_VAR(MayUnrollInner)
|
---|
121 | EIGEN_DEBUG_VAR(Unrolling)
|
---|
122 | }
|
---|
123 | #endif
|
---|
124 | };
|
---|
125 |
|
---|
126 | /***************************************************************************
|
---|
127 | * Part 2 : meta-unrollers
|
---|
128 | ***************************************************************************/
|
---|
129 |
|
---|
130 | /************************
|
---|
131 | *** Default traversal ***
|
---|
132 | ************************/
|
---|
133 |
|
---|
134 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
135 | struct assign_DefaultTraversal_CompleteUnrolling
|
---|
136 | {
|
---|
137 | enum {
|
---|
138 | outer = Index / Derived1::InnerSizeAtCompileTime,
|
---|
139 | inner = Index % Derived1::InnerSizeAtCompileTime
|
---|
140 | };
|
---|
141 |
|
---|
142 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
143 | {
|
---|
144 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
145 | assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, Index+1, Stop>::run(dst, src);
|
---|
146 | }
|
---|
147 | };
|
---|
148 |
|
---|
149 | template<typename Derived1, typename Derived2, int Stop>
|
---|
150 | struct assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
151 | {
|
---|
152 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &) {}
|
---|
153 | };
|
---|
154 |
|
---|
155 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
156 | struct assign_DefaultTraversal_InnerUnrolling
|
---|
157 | {
|
---|
158 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src, typename Derived1::Index outer)
|
---|
159 | {
|
---|
160 | dst.copyCoeffByOuterInner(outer, Index, src);
|
---|
161 | assign_DefaultTraversal_InnerUnrolling<Derived1, Derived2, Index+1, Stop>::run(dst, src, outer);
|
---|
162 | }
|
---|
163 | };
|
---|
164 |
|
---|
165 | template<typename Derived1, typename Derived2, int Stop>
|
---|
166 | struct assign_DefaultTraversal_InnerUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
167 | {
|
---|
168 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &, typename Derived1::Index) {}
|
---|
169 | };
|
---|
170 |
|
---|
171 | /***********************
|
---|
172 | *** Linear traversal ***
|
---|
173 | ***********************/
|
---|
174 |
|
---|
175 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
176 | struct assign_LinearTraversal_CompleteUnrolling
|
---|
177 | {
|
---|
178 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
179 | {
|
---|
180 | dst.copyCoeff(Index, src);
|
---|
181 | assign_LinearTraversal_CompleteUnrolling<Derived1, Derived2, Index+1, Stop>::run(dst, src);
|
---|
182 | }
|
---|
183 | };
|
---|
184 |
|
---|
185 | template<typename Derived1, typename Derived2, int Stop>
|
---|
186 | struct assign_LinearTraversal_CompleteUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
187 | {
|
---|
188 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &) {}
|
---|
189 | };
|
---|
190 |
|
---|
191 | /**************************
|
---|
192 | *** Inner vectorization ***
|
---|
193 | **************************/
|
---|
194 |
|
---|
195 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
196 | struct assign_innervec_CompleteUnrolling
|
---|
197 | {
|
---|
198 | enum {
|
---|
199 | outer = Index / Derived1::InnerSizeAtCompileTime,
|
---|
200 | inner = Index % Derived1::InnerSizeAtCompileTime,
|
---|
201 | JointAlignment = assign_traits<Derived1,Derived2>::JointAlignment
|
---|
202 | };
|
---|
203 |
|
---|
204 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
205 | {
|
---|
206 | dst.template copyPacketByOuterInner<Derived2, Aligned, JointAlignment>(outer, inner, src);
|
---|
207 | assign_innervec_CompleteUnrolling<Derived1, Derived2,
|
---|
208 | Index+packet_traits<typename Derived1::Scalar>::size, Stop>::run(dst, src);
|
---|
209 | }
|
---|
210 | };
|
---|
211 |
|
---|
212 | template<typename Derived1, typename Derived2, int Stop>
|
---|
213 | struct assign_innervec_CompleteUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
214 | {
|
---|
215 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &) {}
|
---|
216 | };
|
---|
217 |
|
---|
218 | template<typename Derived1, typename Derived2, int Index, int Stop>
|
---|
219 | struct assign_innervec_InnerUnrolling
|
---|
220 | {
|
---|
221 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src, typename Derived1::Index outer)
|
---|
222 | {
|
---|
223 | dst.template copyPacketByOuterInner<Derived2, Aligned, Aligned>(outer, Index, src);
|
---|
224 | assign_innervec_InnerUnrolling<Derived1, Derived2,
|
---|
225 | Index+packet_traits<typename Derived1::Scalar>::size, Stop>::run(dst, src, outer);
|
---|
226 | }
|
---|
227 | };
|
---|
228 |
|
---|
229 | template<typename Derived1, typename Derived2, int Stop>
|
---|
230 | struct assign_innervec_InnerUnrolling<Derived1, Derived2, Stop, Stop>
|
---|
231 | {
|
---|
232 | static EIGEN_STRONG_INLINE void run(Derived1 &, const Derived2 &, typename Derived1::Index) {}
|
---|
233 | };
|
---|
234 |
|
---|
235 | /***************************************************************************
|
---|
236 | * Part 3 : implementation of all cases
|
---|
237 | ***************************************************************************/
|
---|
238 |
|
---|
239 | template<typename Derived1, typename Derived2,
|
---|
240 | int Traversal = assign_traits<Derived1, Derived2>::Traversal,
|
---|
241 | int Unrolling = assign_traits<Derived1, Derived2>::Unrolling,
|
---|
242 | int Version = Specialized>
|
---|
243 | struct assign_impl;
|
---|
244 |
|
---|
245 | /************************
|
---|
246 | *** Default traversal ***
|
---|
247 | ************************/
|
---|
248 |
|
---|
249 | template<typename Derived1, typename Derived2, int Unrolling, int Version>
|
---|
250 | struct assign_impl<Derived1, Derived2, InvalidTraversal, Unrolling, Version>
|
---|
251 | {
|
---|
252 | static inline void run(Derived1 &, const Derived2 &) { }
|
---|
253 | };
|
---|
254 |
|
---|
255 | template<typename Derived1, typename Derived2, int Version>
|
---|
256 | struct assign_impl<Derived1, Derived2, DefaultTraversal, NoUnrolling, Version>
|
---|
257 | {
|
---|
258 | typedef typename Derived1::Index Index;
|
---|
259 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
260 | {
|
---|
261 | const Index innerSize = dst.innerSize();
|
---|
262 | const Index outerSize = dst.outerSize();
|
---|
263 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
264 | for(Index inner = 0; inner < innerSize; ++inner)
|
---|
265 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
266 | }
|
---|
267 | };
|
---|
268 |
|
---|
269 | template<typename Derived1, typename Derived2, int Version>
|
---|
270 | struct assign_impl<Derived1, Derived2, DefaultTraversal, CompleteUnrolling, Version>
|
---|
271 | {
|
---|
272 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
273 | {
|
---|
274 | assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, 0, Derived1::SizeAtCompileTime>
|
---|
275 | ::run(dst, src);
|
---|
276 | }
|
---|
277 | };
|
---|
278 |
|
---|
279 | template<typename Derived1, typename Derived2, int Version>
|
---|
280 | struct assign_impl<Derived1, Derived2, DefaultTraversal, InnerUnrolling, Version>
|
---|
281 | {
|
---|
282 | typedef typename Derived1::Index Index;
|
---|
283 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
284 | {
|
---|
285 | const Index outerSize = dst.outerSize();
|
---|
286 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
287 | assign_DefaultTraversal_InnerUnrolling<Derived1, Derived2, 0, Derived1::InnerSizeAtCompileTime>
|
---|
288 | ::run(dst, src, outer);
|
---|
289 | }
|
---|
290 | };
|
---|
291 |
|
---|
292 | /***********************
|
---|
293 | *** Linear traversal ***
|
---|
294 | ***********************/
|
---|
295 |
|
---|
296 | template<typename Derived1, typename Derived2, int Version>
|
---|
297 | struct assign_impl<Derived1, Derived2, LinearTraversal, NoUnrolling, Version>
|
---|
298 | {
|
---|
299 | typedef typename Derived1::Index Index;
|
---|
300 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
301 | {
|
---|
302 | const Index size = dst.size();
|
---|
303 | for(Index i = 0; i < size; ++i)
|
---|
304 | dst.copyCoeff(i, src);
|
---|
305 | }
|
---|
306 | };
|
---|
307 |
|
---|
308 | template<typename Derived1, typename Derived2, int Version>
|
---|
309 | struct assign_impl<Derived1, Derived2, LinearTraversal, CompleteUnrolling, Version>
|
---|
310 | {
|
---|
311 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
312 | {
|
---|
313 | assign_LinearTraversal_CompleteUnrolling<Derived1, Derived2, 0, Derived1::SizeAtCompileTime>
|
---|
314 | ::run(dst, src);
|
---|
315 | }
|
---|
316 | };
|
---|
317 |
|
---|
318 | /**************************
|
---|
319 | *** Inner vectorization ***
|
---|
320 | **************************/
|
---|
321 |
|
---|
322 | template<typename Derived1, typename Derived2, int Version>
|
---|
323 | struct assign_impl<Derived1, Derived2, InnerVectorizedTraversal, NoUnrolling, Version>
|
---|
324 | {
|
---|
325 | typedef typename Derived1::Index Index;
|
---|
326 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
327 | {
|
---|
328 | const Index innerSize = dst.innerSize();
|
---|
329 | const Index outerSize = dst.outerSize();
|
---|
330 | const Index packetSize = packet_traits<typename Derived1::Scalar>::size;
|
---|
331 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
332 | for(Index inner = 0; inner < innerSize; inner+=packetSize)
|
---|
333 | dst.template copyPacketByOuterInner<Derived2, Aligned, Aligned>(outer, inner, src);
|
---|
334 | }
|
---|
335 | };
|
---|
336 |
|
---|
337 | template<typename Derived1, typename Derived2, int Version>
|
---|
338 | struct assign_impl<Derived1, Derived2, InnerVectorizedTraversal, CompleteUnrolling, Version>
|
---|
339 | {
|
---|
340 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
341 | {
|
---|
342 | assign_innervec_CompleteUnrolling<Derived1, Derived2, 0, Derived1::SizeAtCompileTime>
|
---|
343 | ::run(dst, src);
|
---|
344 | }
|
---|
345 | };
|
---|
346 |
|
---|
347 | template<typename Derived1, typename Derived2, int Version>
|
---|
348 | struct assign_impl<Derived1, Derived2, InnerVectorizedTraversal, InnerUnrolling, Version>
|
---|
349 | {
|
---|
350 | typedef typename Derived1::Index Index;
|
---|
351 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
352 | {
|
---|
353 | const Index outerSize = dst.outerSize();
|
---|
354 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
355 | assign_innervec_InnerUnrolling<Derived1, Derived2, 0, Derived1::InnerSizeAtCompileTime>
|
---|
356 | ::run(dst, src, outer);
|
---|
357 | }
|
---|
358 | };
|
---|
359 |
|
---|
360 | /***************************
|
---|
361 | *** Linear vectorization ***
|
---|
362 | ***************************/
|
---|
363 |
|
---|
364 | template <bool IsAligned = false>
|
---|
365 | struct unaligned_assign_impl
|
---|
366 | {
|
---|
367 | template <typename Derived, typename OtherDerived>
|
---|
368 | static EIGEN_STRONG_INLINE void run(const Derived&, OtherDerived&, typename Derived::Index, typename Derived::Index) {}
|
---|
369 | };
|
---|
370 |
|
---|
371 | template <>
|
---|
372 | struct unaligned_assign_impl<false>
|
---|
373 | {
|
---|
374 | // MSVC must not inline this functions. If it does, it fails to optimize the
|
---|
375 | // packet access path.
|
---|
376 | #ifdef _MSC_VER
|
---|
377 | template <typename Derived, typename OtherDerived>
|
---|
378 | static EIGEN_DONT_INLINE void run(const Derived& src, OtherDerived& dst, typename Derived::Index start, typename Derived::Index end)
|
---|
379 | #else
|
---|
380 | template <typename Derived, typename OtherDerived>
|
---|
381 | static EIGEN_STRONG_INLINE void run(const Derived& src, OtherDerived& dst, typename Derived::Index start, typename Derived::Index end)
|
---|
382 | #endif
|
---|
383 | {
|
---|
384 | for (typename Derived::Index index = start; index < end; ++index)
|
---|
385 | dst.copyCoeff(index, src);
|
---|
386 | }
|
---|
387 | };
|
---|
388 |
|
---|
389 | template<typename Derived1, typename Derived2, int Version>
|
---|
390 | struct assign_impl<Derived1, Derived2, LinearVectorizedTraversal, NoUnrolling, Version>
|
---|
391 | {
|
---|
392 | typedef typename Derived1::Index Index;
|
---|
393 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
394 | {
|
---|
395 | const Index size = dst.size();
|
---|
396 | typedef packet_traits<typename Derived1::Scalar> PacketTraits;
|
---|
397 | enum {
|
---|
398 | packetSize = PacketTraits::size,
|
---|
399 | dstAlignment = PacketTraits::AlignedOnScalar ? Aligned : int(assign_traits<Derived1,Derived2>::DstIsAligned) ,
|
---|
400 | srcAlignment = assign_traits<Derived1,Derived2>::JointAlignment
|
---|
401 | };
|
---|
402 | const Index alignedStart = assign_traits<Derived1,Derived2>::DstIsAligned ? 0
|
---|
403 | : internal::first_aligned(&dst.coeffRef(0), size);
|
---|
404 | const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
|
---|
405 |
|
---|
406 | unaligned_assign_impl<assign_traits<Derived1,Derived2>::DstIsAligned!=0>::run(src,dst,0,alignedStart);
|
---|
407 |
|
---|
408 | for(Index index = alignedStart; index < alignedEnd; index += packetSize)
|
---|
409 | {
|
---|
410 | dst.template copyPacket<Derived2, dstAlignment, srcAlignment>(index, src);
|
---|
411 | }
|
---|
412 |
|
---|
413 | unaligned_assign_impl<>::run(src,dst,alignedEnd,size);
|
---|
414 | }
|
---|
415 | };
|
---|
416 |
|
---|
417 | template<typename Derived1, typename Derived2, int Version>
|
---|
418 | struct assign_impl<Derived1, Derived2, LinearVectorizedTraversal, CompleteUnrolling, Version>
|
---|
419 | {
|
---|
420 | typedef typename Derived1::Index Index;
|
---|
421 | static EIGEN_STRONG_INLINE void run(Derived1 &dst, const Derived2 &src)
|
---|
422 | {
|
---|
423 | enum { size = Derived1::SizeAtCompileTime,
|
---|
424 | packetSize = packet_traits<typename Derived1::Scalar>::size,
|
---|
425 | alignedSize = (size/packetSize)*packetSize };
|
---|
426 |
|
---|
427 | assign_innervec_CompleteUnrolling<Derived1, Derived2, 0, alignedSize>::run(dst, src);
|
---|
428 | assign_DefaultTraversal_CompleteUnrolling<Derived1, Derived2, alignedSize, size>::run(dst, src);
|
---|
429 | }
|
---|
430 | };
|
---|
431 |
|
---|
432 | /**************************
|
---|
433 | *** Slice vectorization ***
|
---|
434 | ***************************/
|
---|
435 |
|
---|
436 | template<typename Derived1, typename Derived2, int Version>
|
---|
437 | struct assign_impl<Derived1, Derived2, SliceVectorizedTraversal, NoUnrolling, Version>
|
---|
438 | {
|
---|
439 | typedef typename Derived1::Index Index;
|
---|
440 | static inline void run(Derived1 &dst, const Derived2 &src)
|
---|
441 | {
|
---|
442 | typedef typename Derived1::Scalar Scalar;
|
---|
443 | typedef packet_traits<Scalar> PacketTraits;
|
---|
444 | enum {
|
---|
445 | packetSize = PacketTraits::size,
|
---|
446 | alignable = PacketTraits::AlignedOnScalar,
|
---|
447 | dstIsAligned = assign_traits<Derived1,Derived2>::DstIsAligned,
|
---|
448 | dstAlignment = alignable ? Aligned : int(dstIsAligned),
|
---|
449 | srcAlignment = assign_traits<Derived1,Derived2>::JointAlignment
|
---|
450 | };
|
---|
451 | const Scalar *dst_ptr = &dst.coeffRef(0,0);
|
---|
452 | if((!bool(dstIsAligned)) && (size_t(dst_ptr) % sizeof(Scalar))>0)
|
---|
453 | {
|
---|
454 | // the pointer is not aligend-on scalar, so alignment is not possible
|
---|
455 | return assign_impl<Derived1,Derived2,DefaultTraversal,NoUnrolling>::run(dst, src);
|
---|
456 | }
|
---|
457 | const Index packetAlignedMask = packetSize - 1;
|
---|
458 | const Index innerSize = dst.innerSize();
|
---|
459 | const Index outerSize = dst.outerSize();
|
---|
460 | const Index alignedStep = alignable ? (packetSize - dst.outerStride() % packetSize) & packetAlignedMask : 0;
|
---|
461 | Index alignedStart = ((!alignable) || bool(dstIsAligned)) ? 0 : internal::first_aligned(dst_ptr, innerSize);
|
---|
462 |
|
---|
463 | for(Index outer = 0; outer < outerSize; ++outer)
|
---|
464 | {
|
---|
465 | const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
|
---|
466 | // do the non-vectorizable part of the assignment
|
---|
467 | for(Index inner = 0; inner<alignedStart ; ++inner)
|
---|
468 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
469 |
|
---|
470 | // do the vectorizable part of the assignment
|
---|
471 | for(Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
|
---|
472 | dst.template copyPacketByOuterInner<Derived2, dstAlignment, Unaligned>(outer, inner, src);
|
---|
473 |
|
---|
474 | // do the non-vectorizable part of the assignment
|
---|
475 | for(Index inner = alignedEnd; inner<innerSize ; ++inner)
|
---|
476 | dst.copyCoeffByOuterInner(outer, inner, src);
|
---|
477 |
|
---|
478 | alignedStart = std::min<Index>((alignedStart+alignedStep)%packetSize, innerSize);
|
---|
479 | }
|
---|
480 | }
|
---|
481 | };
|
---|
482 |
|
---|
483 | } // end namespace internal
|
---|
484 |
|
---|
485 | /***************************************************************************
|
---|
486 | * Part 4 : implementation of DenseBase methods
|
---|
487 | ***************************************************************************/
|
---|
488 |
|
---|
489 | template<typename Derived>
|
---|
490 | template<typename OtherDerived>
|
---|
491 | EIGEN_STRONG_INLINE Derived& DenseBase<Derived>
|
---|
492 | ::lazyAssign(const DenseBase<OtherDerived>& other)
|
---|
493 | {
|
---|
494 | enum{
|
---|
495 | SameType = internal::is_same<typename Derived::Scalar,typename OtherDerived::Scalar>::value
|
---|
496 | };
|
---|
497 |
|
---|
498 | EIGEN_STATIC_ASSERT_LVALUE(Derived)
|
---|
499 | EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Derived,OtherDerived)
|
---|
500 | EIGEN_STATIC_ASSERT(SameType,YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY)
|
---|
501 |
|
---|
502 | #ifdef EIGEN_DEBUG_ASSIGN
|
---|
503 | internal::assign_traits<Derived, OtherDerived>::debug();
|
---|
504 | #endif
|
---|
505 | eigen_assert(rows() == other.rows() && cols() == other.cols());
|
---|
506 | internal::assign_impl<Derived, OtherDerived, int(SameType) ? int(internal::assign_traits<Derived, OtherDerived>::Traversal)
|
---|
507 | : int(InvalidTraversal)>::run(derived(),other.derived());
|
---|
508 | #ifndef EIGEN_NO_DEBUG
|
---|
509 | checkTransposeAliasing(other.derived());
|
---|
510 | #endif
|
---|
511 | return derived();
|
---|
512 | }
|
---|
513 |
|
---|
514 | namespace internal {
|
---|
515 |
|
---|
516 | template<typename Derived, typename OtherDerived,
|
---|
517 | bool EvalBeforeAssigning = (int(internal::traits<OtherDerived>::Flags) & EvalBeforeAssigningBit) != 0,
|
---|
518 | bool NeedToTranspose = ((int(Derived::RowsAtCompileTime) == 1 && int(OtherDerived::ColsAtCompileTime) == 1)
|
---|
519 | | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&".
|
---|
520 | // revert to || as soon as not needed anymore.
|
---|
521 | (int(Derived::ColsAtCompileTime) == 1 && int(OtherDerived::RowsAtCompileTime) == 1))
|
---|
522 | && int(Derived::SizeAtCompileTime) != 1>
|
---|
523 | struct assign_selector;
|
---|
524 |
|
---|
525 | template<typename Derived, typename OtherDerived>
|
---|
526 | struct assign_selector<Derived,OtherDerived,false,false> {
|
---|
527 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.derived()); }
|
---|
528 | template<typename ActualDerived, typename ActualOtherDerived>
|
---|
529 | static EIGEN_STRONG_INLINE Derived& evalTo(ActualDerived& dst, const ActualOtherDerived& other) { other.evalTo(dst); return dst; }
|
---|
530 | };
|
---|
531 | template<typename Derived, typename OtherDerived>
|
---|
532 | struct assign_selector<Derived,OtherDerived,true,false> {
|
---|
533 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.eval()); }
|
---|
534 | };
|
---|
535 | template<typename Derived, typename OtherDerived>
|
---|
536 | struct assign_selector<Derived,OtherDerived,false,true> {
|
---|
537 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose()); }
|
---|
538 | template<typename ActualDerived, typename ActualOtherDerived>
|
---|
539 | static EIGEN_STRONG_INLINE Derived& evalTo(ActualDerived& dst, const ActualOtherDerived& other) { Transpose<ActualDerived> dstTrans(dst); other.evalTo(dstTrans); return dst; }
|
---|
540 | };
|
---|
541 | template<typename Derived, typename OtherDerived>
|
---|
542 | struct assign_selector<Derived,OtherDerived,true,true> {
|
---|
543 | static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose().eval()); }
|
---|
544 | };
|
---|
545 |
|
---|
546 | } // end namespace internal
|
---|
547 |
|
---|
548 | template<typename Derived>
|
---|
549 | template<typename OtherDerived>
|
---|
550 | EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase<OtherDerived>& other)
|
---|
551 | {
|
---|
552 | return internal::assign_selector<Derived,OtherDerived>::run(derived(), other.derived());
|
---|
553 | }
|
---|
554 |
|
---|
555 | template<typename Derived>
|
---|
556 | EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase& other)
|
---|
557 | {
|
---|
558 | return internal::assign_selector<Derived,Derived>::run(derived(), other.derived());
|
---|
559 | }
|
---|
560 |
|
---|
561 | template<typename Derived>
|
---|
562 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const MatrixBase& other)
|
---|
563 | {
|
---|
564 | return internal::assign_selector<Derived,Derived>::run(derived(), other.derived());
|
---|
565 | }
|
---|
566 |
|
---|
567 | template<typename Derived>
|
---|
568 | template <typename OtherDerived>
|
---|
569 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const DenseBase<OtherDerived>& other)
|
---|
570 | {
|
---|
571 | return internal::assign_selector<Derived,OtherDerived>::run(derived(), other.derived());
|
---|
572 | }
|
---|
573 |
|
---|
574 | template<typename Derived>
|
---|
575 | template <typename OtherDerived>
|
---|
576 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const EigenBase<OtherDerived>& other)
|
---|
577 | {
|
---|
578 | return internal::assign_selector<Derived,OtherDerived,false>::evalTo(derived(), other.derived());
|
---|
579 | }
|
---|
580 |
|
---|
581 | template<typename Derived>
|
---|
582 | template<typename OtherDerived>
|
---|
583 | EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other)
|
---|
584 | {
|
---|
585 | return internal::assign_selector<Derived,OtherDerived,false>::evalTo(derived(), other.derived());
|
---|
586 | }
|
---|
587 |
|
---|
588 | } // end namespace Eigen
|
---|
589 |
|
---|
590 | #endif // EIGEN_ASSIGN_H
|
---|