Eigen  3.4.90 (git rev 5a9f66fb35d03a4da9ef8976e67a61b30aa16dcf)
 
Loading...
Searching...
No Matches
CoreEvaluators.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2011 Benoit Jacob <[email protected]>
5// Copyright (C) 2011-2014 Gael Guennebaud <[email protected]>
6// Copyright (C) 2011-2012 Jitse Niesen <[email protected]>
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11
12#ifndef EIGEN_COREEVALUATORS_H
13#define EIGEN_COREEVALUATORS_H
14
15// IWYU pragma: private
16#include "./InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22// This class returns the evaluator kind from the expression storage kind.
23// Default assumes index based accessors
24template <typename StorageKind>
25struct storage_kind_to_evaluator_kind {
26 typedef IndexBased Kind;
27};
28
29// This class returns the evaluator shape from the expression storage kind.
30// It can be Dense, Sparse, Triangular, Diagonal, SelfAdjoint, Band, etc.
31template <typename StorageKind>
32struct storage_kind_to_shape;
33
34template <>
35struct storage_kind_to_shape<Dense> {
36 typedef DenseShape Shape;
37};
38template <>
39struct storage_kind_to_shape<SolverStorage> {
40 typedef SolverShape Shape;
41};
42template <>
43struct storage_kind_to_shape<PermutationStorage> {
44 typedef PermutationShape Shape;
45};
46template <>
47struct storage_kind_to_shape<TranspositionsStorage> {
48 typedef TranspositionsShape Shape;
49};
50
51// Evaluators have to be specialized with respect to various criteria such as:
52// - storage/structure/shape
53// - scalar type
54// - etc.
55// Therefore, we need specialization of evaluator providing additional template arguments for each kind of evaluators.
56// We currently distinguish the following kind of evaluators:
57// - unary_evaluator for expressions taking only one arguments (CwiseUnaryOp, CwiseUnaryView, Transpose,
58// MatrixWrapper, ArrayWrapper, Reverse, Replicate)
59// - binary_evaluator for expression taking two arguments (CwiseBinaryOp)
60// - ternary_evaluator for expression taking three arguments (CwiseTernaryOp)
61// - product_evaluator for linear algebra products (Product); special case of binary_evaluator because it requires
62// additional tags for dispatching.
63// - mapbase_evaluator for Map, Block, Ref
64// - block_evaluator for Block (special dispatching to a mapbase_evaluator or unary_evaluator)
65
66template <typename T, typename Arg1Kind = typename evaluator_traits<typename T::Arg1>::Kind,
67 typename Arg2Kind = typename evaluator_traits<typename T::Arg2>::Kind,
68 typename Arg3Kind = typename evaluator_traits<typename T::Arg3>::Kind,
69 typename Arg1Scalar = typename traits<typename T::Arg1>::Scalar,
70 typename Arg2Scalar = typename traits<typename T::Arg2>::Scalar,
71 typename Arg3Scalar = typename traits<typename T::Arg3>::Scalar>
72struct ternary_evaluator;
73
74template <typename T, typename LhsKind = typename evaluator_traits<typename T::Lhs>::Kind,
75 typename RhsKind = typename evaluator_traits<typename T::Rhs>::Kind,
76 typename LhsScalar = typename traits<typename T::Lhs>::Scalar,
77 typename RhsScalar = typename traits<typename T::Rhs>::Scalar>
78struct binary_evaluator;
79
80template <typename T, typename Kind = typename evaluator_traits<typename T::NestedExpression>::Kind,
81 typename Scalar = typename T::Scalar>
82struct unary_evaluator;
83
84// evaluator_traits<T> contains traits for evaluator<T>
85
86template <typename T>
87struct evaluator_traits_base {
88 // by default, get evaluator kind and shape from storage
89 typedef typename storage_kind_to_evaluator_kind<typename traits<T>::StorageKind>::Kind Kind;
90 typedef typename storage_kind_to_shape<typename traits<T>::StorageKind>::Shape Shape;
91};
92
93// Default evaluator traits
94template <typename T>
95struct evaluator_traits : public evaluator_traits_base<T> {};
96
97template <typename T, typename Shape = typename evaluator_traits<T>::Shape>
98struct evaluator_assume_aliasing {
99 static const bool value = false;
100};
101
102// By default, we assume a unary expression:
103template <typename T>
104struct evaluator : public unary_evaluator<T> {
105 typedef unary_evaluator<T> Base;
106 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const T& xpr) : Base(xpr) {}
107};
108
109// TODO: Think about const-correctness
110template <typename T>
111struct evaluator<const T> : evaluator<T> {
112 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const T& xpr) : evaluator<T>(xpr) {}
113};
114
115// ---------- base class for all evaluators ----------
116
117template <typename ExpressionType>
118struct evaluator_base {
119 // TODO that's not very nice to have to propagate all these traits. They are currently only needed to handle
120 // outer,inner indices.
121 typedef traits<ExpressionType> ExpressionTraits;
122
123 enum { Alignment = 0 };
124 // noncopyable:
125 // Don't make this class inherit noncopyable as this kills EBO (Empty Base Optimization)
126 // and make complex evaluator much larger than then should do.
127 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator_base() {}
128 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~evaluator_base() {}
129
130 private:
131 EIGEN_DEVICE_FUNC evaluator_base(const evaluator_base&);
132 EIGEN_DEVICE_FUNC const evaluator_base& operator=(const evaluator_base&);
133};
134
135// -------------------- Matrix and Array --------------------
136//
137// evaluator<PlainObjectBase> is a common base class for the
138// Matrix and Array evaluators.
139// Here we directly specialize evaluator. This is not really a unary expression, and it is, by definition, dense,
140// so no need for more sophisticated dispatching.
141
142// this helper permits to completely eliminate m_outerStride if it is known at compiletime.
143template <typename Scalar, int OuterStride>
144class plainobjectbase_evaluator_data {
145 public:
146 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE plainobjectbase_evaluator_data(const Scalar* ptr, Index outerStride)
147 : data(ptr) {
148#ifndef EIGEN_INTERNAL_DEBUGGING
149 EIGEN_UNUSED_VARIABLE(outerStride);
150#endif
151 eigen_internal_assert(outerStride == OuterStride);
152 }
153 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index outerStride() const EIGEN_NOEXCEPT { return OuterStride; }
154 const Scalar* data;
155};
156
157template <typename Scalar>
158class plainobjectbase_evaluator_data<Scalar, Dynamic> {
159 public:
160 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE plainobjectbase_evaluator_data(const Scalar* ptr, Index outerStride)
161 : data(ptr), m_outerStride(outerStride) {}
162 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outerStride() const { return m_outerStride; }
163 const Scalar* data;
164
165 protected:
166 Index m_outerStride;
167};
168
169template <typename Derived>
170struct evaluator<PlainObjectBase<Derived>> : evaluator_base<Derived> {
171 typedef PlainObjectBase<Derived> PlainObjectType;
172 typedef typename PlainObjectType::Scalar Scalar;
173 typedef typename PlainObjectType::CoeffReturnType CoeffReturnType;
174
175 enum {
176 IsRowMajor = PlainObjectType::IsRowMajor,
177 IsVectorAtCompileTime = PlainObjectType::IsVectorAtCompileTime,
178 RowsAtCompileTime = PlainObjectType::RowsAtCompileTime,
179 ColsAtCompileTime = PlainObjectType::ColsAtCompileTime,
180
181 CoeffReadCost = NumTraits<Scalar>::ReadCost,
182 Flags = traits<Derived>::EvaluatorFlags,
183 Alignment = traits<Derived>::Alignment
184 };
185 enum {
186 // We do not need to know the outer stride for vectors
187 OuterStrideAtCompileTime = IsVectorAtCompileTime ? 0
188 : int(IsRowMajor) ? ColsAtCompileTime
189 : RowsAtCompileTime
190 };
191
192 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator() : m_d(0, OuterStrideAtCompileTime) {
193 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
194 }
195
196 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const PlainObjectType& m)
197 : m_d(m.data(), IsVectorAtCompileTime ? 0 : m.outerStride()) {
198 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
199 }
200
201 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
202 if (IsRowMajor)
203 return m_d.data[row * m_d.outerStride() + col];
204 else
205 return m_d.data[row + col * m_d.outerStride()];
206 }
207
208 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_d.data[index]; }
209
210 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
211 if (IsRowMajor)
212 return const_cast<Scalar*>(m_d.data)[row * m_d.outerStride() + col];
213 else
214 return const_cast<Scalar*>(m_d.data)[row + col * m_d.outerStride()];
215 }
216
217 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return const_cast<Scalar*>(m_d.data)[index]; }
218
219 template <int LoadMode, typename PacketType>
220 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
221 if (IsRowMajor)
222 return ploadt<PacketType, LoadMode>(m_d.data + row * m_d.outerStride() + col);
223 else
224 return ploadt<PacketType, LoadMode>(m_d.data + row + col * m_d.outerStride());
225 }
226
227 template <int LoadMode, typename PacketType>
228 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
229 return ploadt<PacketType, LoadMode>(m_d.data + index);
230 }
231
232 template <int StoreMode, typename PacketType>
233 EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) {
234 if (IsRowMajor)
235 return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_d.data) + row * m_d.outerStride() + col, x);
236 else
237 return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_d.data) + row + col * m_d.outerStride(), x);
238 }
239
240 template <int StoreMode, typename PacketType>
241 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
242 return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_d.data) + index, x);
243 }
244
245 protected:
246 plainobjectbase_evaluator_data<Scalar, OuterStrideAtCompileTime> m_d;
247};
248
249template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols>
250struct evaluator<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols>>
251 : evaluator<PlainObjectBase<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols>>> {
252 typedef Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> XprType;
253
254 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator() {}
255
256 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& m) : evaluator<PlainObjectBase<XprType>>(m) {}
257};
258
259template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols>
260struct evaluator<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols>>
261 : evaluator<PlainObjectBase<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols>>> {
262 typedef Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> XprType;
263
264 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE evaluator() {}
265
266 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& m) : evaluator<PlainObjectBase<XprType>>(m) {}
267};
268
269// -------------------- Transpose --------------------
270
271template <typename ArgType>
272struct unary_evaluator<Transpose<ArgType>, IndexBased> : evaluator_base<Transpose<ArgType>> {
273 typedef Transpose<ArgType> XprType;
274
275 enum {
276 CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
277 Flags = evaluator<ArgType>::Flags ^ RowMajorBit,
278 Alignment = evaluator<ArgType>::Alignment
279 };
280
281 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& t) : m_argImpl(t.nestedExpression()) {}
282
283 typedef typename XprType::Scalar Scalar;
284 typedef typename XprType::CoeffReturnType CoeffReturnType;
285
286 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
287 return m_argImpl.coeff(col, row);
288 }
289
290 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_argImpl.coeff(index); }
291
292 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(col, row); }
293
294 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename XprType::Scalar& coeffRef(Index index) {
295 return m_argImpl.coeffRef(index);
296 }
297
298 template <int LoadMode, typename PacketType>
299 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
300 return m_argImpl.template packet<LoadMode, PacketType>(col, row);
301 }
302
303 template <int LoadMode, typename PacketType>
304 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
305 return m_argImpl.template packet<LoadMode, PacketType>(index);
306 }
307
308 template <int StoreMode, typename PacketType>
309 EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) {
310 m_argImpl.template writePacket<StoreMode, PacketType>(col, row, x);
311 }
312
313 template <int StoreMode, typename PacketType>
314 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
315 m_argImpl.template writePacket<StoreMode, PacketType>(index, x);
316 }
317
318 protected:
319 evaluator<ArgType> m_argImpl;
320};
321
322// -------------------- CwiseNullaryOp --------------------
323// Like Matrix and Array, this is not really a unary expression, so we directly specialize evaluator.
324// Likewise, there is not need to more sophisticated dispatching here.
325
326template <typename Scalar, typename NullaryOp, bool has_nullary = has_nullary_operator<NullaryOp>::value,
327 bool has_unary = has_unary_operator<NullaryOp>::value,
328 bool has_binary = has_binary_operator<NullaryOp>::value>
329struct nullary_wrapper {
330 template <typename IndexType>
331 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const {
332 return op(i, j);
333 }
334 template <typename IndexType>
335 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const {
336 return op(i);
337 }
338
339 template <typename T, typename IndexType>
340 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const {
341 return op.template packetOp<T>(i, j);
342 }
343 template <typename T, typename IndexType>
344 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const {
345 return op.template packetOp<T>(i);
346 }
347};
348
349template <typename Scalar, typename NullaryOp>
350struct nullary_wrapper<Scalar, NullaryOp, true, false, false> {
351 template <typename IndexType>
352 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType = 0, IndexType = 0) const {
353 return op();
354 }
355 template <typename T, typename IndexType>
356 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType = 0, IndexType = 0) const {
357 return op.template packetOp<T>();
358 }
359};
360
361template <typename Scalar, typename NullaryOp>
362struct nullary_wrapper<Scalar, NullaryOp, false, false, true> {
363 template <typename IndexType>
364 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j = 0) const {
365 return op(i, j);
366 }
367 template <typename T, typename IndexType>
368 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j = 0) const {
369 return op.template packetOp<T>(i, j);
370 }
371};
372
373// We need the following specialization for vector-only functors assigned to a runtime vector,
374// for instance, using linspace and assigning a RowVectorXd to a MatrixXd or even a row of a MatrixXd.
375// In this case, i==0 and j is used for the actual iteration.
376template <typename Scalar, typename NullaryOp>
377struct nullary_wrapper<Scalar, NullaryOp, false, true, false> {
378 template <typename IndexType>
379 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const {
380 eigen_assert(i == 0 || j == 0);
381 return op(i + j);
382 }
383 template <typename T, typename IndexType>
384 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const {
385 eigen_assert(i == 0 || j == 0);
386 return op.template packetOp<T>(i + j);
387 }
388
389 template <typename IndexType>
390 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const {
391 return op(i);
392 }
393 template <typename T, typename IndexType>
394 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const {
395 return op.template packetOp<T>(i);
396 }
397};
398
399template <typename Scalar, typename NullaryOp>
400struct nullary_wrapper<Scalar, NullaryOp, false, false, false> {};
401
402#if 0 && EIGEN_COMP_MSVC > 0
403// Disable this ugly workaround. This is now handled in traits<Ref>::match,
404// but this piece of code might still become handly if some other weird compilation
405// erros pop up again.
406
407// MSVC exhibits a weird compilation error when
408// compiling:
409// Eigen::MatrixXf A = MatrixXf::Random(3,3);
410// Ref<const MatrixXf> R = 2.f*A;
411// and that has_*ary_operator<scalar_constant_op<float>> have not been instantiated yet.
412// The "problem" is that evaluator<2.f*A> is instantiated by traits<Ref>::match<2.f*A>
413// and at that time has_*ary_operator<T> returns true regardless of T.
414// Then nullary_wrapper is badly instantiated as nullary_wrapper<.,.,true,true,true>.
415// The trick is thus to defer the proper instantiation of nullary_wrapper when coeff(),
416// and packet() are really instantiated as implemented below:
417
418// This is a simple wrapper around Index to enforce the re-instantiation of
419// has_*ary_operator when needed.
420template<typename T> struct nullary_wrapper_workaround_msvc {
421 nullary_wrapper_workaround_msvc(const T&);
422 operator T()const;
423};
424
425template<typename Scalar,typename NullaryOp>
426struct nullary_wrapper<Scalar,NullaryOp,true,true,true>
427{
428 template <typename IndexType>
429 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i, IndexType j) const {
430 return nullary_wrapper<Scalar,NullaryOp,
431 has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
432 has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
433 has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().operator()(op,i,j);
434 }
435 template <typename IndexType>
436 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const NullaryOp& op, IndexType i) const {
437 return nullary_wrapper<Scalar,NullaryOp,
438 has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
439 has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
440 has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().operator()(op,i);
441 }
442
443 template <typename T, typename IndexType>
444 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i, IndexType j) const {
445 return nullary_wrapper<Scalar,NullaryOp,
446 has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
447 has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
448 has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().template packetOp<T>(op,i,j);
449 }
450 template <typename T, typename IndexType>
451 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T packetOp(const NullaryOp& op, IndexType i) const {
452 return nullary_wrapper<Scalar,NullaryOp,
453 has_nullary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
454 has_unary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value,
455 has_binary_operator<NullaryOp,nullary_wrapper_workaround_msvc<IndexType> >::value>().template packetOp<T>(op,i);
456 }
457};
458#endif // MSVC workaround
459
460template <typename NullaryOp, typename PlainObjectType>
461struct evaluator<CwiseNullaryOp<NullaryOp, PlainObjectType>>
462 : evaluator_base<CwiseNullaryOp<NullaryOp, PlainObjectType>> {
463 typedef CwiseNullaryOp<NullaryOp, PlainObjectType> XprType;
464 typedef internal::remove_all_t<PlainObjectType> PlainObjectTypeCleaned;
465
466 enum {
467 CoeffReadCost = internal::functor_traits<NullaryOp>::Cost,
468
469 Flags = (evaluator<PlainObjectTypeCleaned>::Flags &
470 (HereditaryBits | (functor_has_linear_access<NullaryOp>::ret ? LinearAccessBit : 0) |
471 (functor_traits<NullaryOp>::PacketAccess ? PacketAccessBit : 0))) |
472 (functor_traits<NullaryOp>::IsRepeatable ? 0 : EvalBeforeNestingBit),
473 Alignment = AlignedMax
474 };
475
476 EIGEN_DEVICE_FUNC explicit evaluator(const XprType& n) : m_functor(n.functor()), m_wrapper() {
477 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
478 }
479
480 typedef typename XprType::CoeffReturnType CoeffReturnType;
481
482 template <typename IndexType>
483 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(IndexType row, IndexType col) const {
484 return m_wrapper(m_functor, row, col);
485 }
486
487 template <typename IndexType>
488 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(IndexType index) const {
489 return m_wrapper(m_functor, index);
490 }
491
492 template <int LoadMode, typename PacketType, typename IndexType>
493 EIGEN_STRONG_INLINE PacketType packet(IndexType row, IndexType col) const {
494 return m_wrapper.template packetOp<PacketType>(m_functor, row, col);
495 }
496
497 template <int LoadMode, typename PacketType, typename IndexType>
498 EIGEN_STRONG_INLINE PacketType packet(IndexType index) const {
499 return m_wrapper.template packetOp<PacketType>(m_functor, index);
500 }
501
502 protected:
503 const NullaryOp m_functor;
504 const internal::nullary_wrapper<CoeffReturnType, NullaryOp> m_wrapper;
505};
506
507// -------------------- CwiseUnaryOp --------------------
508
509template <typename UnaryOp, typename ArgType>
510struct unary_evaluator<CwiseUnaryOp<UnaryOp, ArgType>, IndexBased> : evaluator_base<CwiseUnaryOp<UnaryOp, ArgType>> {
511 typedef CwiseUnaryOp<UnaryOp, ArgType> XprType;
512
513 enum {
514 CoeffReadCost = int(evaluator<ArgType>::CoeffReadCost) + int(functor_traits<UnaryOp>::Cost),
515
516 Flags = evaluator<ArgType>::Flags &
517 (HereditaryBits | LinearAccessBit | (functor_traits<UnaryOp>::PacketAccess ? PacketAccessBit : 0)),
518 Alignment = evaluator<ArgType>::Alignment
519 };
520
521 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& op) : m_d(op) {
522 EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<UnaryOp>::Cost);
523 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
524 }
525
526 typedef typename XprType::CoeffReturnType CoeffReturnType;
527
528 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
529 return m_d.func()(m_d.argImpl.coeff(row, col));
530 }
531
532 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
533 return m_d.func()(m_d.argImpl.coeff(index));
534 }
535
536 template <int LoadMode, typename PacketType>
537 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
538 return m_d.func().packetOp(m_d.argImpl.template packet<LoadMode, PacketType>(row, col));
539 }
540
541 template <int LoadMode, typename PacketType>
542 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
543 return m_d.func().packetOp(m_d.argImpl.template packet<LoadMode, PacketType>(index));
544 }
545
546 protected:
547 // this helper permits to completely eliminate the functor if it is empty
548 struct Data {
549 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr)
550 : op(xpr.functor()), argImpl(xpr.nestedExpression()) {}
551 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryOp& func() const { return op; }
552 UnaryOp op;
553 evaluator<ArgType> argImpl;
554 };
555
556 Data m_d;
557};
558
559// ----------------------- Casting ---------------------
560
561template <typename SrcType, typename DstType, typename ArgType>
562struct unary_evaluator<CwiseUnaryOp<core_cast_op<SrcType, DstType>, ArgType>, IndexBased> {
563 using CastOp = core_cast_op<SrcType, DstType>;
564 using XprType = CwiseUnaryOp<CastOp, ArgType>;
565
566 // Use the largest packet type by default
567 using SrcPacketType = typename packet_traits<SrcType>::type;
568 static constexpr int SrcPacketSize = unpacket_traits<SrcPacketType>::size;
569 static constexpr int SrcPacketBytes = SrcPacketSize * sizeof(SrcType);
570
571 enum {
572 CoeffReadCost = int(evaluator<ArgType>::CoeffReadCost) + int(functor_traits<CastOp>::Cost),
573 PacketAccess = functor_traits<CastOp>::PacketAccess,
574 ActualPacketAccessBit = PacketAccess ? PacketAccessBit : 0,
575 Flags = evaluator<ArgType>::Flags & (HereditaryBits | LinearAccessBit | ActualPacketAccessBit),
576 IsRowMajor = (evaluator<ArgType>::Flags & RowMajorBit),
577 Alignment = evaluator<ArgType>::Alignment
578 };
579
580 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& xpr)
581 : m_argImpl(xpr.nestedExpression()), m_rows(xpr.rows()), m_cols(xpr.cols()) {
582 EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<CastOp>::Cost);
583 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
584 }
585
586 template <typename DstPacketType>
587 using AltSrcScalarOp = std::enable_if_t<(unpacket_traits<DstPacketType>::size < SrcPacketSize &&
588 !find_packet_by_size<SrcType, unpacket_traits<DstPacketType>::size>::value),
589 bool>;
590 template <typename DstPacketType>
591 using SrcPacketArgs1 =
592 std::enable_if_t<(find_packet_by_size<SrcType, unpacket_traits<DstPacketType>::size>::value), bool>;
593 template <typename DstPacketType>
594 using SrcPacketArgs2 = std::enable_if_t<(unpacket_traits<DstPacketType>::size) == (2 * SrcPacketSize), bool>;
595 template <typename DstPacketType>
596 using SrcPacketArgs4 = std::enable_if_t<(unpacket_traits<DstPacketType>::size) == (4 * SrcPacketSize), bool>;
597 template <typename DstPacketType>
598 using SrcPacketArgs8 = std::enable_if_t<(unpacket_traits<DstPacketType>::size) == (8 * SrcPacketSize), bool>;
599
600 template <bool UseRowMajor = IsRowMajor, std::enable_if_t<UseRowMajor, bool> = true>
601 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_array_bounds(Index, Index col, Index packetSize) const {
602 return col + packetSize <= cols();
603 }
604 template <bool UseRowMajor = IsRowMajor, std::enable_if_t<!UseRowMajor, bool> = true>
605 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_array_bounds(Index row, Index, Index packetSize) const {
606 return row + packetSize <= rows();
607 }
608 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool check_array_bounds(Index index, Index packetSize) const {
609 return index + packetSize <= size();
610 }
611
612 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE SrcType srcCoeff(Index row, Index col, Index offset) const {
613 Index actualRow = IsRowMajor ? row : row + offset;
614 Index actualCol = IsRowMajor ? col + offset : col;
615 return m_argImpl.coeff(actualRow, actualCol);
616 }
617 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE SrcType srcCoeff(Index index, Index offset) const {
618 Index actualIndex = index + offset;
619 return m_argImpl.coeff(actualIndex);
620 }
621
622 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstType coeff(Index row, Index col) const {
623 return cast<SrcType, DstType>(srcCoeff(row, col, 0));
624 }
625 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstType coeff(Index index) const {
626 return cast<SrcType, DstType>(srcCoeff(index, 0));
627 }
628
629 template <int LoadMode, typename PacketType = SrcPacketType>
630 EIGEN_STRONG_INLINE PacketType srcPacket(Index row, Index col, Index offset) const {
631 constexpr int PacketSize = unpacket_traits<PacketType>::size;
632 Index actualRow = IsRowMajor ? row : row + (offset * PacketSize);
633 Index actualCol = IsRowMajor ? col + (offset * PacketSize) : col;
634 eigen_assert(check_array_bounds(actualRow, actualCol, PacketSize) && "Array index out of bounds");
635 return m_argImpl.template packet<LoadMode, PacketType>(actualRow, actualCol);
636 }
637 template <int LoadMode, typename PacketType = SrcPacketType>
638 EIGEN_STRONG_INLINE PacketType srcPacket(Index index, Index offset) const {
639 constexpr int PacketSize = unpacket_traits<PacketType>::size;
640 Index actualIndex = index + (offset * PacketSize);
641 eigen_assert(check_array_bounds(actualIndex, PacketSize) && "Array index out of bounds");
642 return m_argImpl.template packet<LoadMode, PacketType>(actualIndex);
643 }
644
645 // There is no source packet type with equal or fewer elements than DstPacketType.
646 // This is problematic as the evaluation loop may attempt to access data outside the bounds of the array.
647 // For example, consider the cast utilizing pcast<Packet4f,Packet2d> with an array of size 4: {0.0f,1.0f,2.0f,3.0f}.
648 // The first iteration of the evaulation loop will load 16 bytes: {0.0f,1.0f,2.0f,3.0f} and cast to {0.0,1.0}, which
649 // is acceptable. The second iteration will load 16 bytes: {2.0f,3.0f,?,?}, which is outside the bounds of the array.
650
651 // Instead, perform runtime check to determine if the load would access data outside the bounds of the array.
652 // If not, perform full load. Otherwise, revert to a scalar loop to perform a partial load.
653 // In either case, perform a vectorized cast of the source packet.
654 template <int LoadMode, typename DstPacketType, AltSrcScalarOp<DstPacketType> = true>
655 EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const {
656 constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size;
657 constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType);
658 constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode);
659 SrcPacketType src;
660 if (EIGEN_PREDICT_TRUE(check_array_bounds(row, col, SrcPacketSize))) {
661 src = srcPacket<SrcLoadMode>(row, col, 0);
662 } else {
663 Array<SrcType, SrcPacketSize, 1> srcArray;
664 for (size_t k = 0; k < DstPacketSize; k++) srcArray[k] = srcCoeff(row, col, k);
665 for (size_t k = DstPacketSize; k < SrcPacketSize; k++) srcArray[k] = SrcType(0);
666 src = pload<SrcPacketType>(srcArray.data());
667 }
668 return pcast<SrcPacketType, DstPacketType>(src);
669 }
670 // Use the source packet type with the same size as DstPacketType, if it exists
671 template <int LoadMode, typename DstPacketType, SrcPacketArgs1<DstPacketType> = true>
672 EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const {
673 constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size;
674 using SizedSrcPacketType = typename find_packet_by_size<SrcType, DstPacketSize>::type;
675 constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType);
676 constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode);
677 return pcast<SizedSrcPacketType, DstPacketType>(srcPacket<SrcLoadMode, SizedSrcPacketType>(row, col, 0));
678 }
679 // unpacket_traits<DstPacketType>::size == 2 * SrcPacketSize
680 template <int LoadMode, typename DstPacketType, SrcPacketArgs2<DstPacketType> = true>
681 EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const {
682 constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode);
683 return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(row, col, 0),
684 srcPacket<SrcLoadMode>(row, col, 1));
685 }
686 // unpacket_traits<DstPacketType>::size == 4 * SrcPacketSize
687 template <int LoadMode, typename DstPacketType, SrcPacketArgs4<DstPacketType> = true>
688 EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const {
689 constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode);
690 return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(row, col, 0), srcPacket<SrcLoadMode>(row, col, 1),
691 srcPacket<SrcLoadMode>(row, col, 2),
692 srcPacket<SrcLoadMode>(row, col, 3));
693 }
694 // unpacket_traits<DstPacketType>::size == 8 * SrcPacketSize
695 template <int LoadMode, typename DstPacketType, SrcPacketArgs8<DstPacketType> = true>
696 EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const {
697 constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode);
698 return pcast<SrcPacketType, DstPacketType>(
699 srcPacket<SrcLoadMode>(row, col, 0), srcPacket<SrcLoadMode>(row, col, 1), srcPacket<SrcLoadMode>(row, col, 2),
700 srcPacket<SrcLoadMode>(row, col, 3), srcPacket<SrcLoadMode>(row, col, 4), srcPacket<SrcLoadMode>(row, col, 5),
701 srcPacket<SrcLoadMode>(row, col, 6), srcPacket<SrcLoadMode>(row, col, 7));
702 }
703
704 // Analagous routines for linear access.
705 template <int LoadMode, typename DstPacketType, AltSrcScalarOp<DstPacketType> = true>
706 EIGEN_STRONG_INLINE DstPacketType packet(Index index) const {
707 constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size;
708 constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType);
709 constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode);
710 SrcPacketType src;
711 if (EIGEN_PREDICT_TRUE(check_array_bounds(index, SrcPacketSize))) {
712 src = srcPacket<SrcLoadMode>(index, 0);
713 } else {
714 Array<SrcType, SrcPacketSize, 1> srcArray;
715 for (size_t k = 0; k < DstPacketSize; k++) srcArray[k] = srcCoeff(index, k);
716 for (size_t k = DstPacketSize; k < SrcPacketSize; k++) srcArray[k] = SrcType(0);
717 src = pload<SrcPacketType>(srcArray.data());
718 }
719 return pcast<SrcPacketType, DstPacketType>(src);
720 }
721 template <int LoadMode, typename DstPacketType, SrcPacketArgs1<DstPacketType> = true>
722 EIGEN_STRONG_INLINE DstPacketType packet(Index index) const {
723 constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size;
724 using SizedSrcPacketType = typename find_packet_by_size<SrcType, DstPacketSize>::type;
725 constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType);
726 constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode);
727 return pcast<SizedSrcPacketType, DstPacketType>(srcPacket<SrcLoadMode, SizedSrcPacketType>(index, 0));
728 }
729 template <int LoadMode, typename DstPacketType, SrcPacketArgs2<DstPacketType> = true>
730 EIGEN_STRONG_INLINE DstPacketType packet(Index index) const {
731 constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode);
732 return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(index, 0), srcPacket<SrcLoadMode>(index, 1));
733 }
734 template <int LoadMode, typename DstPacketType, SrcPacketArgs4<DstPacketType> = true>
735 EIGEN_STRONG_INLINE DstPacketType packet(Index index) const {
736 constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode);
737 return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(index, 0), srcPacket<SrcLoadMode>(index, 1),
738 srcPacket<SrcLoadMode>(index, 2), srcPacket<SrcLoadMode>(index, 3));
739 }
740 template <int LoadMode, typename DstPacketType, SrcPacketArgs8<DstPacketType> = true>
741 EIGEN_STRONG_INLINE DstPacketType packet(Index index) const {
742 constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode);
743 return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(index, 0), srcPacket<SrcLoadMode>(index, 1),
744 srcPacket<SrcLoadMode>(index, 2), srcPacket<SrcLoadMode>(index, 3),
745 srcPacket<SrcLoadMode>(index, 4), srcPacket<SrcLoadMode>(index, 5),
746 srcPacket<SrcLoadMode>(index, 6), srcPacket<SrcLoadMode>(index, 7));
747 }
748
749 constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rows() const { return m_rows; }
750 constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index cols() const { return m_cols; }
751 constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_rows * m_cols; }
752
753 protected:
754 const evaluator<ArgType> m_argImpl;
755 const variable_if_dynamic<Index, XprType::RowsAtCompileTime> m_rows;
756 const variable_if_dynamic<Index, XprType::ColsAtCompileTime> m_cols;
757};
758
759// -------------------- CwiseTernaryOp --------------------
760
761// this is a ternary expression
762template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3>
763struct evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>>
764 : public ternary_evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>> {
765 typedef CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> XprType;
766 typedef ternary_evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>> Base;
767
768 EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : Base(xpr) {}
769};
770
771template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3>
772struct ternary_evaluator<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>, IndexBased, IndexBased>
773 : evaluator_base<CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>> {
774 typedef CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3> XprType;
775
776 enum {
777 CoeffReadCost = int(evaluator<Arg1>::CoeffReadCost) + int(evaluator<Arg2>::CoeffReadCost) +
778 int(evaluator<Arg3>::CoeffReadCost) + int(functor_traits<TernaryOp>::Cost),
779
780 Arg1Flags = evaluator<Arg1>::Flags,
781 Arg2Flags = evaluator<Arg2>::Flags,
782 Arg3Flags = evaluator<Arg3>::Flags,
783 SameType = is_same<typename Arg1::Scalar, typename Arg2::Scalar>::value &&
784 is_same<typename Arg1::Scalar, typename Arg3::Scalar>::value,
785 StorageOrdersAgree = (int(Arg1Flags) & RowMajorBit) == (int(Arg2Flags) & RowMajorBit) &&
786 (int(Arg1Flags) & RowMajorBit) == (int(Arg3Flags) & RowMajorBit),
787 Flags0 = (int(Arg1Flags) | int(Arg2Flags) | int(Arg3Flags)) &
788 (HereditaryBits |
789 (int(Arg1Flags) & int(Arg2Flags) & int(Arg3Flags) &
790 ((StorageOrdersAgree ? LinearAccessBit : 0) |
791 (functor_traits<TernaryOp>::PacketAccess && StorageOrdersAgree && SameType ? PacketAccessBit : 0)))),
792 Flags = (Flags0 & ~RowMajorBit) | (Arg1Flags & RowMajorBit),
793 Alignment = plain_enum_min(plain_enum_min(evaluator<Arg1>::Alignment, evaluator<Arg2>::Alignment),
794 evaluator<Arg3>::Alignment)
795 };
796
797 EIGEN_DEVICE_FUNC explicit ternary_evaluator(const XprType& xpr) : m_d(xpr) {
798 EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<TernaryOp>::Cost);
799 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
800 }
801
802 typedef typename XprType::CoeffReturnType CoeffReturnType;
803
804 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
805 return m_d.func()(m_d.arg1Impl.coeff(row, col), m_d.arg2Impl.coeff(row, col), m_d.arg3Impl.coeff(row, col));
806 }
807
808 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
809 return m_d.func()(m_d.arg1Impl.coeff(index), m_d.arg2Impl.coeff(index), m_d.arg3Impl.coeff(index));
810 }
811
812 template <int LoadMode, typename PacketType>
813 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
814 return m_d.func().packetOp(m_d.arg1Impl.template packet<LoadMode, PacketType>(row, col),
815 m_d.arg2Impl.template packet<LoadMode, PacketType>(row, col),
816 m_d.arg3Impl.template packet<LoadMode, PacketType>(row, col));
817 }
818
819 template <int LoadMode, typename PacketType>
820 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
821 return m_d.func().packetOp(m_d.arg1Impl.template packet<LoadMode, PacketType>(index),
822 m_d.arg2Impl.template packet<LoadMode, PacketType>(index),
823 m_d.arg3Impl.template packet<LoadMode, PacketType>(index));
824 }
825
826 protected:
827 // this helper permits to completely eliminate the functor if it is empty
828 struct Data {
829 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr)
830 : op(xpr.functor()), arg1Impl(xpr.arg1()), arg2Impl(xpr.arg2()), arg3Impl(xpr.arg3()) {}
831 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TernaryOp& func() const { return op; }
832 TernaryOp op;
833 evaluator<Arg1> arg1Impl;
834 evaluator<Arg2> arg2Impl;
835 evaluator<Arg3> arg3Impl;
836 };
837
838 Data m_d;
839};
840
841// specialization for expresions like (a < b).select(c, d) to enable full vectorization
842template <typename Arg1, typename Arg2, typename Scalar, typename CmpLhsType, typename CmpRhsType, ComparisonName cmp>
843struct evaluator<CwiseTernaryOp<scalar_boolean_select_op<Scalar, Scalar, bool>, Arg1, Arg2,
844 CwiseBinaryOp<scalar_cmp_op<Scalar, Scalar, cmp, false>, CmpLhsType, CmpRhsType>>>
845 : public ternary_evaluator<
846 CwiseTernaryOp<scalar_boolean_select_op<Scalar, Scalar, Scalar>, Arg1, Arg2,
847 CwiseBinaryOp<scalar_cmp_op<Scalar, Scalar, cmp, true>, CmpLhsType, CmpRhsType>>> {
848 using DummyTernaryOp = scalar_boolean_select_op<Scalar, Scalar, bool>;
849 using DummyArg3 = CwiseBinaryOp<scalar_cmp_op<Scalar, Scalar, cmp, false>, CmpLhsType, CmpRhsType>;
850 using DummyXprType = CwiseTernaryOp<DummyTernaryOp, Arg1, Arg2, DummyArg3>;
851
852 using TernaryOp = scalar_boolean_select_op<Scalar, Scalar, Scalar>;
853 using Arg3 = CwiseBinaryOp<scalar_cmp_op<Scalar, Scalar, cmp, true>, CmpLhsType, CmpRhsType>;
854 using XprType = CwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>;
855
856 using Base = ternary_evaluator<XprType>;
857
858 EIGEN_DEVICE_FUNC explicit evaluator(const DummyXprType& xpr)
859 : Base(XprType(xpr.arg1(), xpr.arg2(), Arg3(xpr.arg3().lhs(), xpr.arg3().rhs()))) {}
860};
861
862// -------------------- CwiseBinaryOp --------------------
863
864// this is a binary expression
865template <typename BinaryOp, typename Lhs, typename Rhs>
866struct evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>> : public binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>> {
867 typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType;
868 typedef binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>> Base;
869
870 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) : Base(xpr) {}
871};
872
873template <typename BinaryOp, typename Lhs, typename Rhs>
874struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IndexBased, IndexBased>
875 : evaluator_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs>> {
876 typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType;
877
878 enum {
879 CoeffReadCost =
880 int(evaluator<Lhs>::CoeffReadCost) + int(evaluator<Rhs>::CoeffReadCost) + int(functor_traits<BinaryOp>::Cost),
881
882 LhsFlags = evaluator<Lhs>::Flags,
883 RhsFlags = evaluator<Rhs>::Flags,
884 SameType = is_same<typename Lhs::Scalar, typename Rhs::Scalar>::value,
885 StorageOrdersAgree = (int(LhsFlags) & RowMajorBit) == (int(RhsFlags) & RowMajorBit),
886 Flags0 = (int(LhsFlags) | int(RhsFlags)) &
887 (HereditaryBits |
888 (int(LhsFlags) & int(RhsFlags) &
889 ((StorageOrdersAgree ? LinearAccessBit : 0) |
890 (functor_traits<BinaryOp>::PacketAccess && StorageOrdersAgree && SameType ? PacketAccessBit : 0)))),
891 Flags = (Flags0 & ~RowMajorBit) | (LhsFlags & RowMajorBit),
892 Alignment = plain_enum_min(evaluator<Lhs>::Alignment, evaluator<Rhs>::Alignment)
893 };
894
895 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit binary_evaluator(const XprType& xpr) : m_d(xpr) {
896 EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<BinaryOp>::Cost);
897 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
898 }
899
900 typedef typename XprType::CoeffReturnType CoeffReturnType;
901
902 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
903 return m_d.func()(m_d.lhsImpl.coeff(row, col), m_d.rhsImpl.coeff(row, col));
904 }
905
906 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
907 return m_d.func()(m_d.lhsImpl.coeff(index), m_d.rhsImpl.coeff(index));
908 }
909
910 template <int LoadMode, typename PacketType>
911 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
912 return m_d.func().packetOp(m_d.lhsImpl.template packet<LoadMode, PacketType>(row, col),
913 m_d.rhsImpl.template packet<LoadMode, PacketType>(row, col));
914 }
915
916 template <int LoadMode, typename PacketType>
917 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
918 return m_d.func().packetOp(m_d.lhsImpl.template packet<LoadMode, PacketType>(index),
919 m_d.rhsImpl.template packet<LoadMode, PacketType>(index));
920 }
921
922 protected:
923 // this helper permits to completely eliminate the functor if it is empty
924 struct Data {
925 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr)
926 : op(xpr.functor()), lhsImpl(xpr.lhs()), rhsImpl(xpr.rhs()) {}
927 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const BinaryOp& func() const { return op; }
928 BinaryOp op;
929 evaluator<Lhs> lhsImpl;
930 evaluator<Rhs> rhsImpl;
931 };
932
933 Data m_d;
934};
935
936// -------------------- CwiseUnaryView --------------------
937
938template <typename UnaryOp, typename ArgType, typename StrideType>
939struct unary_evaluator<CwiseUnaryView<UnaryOp, ArgType, StrideType>, IndexBased>
940 : evaluator_base<CwiseUnaryView<UnaryOp, ArgType, StrideType>> {
941 typedef CwiseUnaryView<UnaryOp, ArgType, StrideType> XprType;
942
943 enum {
944 CoeffReadCost = int(evaluator<ArgType>::CoeffReadCost) + int(functor_traits<UnaryOp>::Cost),
945
946 Flags = (evaluator<ArgType>::Flags & (HereditaryBits | LinearAccessBit | DirectAccessBit)),
947
948 Alignment = 0 // FIXME it is not very clear why alignment is necessarily lost...
949 };
950
951 EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op) : m_d(op) {
952 EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<UnaryOp>::Cost);
953 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
954 }
955
956 typedef typename XprType::Scalar Scalar;
957 typedef typename XprType::CoeffReturnType CoeffReturnType;
958
959 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
960 return m_d.func()(m_d.argImpl.coeff(row, col));
961 }
962
963 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
964 return m_d.func()(m_d.argImpl.coeff(index));
965 }
966
967 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
968 return m_d.func()(m_d.argImpl.coeffRef(row, col));
969 }
970
971 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
972 return m_d.func()(m_d.argImpl.coeffRef(index));
973 }
974
975 protected:
976 // this helper permits to completely eliminate the functor if it is empty
977 struct Data {
978 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Data(const XprType& xpr)
979 : op(xpr.functor()), argImpl(xpr.nestedExpression()) {}
980 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const UnaryOp& func() const { return op; }
981 UnaryOp op;
982 evaluator<ArgType> argImpl;
983 };
984
985 Data m_d;
986};
987
988// -------------------- Map --------------------
989
990// FIXME perhaps the PlainObjectType could be provided by Derived::PlainObject ?
991// but that might complicate template specialization
992template <typename Derived, typename PlainObjectType>
993struct mapbase_evaluator;
994
995template <typename Derived, typename PlainObjectType>
996struct mapbase_evaluator : evaluator_base<Derived> {
997 typedef Derived XprType;
998 typedef typename XprType::PointerType PointerType;
999 typedef typename XprType::Scalar Scalar;
1000 typedef typename XprType::CoeffReturnType CoeffReturnType;
1001
1002 enum {
1003 IsRowMajor = XprType::RowsAtCompileTime,
1004 ColsAtCompileTime = XprType::ColsAtCompileTime,
1005 CoeffReadCost = NumTraits<Scalar>::ReadCost
1006 };
1007
1008 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit mapbase_evaluator(const XprType& map)
1009 : m_data(const_cast<PointerType>(map.data())),
1010 m_innerStride(map.innerStride()),
1011 m_outerStride(map.outerStride()) {
1012 EIGEN_STATIC_ASSERT(check_implication((evaluator<Derived>::Flags & PacketAccessBit) != 0,
1013 internal::inner_stride_at_compile_time<Derived>::ret == 1),
1014 PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1);
1015 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
1016 }
1017
1018 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
1019 return m_data[col * colStride() + row * rowStride()];
1020 }
1021
1022 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1023 return m_data[index * m_innerStride.value()];
1024 }
1025
1026 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
1027 return m_data[col * colStride() + row * rowStride()];
1028 }
1029
1030 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_data[index * m_innerStride.value()]; }
1031
1032 template <int LoadMode, typename PacketType>
1033 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
1034 PointerType ptr = m_data + row * rowStride() + col * colStride();
1035 return internal::ploadt<PacketType, LoadMode>(ptr);
1036 }
1037
1038 template <int LoadMode, typename PacketType>
1039 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
1040 return internal::ploadt<PacketType, LoadMode>(m_data + index * m_innerStride.value());
1041 }
1042
1043 template <int StoreMode, typename PacketType>
1044 EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) {
1045 PointerType ptr = m_data + row * rowStride() + col * colStride();
1046 return internal::pstoret<Scalar, PacketType, StoreMode>(ptr, x);
1047 }
1048
1049 template <int StoreMode, typename PacketType>
1050 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
1051 internal::pstoret<Scalar, PacketType, StoreMode>(m_data + index * m_innerStride.value(), x);
1052 }
1053
1054 protected:
1055 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rowStride() const EIGEN_NOEXCEPT {
1056 return XprType::IsRowMajor ? m_outerStride.value() : m_innerStride.value();
1057 }
1058 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index colStride() const EIGEN_NOEXCEPT {
1059 return XprType::IsRowMajor ? m_innerStride.value() : m_outerStride.value();
1060 }
1061
1062 PointerType m_data;
1063 const internal::variable_if_dynamic<Index, XprType::InnerStrideAtCompileTime> m_innerStride;
1064 const internal::variable_if_dynamic<Index, XprType::OuterStrideAtCompileTime> m_outerStride;
1065};
1066
1067template <typename PlainObjectType, int MapOptions, typename StrideType>
1068struct evaluator<Map<PlainObjectType, MapOptions, StrideType>>
1069 : public mapbase_evaluator<Map<PlainObjectType, MapOptions, StrideType>, PlainObjectType> {
1070 typedef Map<PlainObjectType, MapOptions, StrideType> XprType;
1071 typedef typename XprType::Scalar Scalar;
1072 // TODO: should check for smaller packet types once we can handle multi-sized packet types
1073 typedef typename packet_traits<Scalar>::type PacketScalar;
1074
1075 enum {
1076 InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0
1077 ? int(PlainObjectType::InnerStrideAtCompileTime)
1078 : int(StrideType::InnerStrideAtCompileTime),
1079 OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0
1080 ? int(PlainObjectType::OuterStrideAtCompileTime)
1081 : int(StrideType::OuterStrideAtCompileTime),
1082 HasNoInnerStride = InnerStrideAtCompileTime == 1,
1083 HasNoOuterStride = StrideType::OuterStrideAtCompileTime == 0,
1084 HasNoStride = HasNoInnerStride && HasNoOuterStride,
1085 IsDynamicSize = PlainObjectType::SizeAtCompileTime == Dynamic,
1086
1087 PacketAccessMask = bool(HasNoInnerStride) ? ~int(0) : ~int(PacketAccessBit),
1088 LinearAccessMask =
1089 bool(HasNoStride) || bool(PlainObjectType::IsVectorAtCompileTime) ? ~int(0) : ~int(LinearAccessBit),
1090 Flags = int(evaluator<PlainObjectType>::Flags) & (LinearAccessMask & PacketAccessMask),
1091
1092 Alignment = int(MapOptions) & int(AlignedMask)
1093 };
1094
1095 EIGEN_DEVICE_FUNC explicit evaluator(const XprType& map) : mapbase_evaluator<XprType, PlainObjectType>(map) {}
1096};
1097
1098// -------------------- Ref --------------------
1099
1100template <typename PlainObjectType, int RefOptions, typename StrideType>
1101struct evaluator<Ref<PlainObjectType, RefOptions, StrideType>>
1102 : public mapbase_evaluator<Ref<PlainObjectType, RefOptions, StrideType>, PlainObjectType> {
1103 typedef Ref<PlainObjectType, RefOptions, StrideType> XprType;
1104
1105 enum {
1106 Flags = evaluator<Map<PlainObjectType, RefOptions, StrideType>>::Flags,
1107 Alignment = evaluator<Map<PlainObjectType, RefOptions, StrideType>>::Alignment
1108 };
1109
1110 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& ref)
1111 : mapbase_evaluator<XprType, PlainObjectType>(ref) {}
1112};
1113
1114// -------------------- Block --------------------
1115
1116template <typename ArgType, int BlockRows, int BlockCols, bool InnerPanel,
1117 bool HasDirectAccess = internal::has_direct_access<ArgType>::ret>
1118struct block_evaluator;
1119
1120template <typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
1121struct evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>>
1122 : block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel> {
1123 typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType;
1124 typedef typename XprType::Scalar Scalar;
1125 // TODO: should check for smaller packet types once we can handle multi-sized packet types
1126 typedef typename packet_traits<Scalar>::type PacketScalar;
1127
1128 enum {
1129 CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
1130
1131 RowsAtCompileTime = traits<XprType>::RowsAtCompileTime,
1132 ColsAtCompileTime = traits<XprType>::ColsAtCompileTime,
1133 MaxRowsAtCompileTime = traits<XprType>::MaxRowsAtCompileTime,
1134 MaxColsAtCompileTime = traits<XprType>::MaxColsAtCompileTime,
1135
1136 ArgTypeIsRowMajor = (int(evaluator<ArgType>::Flags) & RowMajorBit) != 0,
1137 IsRowMajor = (MaxRowsAtCompileTime == 1 && MaxColsAtCompileTime != 1) ? 1
1138 : (MaxColsAtCompileTime == 1 && MaxRowsAtCompileTime != 1) ? 0
1139 : ArgTypeIsRowMajor,
1140 HasSameStorageOrderAsArgType = (IsRowMajor == ArgTypeIsRowMajor),
1141 InnerSize = IsRowMajor ? int(ColsAtCompileTime) : int(RowsAtCompileTime),
1142 InnerStrideAtCompileTime = HasSameStorageOrderAsArgType ? int(inner_stride_at_compile_time<ArgType>::ret)
1143 : int(outer_stride_at_compile_time<ArgType>::ret),
1144 OuterStrideAtCompileTime = HasSameStorageOrderAsArgType ? int(outer_stride_at_compile_time<ArgType>::ret)
1145 : int(inner_stride_at_compile_time<ArgType>::ret),
1146 MaskPacketAccessBit = (InnerStrideAtCompileTime == 1 || HasSameStorageOrderAsArgType) ? PacketAccessBit : 0,
1147
1148 FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1 ||
1149 (InnerPanel && (evaluator<ArgType>::Flags & LinearAccessBit)))
1150 ? LinearAccessBit
1151 : 0,
1152 FlagsRowMajorBit = XprType::Flags & RowMajorBit,
1153 Flags0 = evaluator<ArgType>::Flags & ((HereditaryBits & ~RowMajorBit) | DirectAccessBit | MaskPacketAccessBit),
1154 Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit,
1155
1156 PacketAlignment = unpacket_traits<PacketScalar>::alignment,
1157 Alignment0 = (InnerPanel && (OuterStrideAtCompileTime != Dynamic) && (OuterStrideAtCompileTime != 0) &&
1158 (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % int(PacketAlignment)) == 0))
1159 ? int(PacketAlignment)
1160 : 0,
1161 Alignment = plain_enum_min(evaluator<ArgType>::Alignment, Alignment0)
1162 };
1163 typedef block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel> block_evaluator_type;
1164 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& block) : block_evaluator_type(block) {
1165 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
1166 }
1167};
1168
1169// no direct-access => dispatch to a unary evaluator
1170template <typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
1171struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /*HasDirectAccess*/ false>
1172 : unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>> {
1173 typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType;
1174
1175 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit block_evaluator(const XprType& block)
1176 : unary_evaluator<XprType>(block) {}
1177};
1178
1179template <typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
1180struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBased>
1181 : evaluator_base<Block<ArgType, BlockRows, BlockCols, InnerPanel>> {
1182 typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType;
1183
1184 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& block)
1185 : m_argImpl(block.nestedExpression()),
1186 m_startRow(block.startRow()),
1187 m_startCol(block.startCol()),
1188 m_linear_offset(ForwardLinearAccess
1189 ? (ArgType::IsRowMajor
1190 ? block.startRow() * block.nestedExpression().cols() + block.startCol()
1191 : block.startCol() * block.nestedExpression().rows() + block.startRow())
1192 : 0) {}
1193
1194 typedef typename XprType::Scalar Scalar;
1195 typedef typename XprType::CoeffReturnType CoeffReturnType;
1196
1197 enum {
1198 RowsAtCompileTime = XprType::RowsAtCompileTime,
1199 ForwardLinearAccess = (InnerPanel || int(XprType::IsRowMajor) == int(ArgType::IsRowMajor)) &&
1200 bool(evaluator<ArgType>::Flags & LinearAccessBit)
1201 };
1202
1203 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
1204 return m_argImpl.coeff(m_startRow.value() + row, m_startCol.value() + col);
1205 }
1206
1207 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1208 return linear_coeff_impl(index, bool_constant<ForwardLinearAccess>());
1209 }
1210
1211 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
1212 return m_argImpl.coeffRef(m_startRow.value() + row, m_startCol.value() + col);
1213 }
1214
1215 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
1216 return linear_coeffRef_impl(index, bool_constant<ForwardLinearAccess>());
1217 }
1218
1219 template <int LoadMode, typename PacketType>
1220 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
1221 return m_argImpl.template packet<LoadMode, PacketType>(m_startRow.value() + row, m_startCol.value() + col);
1222 }
1223
1224 template <int LoadMode, typename PacketType>
1225 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
1226 if (ForwardLinearAccess)
1227 return m_argImpl.template packet<LoadMode, PacketType>(m_linear_offset.value() + index);
1228 else
1229 return packet<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
1230 }
1231
1232 template <int StoreMode, typename PacketType>
1233 EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) {
1234 return m_argImpl.template writePacket<StoreMode, PacketType>(m_startRow.value() + row, m_startCol.value() + col, x);
1235 }
1236
1237 template <int StoreMode, typename PacketType>
1238 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
1239 if (ForwardLinearAccess)
1240 return m_argImpl.template writePacket<StoreMode, PacketType>(m_linear_offset.value() + index, x);
1241 else
1242 return writePacket<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0,
1243 x);
1244 }
1245
1246 protected:
1247 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType
1248 linear_coeff_impl(Index index, internal::true_type /* ForwardLinearAccess */) const {
1249 return m_argImpl.coeff(m_linear_offset.value() + index);
1250 }
1251 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType
1252 linear_coeff_impl(Index index, internal::false_type /* not ForwardLinearAccess */) const {
1253 return coeff(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
1254 }
1255
1256 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& linear_coeffRef_impl(Index index,
1257 internal::true_type /* ForwardLinearAccess */) {
1258 return m_argImpl.coeffRef(m_linear_offset.value() + index);
1259 }
1260 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& linear_coeffRef_impl(
1261 Index index, internal::false_type /* not ForwardLinearAccess */) {
1262 return coeffRef(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
1263 }
1264
1265 evaluator<ArgType> m_argImpl;
1266 const variable_if_dynamic<Index, (ArgType::RowsAtCompileTime == 1 && BlockRows == 1) ? 0 : Dynamic> m_startRow;
1267 const variable_if_dynamic<Index, (ArgType::ColsAtCompileTime == 1 && BlockCols == 1) ? 0 : Dynamic> m_startCol;
1268 const variable_if_dynamic<Index, ForwardLinearAccess ? Dynamic : 0> m_linear_offset;
1269};
1270
1271// TODO: This evaluator does not actually use the child evaluator;
1272// all action is via the data() as returned by the Block expression.
1273
1274template <typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
1275struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /* HasDirectAccess */ true>
1276 : mapbase_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>,
1277 typename Block<ArgType, BlockRows, BlockCols, InnerPanel>::PlainObject> {
1278 typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType;
1279 typedef typename XprType::Scalar Scalar;
1280
1281 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit block_evaluator(const XprType& block)
1282 : mapbase_evaluator<XprType, typename XprType::PlainObject>(block) {
1283 eigen_internal_assert((internal::is_constant_evaluated() ||
1284 (std::uintptr_t(block.data()) % plain_enum_max(1, evaluator<XprType>::Alignment)) == 0) &&
1285 "data is not aligned");
1286 }
1287};
1288
1289// -------------------- Select --------------------
1290// NOTE shall we introduce a ternary_evaluator?
1291
1292// TODO enable vectorization for Select
1293template <typename ConditionMatrixType, typename ThenMatrixType, typename ElseMatrixType>
1294struct evaluator<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType>>
1295 : evaluator_base<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType>> {
1296 typedef Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> XprType;
1297 enum {
1298 CoeffReadCost = evaluator<ConditionMatrixType>::CoeffReadCost +
1299 plain_enum_max(evaluator<ThenMatrixType>::CoeffReadCost, evaluator<ElseMatrixType>::CoeffReadCost),
1300
1301 Flags = (unsigned int)evaluator<ThenMatrixType>::Flags & evaluator<ElseMatrixType>::Flags & HereditaryBits,
1302
1303 Alignment = plain_enum_min(evaluator<ThenMatrixType>::Alignment, evaluator<ElseMatrixType>::Alignment)
1304 };
1305
1306 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& select)
1307 : m_conditionImpl(select.conditionMatrix()), m_thenImpl(select.thenMatrix()), m_elseImpl(select.elseMatrix()) {
1308 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
1309 }
1310
1311 typedef typename XprType::CoeffReturnType CoeffReturnType;
1312
1313 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
1314 if (m_conditionImpl.coeff(row, col))
1315 return m_thenImpl.coeff(row, col);
1316 else
1317 return m_elseImpl.coeff(row, col);
1318 }
1319
1320 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1321 if (m_conditionImpl.coeff(index))
1322 return m_thenImpl.coeff(index);
1323 else
1324 return m_elseImpl.coeff(index);
1325 }
1326
1327 protected:
1328 evaluator<ConditionMatrixType> m_conditionImpl;
1329 evaluator<ThenMatrixType> m_thenImpl;
1330 evaluator<ElseMatrixType> m_elseImpl;
1331};
1332
1333// -------------------- Replicate --------------------
1334
1335template <typename ArgType, int RowFactor, int ColFactor>
1336struct unary_evaluator<Replicate<ArgType, RowFactor, ColFactor>>
1337 : evaluator_base<Replicate<ArgType, RowFactor, ColFactor>> {
1338 typedef Replicate<ArgType, RowFactor, ColFactor> XprType;
1339 typedef typename XprType::CoeffReturnType CoeffReturnType;
1340 enum { Factor = (RowFactor == Dynamic || ColFactor == Dynamic) ? Dynamic : RowFactor * ColFactor };
1341 typedef typename internal::nested_eval<ArgType, Factor>::type ArgTypeNested;
1342 typedef internal::remove_all_t<ArgTypeNested> ArgTypeNestedCleaned;
1343
1344 enum {
1345 CoeffReadCost = evaluator<ArgTypeNestedCleaned>::CoeffReadCost,
1346 LinearAccessMask = XprType::IsVectorAtCompileTime ? LinearAccessBit : 0,
1347 Flags = (evaluator<ArgTypeNestedCleaned>::Flags & (HereditaryBits | LinearAccessMask) & ~RowMajorBit) |
1348 (traits<XprType>::Flags & RowMajorBit),
1349
1350 Alignment = evaluator<ArgTypeNestedCleaned>::Alignment
1351 };
1352
1353 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& replicate)
1354 : m_arg(replicate.nestedExpression()),
1355 m_argImpl(m_arg),
1356 m_rows(replicate.nestedExpression().rows()),
1357 m_cols(replicate.nestedExpression().cols()) {}
1358
1359 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
1360 // try to avoid using modulo; this is a pure optimization strategy
1361 const Index actual_row = internal::traits<XprType>::RowsAtCompileTime == 1 ? 0
1362 : RowFactor == 1 ? row
1363 : row % m_rows.value();
1364 const Index actual_col = internal::traits<XprType>::ColsAtCompileTime == 1 ? 0
1365 : ColFactor == 1 ? col
1366 : col % m_cols.value();
1367
1368 return m_argImpl.coeff(actual_row, actual_col);
1369 }
1370
1371 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1372 // try to avoid using modulo; this is a pure optimization strategy
1373 const Index actual_index = internal::traits<XprType>::RowsAtCompileTime == 1
1374 ? (ColFactor == 1 ? index : index % m_cols.value())
1375 : (RowFactor == 1 ? index : index % m_rows.value());
1376
1377 return m_argImpl.coeff(actual_index);
1378 }
1379
1380 template <int LoadMode, typename PacketType>
1381 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
1382 const Index actual_row = internal::traits<XprType>::RowsAtCompileTime == 1 ? 0
1383 : RowFactor == 1 ? row
1384 : row % m_rows.value();
1385 const Index actual_col = internal::traits<XprType>::ColsAtCompileTime == 1 ? 0
1386 : ColFactor == 1 ? col
1387 : col % m_cols.value();
1388
1389 return m_argImpl.template packet<LoadMode, PacketType>(actual_row, actual_col);
1390 }
1391
1392 template <int LoadMode, typename PacketType>
1393 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
1394 const Index actual_index = internal::traits<XprType>::RowsAtCompileTime == 1
1395 ? (ColFactor == 1 ? index : index % m_cols.value())
1396 : (RowFactor == 1 ? index : index % m_rows.value());
1397
1398 return m_argImpl.template packet<LoadMode, PacketType>(actual_index);
1399 }
1400
1401 protected:
1402 const ArgTypeNested m_arg;
1403 evaluator<ArgTypeNestedCleaned> m_argImpl;
1404 const variable_if_dynamic<Index, ArgType::RowsAtCompileTime> m_rows;
1405 const variable_if_dynamic<Index, ArgType::ColsAtCompileTime> m_cols;
1406};
1407
1408// -------------------- MatrixWrapper and ArrayWrapper --------------------
1409//
1410// evaluator_wrapper_base<T> is a common base class for the
1411// MatrixWrapper and ArrayWrapper evaluators.
1412
1413template <typename XprType>
1414struct evaluator_wrapper_base : evaluator_base<XprType> {
1415 typedef remove_all_t<typename XprType::NestedExpressionType> ArgType;
1416 enum {
1417 CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
1418 Flags = evaluator<ArgType>::Flags,
1419 Alignment = evaluator<ArgType>::Alignment
1420 };
1421
1422 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator_wrapper_base(const ArgType& arg) : m_argImpl(arg) {}
1423
1424 typedef typename ArgType::Scalar Scalar;
1425 typedef typename ArgType::CoeffReturnType CoeffReturnType;
1426
1427 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
1428 return m_argImpl.coeff(row, col);
1429 }
1430
1431 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { return m_argImpl.coeff(index); }
1432
1433 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(row, col); }
1434
1435 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(index); }
1436
1437 template <int LoadMode, typename PacketType>
1438 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
1439 return m_argImpl.template packet<LoadMode, PacketType>(row, col);
1440 }
1441
1442 template <int LoadMode, typename PacketType>
1443 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
1444 return m_argImpl.template packet<LoadMode, PacketType>(index);
1445 }
1446
1447 template <int StoreMode, typename PacketType>
1448 EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) {
1449 m_argImpl.template writePacket<StoreMode>(row, col, x);
1450 }
1451
1452 template <int StoreMode, typename PacketType>
1453 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
1454 m_argImpl.template writePacket<StoreMode>(index, x);
1455 }
1456
1457 protected:
1458 evaluator<ArgType> m_argImpl;
1459};
1460
1461template <typename TArgType>
1462struct unary_evaluator<MatrixWrapper<TArgType>> : evaluator_wrapper_base<MatrixWrapper<TArgType>> {
1463 typedef MatrixWrapper<TArgType> XprType;
1464
1465 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& wrapper)
1466 : evaluator_wrapper_base<MatrixWrapper<TArgType>>(wrapper.nestedExpression()) {}
1467};
1468
1469template <typename TArgType>
1470struct unary_evaluator<ArrayWrapper<TArgType>> : evaluator_wrapper_base<ArrayWrapper<TArgType>> {
1471 typedef ArrayWrapper<TArgType> XprType;
1472
1473 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& wrapper)
1474 : evaluator_wrapper_base<ArrayWrapper<TArgType>>(wrapper.nestedExpression()) {}
1475};
1476
1477// -------------------- Reverse --------------------
1478
1479// defined in Reverse.h:
1480template <typename PacketType, bool ReversePacket>
1481struct reverse_packet_cond;
1482
1483template <typename ArgType, int Direction>
1484struct unary_evaluator<Reverse<ArgType, Direction>> : evaluator_base<Reverse<ArgType, Direction>> {
1485 typedef Reverse<ArgType, Direction> XprType;
1486 typedef typename XprType::Scalar Scalar;
1487 typedef typename XprType::CoeffReturnType CoeffReturnType;
1488
1489 enum {
1490 IsRowMajor = XprType::IsRowMajor,
1491 IsColMajor = !IsRowMajor,
1492 ReverseRow = (Direction == Vertical) || (Direction == BothDirections),
1493 ReverseCol = (Direction == Horizontal) || (Direction == BothDirections),
1494 ReversePacket = (Direction == BothDirections) || ((Direction == Vertical) && IsColMajor) ||
1495 ((Direction == Horizontal) && IsRowMajor),
1496
1497 CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
1498
1499 // let's enable LinearAccess only with vectorization because of the product overhead
1500 // FIXME enable DirectAccess with negative strides?
1501 Flags0 = evaluator<ArgType>::Flags,
1502 LinearAccess =
1503 ((Direction == BothDirections) && (int(Flags0) & PacketAccessBit)) ||
1504 ((ReverseRow && XprType::ColsAtCompileTime == 1) || (ReverseCol && XprType::RowsAtCompileTime == 1))
1505 ? LinearAccessBit
1506 : 0,
1507
1508 Flags = int(Flags0) & (HereditaryBits | PacketAccessBit | LinearAccess),
1509
1510 Alignment = 0 // FIXME in some rare cases, Alignment could be preserved, like a Vector4f.
1511 };
1512
1513 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit unary_evaluator(const XprType& reverse)
1514 : m_argImpl(reverse.nestedExpression()),
1515 m_rows(ReverseRow ? reverse.nestedExpression().rows() : 1),
1516 m_cols(ReverseCol ? reverse.nestedExpression().cols() : 1) {}
1517
1518 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
1519 return m_argImpl.coeff(ReverseRow ? m_rows.value() - row - 1 : row, ReverseCol ? m_cols.value() - col - 1 : col);
1520 }
1521
1522 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1523 return m_argImpl.coeff(m_rows.value() * m_cols.value() - index - 1);
1524 }
1525
1526 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) {
1527 return m_argImpl.coeffRef(ReverseRow ? m_rows.value() - row - 1 : row, ReverseCol ? m_cols.value() - col - 1 : col);
1528 }
1529
1530 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
1531 return m_argImpl.coeffRef(m_rows.value() * m_cols.value() - index - 1);
1532 }
1533
1534 template <int LoadMode, typename PacketType>
1535 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
1536 enum {
1537 PacketSize = unpacket_traits<PacketType>::size,
1538 OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1,
1539 OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1
1540 };
1541 typedef internal::reverse_packet_cond<PacketType, ReversePacket> reverse_packet;
1542 return reverse_packet::run(m_argImpl.template packet<LoadMode, PacketType>(
1543 ReverseRow ? m_rows.value() - row - OffsetRow : row, ReverseCol ? m_cols.value() - col - OffsetCol : col));
1544 }
1545
1546 template <int LoadMode, typename PacketType>
1547 EIGEN_STRONG_INLINE PacketType packet(Index index) const {
1548 enum { PacketSize = unpacket_traits<PacketType>::size };
1549 return preverse(
1550 m_argImpl.template packet<LoadMode, PacketType>(m_rows.value() * m_cols.value() - index - PacketSize));
1551 }
1552
1553 template <int LoadMode, typename PacketType>
1554 EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) {
1555 // FIXME we could factorize some code with packet(i,j)
1556 enum {
1557 PacketSize = unpacket_traits<PacketType>::size,
1558 OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1,
1559 OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1
1560 };
1561 typedef internal::reverse_packet_cond<PacketType, ReversePacket> reverse_packet;
1562 m_argImpl.template writePacket<LoadMode>(ReverseRow ? m_rows.value() - row - OffsetRow : row,
1563 ReverseCol ? m_cols.value() - col - OffsetCol : col,
1564 reverse_packet::run(x));
1565 }
1566
1567 template <int LoadMode, typename PacketType>
1568 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
1569 enum { PacketSize = unpacket_traits<PacketType>::size };
1570 m_argImpl.template writePacket<LoadMode>(m_rows.value() * m_cols.value() - index - PacketSize, preverse(x));
1571 }
1572
1573 protected:
1574 evaluator<ArgType> m_argImpl;
1575
1576 // If we do not reverse rows, then we do not need to know the number of rows; same for columns
1577 // Nonetheless, in this case it is important to set to 1 such that the coeff(index) method works fine for vectors.
1578 const variable_if_dynamic<Index, ReverseRow ? ArgType::RowsAtCompileTime : 1> m_rows;
1579 const variable_if_dynamic<Index, ReverseCol ? ArgType::ColsAtCompileTime : 1> m_cols;
1580};
1581
1582// -------------------- Diagonal --------------------
1583
1584template <typename ArgType, int DiagIndex>
1585struct evaluator<Diagonal<ArgType, DiagIndex>> : evaluator_base<Diagonal<ArgType, DiagIndex>> {
1586 typedef Diagonal<ArgType, DiagIndex> XprType;
1587
1588 enum {
1589 CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
1590
1591 Flags =
1592 (unsigned int)(evaluator<ArgType>::Flags & (HereditaryBits | DirectAccessBit) & ~RowMajorBit) | LinearAccessBit,
1593
1594 Alignment = 0
1595 };
1596
1597 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& diagonal)
1598 : m_argImpl(diagonal.nestedExpression()), m_index(diagonal.index()) {}
1599
1600 typedef typename XprType::Scalar Scalar;
1601 typedef typename XprType::CoeffReturnType CoeffReturnType;
1602
1603 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index) const {
1604 return m_argImpl.coeff(row + rowOffset(), row + colOffset());
1605 }
1606
1607 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1608 return m_argImpl.coeff(index + rowOffset(), index + colOffset());
1609 }
1610
1611 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index) {
1612 return m_argImpl.coeffRef(row + rowOffset(), row + colOffset());
1613 }
1614
1615 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) {
1616 return m_argImpl.coeffRef(index + rowOffset(), index + colOffset());
1617 }
1618
1619 protected:
1620 evaluator<ArgType> m_argImpl;
1621 const internal::variable_if_dynamicindex<Index, XprType::DiagIndex> m_index;
1622
1623 private:
1624 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index rowOffset() const {
1625 return m_index.value() > 0 ? 0 : -m_index.value();
1626 }
1627 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE EIGEN_CONSTEXPR Index colOffset() const {
1628 return m_index.value() > 0 ? m_index.value() : 0;
1629 }
1630};
1631
1632//----------------------------------------------------------------------
1633// deprecated code
1634//----------------------------------------------------------------------
1635
1636// -------------------- EvalToTemp --------------------
1637
1638// expression class for evaluating nested expression to a temporary
1639
1640template <typename ArgType>
1641class EvalToTemp;
1642
1643template <typename ArgType>
1644struct traits<EvalToTemp<ArgType>> : public traits<ArgType> {};
1645
1646template <typename ArgType>
1647class EvalToTemp : public dense_xpr_base<EvalToTemp<ArgType>>::type {
1648 public:
1649 typedef typename dense_xpr_base<EvalToTemp>::type Base;
1650 EIGEN_GENERIC_PUBLIC_INTERFACE(EvalToTemp)
1651
1652 explicit EvalToTemp(const ArgType& arg) : m_arg(arg) {}
1653
1654 const ArgType& arg() const { return m_arg; }
1655
1656 EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_arg.rows(); }
1657
1658 EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_arg.cols(); }
1659
1660 private:
1661 const ArgType& m_arg;
1662};
1663
1664template <typename ArgType>
1665struct evaluator<EvalToTemp<ArgType>> : public evaluator<typename ArgType::PlainObject> {
1666 typedef EvalToTemp<ArgType> XprType;
1667 typedef typename ArgType::PlainObject PlainObject;
1668 typedef evaluator<PlainObject> Base;
1669
1670 EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : m_result(xpr.arg()) {
1671 internal::construct_at<Base>(this, m_result);
1672 }
1673
1674 // This constructor is used when nesting an EvalTo evaluator in another evaluator
1675 EIGEN_DEVICE_FUNC evaluator(const ArgType& arg) : m_result(arg) { internal::construct_at<Base>(this, m_result); }
1676
1677 protected:
1678 PlainObject m_result;
1679};
1680
1681} // namespace internal
1682
1683} // end namespace Eigen
1684
1685#endif // EIGEN_COREEVALUATORS_H
Eigen::BothDirections
@ BothDirections
Definition Constants.h:272
Eigen::Horizontal
@ Horizontal
Definition Constants.h:269
Eigen::Vertical
@ Vertical
Definition Constants.h:266
Eigen::ActualPacketAccessBit
const unsigned int ActualPacketAccessBit
Definition Constants.h:108
Eigen::PacketAccessBit
const unsigned int PacketAccessBit
Definition Constants.h:97
Eigen::LinearAccessBit
const unsigned int LinearAccessBit
Definition Constants.h:133
Eigen::EvalBeforeNestingBit
const unsigned int EvalBeforeNestingBit
Definition Constants.h:74
Eigen::DirectAccessBit
const unsigned int DirectAccessBit
Definition Constants.h:159
Eigen::RowMajorBit
const unsigned int RowMajorBit
Definition Constants.h:70
Eigen
Namespace containing all symbols from the Eigen library.
Definition Core:137
Eigen::arg
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_arg_op< typename Derived::Scalar >, const Derived > arg(const Eigen::ArrayBase< Derived > &x)
Eigen::Dynamic
const int Dynamic
Definition Constants.h:25