Eigen  3.4.90 (git rev 5a9f66fb35d03a4da9ef8976e67a61b30aa16dcf)
 
Loading...
Searching...
No Matches
ZVector/PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Konstantinos Margaritis <[email protected]>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_ZVECTOR_H
11#define EIGEN_PACKET_MATH_ZVECTOR_H
12
13// IWYU pragma: private
14#include "../../InternalHeaderCheck.h"
15
16namespace Eigen {
17
18namespace internal {
19
20#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
21#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 16
22#endif
23
24#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
25#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
26#endif
27
28#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
29#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
30#endif
31
32typedef __vector int Packet4i;
33typedef __vector unsigned int Packet4ui;
34typedef __vector __bool int Packet4bi;
35typedef __vector short int Packet8i;
36typedef __vector unsigned char Packet16uc;
37typedef __vector double Packet2d;
38typedef __vector unsigned long long Packet2ul;
39typedef __vector long long Packet2l;
40
41// Z14 has builtin support for float vectors
42#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
43typedef __vector float Packet4f;
44#else
45typedef struct {
46 Packet2d v4f[2];
47} Packet4f;
48#endif
49
50typedef union {
51 numext::int32_t i[4];
52 numext::uint32_t ui[4];
53 numext::int64_t l[2];
54 numext::uint64_t ul[2];
55 double d[2];
56 float f[4];
57 Packet4i v4i;
58 Packet4ui v4ui;
59 Packet2l v2l;
60 Packet2ul v2ul;
61 Packet2d v2d;
62#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
63 Packet4f v4f;
64#endif
65} Packet;
66
67// We don't want to write the same code all the time, but we need to reuse the constants
68// and it doesn't really work to declare them global, so we define macros instead
69
70#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME, X) Packet4i p4i_##NAME = reinterpret_cast<Packet4i>(vec_splat_s32(X))
71
72#define EIGEN_DECLARE_CONST_FAST_Packet2d(NAME, X) Packet2d p2d_##NAME = reinterpret_cast<Packet2d>(vec_splat_s64(X))
73
74#define EIGEN_DECLARE_CONST_FAST_Packet2l(NAME, X) Packet2l p2l_##NAME = reinterpret_cast<Packet2l>(vec_splat_s64(X))
75
76#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) Packet4i p4i_##NAME = pset1<Packet4i>(X)
77
78#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) Packet2d p2d_##NAME = pset1<Packet2d>(X)
79
80#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) Packet2l p2l_##NAME = pset1<Packet2l>(X)
81
82// These constants are endian-agnostic
83static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,}
84static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1}
85
86static EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0);
87static EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0);
88static EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1);
89
90static Packet2d p2d_ONE = {1.0, 1.0};
91static Packet2d p2d_ZERO_ = {numext::bit_cast<double>(0x8000000000000000ull),
92 numext::bit_cast<double>(0x8000000000000000ull)};
93
94#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
95#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME, X) Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X))
96
97#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) Packet4f p4f_##NAME = pset1<Packet4f>(X)
98
99#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
100 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
101
102static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0}
103static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1, -1); //{ -1, -1, -1, -1}
104static Packet4f p4f_MZERO = {0x80000000, 0x80000000, 0x80000000, 0x80000000};
105#endif
106
107static Packet4i p4i_COUNTDOWN = {0, 1, 2, 3};
108static Packet4f p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0};
109static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(
110 vec_sld(reinterpret_cast<Packet16uc>(p2d_ZERO), reinterpret_cast<Packet16uc>(p2d_ONE), 8));
111
112static Packet16uc p16uc_PSET64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7};
113static Packet16uc p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7};
114
115// Mask alignment
116#define EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0
117
118#define EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & EIGEN_MASK_ALIGNMENT)
119
120// Handle endianness properly while loading constants
121// Define global static constants:
122
123static Packet16uc p16uc_FORWARD = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
124static Packet16uc p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3};
125static Packet16uc p16uc_REVERSE64 = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
126
127static Packet16uc p16uc_PSET32_WODD =
128 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
129 8); //{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
130static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
131 8); //{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
132/*static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3),
1338); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
134
135static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD,
136(Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };*/
137static Packet16uc p16uc_PSET64_LO = (Packet16uc)vec_mergel(
138 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
139/*static Packet16uc p16uc_TRANSPOSE64_HI = vec_add(p16uc_PSET64_HI, p16uc_HALF64_0_16); //{ 0,1,2,3, 4,5,6,7,
14016,17,18,19, 20,21,22,23}; static Packet16uc p16uc_TRANSPOSE64_LO = vec_add(p16uc_PSET64_LO, p16uc_HALF64_0_16); //{
1418,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};*/
142static Packet16uc p16uc_TRANSPOSE64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
143static Packet16uc p16uc_TRANSPOSE64_LO = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
144
145static Packet16uc p16uc_COMPLEX32_REV =
146 vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
147
148static Packet16uc p16uc_COMPLEX32_REV2 =
149 vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
150
151#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
152#define EIGEN_ZVECTOR_PREFETCH(ADDR) __builtin_prefetch(ADDR);
153#else
154#define EIGEN_ZVECTOR_PREFETCH(ADDR) asm(" pfd [%[addr]]\n" ::[addr] "r"(ADDR) : "cc");
155#endif
156
157template <>
158struct packet_traits<int> : default_packet_traits {
159 typedef Packet4i type;
160 typedef Packet4i half;
161 enum {
162 Vectorizable = 1,
163 AlignedOnScalar = 1,
164 size = 4,
165
166 HasAdd = 1,
167 HasSub = 1,
168 HasMul = 1,
169 HasDiv = 1,
170 HasBlend = 1
171 };
172};
173
174template <>
175struct packet_traits<float> : default_packet_traits {
176 typedef Packet4f type;
177 typedef Packet4f half;
178 enum {
179 Vectorizable = 1,
180 AlignedOnScalar = 1,
181 size = 4,
182
183 HasAdd = 1,
184 HasSub = 1,
185 HasMul = 1,
186 HasDiv = 1,
187 HasMin = 1,
188 HasMax = 1,
189 HasAbs = 1,
190 HasSin = 0,
191 HasCos = 0,
192 HasLog = 0,
193 HasExp = 1,
194 HasSqrt = 1,
195 HasRsqrt = 1,
196 HasTanh = 1,
197 HasErf = 1,
198 HasNegate = 1,
199 HasBlend = 1
200 };
201};
202
203template <>
204struct packet_traits<double> : default_packet_traits {
205 typedef Packet2d type;
206 typedef Packet2d half;
207 enum {
208 Vectorizable = 1,
209 AlignedOnScalar = 1,
210 size = 2,
211
212 HasAdd = 1,
213 HasSub = 1,
214 HasMul = 1,
215 HasDiv = 1,
216 HasMin = 1,
217 HasMax = 1,
218 HasAbs = 1,
219 HasSin = 0,
220 HasCos = 0,
221 HasLog = 0,
222 HasExp = 1,
223 HasSqrt = 1,
224 HasRsqrt = 1,
225 HasNegate = 1,
226 HasBlend = 1
227 };
228};
229
230template <>
231struct unpacket_traits<Packet4i> {
232 typedef int type;
233 enum {
234 size = 4,
235 alignment = Aligned16,
236 vectorizable = true,
237 masked_load_available = false,
238 masked_store_available = false
239 };
240 typedef Packet4i half;
241};
242template <>
243struct unpacket_traits<Packet4f> {
244 typedef float type;
245 enum {
246 size = 4,
247 alignment = Aligned16,
248 vectorizable = true,
249 masked_load_available = false,
250 masked_store_available = false
251 };
252 typedef Packet4f half;
253};
254template <>
255struct unpacket_traits<Packet2d> {
256 typedef double type;
257 enum {
258 size = 2,
259 alignment = Aligned16,
260 vectorizable = true,
261 masked_load_available = false,
262 masked_store_available = false
263 };
264 typedef Packet2d half;
265};
266
267/* Forward declaration */
268EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel);
269
270inline std::ostream& operator<<(std::ostream& s, const Packet4i& v) {
271 Packet vt;
272 vt.v4i = v;
273 s << vt.i[0] << ", " << vt.i[1] << ", " << vt.i[2] << ", " << vt.i[3];
274 return s;
275}
276
277inline std::ostream& operator<<(std::ostream& s, const Packet4ui& v) {
278 Packet vt;
279 vt.v4ui = v;
280 s << vt.ui[0] << ", " << vt.ui[1] << ", " << vt.ui[2] << ", " << vt.ui[3];
281 return s;
282}
283
284inline std::ostream& operator<<(std::ostream& s, const Packet2l& v) {
285 Packet vt;
286 vt.v2l = v;
287 s << vt.l[0] << ", " << vt.l[1];
288 return s;
289}
290
291inline std::ostream& operator<<(std::ostream& s, const Packet2ul& v) {
292 Packet vt;
293 vt.v2ul = v;
294 s << vt.ul[0] << ", " << vt.ul[1];
295 return s;
296}
297
298inline std::ostream& operator<<(std::ostream& s, const Packet2d& v) {
299 Packet vt;
300 vt.v2d = v;
301 s << vt.d[0] << ", " << vt.d[1];
302 return s;
303}
304
305#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ >= 12)
306inline std::ostream& operator<<(std::ostream& s, const Packet4f& v) {
307 Packet vt;
308 vt.v4f = v;
309 s << vt.f[0] << ", " << vt.f[1] << ", " << vt.f[2] << ", " << vt.f[3];
310 return s;
311}
312#endif
313
314template <>
315EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) {
316 // FIXME: No intrinsic yet
317 EIGEN_DEBUG_ALIGNED_LOAD
318 Packet* vfrom;
319 vfrom = (Packet*)from;
320 return vfrom->v4i;
321}
322
323template <>
324EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
325 // FIXME: No intrinsic yet
326 EIGEN_DEBUG_ALIGNED_LOAD
327 Packet* vfrom;
328 vfrom = (Packet*)from;
329 return vfrom->v2d;
330}
331
332template <>
333EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) {
334 // FIXME: No intrinsic yet
335 EIGEN_DEBUG_ALIGNED_STORE
336 Packet* vto;
337 vto = (Packet*)to;
338 vto->v4i = from;
339}
340
341template <>
342EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
343 // FIXME: No intrinsic yet
344 EIGEN_DEBUG_ALIGNED_STORE
345 Packet* vto;
346 vto = (Packet*)to;
347 vto->v2d = from;
348}
349
350template <>
351EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) {
352 return vec_splats(from);
353}
354template <>
355EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
356 return vec_splats(from);
357}
358
359template <>
360EIGEN_STRONG_INLINE void pbroadcast4<Packet4i>(const int* a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) {
361 a3 = pload<Packet4i>(a);
362 a0 = vec_splat(a3, 0);
363 a1 = vec_splat(a3, 1);
364 a2 = vec_splat(a3, 2);
365 a3 = vec_splat(a3, 3);
366}
367
368template <>
369EIGEN_STRONG_INLINE void pbroadcast4<Packet2d>(const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2,
370 Packet2d& a3) {
371 a1 = pload<Packet2d>(a);
372 a0 = vec_splat(a1, 0);
373 a1 = vec_splat(a1, 1);
374 a3 = pload<Packet2d>(a + 2);
375 a2 = vec_splat(a3, 0);
376 a3 = vec_splat(a3, 1);
377}
378
379template <>
380EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride) {
381 EIGEN_ALIGN16 int ai[4];
382 ai[0] = from[0 * stride];
383 ai[1] = from[1 * stride];
384 ai[2] = from[2 * stride];
385 ai[3] = from[3 * stride];
386 return pload<Packet4i>(ai);
387}
388
389template <>
390EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
391 EIGEN_ALIGN16 double af[2];
392 af[0] = from[0 * stride];
393 af[1] = from[1 * stride];
394 return pload<Packet2d>(af);
395}
396
397template <>
398EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) {
399 EIGEN_ALIGN16 int ai[4];
400 pstore<int>((int*)ai, from);
401 to[0 * stride] = ai[0];
402 to[1 * stride] = ai[1];
403 to[2 * stride] = ai[2];
404 to[3 * stride] = ai[3];
405}
406
407template <>
408EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
409 EIGEN_ALIGN16 double af[2];
410 pstore<double>(af, from);
411 to[0 * stride] = af[0];
412 to[1 * stride] = af[1];
413}
414
415template <>
416EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
417 return (a + b);
418}
419template <>
420EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
421 return (a + b);
422}
423
424template <>
425EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
426 return (a - b);
427}
428template <>
429EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
430 return (a - b);
431}
432
433template <>
434EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
435 return (a * b);
436}
437template <>
438EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
439 return (a * b);
440}
441
442template <>
443EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
444 return (a / b);
445}
446template <>
447EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
448 return (a / b);
449}
450
451template <>
452EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
453 return (-a);
454}
455template <>
456EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
457 return (-a);
458}
459
460template <>
461EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) {
462 return a;
463}
464template <>
465EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) {
466 return a;
467}
468
469template <>
470EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
471 return padd<Packet4i>(pmul<Packet4i>(a, b), c);
472}
473template <>
474EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
475 return vec_madd(a, b, c);
476}
477
478template <>
479EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) {
480 return padd<Packet4i>(pset1<Packet4i>(a), p4i_COUNTDOWN);
481}
482template <>
483EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
484 return padd<Packet2d>(pset1<Packet2d>(a), p2d_COUNTDOWN);
485}
486
487template <>
488EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
489 return vec_min(a, b);
490}
491template <>
492EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
493 return vec_min(a, b);
494}
495
496template <>
497EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
498 return vec_max(a, b);
499}
500template <>
501EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
502 return vec_max(a, b);
503}
504
505template <>
506EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
507 return vec_and(a, b);
508}
509template <>
510EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
511 return vec_and(a, b);
512}
513
514template <>
515EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
516 return vec_or(a, b);
517}
518template <>
519EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
520 return vec_or(a, b);
521}
522
523template <>
524EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
525 return vec_xor(a, b);
526}
527template <>
528EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
529 return vec_xor(a, b);
530}
531
532template <>
533EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
534 return pand<Packet4i>(a, vec_nor(b, b));
535}
536template <>
537EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
538 return vec_and(a, vec_nor(b, b));
539}
540
541template <>
542EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) {
543 return vec_round(a);
544}
545template <>
546EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
547 return vec_ceil(a);
548}
549template <>
550EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
551 return vec_floor(a);
552}
553
554template <>
555EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) {
556 return pload<Packet4i>(from);
557}
558template <>
559EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
560 return pload<Packet2d>(from);
561}
562
563template <>
564EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) {
565 Packet4i p = pload<Packet4i>(from);
566 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
567}
568
569template <>
570EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) {
571 Packet2d p = pload<Packet2d>(from);
572 return vec_perm(p, p, p16uc_PSET64_HI);
573}
574
575template <>
576EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) {
577 pstore<int>(to, from);
578}
579template <>
580EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
581 pstore<double>(to, from);
582}
583
584template <>
585EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
586 EIGEN_ZVECTOR_PREFETCH(addr);
587}
588template <>
589EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
590 EIGEN_ZVECTOR_PREFETCH(addr);
591}
592
593template <>
594EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) {
595 EIGEN_ALIGN16 int x[4];
596 pstore(x, a);
597 return x[0];
598}
599template <>
600EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
601 EIGEN_ALIGN16 double x[2];
602 pstore(x, a);
603 return x[0];
604}
605
606template <>
607EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
608 return reinterpret_cast<Packet4i>(
609 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
610}
611
612template <>
613EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
614 return reinterpret_cast<Packet2d>(
615 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64));
616}
617
618template <>
619EIGEN_STRONG_INLINE Packet4i pabs<Packet4i>(const Packet4i& a) {
620 return vec_abs(a);
621}
622template <>
623EIGEN_STRONG_INLINE Packet2d pabs<Packet2d>(const Packet2d& a) {
624 return vec_abs(a);
625}
626
627template <>
628EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) {
629 Packet4i b, sum;
630 b = vec_sld(a, a, 8);
631 sum = padd<Packet4i>(a, b);
632 b = vec_sld(sum, sum, 4);
633 sum = padd<Packet4i>(sum, b);
634 return pfirst(sum);
635}
636
637template <>
638EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
639 Packet2d b, sum;
640 b = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8));
641 sum = padd<Packet2d>(a, b);
642 return pfirst(sum);
643}
644
645// Other reduction functions:
646// mul
647template <>
648EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) {
649 EIGEN_ALIGN16 int aux[4];
650 pstore(aux, a);
651 return aux[0] * aux[1] * aux[2] * aux[3];
652}
653
654template <>
655EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
656 return pfirst(
657 pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
658}
659
660// min
661template <>
662EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) {
663 Packet4i b, res;
664 b = pmin<Packet4i>(a, vec_sld(a, a, 8));
665 res = pmin<Packet4i>(b, vec_sld(b, b, 4));
666 return pfirst(res);
667}
668
669template <>
670EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
671 return pfirst(pmin<Packet2d>(
672 a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
673}
674
675// max
676template <>
677EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) {
678 Packet4i b, res;
679 b = pmax<Packet4i>(a, vec_sld(a, a, 8));
680 res = pmax<Packet4i>(b, vec_sld(b, b, 4));
681 return pfirst(res);
682}
683
684// max
685template <>
686EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
687 return pfirst(pmax<Packet2d>(
688 a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
689}
690
691EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
692 Packet4i t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
693 Packet4i t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
694 Packet4i t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
695 Packet4i t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
696 kernel.packet[0] = vec_mergeh(t0, t2);
697 kernel.packet[1] = vec_mergel(t0, t2);
698 kernel.packet[2] = vec_mergeh(t1, t3);
699 kernel.packet[3] = vec_mergel(t1, t3);
700}
701
702EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
703 Packet2d t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
704 Packet2d t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
705 kernel.packet[0] = t0;
706 kernel.packet[1] = t1;
707}
708
709template <>
710EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket,
711 const Packet4i& elsePacket) {
712 Packet4ui select = {ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3]};
713 Packet4ui mask = vec_cmpeq(select, reinterpret_cast<Packet4ui>(p4i_ONE));
714 return vec_sel(elsePacket, thenPacket, mask);
715}
716
717template <>
718EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket,
719 const Packet2d& elsePacket) {
720 Packet2ul select = {ifPacket.select[0], ifPacket.select[1]};
721 Packet2ul mask = vec_cmpeq(select, reinterpret_cast<Packet2ul>(p2l_ONE));
722 return vec_sel(elsePacket, thenPacket, mask);
723}
724
725/* z13 has no vector float support so we emulate that with double
726 z14 has proper vector float support.
727*/
728#if !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 12)
729/* Helper function to simulate a vec_splat_packet4f
730 */
731template <int element>
732EIGEN_STRONG_INLINE Packet4f vec_splat_packet4f(const Packet4f& from) {
733 Packet4f splat;
734 switch (element) {
735 case 0:
736 splat.v4f[0] = vec_splat(from.v4f[0], 0);
737 splat.v4f[1] = splat.v4f[0];
738 break;
739 case 1:
740 splat.v4f[0] = vec_splat(from.v4f[0], 1);
741 splat.v4f[1] = splat.v4f[0];
742 break;
743 case 2:
744 splat.v4f[0] = vec_splat(from.v4f[1], 0);
745 splat.v4f[1] = splat.v4f[0];
746 break;
747 case 3:
748 splat.v4f[0] = vec_splat(from.v4f[1], 1);
749 splat.v4f[1] = splat.v4f[0];
750 break;
751 }
752 return splat;
753}
754
755template <>
756EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
757 // FIXME: No intrinsic yet
758 EIGEN_DEBUG_ALIGNED_LOAD
759 Packet4f vfrom;
760 vfrom.v4f[0] = vec_ld2f(&from[0]);
761 vfrom.v4f[1] = vec_ld2f(&from[2]);
762 return vfrom;
763}
764
765template <>
766EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
767 // FIXME: No intrinsic yet
768 EIGEN_DEBUG_ALIGNED_STORE
769 vec_st2f(from.v4f[0], &to[0]);
770 vec_st2f(from.v4f[1], &to[2]);
771}
772
773template <>
774EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
775 Packet4f to;
776 to.v4f[0] = pset1<Packet2d>(static_cast<const double&>(from));
777 to.v4f[1] = to.v4f[0];
778 return to;
779}
780
781template <>
782EIGEN_STRONG_INLINE void pbroadcast4<Packet4f>(const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
783 a3 = pload<Packet4f>(a);
784 a0 = vec_splat_packet4f<0>(a3);
785 a1 = vec_splat_packet4f<1>(a3);
786 a2 = vec_splat_packet4f<2>(a3);
787 a3 = vec_splat_packet4f<3>(a3);
788}
789
790template <>
791EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
792 EIGEN_ALIGN16 float ai[4];
793 ai[0] = from[0 * stride];
794 ai[1] = from[1 * stride];
795 ai[2] = from[2 * stride];
796 ai[3] = from[3 * stride];
797 return pload<Packet4f>(ai);
798}
799
800template <>
801EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
802 EIGEN_ALIGN16 float ai[4];
803 pstore<float>((float*)ai, from);
804 to[0 * stride] = ai[0];
805 to[1 * stride] = ai[1];
806 to[2 * stride] = ai[2];
807 to[3 * stride] = ai[3];
808}
809
810template <>
811EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
812 Packet4f c;
813 c.v4f[0] = a.v4f[0] + b.v4f[0];
814 c.v4f[1] = a.v4f[1] + b.v4f[1];
815 return c;
816}
817
818template <>
819EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
820 Packet4f c;
821 c.v4f[0] = a.v4f[0] - b.v4f[0];
822 c.v4f[1] = a.v4f[1] - b.v4f[1];
823 return c;
824}
825
826template <>
827EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
828 Packet4f c;
829 c.v4f[0] = a.v4f[0] * b.v4f[0];
830 c.v4f[1] = a.v4f[1] * b.v4f[1];
831 return c;
832}
833
834template <>
835EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
836 Packet4f c;
837 c.v4f[0] = a.v4f[0] / b.v4f[0];
838 c.v4f[1] = a.v4f[1] / b.v4f[1];
839 return c;
840}
841
842template <>
843EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
844 Packet4f c;
845 c.v4f[0] = -a.v4f[0];
846 c.v4f[1] = -a.v4f[1];
847 return c;
848}
849
850template <>
851EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
852 Packet4f res;
853 res.v4f[0] = vec_madd(a.v4f[0], b.v4f[0], c.v4f[0]);
854 res.v4f[1] = vec_madd(a.v4f[1], b.v4f[1], c.v4f[1]);
855 return res;
856}
857
858template <>
859EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
860 Packet4f res;
861 res.v4f[0] = pmin(a.v4f[0], b.v4f[0]);
862 res.v4f[1] = pmin(a.v4f[1], b.v4f[1]);
863 return res;
864}
865
866template <>
867EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
868 Packet4f res;
869 res.v4f[0] = pmax(a.v4f[0], b.v4f[0]);
870 res.v4f[1] = pmax(a.v4f[1], b.v4f[1]);
871 return res;
872}
873
874template <>
875EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
876 Packet4f res;
877 res.v4f[0] = pand(a.v4f[0], b.v4f[0]);
878 res.v4f[1] = pand(a.v4f[1], b.v4f[1]);
879 return res;
880}
881
882template <>
883EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
884 Packet4f res;
885 res.v4f[0] = por(a.v4f[0], b.v4f[0]);
886 res.v4f[1] = por(a.v4f[1], b.v4f[1]);
887 return res;
888}
889
890template <>
891EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
892 Packet4f res;
893 res.v4f[0] = pxor(a.v4f[0], b.v4f[0]);
894 res.v4f[1] = pxor(a.v4f[1], b.v4f[1]);
895 return res;
896}
897
898template <>
899EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
900 Packet4f res;
901 res.v4f[0] = pandnot(a.v4f[0], b.v4f[0]);
902 res.v4f[1] = pandnot(a.v4f[1], b.v4f[1]);
903 return res;
904}
905
906template <>
907EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
908 Packet4f res;
909 res.v4f[0] = vec_round(a.v4f[0]);
910 res.v4f[1] = vec_round(a.v4f[1]);
911 return res;
912}
913
914template <>
915EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
916 Packet4f res;
917 res.v4f[0] = vec_ceil(a.v4f[0]);
918 res.v4f[1] = vec_ceil(a.v4f[1]);
919 return res;
920}
921
922template <>
923EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
924 Packet4f res;
925 res.v4f[0] = vec_floor(a.v4f[0]);
926 res.v4f[1] = vec_floor(a.v4f[1]);
927 return res;
928}
929
930template <>
931EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
932 Packet4f p = pload<Packet4f>(from);
933 p.v4f[1] = vec_splat(p.v4f[0], 1);
934 p.v4f[0] = vec_splat(p.v4f[0], 0);
935 return p;
936}
937
938template <>
939EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
940 EIGEN_ALIGN16 float x[2];
941 vec_st2f(a.v4f[0], &x[0]);
942 return x[0];
943}
944
945template <>
946EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
947 Packet4f rev;
948 rev.v4f[0] = preverse<Packet2d>(a.v4f[1]);
949 rev.v4f[1] = preverse<Packet2d>(a.v4f[0]);
950 return rev;
951}
952
953template <>
954EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(const Packet4f& a) {
955 Packet4f res;
956 res.v4f[0] = pabs(a.v4f[0]);
957 res.v4f[1] = pabs(a.v4f[1]);
958 return res;
959}
960
961template <>
962EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
963 Packet2d sum;
964 sum = padd<Packet2d>(a.v4f[0], a.v4f[1]);
965 double first = predux<Packet2d>(sum);
966 return static_cast<float>(first);
967}
968
969template <>
970EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
971 // Return predux_mul<Packet2d> of the subvectors product
972 return static_cast<float>(pfirst(predux_mul(pmul(a.v4f[0], a.v4f[1]))));
973}
974
975template <>
976EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
977 Packet2d b, res;
978 b = pmin<Packet2d>(a.v4f[0], a.v4f[1]);
979 res = pmin<Packet2d>(
980 b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8)));
981 return static_cast<float>(pfirst(res));
982}
983
984template <>
985EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
986 Packet2d b, res;
987 b = pmax<Packet2d>(a.v4f[0], a.v4f[1]);
988 res = pmax<Packet2d>(
989 b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8)));
990 return static_cast<float>(pfirst(res));
991}
992
993/* Split the Packet4f PacketBlock into 4 Packet2d PacketBlocks and transpose each one
994 */
995EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
996 PacketBlock<Packet2d, 2> t0, t1, t2, t3;
997 // copy top-left 2x2 Packet2d block
998 t0.packet[0] = kernel.packet[0].v4f[0];
999 t0.packet[1] = kernel.packet[1].v4f[0];
1000
1001 // copy top-right 2x2 Packet2d block
1002 t1.packet[0] = kernel.packet[0].v4f[1];
1003 t1.packet[1] = kernel.packet[1].v4f[1];
1004
1005 // copy bottom-left 2x2 Packet2d block
1006 t2.packet[0] = kernel.packet[2].v4f[0];
1007 t2.packet[1] = kernel.packet[3].v4f[0];
1008
1009 // copy bottom-right 2x2 Packet2d block
1010 t3.packet[0] = kernel.packet[2].v4f[1];
1011 t3.packet[1] = kernel.packet[3].v4f[1];
1012
1013 // Transpose all 2x2 blocks
1014 ptranspose(t0);
1015 ptranspose(t1);
1016 ptranspose(t2);
1017 ptranspose(t3);
1018
1019 // Copy back transposed blocks, but exchange t1 and t2 due to transposition
1020 kernel.packet[0].v4f[0] = t0.packet[0];
1021 kernel.packet[0].v4f[1] = t2.packet[0];
1022 kernel.packet[1].v4f[0] = t0.packet[1];
1023 kernel.packet[1].v4f[1] = t2.packet[1];
1024 kernel.packet[2].v4f[0] = t1.packet[0];
1025 kernel.packet[2].v4f[1] = t3.packet[0];
1026 kernel.packet[3].v4f[0] = t1.packet[1];
1027 kernel.packet[3].v4f[1] = t3.packet[1];
1028}
1029
1030template <>
1031EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket,
1032 const Packet4f& elsePacket) {
1033 Packet2ul select_hi = {ifPacket.select[0], ifPacket.select[1]};
1034 Packet2ul select_lo = {ifPacket.select[2], ifPacket.select[3]};
1035 Packet2ul mask_hi = vec_cmpeq(select_hi, reinterpret_cast<Packet2ul>(p2l_ONE));
1036 Packet2ul mask_lo = vec_cmpeq(select_lo, reinterpret_cast<Packet2ul>(p2l_ONE));
1037 Packet4f result;
1038 result.v4f[0] = vec_sel(elsePacket.v4f[0], thenPacket.v4f[0], mask_hi);
1039 result.v4f[1] = vec_sel(elsePacket.v4f[1], thenPacket.v4f[1], mask_lo);
1040 return result;
1041}
1042
1043template <>
1044Packet4f EIGEN_STRONG_INLINE pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b) {
1045 Packet4f res;
1046 res.v4f[0] = pcmp_le(a.v4f[0], b.v4f[0]);
1047 res.v4f[1] = pcmp_le(a.v4f[1], b.v4f[1]);
1048 return res;
1049}
1050
1051template <>
1052Packet4f EIGEN_STRONG_INLINE pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b) {
1053 Packet4f res;
1054 res.v4f[0] = pcmp_lt(a.v4f[0], b.v4f[0]);
1055 res.v4f[1] = pcmp_lt(a.v4f[1], b.v4f[1]);
1056 return res;
1057}
1058
1059template <>
1060Packet4f EIGEN_STRONG_INLINE pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b) {
1061 Packet4f res;
1062 res.v4f[0] = pcmp_eq(a.v4f[0], b.v4f[0]);
1063 res.v4f[1] = pcmp_eq(a.v4f[1], b.v4f[1]);
1064 return res;
1065}
1066
1067#else
1068template <>
1069EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
1070 // FIXME: No intrinsic yet
1071 EIGEN_DEBUG_ALIGNED_LOAD
1072 Packet* vfrom;
1073 vfrom = (Packet*)from;
1074 return vfrom->v4f;
1075}
1076
1077template <>
1078EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
1079 // FIXME: No intrinsic yet
1080 EIGEN_DEBUG_ALIGNED_STORE
1081 Packet* vto;
1082 vto = (Packet*)to;
1083 vto->v4f = from;
1084}
1085
1086template <>
1087EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
1088 return vec_splats(from);
1089}
1090
1091template <>
1092EIGEN_STRONG_INLINE void pbroadcast4<Packet4f>(const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
1093 a3 = pload<Packet4f>(a);
1094 a0 = vec_splat(a3, 0);
1095 a1 = vec_splat(a3, 1);
1096 a2 = vec_splat(a3, 2);
1097 a3 = vec_splat(a3, 3);
1098}
1099
1100template <>
1101EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
1102 EIGEN_ALIGN16 float af[4];
1103 af[0] = from[0 * stride];
1104 af[1] = from[1 * stride];
1105 af[2] = from[2 * stride];
1106 af[3] = from[3 * stride];
1107 return pload<Packet4f>(af);
1108}
1109
1110template <>
1111EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
1112 EIGEN_ALIGN16 float af[4];
1113 pstore<float>((float*)af, from);
1114 to[0 * stride] = af[0];
1115 to[1 * stride] = af[1];
1116 to[2 * stride] = af[2];
1117 to[3 * stride] = af[3];
1118}
1119
1120template <>
1121EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
1122 return (a + b);
1123}
1124template <>
1125EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
1126 return (a - b);
1127}
1128template <>
1129EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
1130 return (a * b);
1131}
1132template <>
1133EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
1134 return (a / b);
1135}
1136template <>
1137EIGEN_STRONG_INLINE Packet4f pnegate<Packet4f>(const Packet4f& a) {
1138 return (-a);
1139}
1140template <>
1141EIGEN_STRONG_INLINE Packet4f pconj<Packet4f>(const Packet4f& a) {
1142 return a;
1143}
1144template <>
1145EIGEN_STRONG_INLINE Packet4f pmadd<Packet4f>(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1146 return vec_madd(a, b, c);
1147}
1148template <>
1149EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
1150 return vec_min(a, b);
1151}
1152template <>
1153EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
1154 return vec_max(a, b);
1155}
1156template <>
1157EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
1158 return vec_and(a, b);
1159}
1160template <>
1161EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
1162 return vec_or(a, b);
1163}
1164template <>
1165EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
1166 return vec_xor(a, b);
1167}
1168template <>
1169EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
1170 return vec_and(a, vec_nor(b, b));
1171}
1172template <>
1173EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
1174 return vec_round(a);
1175}
1176template <>
1177EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
1178 return vec_ceil(a);
1179}
1180template <>
1181EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
1182 return vec_floor(a);
1183}
1184template <>
1185EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(const Packet4f& a) {
1186 return vec_abs(a);
1187}
1188template <>
1189EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
1190 EIGEN_ALIGN16 float x[4];
1191 pstore(x, a);
1192 return x[0];
1193}
1194
1195template <>
1196EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
1197 Packet4f p = pload<Packet4f>(from);
1198 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
1199}
1200
1201template <>
1202EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
1203 return reinterpret_cast<Packet4f>(
1204 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
1205}
1206
1207template <>
1208EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
1209 Packet4f b, sum;
1210 b = vec_sld(a, a, 8);
1211 sum = padd<Packet4f>(a, b);
1212 b = vec_sld(sum, sum, 4);
1213 sum = padd<Packet4f>(sum, b);
1214 return pfirst(sum);
1215}
1216
1217// Other reduction functions:
1218// mul
1219template <>
1220EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
1221 Packet4f prod;
1222 prod = pmul(a, vec_sld(a, a, 8));
1223 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
1224}
1225
1226// min
1227template <>
1228EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
1229 Packet4f b, res;
1230 b = pmin<Packet4f>(a, vec_sld(a, a, 8));
1231 res = pmin<Packet4f>(b, vec_sld(b, b, 4));
1232 return pfirst(res);
1233}
1234
1235// max
1236template <>
1237EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
1238 Packet4f b, res;
1239 b = pmax<Packet4f>(a, vec_sld(a, a, 8));
1240 res = pmax<Packet4f>(b, vec_sld(b, b, 4));
1241 return pfirst(res);
1242}
1243
1244EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
1245 Packet4f t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
1246 Packet4f t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
1247 Packet4f t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
1248 Packet4f t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
1249 kernel.packet[0] = vec_mergeh(t0, t2);
1250 kernel.packet[1] = vec_mergel(t0, t2);
1251 kernel.packet[2] = vec_mergeh(t1, t3);
1252 kernel.packet[3] = vec_mergel(t1, t3);
1253}
1254
1255template <>
1256EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket,
1257 const Packet4f& elsePacket) {
1258 Packet4ui select = {ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3]};
1259 Packet4ui mask = vec_cmpeq(select, reinterpret_cast<Packet4ui>(p4i_ONE));
1260 return vec_sel(elsePacket, thenPacket, mask);
1261}
1262
1263#endif
1264
1265template <>
1266EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1267 EIGEN_ZVECTOR_PREFETCH(addr);
1268}
1269template <>
1270EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
1271 return pload<Packet4f>(from);
1272}
1273template <>
1274EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
1275 pstore<float>(to, from);
1276}
1277template <>
1278EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
1279 return padd<Packet4f>(pset1<Packet4f>(a), p4f_COUNTDOWN);
1280}
1281
1282} // end namespace internal
1283
1284} // end namespace Eigen
1285
1286#endif // EIGEN_PACKET_MATH_ZVECTOR_H
Eigen::Aligned16
@ Aligned16
Definition Constants.h:237
Eigen
Namespace containing all symbols from the Eigen library.
Definition Core:137