Eigen  3.4.90 (git rev 5a9f66fb35d03a4da9ef8976e67a61b30aa16dcf)
 
Loading...
Searching...
No Matches
NEON/TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Rasmus Munk Larsen <[email protected]>
5// Copyright (C) 2020 Antonio Sanchez <[email protected]>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_TYPE_CASTING_NEON_H
12#define EIGEN_TYPE_CASTING_NEON_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21//==============================================================================
22// preinterpret (truncation operations)
23//==============================================================================
24
25template <>
26EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet16c>(const Packet16c& a) {
27 return Packet8c(vget_low_s8(a));
28}
29template <>
30EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet8c>(const Packet8c& a) {
31 return Packet4c(vget_lane_s32(vreinterpret_s32_s8(a), 0));
32}
33template <>
34EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet16c>(const Packet16c& a) {
35 return preinterpret<Packet4c>(preinterpret<Packet8c>(a));
36}
37
38template <>
39EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet16uc>(const Packet16uc& a) {
40 return Packet8uc(vget_low_u8(a));
41}
42template <>
43EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet8uc>(const Packet8uc& a) {
44 return Packet4uc(vget_lane_u32(vreinterpret_u32_u8(a), 0));
45}
46template <>
47EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet16uc>(const Packet16uc& a) {
48 return preinterpret<Packet4uc>(preinterpret<Packet8uc>(a));
49}
50
51template <>
52EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet8s>(const Packet8s& a) {
53 return Packet4s(vget_low_s16(a));
54}
55
56template <>
57EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet8us>(const Packet8us& a) {
58 return Packet4us(vget_low_u16(a));
59}
60
61template <>
62EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet4i>(const Packet4i& a) {
63 return Packet2i(vget_low_s32(a));
64}
65template <>
66EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet4ui>(const Packet4ui& a) {
67 return Packet2ui(vget_low_u32(a));
68}
69
70template <>
71EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet4f>(const Packet4f& a) {
72 return Packet2f(vget_low_f32(a));
73}
74
75//==============================================================================
76// preinterpret
77//==============================================================================
78template <>
79EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
80 return Packet2f(vreinterpret_f32_s32(a));
81}
82template <>
83EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
84 return Packet2f(vreinterpret_f32_u32(a));
85}
86template <>
87EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
88 return Packet4f(vreinterpretq_f32_s32(a));
89}
90template <>
91EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
92 return Packet4f(vreinterpretq_f32_u32(a));
93}
94
95template <>
96EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
97 return static_cast<Packet4c>(a);
98}
99template <>
100EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
101 return Packet8c(vreinterpret_s8_u8(a));
102}
103template <>
104EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
105 return Packet16c(vreinterpretq_s8_u8(a));
106}
107
108template <>
109EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
110 return static_cast<Packet4uc>(a);
111}
112template <>
113EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
114 return Packet8uc(vreinterpret_u8_s8(a));
115}
116template <>
117EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
118 return Packet16uc(vreinterpretq_u8_s8(a));
119}
120
121template <>
122EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
123 return Packet4s(vreinterpret_s16_u16(a));
124}
125template <>
126EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
127 return Packet8s(vreinterpretq_s16_u16(a));
128}
129template <>
130EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
131 return Packet4us(vreinterpret_u16_s16(a));
132}
133template <>
134EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
135 return Packet8us(vreinterpretq_u16_s16(a));
136}
137
138template <>
139EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
140 return Packet2i(vreinterpret_s32_f32(a));
141}
142template <>
143EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
144 return Packet2i(vreinterpret_s32_u32(a));
145}
146template <>
147EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
148 return Packet4i(vreinterpretq_s32_f32(a));
149}
150template <>
151EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
152 return Packet4i(vreinterpretq_s32_u32(a));
153}
154
155template <>
156EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
157 return Packet2ui(vreinterpret_u32_f32(a));
158}
159template <>
160EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
161 return Packet2ui(vreinterpret_u32_s32(a));
162}
163template <>
164EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
165 return Packet4ui(vreinterpretq_u32_f32(a));
166}
167template <>
168EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
169 return Packet4ui(vreinterpretq_u32_s32(a));
170}
171
172template <>
173EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
174 return Packet2l(vreinterpretq_s64_u64(a));
175}
176template <>
177EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
178 return Packet2ul(vreinterpretq_u64_s64(a));
179}
180
181//==============================================================================
182// pcast, SrcType = float
183//==============================================================================
184
185template <>
186struct type_casting_traits<float, numext::int64_t> {
187 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
188};
189template <>
190struct type_casting_traits<float, numext::uint64_t> {
191 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
192};
193// If float64 exists, first convert to that to keep as much precision as possible.
194#if EIGEN_ARCH_ARM64
195template <>
196EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
197 // Discard second half of input.
198 return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
199}
200template <>
201EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
202 return vcvtq_s64_f64(vcvt_f64_f32(a));
203}
204template <>
205EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
206 // Discard second half of input.
207 return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
208}
209template <>
210EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
211 return vcvtq_u64_f64(vcvt_f64_f32(a));
212}
213#else
214template <>
215EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
216 // Discard second half of input.
217 return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a)));
218}
219template <>
220EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
221 return vmovl_s32(vcvt_s32_f32(a));
222}
223template <>
224EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
225 // Discard second half of input.
226 return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a)));
227}
228template <>
229EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
230 // Discard second half of input.
231 return vmovl_u32(vcvt_u32_f32(a));
232}
233#endif // EIGEN_ARCH_ARM64
234
235template <>
236struct type_casting_traits<float, numext::int32_t> {
237 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
238};
239template <>
240EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
241 return vcvtq_s32_f32(a);
242}
243template <>
244EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
245 return vcvt_s32_f32(a);
246}
247
248template <>
249struct type_casting_traits<float, numext::uint32_t> {
250 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
251};
252template <>
253EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
254 return vcvtq_u32_f32(a);
255}
256template <>
257EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
258 return vcvt_u32_f32(a);
259}
260
261template <>
262struct type_casting_traits<float, numext::int16_t> {
263 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
264};
265template <>
266EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
267 return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
268}
269template <>
270EIGEN_STRONG_INLINE Packet4s pcast<Packet4f, Packet4s>(const Packet4f& a) {
271 return vmovn_s32(vcvtq_s32_f32(a));
272}
273template <>
274EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
275 return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
276}
277
278template <>
279struct type_casting_traits<float, numext::uint16_t> {
280 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
281};
282template <>
283EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
284 return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
285}
286template <>
287EIGEN_STRONG_INLINE Packet4us pcast<Packet4f, Packet4us>(const Packet4f& a) {
288 return vmovn_u32(vcvtq_u32_f32(a));
289}
290template <>
291EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
292 return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
293}
294
295template <>
296struct type_casting_traits<float, numext::int8_t> {
297 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
298};
299template <>
300EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
301 const Packet4f& d) {
302 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
303 const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
304 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
305}
306template <>
307EIGEN_STRONG_INLINE Packet8c pcast<Packet4f, Packet8c>(const Packet4f& a, const Packet4f& b) {
308 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
309 return vmovn_s16(ab_s16);
310}
311template <>
312EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
313 const Packet2f& d) {
314 const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
315 const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
316 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
317}
318template <>
319EIGEN_STRONG_INLINE Packet4c pcast<Packet4f, Packet4c>(const Packet4f& a) {
320 const int32x4_t a_s32x4 = vcvtq_s32_f32(a);
321 const int16x4_t a_s16x4 = vmovn_s32(a_s32x4);
322 const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
323 const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
324 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
325}
326
327template <>
328struct type_casting_traits<float, numext::uint8_t> {
329 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
330};
331template <>
332EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
333 const Packet4f& d) {
334 return preinterpret<Packet16uc>(pcast<Packet4f, Packet16c>(a, b, c, d));
335}
336template <>
337EIGEN_STRONG_INLINE Packet8uc pcast<Packet4f, Packet8uc>(const Packet4f& a, const Packet4f& b) {
338 return preinterpret<Packet8uc>(pcast<Packet4f, Packet8c>(a, b));
339}
340template <>
341EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
342 const Packet2f& d) {
343 return preinterpret<Packet8uc>(pcast<Packet2f, Packet8c>(a, b, c, d));
344}
345template <>
346EIGEN_STRONG_INLINE Packet4uc pcast<Packet4f, Packet4uc>(const Packet4f& a) {
347 return static_cast<Packet4uc>(pcast<Packet4f, Packet4c>(a));
348}
349
350//==============================================================================
351// pcast, SrcType = int8_t
352//==============================================================================
353template <>
354struct type_casting_traits<numext::int8_t, float> {
355 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
356};
357template <>
358EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
359 // Discard all but first 4 bytes.
360 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
361}
362template <>
363EIGEN_STRONG_INLINE Packet4f pcast<Packet4c, Packet4f>(const Packet4c& a) {
364 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(a))))));
365}
366template <>
367EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
368 // Discard all but first 2 bytes.
369 return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
370}
371
372template <>
373struct type_casting_traits<numext::int8_t, numext::int64_t> {
374 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
375};
376template <>
377EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
378 // Discard all but first two bytes.
379 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
380}
381
382template <>
383struct type_casting_traits<numext::int8_t, numext::uint64_t> {
384 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
385};
386template <>
387EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
388 return preinterpret<Packet2ul>(pcast<Packet16c, Packet2l>(a));
389}
390
391template <>
392struct type_casting_traits<numext::int8_t, numext::int32_t> {
393 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
394};
395template <>
396EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
397 // Discard all but first 4 bytes.
398 return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
399}
400template <>
401EIGEN_STRONG_INLINE Packet4i pcast<Packet8c, Packet4i>(const Packet8c& a) {
402 return vmovl_s16(vget_low_s16(vmovl_s8(a)));
403}
404template <>
405EIGEN_STRONG_INLINE Packet4i pcast<Packet4c, Packet4i>(const Packet4c& a) {
406 return pcast<Packet8c, Packet4i>(vreinterpret_s8_s32(vdup_n_s32(a)));
407}
408template <>
409EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
410 // Discard all but first 2 bytes.
411 return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
412}
413
414template <>
415struct type_casting_traits<numext::int8_t, numext::uint32_t> {
416 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
417};
418template <>
419EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
420 return preinterpret<Packet4ui>(pcast<Packet16c, Packet4i>(a));
421}
422template <>
423EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
424 return preinterpret<Packet2ui>(pcast<Packet8c, Packet2i>(a));
425}
426template <>
427EIGEN_STRONG_INLINE Packet4ui pcast<Packet4c, Packet4ui>(const Packet4c& a) {
428 return preinterpret<Packet4ui>(pcast<Packet4c, Packet4i>(a));
429}
430
431template <>
432struct type_casting_traits<numext::int8_t, numext::int16_t> {
433 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
434};
435template <>
436EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
437 // Discard second half of input.
438 return vmovl_s8(vget_low_s8(a));
439}
440template <>
441EIGEN_STRONG_INLINE Packet8s pcast<Packet8c, Packet8s>(const Packet8c& a) {
442 return vmovl_s8(a);
443}
444template <>
445EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
446 // Discard second half of input.
447 return vget_low_s16(vmovl_s8(a));
448}
449template <>
450EIGEN_STRONG_INLINE Packet4s pcast<Packet4c, Packet4s>(const Packet4c& a) {
451 return pcast<Packet8c, Packet4s>(vreinterpret_s8_s32(vdup_n_s32(a)));
452}
453
454template <>
455struct type_casting_traits<numext::int8_t, numext::uint16_t> {
456 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
457};
458template <>
459EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
460 return preinterpret<Packet8us>(pcast<Packet16c, Packet8s>(a));
461}
462template <>
463EIGEN_STRONG_INLINE Packet8us pcast<Packet8c, Packet8us>(const Packet8c& a) {
464 return preinterpret<Packet8us>(pcast<Packet8c, Packet8s>(a));
465}
466template <>
467EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
468 return preinterpret<Packet4us>(pcast<Packet8c, Packet4s>(a));
469}
470template <>
471EIGEN_STRONG_INLINE Packet4us pcast<Packet4c, Packet4us>(const Packet4c& a) {
472 return preinterpret<Packet4us>(pcast<Packet4c, Packet4s>(a));
473}
474
475//==============================================================================
476// pcast, SrcType = uint8_t
477//==============================================================================
478template <>
479struct type_casting_traits<numext::uint8_t, float> {
480 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
481};
482template <>
483EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
484 // Discard all but first 4 bytes.
485 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
486}
487template <>
488EIGEN_STRONG_INLINE Packet4f pcast<Packet4uc, Packet4f>(const Packet4uc& a) {
489 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))))));
490}
491template <>
492EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
493 // Discard all but first 2 bytes.
494 return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
495}
496
497template <>
498struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
499 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
500};
501template <>
502EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
503 // Discard all but first two bytes.
504 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
505}
506
507template <>
508struct type_casting_traits<numext::uint8_t, numext::int64_t> {
509 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
510};
511template <>
512EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
513 return preinterpret<Packet2l>(pcast<Packet16uc, Packet2ul>(a));
514}
515
516template <>
517struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
518 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
519};
520template <>
521EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
522 // Discard all but first 4 bytes.
523 return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
524}
525template <>
526EIGEN_STRONG_INLINE Packet4ui pcast<Packet8uc, Packet4ui>(const Packet8uc& a) {
527 return vmovl_u16(vget_low_u16(vmovl_u8(a)));
528}
529template <>
530EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
531 // Discard all but first 2 bytes.
532 return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
533}
534template <>
535EIGEN_STRONG_INLINE Packet4ui pcast<Packet4uc, Packet4ui>(const Packet4uc& a) {
536 return pcast<Packet8uc, Packet4ui>(vreinterpret_u8_u32(vdup_n_u32(a)));
537}
538
539template <>
540struct type_casting_traits<numext::uint8_t, numext::int32_t> {
541 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
542};
543template <>
544EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
545 return preinterpret<Packet4i>(pcast<Packet16uc, Packet4ui>(a));
546}
547template <>
548EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
549 return preinterpret<Packet2i>(pcast<Packet8uc, Packet2ui>(a));
550}
551template <>
552EIGEN_STRONG_INLINE Packet4i pcast<Packet4uc, Packet4i>(const Packet4uc& a) {
553 return preinterpret<Packet4i>(pcast<Packet4uc, Packet4ui>(a));
554}
555
556template <>
557struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
558 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
559};
560template <>
561EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
562 // Discard second half of input.
563 return vmovl_u8(vget_low_u8(a));
564}
565template <>
566EIGEN_STRONG_INLINE Packet8us pcast<Packet8uc, Packet8us>(const Packet8uc& a) {
567 return vmovl_u8(a);
568}
569template <>
570EIGEN_STRONG_INLINE Packet4us pcast<Packet4uc, Packet4us>(const Packet4uc& a) {
571 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))));
572}
573
574template <>
575struct type_casting_traits<numext::uint8_t, numext::int16_t> {
576 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
577};
578template <>
579EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
580 return preinterpret<Packet8s>(pcast<Packet16uc, Packet8us>(a));
581}
582template <>
583EIGEN_STRONG_INLINE Packet8s pcast<Packet8uc, Packet8s>(const Packet8uc& a) {
584 return preinterpret<Packet8s>(pcast<Packet8uc, Packet8us>(a));
585}
586template <>
587EIGEN_STRONG_INLINE Packet4s pcast<Packet4uc, Packet4s>(const Packet4uc& a) {
588 return preinterpret<Packet4s>(pcast<Packet4uc, Packet4us>(a));
589}
590
591//==============================================================================
592// pcast, SrcType = int16_t
593//==============================================================================
594template <>
595struct type_casting_traits<numext::int16_t, float> {
596 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
597};
598template <>
599EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
600 // Discard second half of input.
601 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
602}
603template <>
604EIGEN_STRONG_INLINE Packet4f pcast<Packet4s, Packet4f>(const Packet4s& a) {
605 return vcvtq_f32_s32(vmovl_s16(a));
606}
607template <>
608EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
609 // Discard second half of input.
610 return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
611}
612
613template <>
614struct type_casting_traits<numext::int16_t, numext::int64_t> {
615 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
616};
617template <>
618EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
619 // Discard all but first two values.
620 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
621}
622
623template <>
624struct type_casting_traits<numext::int16_t, numext::uint64_t> {
625 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
626};
627template <>
628EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
629 return preinterpret<Packet2ul>(pcast<Packet8s, Packet2l>(a));
630}
631
632template <>
633struct type_casting_traits<numext::int16_t, numext::int32_t> {
634 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
635};
636template <>
637EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
638 // Discard second half of input.
639 return vmovl_s16(vget_low_s16(a));
640}
641template <>
642EIGEN_STRONG_INLINE Packet4i pcast<Packet4s, Packet4i>(const Packet4s& a) {
643 return vmovl_s16(a);
644}
645template <>
646EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
647 // Discard second half of input.
648 return vget_low_s32(vmovl_s16(a));
649}
650
651template <>
652struct type_casting_traits<numext::int16_t, numext::uint32_t> {
653 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
654};
655template <>
656EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
657 return preinterpret<Packet4ui>(pcast<Packet8s, Packet4i>(a));
658}
659template <>
660EIGEN_STRONG_INLINE Packet4ui pcast<Packet4s, Packet4ui>(const Packet4s& a) {
661 return preinterpret<Packet4ui>(pcast<Packet4s, Packet4i>(a));
662}
663template <>
664EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
665 return preinterpret<Packet2ui>(pcast<Packet4s, Packet2i>(a));
666}
667
668template <>
669struct type_casting_traits<numext::int16_t, numext::int8_t> {
670 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
671};
672template <>
673EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
674 return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
675}
676template <>
677EIGEN_STRONG_INLINE Packet8c pcast<Packet8s, Packet8c>(const Packet8s& a) {
678 return vmovn_s16(a);
679}
680template <>
681EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
682 return vmovn_s16(vcombine_s16(a, b));
683}
684template <>
685EIGEN_STRONG_INLINE Packet4c pcast<Packet4s, Packet4c>(const Packet4s& a) {
686 const int8x8_t aa_s8x8 = pcast<Packet4s, Packet8c>(a, a);
687 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
688}
689
690template <>
691struct type_casting_traits<numext::int16_t, numext::uint8_t> {
692 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
693};
694template <>
695EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
696 return preinterpret<Packet16uc>(pcast<Packet8s, Packet16c>(a, b));
697}
698template <>
699EIGEN_STRONG_INLINE Packet8uc pcast<Packet8s, Packet8uc>(const Packet8s& a) {
700 return preinterpret<Packet8uc>(pcast<Packet8s, Packet8c>(a));
701}
702template <>
703EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
704 return preinterpret<Packet8uc>(pcast<Packet4s, Packet8c>(a, b));
705}
706template <>
707EIGEN_STRONG_INLINE Packet4uc pcast<Packet4s, Packet4uc>(const Packet4s& a) {
708 return static_cast<Packet4uc>(pcast<Packet4s, Packet4c>(a));
709}
710
711//==============================================================================
712// pcast, SrcType = uint16_t
713//==============================================================================
714template <>
715struct type_casting_traits<numext::uint16_t, float> {
716 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
717};
718template <>
719EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
720 // Discard second half of input.
721 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
722}
723template <>
724EIGEN_STRONG_INLINE Packet4f pcast<Packet4us, Packet4f>(const Packet4us& a) {
725 return vcvtq_f32_u32(vmovl_u16(a));
726}
727template <>
728EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
729 // Discard second half of input.
730 return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
731}
732
733template <>
734struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
735 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
736};
737template <>
738EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
739 // Discard all but first two values.
740 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
741}
742
743template <>
744struct type_casting_traits<numext::uint16_t, numext::int64_t> {
745 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
746};
747template <>
748EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
749 return preinterpret<Packet2l>(pcast<Packet8us, Packet2ul>(a));
750}
751
752template <>
753struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
754 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
755};
756template <>
757EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
758 // Discard second half of input.
759 return vmovl_u16(vget_low_u16(a));
760}
761template <>
762EIGEN_STRONG_INLINE Packet4ui pcast<Packet4us, Packet4ui>(const Packet4us& a) {
763 return vmovl_u16(a);
764}
765template <>
766EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
767 // Discard second half of input.
768 return vget_low_u32(vmovl_u16(a));
769}
770
771template <>
772struct type_casting_traits<numext::uint16_t, numext::int32_t> {
773 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
774};
775template <>
776EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
777 return preinterpret<Packet4i>(pcast<Packet8us, Packet4ui>(a));
778}
779template <>
780EIGEN_STRONG_INLINE Packet4i pcast<Packet4us, Packet4i>(const Packet4us& a) {
781 return preinterpret<Packet4i>(pcast<Packet4us, Packet4ui>(a));
782}
783template <>
784EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
785 return preinterpret<Packet2i>(pcast<Packet4us, Packet2ui>(a));
786}
787
788template <>
789struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
790 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
791};
792template <>
793EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
794 return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
795}
796template <>
797EIGEN_STRONG_INLINE Packet8uc pcast<Packet8us, Packet8uc>(const Packet8us& a) {
798 return vmovn_u16(a);
799}
800template <>
801EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
802 return vmovn_u16(vcombine_u16(a, b));
803}
804template <>
805EIGEN_STRONG_INLINE Packet4uc pcast<Packet4us, Packet4uc>(const Packet4us& a) {
806 uint8x8_t aa_u8x8 = pcast<Packet4us, Packet8uc>(a, a);
807 return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
808}
809
810template <>
811struct type_casting_traits<numext::uint16_t, numext::int8_t> {
812 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
813};
814template <>
815EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
816 return preinterpret<Packet16c>(pcast<Packet8us, Packet16uc>(a, b));
817}
818template <>
819EIGEN_STRONG_INLINE Packet8c pcast<Packet8us, Packet8c>(const Packet8us& a) {
820 return preinterpret<Packet8c>(pcast<Packet8us, Packet8uc>(a));
821}
822template <>
823EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
824 return preinterpret<Packet8c>(pcast<Packet4us, Packet8uc>(a, b));
825}
826template <>
827EIGEN_STRONG_INLINE Packet4c pcast<Packet4us, Packet4c>(const Packet4us& a) {
828 return static_cast<Packet4c>(pcast<Packet4us, Packet4uc>(a));
829}
830
831//==============================================================================
832// pcast, SrcType = int32_t
833//==============================================================================
834template <>
835struct type_casting_traits<numext::int32_t, float> {
836 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
837};
838template <>
839EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
840 return vcvtq_f32_s32(a);
841}
842template <>
843EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
844 return vcvt_f32_s32(a);
845}
846
847template <>
848struct type_casting_traits<numext::int32_t, numext::int64_t> {
849 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
850};
851template <>
852EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
853 // Discard second half of input.
854 return vmovl_s32(vget_low_s32(a));
855}
856template <>
857EIGEN_STRONG_INLINE Packet2l pcast<Packet2i, Packet2l>(const Packet2i& a) {
858 return vmovl_s32(a);
859}
860
861template <>
862struct type_casting_traits<numext::int32_t, numext::uint64_t> {
863 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
864};
865template <>
866EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
867 return preinterpret<Packet2ul>(pcast<Packet4i, Packet2l>(a));
868}
869template <>
870EIGEN_STRONG_INLINE Packet2ul pcast<Packet2i, Packet2ul>(const Packet2i& a) {
871 return preinterpret<Packet2ul>(pcast<Packet2i, Packet2l>(a));
872}
873
874template <>
875struct type_casting_traits<numext::int32_t, numext::int16_t> {
876 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
877};
878template <>
879EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
880 return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
881}
882template <>
883EIGEN_STRONG_INLINE Packet4s pcast<Packet4i, Packet4s>(const Packet4i& a) {
884 return vmovn_s32(a);
885}
886template <>
887EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
888 return vmovn_s32(vcombine_s32(a, b));
889}
890
891template <>
892struct type_casting_traits<numext::int32_t, numext::uint16_t> {
893 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
894};
895template <>
896EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
897 return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
898}
899template <>
900EIGEN_STRONG_INLINE Packet4us pcast<Packet4i, Packet4us>(const Packet4i& a) {
901 return vmovn_u32(vreinterpretq_u32_s32(a));
902}
903template <>
904EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
905 return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
906}
907
908template <>
909struct type_casting_traits<numext::int32_t, numext::int8_t> {
910 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
911};
912template <>
913EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
914 const Packet4i& d) {
915 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
916 const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
917 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
918}
919template <>
920EIGEN_STRONG_INLINE Packet8c pcast<Packet4i, Packet8c>(const Packet4i& a, const Packet4i& b) {
921 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
922 return vmovn_s16(ab_s16);
923}
924template <>
925EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
926 const Packet2i& d) {
927 const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
928 const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
929 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
930}
931template <>
932EIGEN_STRONG_INLINE Packet4c pcast<Packet4i, Packet4c>(const Packet4i& a) {
933 const int16x4_t a_s16x4 = vmovn_s32(a);
934 const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
935 const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
936 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
937}
938
939template <>
940struct type_casting_traits<numext::int32_t, numext::uint8_t> {
941 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
942};
943template <>
944EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
945 const Packet4i& d) {
946 return preinterpret<Packet16uc>(pcast<Packet4i, Packet16c>(a, b, c, d));
947}
948template <>
949EIGEN_STRONG_INLINE Packet8uc pcast<Packet4i, Packet8uc>(const Packet4i& a, const Packet4i& b) {
950 return preinterpret<Packet8uc>(pcast<Packet4i, Packet8c>(a, b));
951}
952template <>
953EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
954 const Packet2i& d) {
955 return preinterpret<Packet8uc>(pcast<Packet2i, Packet8c>(a, b, c, d));
956}
957template <>
958EIGEN_STRONG_INLINE Packet4uc pcast<Packet4i, Packet4uc>(const Packet4i& a) {
959 return static_cast<Packet4uc>(pcast<Packet4i, Packet4c>(a));
960}
961
962//==============================================================================
963// pcast, SrcType = uint32_t
964//==============================================================================
965template <>
966struct type_casting_traits<numext::uint32_t, float> {
967 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
968};
969template <>
970EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
971 return vcvtq_f32_u32(a);
972}
973template <>
974EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
975 return vcvt_f32_u32(a);
976}
977
978template <>
979struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
980 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
981};
982template <>
983EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
984 // Discard second half of input.
985 return vmovl_u32(vget_low_u32(a));
986}
987template <>
988EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ui, Packet2ul>(const Packet2ui& a) {
989 return vmovl_u32(a);
990}
991
992template <>
993struct type_casting_traits<numext::uint32_t, numext::int64_t> {
994 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
995};
996template <>
997EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
998 return preinterpret<Packet2l>(pcast<Packet4ui, Packet2ul>(a));
999}
1000template <>
1001EIGEN_STRONG_INLINE Packet2l pcast<Packet2ui, Packet2l>(const Packet2ui& a) {
1002 return preinterpret<Packet2l>(pcast<Packet2ui, Packet2ul>(a));
1003}
1004
1005template <>
1006struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
1007 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1008};
1009template <>
1010EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
1011 return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1012}
1013template <>
1014EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
1015 return vmovn_u32(vcombine_u32(a, b));
1016}
1017template <>
1018EIGEN_STRONG_INLINE Packet4us pcast<Packet4ui, Packet4us>(const Packet4ui& a) {
1019 return vmovn_u32(a);
1020}
1021
1022template <>
1023struct type_casting_traits<numext::uint32_t, numext::int16_t> {
1024 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1025};
1026template <>
1027EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
1028 return preinterpret<Packet8s>(pcast<Packet4ui, Packet8us>(a, b));
1029}
1030template <>
1031EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
1032 return preinterpret<Packet4s>(pcast<Packet2ui, Packet4us>(a, b));
1033}
1034template <>
1035EIGEN_STRONG_INLINE Packet4s pcast<Packet4ui, Packet4s>(const Packet4ui& a) {
1036 return preinterpret<Packet4s>(pcast<Packet4ui, Packet4us>(a));
1037}
1038
1039template <>
1040struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
1041 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1042};
1043template <>
1044EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1045 const Packet4ui& d) {
1046 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1047 const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
1048 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
1049}
1050template <>
1051EIGEN_STRONG_INLINE Packet8uc pcast<Packet4ui, Packet8uc>(const Packet4ui& a, const Packet4ui& b) {
1052 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1053 return vmovn_u16(ab_u16);
1054}
1055template <>
1056EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1057 const Packet2ui& d) {
1058 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
1059 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
1060 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
1061}
1062template <>
1063EIGEN_STRONG_INLINE Packet4uc pcast<Packet4ui, Packet4uc>(const Packet4ui& a) {
1064 const uint16x4_t a_u16x4 = vmovn_u32(a);
1065 const uint16x8_t aa_u16x8 = vcombine_u16(a_u16x4, a_u16x4);
1066 const uint8x8_t aa_u8x8 = vmovn_u16(aa_u16x8);
1067 return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
1068}
1069
1070template <>
1071struct type_casting_traits<numext::uint32_t, numext::int8_t> {
1072 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1073};
1074template <>
1075EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1076 const Packet4ui& d) {
1077 return preinterpret<Packet16c>(pcast<Packet4ui, Packet16uc>(a, b, c, d));
1078}
1079template <>
1080EIGEN_STRONG_INLINE Packet8c pcast<Packet4ui, Packet8c>(const Packet4ui& a, const Packet4ui& b) {
1081 return preinterpret<Packet8c>(pcast<Packet4ui, Packet8uc>(a, b));
1082}
1083template <>
1084EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1085 const Packet2ui& d) {
1086 return preinterpret<Packet8c>(pcast<Packet2ui, Packet8uc>(a, b, c, d));
1087}
1088template <>
1089EIGEN_STRONG_INLINE Packet4c pcast<Packet4ui, Packet4c>(const Packet4ui& a) {
1090 return static_cast<Packet4c>(pcast<Packet4ui, Packet4uc>(a));
1091}
1092
1093//==============================================================================
1094// pcast, SrcType = int64_t
1095//==============================================================================
1096template <>
1097struct type_casting_traits<numext::int64_t, float> {
1098 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1099};
1100
1101template <>
1102EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
1103#if EIGEN_ARCH_ARM64
1104 return vcombine_f32(vcvt_f32_f64(vcvtq_f64_s64(a)), vcvt_f32_f64(vcvtq_f64_s64(b)));
1105#else
1106 EIGEN_ALIGN_MAX int64_t lvals[4];
1107 pstore(lvals, a);
1108 pstore(lvals + 2, b);
1109 EIGEN_ALIGN_MAX float fvals[4] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1]),
1110 static_cast<float>(lvals[2]), static_cast<float>(lvals[3])};
1111 return pload<Packet4f>(fvals);
1112#endif
1113}
1114
1115template <>
1116EIGEN_STRONG_INLINE Packet2f pcast<Packet2l, Packet2f>(const Packet2l& a) {
1117#if EIGEN_ARCH_ARM64
1118 return vcvt_f32_f64(vcvtq_f64_s64(a));
1119#else
1120 EIGEN_ALIGN_MAX int64_t lvals[2];
1121 pstore(lvals, a);
1122 EIGEN_ALIGN_MAX float fvals[2] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1])};
1123 return pload<Packet2f>(fvals);
1124#endif
1125}
1126
1127template <>
1128struct type_casting_traits<numext::int64_t, numext::int32_t> {
1129 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1130};
1131template <>
1132EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
1133 return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
1134}
1135template <>
1136EIGEN_STRONG_INLINE Packet2i pcast<Packet2l, Packet2i>(const Packet2l& a) {
1137 return vmovn_s64(a);
1138}
1139
1140template <>
1141struct type_casting_traits<numext::int64_t, numext::uint32_t> {
1142 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1143};
1144template <>
1145EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
1146 return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
1147}
1148template <>
1149EIGEN_STRONG_INLINE Packet2ui pcast<Packet2l, Packet2ui>(const Packet2l& a) {
1150 return vmovn_u64(vreinterpretq_u64_s64(a));
1151}
1152
1153template <>
1154struct type_casting_traits<numext::int64_t, numext::int16_t> {
1155 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1156};
1157template <>
1158EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1159 const Packet2l& d) {
1160 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1161 const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
1162 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1163}
1164template <>
1165EIGEN_STRONG_INLINE Packet4s pcast<Packet2l, Packet4s>(const Packet2l& a, const Packet2l& b) {
1166 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1167 return vmovn_s32(ab_s32);
1168}
1169
1170template <>
1171struct type_casting_traits<numext::int64_t, numext::uint16_t> {
1172 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1173};
1174template <>
1175EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1176 const Packet2l& d) {
1177 return preinterpret<Packet8us>(pcast<Packet2l, Packet8s>(a, b, c, d));
1178}
1179template <>
1180EIGEN_STRONG_INLINE Packet4us pcast<Packet2l, Packet4us>(const Packet2l& a, const Packet2l& b) {
1181 return preinterpret<Packet4us>(pcast<Packet2l, Packet4s>(a, b));
1182}
1183
1184template <>
1185struct type_casting_traits<numext::int64_t, numext::int8_t> {
1186 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1187};
1188template <>
1189EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1190 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1191 const Packet2l& g, const Packet2l& h) {
1192 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1193 const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
1194 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1195}
1196template <>
1197EIGEN_STRONG_INLINE Packet8c pcast<Packet2l, Packet8c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1198 const Packet2l& d) {
1199 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1200 return vmovn_s16(abcd_s16);
1201}
1202template <>
1203EIGEN_STRONG_INLINE Packet4c pcast<Packet2l, Packet4c>(const Packet2l& a, const Packet2l& b) {
1204 const int16x4_t ab_s16 = pcast<Packet2l, Packet4s>(a, b);
1205 const int16x8_t abab_s16 = vcombine_s16(ab_s16, ab_s16);
1206 const int8x8_t abab_s8 = vmovn_s16(abab_s16);
1207 return vget_lane_s32(vreinterpret_s32_s8(abab_s8), 0);
1208}
1209
1210template <>
1211struct type_casting_traits<numext::int64_t, numext::uint8_t> {
1212 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1213};
1214template <>
1215EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1216 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1217 const Packet2l& g, const Packet2l& h) {
1218 const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
1219 const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
1220 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1221}
1222template <>
1223EIGEN_STRONG_INLINE Packet8uc pcast<Packet2l, Packet8uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1224 const Packet2l& d) {
1225 return preinterpret<Packet8uc>(pcast<Packet2l, Packet8c>(a, b, c, d));
1226}
1227template <>
1228EIGEN_STRONG_INLINE Packet4uc pcast<Packet2l, Packet4uc>(const Packet2l& a, const Packet2l& b) {
1229 return static_cast<Packet4uc>(pcast<Packet2l, Packet4c>(a, b));
1230}
1231
1232//==============================================================================
1233// pcast, SrcType = uint64_t
1234//==============================================================================
1235template <>
1236struct type_casting_traits<numext::uint64_t, float> {
1237 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1238};
1239template <>
1240EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
1241#if EIGEN_ARCH_ARM64
1242 return vcombine_f32(vcvt_f32_f64(vcvtq_f64_u64(a)), vcvt_f32_f64(vcvtq_f64_u64(b)));
1243#else
1244 EIGEN_ALIGN_MAX uint64_t uvals[4];
1245 pstore(uvals, a);
1246 pstore(uvals + 2, b);
1247 EIGEN_ALIGN_MAX float fvals[4] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1]),
1248 static_cast<float>(uvals[2]), static_cast<float>(uvals[3])};
1249 return pload<Packet4f>(fvals);
1250#endif
1251}
1252template <>
1253EIGEN_STRONG_INLINE Packet2f pcast<Packet2ul, Packet2f>(const Packet2ul& a) {
1254#if EIGEN_ARCH_ARM64
1255 return vcvt_f32_f64(vcvtq_f64_u64(a));
1256#else
1257 EIGEN_ALIGN_MAX uint64_t uvals[2];
1258 pstore(uvals, a);
1259 EIGEN_ALIGN_MAX float fvals[2] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1])};
1260 return pload<Packet2f>(fvals);
1261#endif
1262}
1263
1264template <>
1265struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1266 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1267};
1268template <>
1269EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1270 return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1271}
1272template <>
1273EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ul, Packet2ui>(const Packet2ul& a) {
1274 return vmovn_u64(a);
1275}
1276
1277template <>
1278struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1279 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1280};
1281template <>
1282EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1283 return preinterpret<Packet4i>(pcast<Packet2ul, Packet4ui>(a, b));
1284}
1285template <>
1286EIGEN_STRONG_INLINE Packet2i pcast<Packet2ul, Packet2i>(const Packet2ul& a) {
1287 return preinterpret<Packet2i>(pcast<Packet2ul, Packet2ui>(a));
1288}
1289
1290template <>
1291struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1292 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1293};
1294template <>
1295EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1296 const Packet2ul& d) {
1297 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1298 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1299 return vcombine_u16(ab_u16, cd_u16);
1300}
1301template <>
1302EIGEN_STRONG_INLINE Packet4us pcast<Packet2ul, Packet4us>(const Packet2ul& a, const Packet2ul& b) {
1303 return vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1304}
1305
1306template <>
1307struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1308 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1309};
1310template <>
1311EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1312 const Packet2ul& d) {
1313 return preinterpret<Packet8s>(pcast<Packet2ul, Packet8us>(a, b, c, d));
1314}
1315template <>
1316EIGEN_STRONG_INLINE Packet4s pcast<Packet2ul, Packet4s>(const Packet2ul& a, const Packet2ul& b) {
1317 return preinterpret<Packet4s>(pcast<Packet2ul, Packet4us>(a, b));
1318}
1319
1320template <>
1321struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1322 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1323};
1324template <>
1325EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1326 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1327 const Packet2ul& g, const Packet2ul& h) {
1328 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1329 const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1330 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1331}
1332template <>
1333EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ul, Packet8uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1334 const Packet2ul& d) {
1335 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1336 return vmovn_u16(abcd_u16);
1337}
1338template <>
1339EIGEN_STRONG_INLINE Packet4uc pcast<Packet2ul, Packet4uc>(const Packet2ul& a, const Packet2ul& b) {
1340 const uint16x4_t ab_u16 = pcast<Packet2ul, Packet4us>(a, b);
1341 const uint16x8_t abab_u16 = vcombine_u16(ab_u16, ab_u16);
1342 const uint8x8_t abab_u8 = vmovn_u16(abab_u16);
1343 return vget_lane_u32(vreinterpret_u32_u8(abab_u8), 0);
1344}
1345
1346template <>
1347struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1348 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1349};
1350template <>
1351EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1352 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1353 const Packet2ul& g, const Packet2ul& h) {
1354 return preinterpret<Packet16c>(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1355}
1356template <>
1357EIGEN_STRONG_INLINE Packet8c pcast<Packet2ul, Packet8c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1358 const Packet2ul& d) {
1359 return preinterpret<Packet8c>(pcast<Packet2ul, Packet8uc>(a, b, c, d));
1360}
1361template <>
1362EIGEN_STRONG_INLINE Packet4c pcast<Packet2ul, Packet4c>(const Packet2ul& a, const Packet2ul& b) {
1363 return static_cast<Packet4c>(pcast<Packet2ul, Packet4uc>(a, b));
1364}
1365
1366#if EIGEN_ARCH_ARM64
1367
1368//==============================================================================
1369// pcast/preinterpret, Double
1370//==============================================================================
1371
1372template <>
1373EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1374 return Packet2d(vreinterpretq_f64_s64(a));
1375}
1376template <>
1377EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1378 return Packet2d(vreinterpretq_f64_u64(a));
1379}
1380template <>
1381EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1382 return Packet2l(vreinterpretq_s64_f64(a));
1383}
1384template <>
1385EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1386 return Packet2ul(vreinterpretq_u64_f64(a));
1387}
1388template <>
1389EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1390 return Packet2d(vreinterpretq_f64_s32(a));
1391}
1392template <>
1393EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1394 return Packet4i(vreinterpretq_s32_f64(a));
1395}
1396
1397template <>
1398struct type_casting_traits<double, float> {
1399 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1400};
1401template <>
1402EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1403 return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1404}
1405template <>
1406EIGEN_STRONG_INLINE Packet2f pcast<Packet2d, Packet2f>(const Packet2d& a) {
1407 return vcvt_f32_f64(a);
1408}
1409
1410template <>
1411struct type_casting_traits<double, numext::int64_t> {
1412 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1413};
1414template <>
1415EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1416 return vcvtq_s64_f64(a);
1417}
1418
1419template <>
1420struct type_casting_traits<double, numext::uint64_t> {
1421 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1422};
1423template <>
1424EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1425 return vcvtq_u64_f64(a);
1426}
1427
1428template <>
1429struct type_casting_traits<double, numext::int32_t> {
1430 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1431};
1432template <>
1433EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1434 return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1435}
1436template <>
1437EIGEN_STRONG_INLINE Packet2i pcast<Packet2d, Packet2i>(const Packet2d& a) {
1438 return vmovn_s64(vcvtq_s64_f64(a));
1439}
1440
1441template <>
1442struct type_casting_traits<double, numext::uint32_t> {
1443 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1444};
1445template <>
1446EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1447 return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1448}
1449template <>
1450EIGEN_STRONG_INLINE Packet2ui pcast<Packet2d, Packet2ui>(const Packet2d& a) {
1451 return vmovn_u64(vcvtq_u64_f64(a));
1452}
1453
1454template <>
1455struct type_casting_traits<double, numext::int16_t> {
1456 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1457};
1458template <>
1459EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1460 const Packet2d& d) {
1461 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1462 const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1463 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1464}
1465template <>
1466EIGEN_STRONG_INLINE Packet4s pcast<Packet2d, Packet4s>(const Packet2d& a, const Packet2d& b) {
1467 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1468 return vmovn_s32(ab_s32);
1469}
1470
1471template <>
1472struct type_casting_traits<double, numext::uint16_t> {
1473 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1474};
1475template <>
1476EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1477 const Packet2d& d) {
1478 return preinterpret<Packet8us>(pcast<Packet2d, Packet8s>(a, b, c, d));
1479}
1480template <>
1481EIGEN_STRONG_INLINE Packet4us pcast<Packet2d, Packet4us>(const Packet2d& a, const Packet2d& b) {
1482 return preinterpret<Packet4us>(pcast<Packet2d, Packet4s>(a, b));
1483}
1484
1485template <>
1486struct type_casting_traits<double, numext::int8_t> {
1487 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1488};
1489template <>
1490EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1491 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1492 const Packet2d& g, const Packet2d& h) {
1493 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1494 const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1495 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1496}
1497template <>
1498EIGEN_STRONG_INLINE Packet8c pcast<Packet2d, Packet8c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1499 const Packet2d& d) {
1500 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1501 return vmovn_s16(abcd_s16);
1502}
1503template <>
1504EIGEN_STRONG_INLINE Packet4c pcast<Packet2d, Packet4c>(const Packet2d& a, const Packet2d& b) {
1505 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1506 return pcast<Packet4i, Packet4c>(ab_s32);
1507}
1508
1509template <>
1510struct type_casting_traits<double, numext::uint8_t> {
1511 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1512};
1513template <>
1514EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1515 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1516 const Packet2d& g, const Packet2d& h) {
1517 const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1518 const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1519 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1520}
1521template <>
1522EIGEN_STRONG_INLINE Packet8uc pcast<Packet2d, Packet8uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1523 const Packet2d& d) {
1524 return preinterpret<Packet8uc>(pcast<Packet2d, Packet8c>(a, b, c, d));
1525}
1526template <>
1527EIGEN_STRONG_INLINE Packet4uc pcast<Packet2d, Packet4uc>(const Packet2d& a, const Packet2d& b) {
1528 return static_cast<Packet4uc>(pcast<Packet2d, Packet4c>(a, b));
1529}
1530
1531template <>
1532struct type_casting_traits<float, double> {
1533 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1534};
1535template <>
1536EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1537 // Discard second-half of input.
1538 return vcvt_f64_f32(vget_low_f32(a));
1539}
1540template <>
1541EIGEN_STRONG_INLINE Packet2d pcast<Packet2f, Packet2d>(const Packet2f& a) {
1542 return vcvt_f64_f32(a);
1543}
1544
1545template <>
1546struct type_casting_traits<numext::int8_t, double> {
1547 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1548};
1549template <>
1550EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1551 // Discard all but first two values.
1552 // MSVC defines most intrinsics as macros, so we need to do this in two lines for portability.
1553 Packet2f tmp = pcast<Packet8c, Packet2f>(vget_low_s8(a));
1554 return vcvt_f64_f32(tmp);
1555}
1556
1557template <>
1558struct type_casting_traits<numext::uint8_t, double> {
1559 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1560};
1561template <>
1562EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1563 // Discard all but first two values.
1564 Packet2f tmp = pcast<Packet8uc, Packet2f>(vget_low_u8(a));
1565 return vcvt_f64_f32(tmp);
1566}
1567
1568template <>
1569struct type_casting_traits<numext::int16_t, double> {
1570 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1571};
1572template <>
1573EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1574 // Discard all but first two values.
1575 Packet2f tmp = pcast<Packet4s, Packet2f>(vget_low_s16(a));
1576 return vcvt_f64_f32(tmp);
1577}
1578
1579template <>
1580struct type_casting_traits<numext::uint16_t, double> {
1581 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1582};
1583template <>
1584EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1585 // Discard all but first two values.
1586 Packet2f tmp = pcast<Packet4us, Packet2f>(vget_low_u16(a));
1587 return vcvt_f64_f32(tmp);
1588}
1589
1590template <>
1591struct type_casting_traits<numext::int32_t, double> {
1592 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1593};
1594template <>
1595EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1596 // Discard second half of input.
1597 return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1598}
1599template <>
1600EIGEN_STRONG_INLINE Packet2d pcast<Packet2i, Packet2d>(const Packet2i& a) {
1601 return vcvtq_f64_s64(vmovl_s32(a));
1602}
1603
1604template <>
1605struct type_casting_traits<numext::uint32_t, double> {
1606 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1607};
1608template <>
1609EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1610 // Discard second half of input.
1611 return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1612}
1613template <>
1614EIGEN_STRONG_INLINE Packet2d pcast<Packet2ui, Packet2d>(const Packet2ui& a) {
1615 return vcvtq_f64_u64(vmovl_u32(a));
1616}
1617
1618template <>
1619struct type_casting_traits<numext::int64_t, double> {
1620 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1621};
1622template <>
1623EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1624 return vcvtq_f64_s64(a);
1625}
1626
1627template <>
1628struct type_casting_traits<numext::uint64_t, double> {
1629 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1630};
1631template <>
1632EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1633 return vcvtq_f64_u64(a);
1634}
1635
1636#endif // EIGEN_ARCH_ARM64
1637
1638} // end namespace internal
1639
1640} // end namespace Eigen
1641
1642#endif // EIGEN_TYPE_CASTING_NEON_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition Core:137