TR-mbed 1.0
Loading...
Searching...
No Matches
PacketMath.h
Go to the documentation of this file.
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org>
6// Heavily based on Gael's SSE version.
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11
12#ifndef EIGEN_PACKET_MATH_NEON_H
13#define EIGEN_PACKET_MATH_NEON_H
14
15namespace Eigen {
16
17namespace internal {
18
19#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
21#endif
22
23#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
24#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
25#endif
26
27#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
28#if EIGEN_ARCH_ARM64
29#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
30#else
31#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
32#endif
33#endif
34
35#if EIGEN_COMP_MSVC_STRICT
36
37// In MSVC's arm_neon.h header file, all NEON vector types
38// are aliases to the same underlying type __n128.
39// We thus have to wrap them to make them different C++ types.
40// (See also bug 1428)
41typedef eigen_packet_wrapper<float32x2_t,0> Packet2f;
42typedef eigen_packet_wrapper<float32x4_t,1> Packet4f;
43typedef eigen_packet_wrapper<int32_t ,2> Packet4c;
44typedef eigen_packet_wrapper<int8x8_t ,3> Packet8c;
45typedef eigen_packet_wrapper<int8x16_t ,4> Packet16c;
46typedef eigen_packet_wrapper<uint32_t ,5> Packet4uc;
47typedef eigen_packet_wrapper<uint8x8_t ,6> Packet8uc;
48typedef eigen_packet_wrapper<uint8x16_t ,7> Packet16uc;
49typedef eigen_packet_wrapper<int16x4_t ,8> Packet4s;
50typedef eigen_packet_wrapper<int16x8_t ,9> Packet8s;
51typedef eigen_packet_wrapper<uint16x4_t ,10> Packet4us;
52typedef eigen_packet_wrapper<uint16x8_t ,11> Packet8us;
53typedef eigen_packet_wrapper<int32x2_t ,12> Packet2i;
54typedef eigen_packet_wrapper<int32x4_t ,13> Packet4i;
55typedef eigen_packet_wrapper<uint32x2_t ,14> Packet2ui;
56typedef eigen_packet_wrapper<uint32x4_t ,15> Packet4ui;
57typedef eigen_packet_wrapper<int64x2_t ,16> Packet2l;
58typedef eigen_packet_wrapper<uint64x2_t ,17> Packet2ul;
59
60#else
61
62typedef float32x2_t Packet2f;
63typedef float32x4_t Packet4f;
64typedef eigen_packet_wrapper<int32_t ,2> Packet4c;
65typedef int8x8_t Packet8c;
66typedef int8x16_t Packet16c;
67typedef eigen_packet_wrapper<uint32_t ,5> Packet4uc;
68typedef uint8x8_t Packet8uc;
69typedef uint8x16_t Packet16uc;
70typedef int16x4_t Packet4s;
71typedef int16x8_t Packet8s;
72typedef uint16x4_t Packet4us;
73typedef uint16x8_t Packet8us;
74typedef int32x2_t Packet2i;
75typedef int32x4_t Packet4i;
76typedef uint32x2_t Packet2ui;
77typedef uint32x4_t Packet4ui;
78typedef int64x2_t Packet2l;
79typedef uint64x2_t Packet2ul;
80
81#endif // EIGEN_COMP_MSVC_STRICT
82
83EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask){
84 const float* a = reinterpret_cast<const float*>(&m);
85 Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3 )), *(a + ((mask >> 6) & 3))};
86 return res;
87}
88
89// fuctionally equivalent to _mm_shuffle_ps in SSE when interleave
90// == false (i.e. shuffle<false>(m, n, mask) equals _mm_shuffle_ps(m, n, mask)),
91// interleave m and n when interleave == true. Currently used in LU/arch/InverseSize4.h
92// to enable a shared implementation for fast inversion of matrices of size 4.
93template<bool interleave>
94EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int mask)
95{
96 const float* a = reinterpret_cast<const float*>(&m);
97 const float* b = reinterpret_cast<const float*>(&n);
98 Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))};
99 return res;
100}
101
102template<>
103EIGEN_STRONG_INLINE Packet4f shuffle2<true>(const Packet4f &m, const Packet4f &n, int mask)
104{
105 const float* a = reinterpret_cast<const float*>(&m);
106 const float* b = reinterpret_cast<const float*>(&n);
107 Packet4f res = {*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))};
108 return res;
109}
110
111EIGEN_STRONG_INLINE static int eigen_neon_shuffle_mask(int p, int q, int r, int s) {return ((s)<<6|(r)<<4|(q)<<2|(p));}
112
113EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s)
114{
115 return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s));
116}
117EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s)
118{
119 return shuffle2<false>(a,b,eigen_neon_shuffle_mask(p, q, r, s));
120}
121EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b)
122{
123 return shuffle2<false>(a,b,eigen_neon_shuffle_mask(0, 1, 0, 1));
124}
125EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b)
126{
127 return shuffle2<false>(b,a,eigen_neon_shuffle_mask(2, 3, 2, 3));
128}
129EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b)
130{
131 return shuffle2<true>(a,b,eigen_neon_shuffle_mask(0, 0, 1, 1));
132}
133EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b)
134{
135 return shuffle2<true>(a,b,eigen_neon_shuffle_mask(2, 2, 3, 3));
136}
137#define vec4f_duplane(a, p) \
138 vdupq_lane_f32(vget_low_f32(a), p)
139
140#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
141 const Packet4f p4f_##NAME = pset1<Packet4f>(X)
142
143#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
144 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X))
145
146#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
147 const Packet4i p4i_##NAME = pset1<Packet4i>(X)
148
149#if EIGEN_ARCH_ARM64
150 // __builtin_prefetch tends to do nothing on ARM64 compilers because the
151 // prefetch instructions there are too detailed for __builtin_prefetch to map
152 // meaningfully to them.
153 #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) : );
154#elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
155 #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR);
156#elif defined __pld
157 #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR)
158#elif EIGEN_ARCH_ARM32
159 #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ("pld [%[addr]]\n" :: [addr] "r" (ADDR) : );
160#else
161 // by default no explicit prefetching
162 #define EIGEN_ARM_PREFETCH(ADDR)
163#endif
164
165template <>
166struct packet_traits<float> : default_packet_traits
167{
168 typedef Packet4f type;
169 typedef Packet2f half;
170 enum
171 {
172 Vectorizable = 1,
173 AlignedOnScalar = 1,
174 size = 4,
175 HasHalfPacket = 1,
176
177 HasAdd = 1,
178 HasSub = 1,
179 HasShift = 1,
180 HasMul = 1,
181 HasNegate = 1,
182 HasAbs = 1,
183 HasArg = 0,
184 HasAbs2 = 1,
185 HasAbsDiff = 1,
186 HasMin = 1,
187 HasMax = 1,
188 HasConj = 1,
189 HasSetLinear = 0,
190 HasBlend = 0,
191
192 HasDiv = 1,
193 HasFloor = 1,
194 HasCeil = 1,
195 HasRint = 1,
196
197 HasSin = EIGEN_FAST_MATH,
198 HasCos = EIGEN_FAST_MATH,
199 HasLog = 1,
200 HasExp = 1,
201 HasSqrt = 1,
202 HasRsqrt = 1,
203 HasTanh = EIGEN_FAST_MATH,
204 HasErf = EIGEN_FAST_MATH,
205 HasBessel = 0, // Issues with accuracy.
206 HasNdtri = 0
207 };
208};
209
210template <>
238
239template <>
269
270template <>
298
299template <>
328
329template <>
330struct packet_traits<int32_t> : default_packet_traits
331{
332 typedef Packet4i type;
333 typedef Packet2i half;
334 enum
335 {
336 Vectorizable = 1,
337 AlignedOnScalar = 1,
338 size = 4,
339 HasHalfPacket = 1,
340
341 HasAdd = 1,
342 HasSub = 1,
343 HasShift = 1,
344 HasMul = 1,
345 HasNegate = 1,
346 HasAbs = 1,
347 HasArg = 0,
348 HasAbs2 = 1,
349 HasAbsDiff = 1,
350 HasMin = 1,
351 HasMax = 1,
352 HasConj = 1,
353 HasSetLinear = 0,
354 HasBlend = 0
355 };
356};
357
358template <>
388
389template <>
418
419template <>
448
449#if EIGEN_GNUC_AT_MOST(4, 4) && !EIGEN_COMP_LLVM
450// workaround gcc 4.2, 4.3 and 4.4 compilation issue
451EIGEN_STRONG_INLINE float32x4_t vld1q_f32(const float* x) { return ::vld1q_f32((const float32_t*)x); }
452EIGEN_STRONG_INLINE float32x2_t vld1_f32(const float* x) { return ::vld1_f32 ((const float32_t*)x); }
453EIGEN_STRONG_INLINE float32x2_t vld1_dup_f32(const float* x) { return ::vld1_dup_f32 ((const float32_t*)x); }
454EIGEN_STRONG_INLINE void vst1q_f32(float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); }
455EIGEN_STRONG_INLINE void vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); }
456#endif
457
458template<> struct unpacket_traits<Packet2f>
459{
460 typedef float type;
461 typedef Packet2f half;
462 typedef Packet2i integer_packet;
463 enum
464 {
465 size = 2,
466 alignment = Aligned16,
467 vectorizable = true,
468 masked_load_available = false,
469 masked_store_available = false
470 };
471};
472template<> struct unpacket_traits<Packet4f>
473{
474 typedef float type;
475 typedef Packet2f half;
476 typedef Packet4i integer_packet;
477 enum
478 {
479 size = 4,
480 alignment = Aligned16,
481 vectorizable = true,
482 masked_load_available = false,
483 masked_store_available = false
484 };
485};
486template<> struct unpacket_traits<Packet4c>
487{
488 typedef int8_t type;
489 typedef Packet4c half;
490 enum
491 {
492 size = 4,
493 alignment = Unaligned,
494 vectorizable = true,
495 masked_load_available = false,
496 masked_store_available = false
497 };
498};
499template<> struct unpacket_traits<Packet8c>
500{
501 typedef int8_t type;
502 typedef Packet4c half;
503 enum
504 {
505 size = 8,
506 alignment = Aligned16,
507 vectorizable = true,
508 masked_load_available = false,
509 masked_store_available = false
510 };
511};
512template<> struct unpacket_traits<Packet16c>
513{
514 typedef int8_t type;
515 typedef Packet8c half;
516 enum
517 {
518 size = 16,
519 alignment = Aligned16,
520 vectorizable = true,
521 masked_load_available = false,
522 masked_store_available = false
523 };
524};
525template<> struct unpacket_traits<Packet4uc>
526{
527 typedef uint8_t type;
528 typedef Packet4uc half;
529 enum
530 {
531 size = 4,
532 alignment = Unaligned,
533 vectorizable = true,
534 masked_load_available = false,
535 masked_store_available = false
536 };
537};
538template<> struct unpacket_traits<Packet8uc>
539{
540 typedef uint8_t type;
541 typedef Packet4uc half;
542 enum
543 {
544 size = 8,
545 alignment = Aligned16,
546 vectorizable = true,
547 masked_load_available = false,
548 masked_store_available = false
549 };
550};
551template<> struct unpacket_traits<Packet16uc>
552{
553 typedef uint8_t type;
554 typedef Packet8uc half;
555 enum
556 {
557 size = 16,
558 alignment = Aligned16,
559 vectorizable = true,
560 masked_load_available = false,
561 masked_store_available = false};
562};
563template<> struct unpacket_traits<Packet4s>
564{
565 typedef int16_t type;
566 typedef Packet4s half;
567 enum
568 {
569 size = 4,
570 alignment = Aligned16,
571 vectorizable = true,
572 masked_load_available = false,
573 masked_store_available = false
574 };
575};
576template<> struct unpacket_traits<Packet8s>
577{
578 typedef int16_t type;
579 typedef Packet4s half;
580 enum
581 {
582 size = 8,
583 alignment = Aligned16,
584 vectorizable = true,
585 masked_load_available = false,
586 masked_store_available = false
587 };
588};
589template<> struct unpacket_traits<Packet4us>
590{
591 typedef uint16_t type;
592 typedef Packet4us half;
593 enum
594 {
595 size = 4,
596 alignment = Aligned16,
597 vectorizable = true,
598 masked_load_available = false,
599 masked_store_available = false
600 };
601};
602template<> struct unpacket_traits<Packet8us>
603{
604 typedef uint16_t type;
605 typedef Packet4us half;
606 enum
607 {
608 size = 8,
609 alignment = Aligned16,
610 vectorizable = true,
611 masked_load_available = false,
612 masked_store_available = false
613 };
614};
615template<> struct unpacket_traits<Packet2i>
616{
617 typedef int32_t type;
618 typedef Packet2i half;
619 enum
620 {
621 size = 2,
622 alignment = Aligned16,
623 vectorizable = true,
624 masked_load_available = false,
625 masked_store_available = false
626 };
627};
628template<> struct unpacket_traits<Packet4i>
629{
630 typedef int32_t type;
631 typedef Packet2i half;
632 enum
633 {
634 size = 4,
635 alignment = Aligned16,
636 vectorizable = true,
637 masked_load_available = false,
638 masked_store_available = false
639 };
640};
641template<> struct unpacket_traits<Packet2ui>
642{
643 typedef uint32_t type;
644 typedef Packet2ui half;
645 enum
646 {
647 size = 2,
648 alignment = Aligned16,
649 vectorizable = true,
650 masked_load_available = false,
651 masked_store_available = false
652 };
653};
654template<> struct unpacket_traits<Packet4ui>
655{
656 typedef uint32_t type;
657 typedef Packet2ui half;
658 enum
659 {
660 size = 4,
661 alignment = Aligned16,
662 vectorizable = true,
663 masked_load_available = false,
664 masked_store_available = false
665 };
666};
667template<> struct unpacket_traits<Packet2l>
668{
669 typedef int64_t type;
670 typedef Packet2l half;
671 enum
672 {
673 size = 2,
674 alignment = Aligned16,
675 vectorizable = true,
676 masked_load_available = false,
677 masked_store_available = false
678 };
679};
680template<> struct unpacket_traits<Packet2ul>
681{
682 typedef uint64_t type;
683 typedef Packet2ul half;
684 enum
685 {
686 size = 2,
687 alignment = Aligned16,
688 vectorizable = true,
689 masked_load_available = false,
690 masked_store_available = false
691 };
692};
693
694template<> EIGEN_STRONG_INLINE Packet2f pset1<Packet2f>(const float& from) { return vdup_n_f32(from); }
695template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return vdupq_n_f32(from); }
698template<> EIGEN_STRONG_INLINE Packet8c pset1<Packet8c>(const int8_t& from) { return vdup_n_s8(from); }
699template<> EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const int8_t& from) { return vdupq_n_s8(from); }
702template<> EIGEN_STRONG_INLINE Packet8uc pset1<Packet8uc>(const uint8_t& from) { return vdup_n_u8(from); }
703template<> EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const uint8_t& from) { return vdupq_n_u8(from); }
704template<> EIGEN_STRONG_INLINE Packet4s pset1<Packet4s>(const int16_t& from) { return vdup_n_s16(from); }
705template<> EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const int16_t& from) { return vdupq_n_s16(from); }
706template<> EIGEN_STRONG_INLINE Packet4us pset1<Packet4us>(const uint16_t& from) { return vdup_n_u16(from); }
707template<> EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const uint16_t& from) { return vdupq_n_u16(from); }
708template<> EIGEN_STRONG_INLINE Packet2i pset1<Packet2i>(const int32_t& from) { return vdup_n_s32(from); }
709template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) { return vdupq_n_s32(from); }
710template<> EIGEN_STRONG_INLINE Packet2ui pset1<Packet2ui>(const uint32_t& from) { return vdup_n_u32(from); }
711template<> EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) { return vdupq_n_u32(from); }
712template<> EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) { return vdupq_n_s64(from); }
713template<> EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(const uint64_t& from) { return vdupq_n_u64(from); }
714
719
720template<> EIGEN_STRONG_INLINE Packet2f plset<Packet2f>(const float& a)
721{
722 const float c[] = {0.0f,1.0f};
723 return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
724}
725template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a)
726{
727 const float c[] = {0.0f,1.0f,2.0f,3.0f};
728 return vaddq_f32(pset1<Packet4f>(a), vld1q_f32(c));
729}
732template<> EIGEN_STRONG_INLINE Packet8c plset<Packet8c>(const int8_t& a)
733{
734 const int8_t c[] = {0,1,2,3,4,5,6,7};
735 return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
736}
737template<> EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const int8_t& a)
738{
739 const int8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
740 return vaddq_s8(pset1<Packet16c>(a), vld1q_s8(c));
741}
744template<> EIGEN_STRONG_INLINE Packet8uc plset<Packet8uc>(const uint8_t& a)
745{
746 const uint8_t c[] = {0,1,2,3,4,5,6,7};
747 return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
748}
749template<> EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const uint8_t& a)
750{
751 const uint8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
752 return vaddq_u8(pset1<Packet16uc>(a), vld1q_u8(c));
753}
754template<> EIGEN_STRONG_INLINE Packet4s plset<Packet4s>(const int16_t& a)
755{
756 const int16_t c[] = {0,1,2,3};
757 return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
758}
759template<> EIGEN_STRONG_INLINE Packet4us plset<Packet4us>(const uint16_t& a)
760{
761 const uint16_t c[] = {0,1,2,3};
762 return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
763}
764template<> EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const int16_t& a)
765{
766 const int16_t c[] = {0,1,2,3,4,5,6,7};
767 return vaddq_s16(pset1<Packet8s>(a), vld1q_s16(c));
768}
769template<> EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const uint16_t& a)
770{
771 const uint16_t c[] = {0,1,2,3,4,5,6,7};
772 return vaddq_u16(pset1<Packet8us>(a), vld1q_u16(c));
773}
774template<> EIGEN_STRONG_INLINE Packet2i plset<Packet2i>(const int32_t& a)
775{
776 const int32_t c[] = {0,1};
777 return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
778}
779template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a)
780{
781 const int32_t c[] = {0,1,2,3};
782 return vaddq_s32(pset1<Packet4i>(a), vld1q_s32(c));
783}
784template<> EIGEN_STRONG_INLINE Packet2ui plset<Packet2ui>(const uint32_t& a)
785{
786 const uint32_t c[] = {0,1};
787 return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
788}
789template<> EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a)
790{
791 const uint32_t c[] = {0,1,2,3};
792 return vaddq_u32(pset1<Packet4ui>(a), vld1q_u32(c));
793}
794template<> EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a)
795{
796 const int64_t c[] = {0,1};
797 return vaddq_s64(pset1<Packet2l>(a), vld1q_s64(c));
798}
799template<> EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(const uint64_t& a)
800{
801 const uint64_t c[] = {0,1};
802 return vaddq_u64(pset1<Packet2ul>(a), vld1q_u64(c));
803}
804
805template<> EIGEN_STRONG_INLINE Packet2f padd<Packet2f>(const Packet2f& a, const Packet2f& b) { return vadd_f32(a,b); }
806template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return vaddq_f32(a,b); }
813template<> EIGEN_STRONG_INLINE Packet8c padd<Packet8c>(const Packet8c& a, const Packet8c& b) { return vadd_s8(a,b); }
814template<> EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) { return vaddq_s8(a,b); }
821template<> EIGEN_STRONG_INLINE Packet8uc padd<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vadd_u8(a,b); }
822template<> EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vaddq_u8(a,b); }
823template<> EIGEN_STRONG_INLINE Packet4s padd<Packet4s>(const Packet4s& a, const Packet4s& b) { return vadd_s16(a,b); }
824template<> EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) { return vaddq_s16(a,b); }
825template<> EIGEN_STRONG_INLINE Packet4us padd<Packet4us>(const Packet4us& a, const Packet4us& b) { return vadd_u16(a,b); }
826template<> EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) { return vaddq_u16(a,b); }
827template<> EIGEN_STRONG_INLINE Packet2i padd<Packet2i>(const Packet2i& a, const Packet2i& b) { return vadd_s32(a,b); }
828template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return vaddq_s32(a,b); }
829template<> EIGEN_STRONG_INLINE Packet2ui padd<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vadd_u32(a,b); }
830template<> EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vaddq_u32(a,b); }
831template<> EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) { return vaddq_s64(a,b); }
832template<> EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(const Packet2ul& a, const Packet2ul& b) { return vaddq_u64(a,b); }
833
834template<> EIGEN_STRONG_INLINE Packet2f psub<Packet2f>(const Packet2f& a, const Packet2f& b) { return vsub_f32(a,b); }
835template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return vsubq_f32(a,b); }
842template<> EIGEN_STRONG_INLINE Packet8c psub<Packet8c>(const Packet8c& a, const Packet8c& b) { return vsub_s8(a,b); }
843template<> EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) { return vsubq_s8(a,b); }
850template<> EIGEN_STRONG_INLINE Packet8uc psub<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vsub_u8(a,b); }
851template<> EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vsubq_u8(a,b); }
852template<> EIGEN_STRONG_INLINE Packet4s psub<Packet4s>(const Packet4s& a, const Packet4s& b) { return vsub_s16(a,b); }
853template<> EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) { return vsubq_s16(a,b); }
854template<> EIGEN_STRONG_INLINE Packet4us psub<Packet4us>(const Packet4us& a, const Packet4us& b) { return vsub_u16(a,b); }
855template<> EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) { return vsubq_u16(a,b); }
856template<> EIGEN_STRONG_INLINE Packet2i psub<Packet2i>(const Packet2i& a, const Packet2i& b) { return vsub_s32(a,b); }
857template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return vsubq_s32(a,b); }
858template<> EIGEN_STRONG_INLINE Packet2ui psub<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vsub_u32(a,b); }
859template<> EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vsubq_u32(a,b); }
860template<> EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) { return vsubq_s64(a,b); }
861template<> EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(const Packet2ul& a, const Packet2ul& b) { return vsubq_u64(a,b); }
862
863template<> EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b);
864template<> EIGEN_STRONG_INLINE Packet2f paddsub<Packet2f>(const Packet2f& a, const Packet2f & b) {
865 Packet2f mask = {numext::bit_cast<float>(0x80000000u), 0.0f};
866 return padd(a, pxor(mask, b));
867}
868template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
869template<> EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
870 Packet4f mask = {numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f};
871 return padd(a, pxor(mask, b));
872}
873
874template<> EIGEN_STRONG_INLINE Packet2f pnegate(const Packet2f& a) { return vneg_f32(a); }
875template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); }
878template<> EIGEN_STRONG_INLINE Packet8c pnegate(const Packet8c& a) { return vneg_s8(a); }
879template<> EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) { return vnegq_s8(a); }
880template<> EIGEN_STRONG_INLINE Packet4s pnegate(const Packet4s& a) { return vneg_s16(a); }
881template<> EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) { return vnegq_s16(a); }
882template<> EIGEN_STRONG_INLINE Packet2i pnegate(const Packet2i& a) { return vneg_s32(a); }
883template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); }
884template<> EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
885#if EIGEN_ARCH_ARM64
886 return vnegq_s64(a);
887#else
888 return vcombine_s64(
889 vdup_n_s64(-vgetq_lane_s64(a, 0)),
890 vdup_n_s64(-vgetq_lane_s64(a, 1)));
891#endif
892}
893
894template<> EIGEN_STRONG_INLINE Packet2f pconj(const Packet2f& a) { return a; }
895template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
896template<> EIGEN_STRONG_INLINE Packet4c pconj(const Packet4c& a) { return a; }
897template<> EIGEN_STRONG_INLINE Packet8c pconj(const Packet8c& a) { return a; }
898template<> EIGEN_STRONG_INLINE Packet16c pconj(const Packet16c& a) { return a; }
899template<> EIGEN_STRONG_INLINE Packet4uc pconj(const Packet4uc& a) { return a; }
900template<> EIGEN_STRONG_INLINE Packet8uc pconj(const Packet8uc& a) { return a; }
901template<> EIGEN_STRONG_INLINE Packet16uc pconj(const Packet16uc& a) { return a; }
902template<> EIGEN_STRONG_INLINE Packet4s pconj(const Packet4s& a) { return a; }
903template<> EIGEN_STRONG_INLINE Packet8s pconj(const Packet8s& a) { return a; }
904template<> EIGEN_STRONG_INLINE Packet4us pconj(const Packet4us& a) { return a; }
905template<> EIGEN_STRONG_INLINE Packet8us pconj(const Packet8us& a) { return a; }
906template<> EIGEN_STRONG_INLINE Packet2i pconj(const Packet2i& a) { return a; }
907template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
908template<> EIGEN_STRONG_INLINE Packet2ui pconj(const Packet2ui& a) { return a; }
909template<> EIGEN_STRONG_INLINE Packet4ui pconj(const Packet4ui& a) { return a; }
910template<> EIGEN_STRONG_INLINE Packet2l pconj(const Packet2l& a) { return a; }
911template<> EIGEN_STRONG_INLINE Packet2ul pconj(const Packet2ul& a) { return a; }
912
913template<> EIGEN_STRONG_INLINE Packet2f pmul<Packet2f>(const Packet2f& a, const Packet2f& b) { return vmul_f32(a,b); }
914template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); }
921template<> EIGEN_STRONG_INLINE Packet8c pmul<Packet8c>(const Packet8c& a, const Packet8c& b) { return vmul_s8(a,b); }
922template<> EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) { return vmulq_s8(a,b); }
929template<> EIGEN_STRONG_INLINE Packet8uc pmul<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vmul_u8(a,b); }
930template<> EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vmulq_u8(a,b); }
931template<> EIGEN_STRONG_INLINE Packet4s pmul<Packet4s>(const Packet4s& a, const Packet4s& b) { return vmul_s16(a,b); }
932template<> EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) { return vmulq_s16(a,b); }
933template<> EIGEN_STRONG_INLINE Packet4us pmul<Packet4us>(const Packet4us& a, const Packet4us& b) { return vmul_u16(a,b); }
934template<> EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) { return vmulq_u16(a,b); }
935template<> EIGEN_STRONG_INLINE Packet2i pmul<Packet2i>(const Packet2i& a, const Packet2i& b) { return vmul_s32(a,b); }
936template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); }
937template<> EIGEN_STRONG_INLINE Packet2ui pmul<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vmul_u32(a,b); }
938template<> EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vmulq_u32(a,b); }
949
950template<> EIGEN_STRONG_INLINE Packet2f pdiv<Packet2f>(const Packet2f& a, const Packet2f& b)
951{
952#if EIGEN_ARCH_ARM64
953 return vdiv_f32(a,b);
954#else
955 Packet2f inv, restep, div;
956
957 // NEON does not offer a divide instruction, we have to do a reciprocal approximation
958 // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers
959 // a reciprocal estimate AND a reciprocal step -which saves a few instructions
960 // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with
961 // Newton-Raphson and vrecpsq_f32()
962 inv = vrecpe_f32(b);
963
964 // This returns a differential, by which we will have to multiply inv to get a better
965 // approximation of 1/b.
966 restep = vrecps_f32(b, inv);
967 inv = vmul_f32(restep, inv);
968
969 // Finally, multiply a by 1/b and get the wanted result of the division.
970 div = vmul_f32(a, inv);
971
972 return div;
973#endif
974}
975template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
976{
977#if EIGEN_ARCH_ARM64
978 return vdivq_f32(a,b);
979#else
980 Packet4f inv, restep, div;
981
982 // NEON does not offer a divide instruction, we have to do a reciprocal approximation
983 // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers
984 // a reciprocal estimate AND a reciprocal step -which saves a few instructions
985 // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with
986 // Newton-Raphson and vrecpsq_f32()
987 inv = vrecpeq_f32(b);
988
989 // This returns a differential, by which we will have to multiply inv to get a better
990 // approximation of 1/b.
991 restep = vrecpsq_f32(b, inv);
992 inv = vmulq_f32(restep, inv);
993
994 // Finally, multiply a by 1/b and get the wanted result of the division.
995 div = vmulq_f32(a, inv);
996
997 return div;
998#endif
999}
1000
1001template<> EIGEN_STRONG_INLINE Packet4c pdiv<Packet4c>(const Packet4c& /*a*/, const Packet4c& /*b*/)
1002{
1003 eigen_assert(false && "packet integer division are not supported by NEON");
1004 return pset1<Packet4c>(0);
1005}
1006template<> EIGEN_STRONG_INLINE Packet8c pdiv<Packet8c>(const Packet8c& /*a*/, const Packet8c& /*b*/)
1007{
1008 eigen_assert(false && "packet integer division are not supported by NEON");
1009 return pset1<Packet8c>(0);
1010}
1011template<> EIGEN_STRONG_INLINE Packet16c pdiv<Packet16c>(const Packet16c& /*a*/, const Packet16c& /*b*/)
1012{
1013 eigen_assert(false && "packet integer division are not supported by NEON");
1014 return pset1<Packet16c>(0);
1015}
1016template<> EIGEN_STRONG_INLINE Packet4uc pdiv<Packet4uc>(const Packet4uc& /*a*/, const Packet4uc& /*b*/)
1017{
1018 eigen_assert(false && "packet integer division are not supported by NEON");
1019 return pset1<Packet4uc>(0);
1020}
1021template<> EIGEN_STRONG_INLINE Packet8uc pdiv<Packet8uc>(const Packet8uc& /*a*/, const Packet8uc& /*b*/)
1022{
1023 eigen_assert(false && "packet integer division are not supported by NEON");
1024 return pset1<Packet8uc>(0);
1025}
1026template<> EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& /*a*/, const Packet16uc& /*b*/)
1027{
1028 eigen_assert(false && "packet integer division are not supported by NEON");
1029 return pset1<Packet16uc>(0);
1030}
1031template<> EIGEN_STRONG_INLINE Packet4s pdiv<Packet4s>(const Packet4s& /*a*/, const Packet4s& /*b*/)
1032{
1033 eigen_assert(false && "packet integer division are not supported by NEON");
1034 return pset1<Packet4s>(0);
1035}
1036template<> EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(const Packet8s& /*a*/, const Packet8s& /*b*/)
1037{
1038 eigen_assert(false && "packet integer division are not supported by NEON");
1039 return pset1<Packet8s>(0);
1040}
1041template<> EIGEN_STRONG_INLINE Packet4us pdiv<Packet4us>(const Packet4us& /*a*/, const Packet4us& /*b*/)
1042{
1043 eigen_assert(false && "packet integer division are not supported by NEON");
1044 return pset1<Packet4us>(0);
1045}
1046template<> EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(const Packet8us& /*a*/, const Packet8us& /*b*/)
1047{
1048 eigen_assert(false && "packet integer division are not supported by NEON");
1049 return pset1<Packet8us>(0);
1050}
1051template<> EIGEN_STRONG_INLINE Packet2i pdiv<Packet2i>(const Packet2i& /*a*/, const Packet2i& /*b*/)
1052{
1053 eigen_assert(false && "packet integer division are not supported by NEON");
1054 return pset1<Packet2i>(0);
1055}
1056template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
1057{
1058 eigen_assert(false && "packet integer division are not supported by NEON");
1059 return pset1<Packet4i>(0);
1060}
1061template<> EIGEN_STRONG_INLINE Packet2ui pdiv<Packet2ui>(const Packet2ui& /*a*/, const Packet2ui& /*b*/)
1062{
1063 eigen_assert(false && "packet integer division are not supported by NEON");
1064 return pset1<Packet2ui>(0);
1065}
1066template<> EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(const Packet4ui& /*a*/, const Packet4ui& /*b*/)
1067{
1068 eigen_assert(false && "packet integer division are not supported by NEON");
1069 return pset1<Packet4ui>(0);
1070}
1071template<> EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(const Packet2l& /*a*/, const Packet2l& /*b*/)
1072{
1073 eigen_assert(false && "packet integer division are not supported by NEON");
1074 return pset1<Packet2l>(0LL);
1075}
1076template<> EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(const Packet2ul& /*a*/, const Packet2ul& /*b*/)
1077{
1078 eigen_assert(false && "packet integer division are not supported by NEON");
1079 return pset1<Packet2ul>(0ULL);
1080}
1081
1082
1083#ifdef __ARM_FEATURE_FMA
1084template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
1085{ return vfmaq_f32(c,a,b); }
1086template<> EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c)
1087{ return vfma_f32(c,a,b); }
1088#else
1089template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
1090{
1091 return vmlaq_f32(c,a,b);
1092}
1093template<> EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c)
1094{
1095 return vmla_f32(c,a,b);
1096}
1097#endif
1098
1099// No FMA instruction for int, so use MLA unconditionally.
1107template<> EIGEN_STRONG_INLINE Packet8c pmadd(const Packet8c& a, const Packet8c& b, const Packet8c& c)
1108{ return vmla_s8(c,a,b); }
1109template<> EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c)
1110{ return vmlaq_s8(c,a,b); }
1118template<> EIGEN_STRONG_INLINE Packet8uc pmadd(const Packet8uc& a, const Packet8uc& b, const Packet8uc& c)
1119{ return vmla_u8(c,a,b); }
1120template<> EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c)
1121{ return vmlaq_u8(c,a,b); }
1122template<> EIGEN_STRONG_INLINE Packet4s pmadd(const Packet4s& a, const Packet4s& b, const Packet4s& c)
1123{ return vmla_s16(c,a,b); }
1124template<> EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c)
1125{ return vmlaq_s16(c,a,b); }
1126template<> EIGEN_STRONG_INLINE Packet4us pmadd(const Packet4us& a, const Packet4us& b, const Packet4us& c)
1127{ return vmla_u16(c,a,b); }
1128template<> EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c)
1129{ return vmlaq_u16(c,a,b); }
1130template<> EIGEN_STRONG_INLINE Packet2i pmadd(const Packet2i& a, const Packet2i& b, const Packet2i& c)
1131{ return vmla_s32(c,a,b); }
1132template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c)
1133{ return vmlaq_s32(c,a,b); }
1134template<> EIGEN_STRONG_INLINE Packet2ui pmadd(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c)
1135{ return vmla_u32(c,a,b); }
1136template<> EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c)
1137{ return vmlaq_u32(c,a,b); }
1138
1139template<> EIGEN_STRONG_INLINE Packet2f pabsdiff<Packet2f>(const Packet2f& a, const Packet2f& b)
1140{ return vabd_f32(a,b); }
1141template<> EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(const Packet4f& a, const Packet4f& b)
1142{ return vabdq_f32(a,b); }
1149template<> EIGEN_STRONG_INLINE Packet8c pabsdiff<Packet8c>(const Packet8c& a, const Packet8c& b)
1150{ return vabd_s8(a,b); }
1159template<> EIGEN_STRONG_INLINE Packet8uc pabsdiff<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1160{ return vabd_u8(a,b); }
1163template<> EIGEN_STRONG_INLINE Packet4s pabsdiff<Packet4s>(const Packet4s& a, const Packet4s& b)
1164{ return vabd_s16(a,b); }
1165template<> EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(const Packet8s& a, const Packet8s& b)
1166{ return vabdq_s16(a,b); }
1171template<> EIGEN_STRONG_INLINE Packet2i pabsdiff<Packet2i>(const Packet2i& a, const Packet2i& b)
1172{ return vabd_s32(a,b); }
1173template<> EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(const Packet4i& a, const Packet4i& b)
1174{ return vabdq_s32(a,b); }
1179
1180template<> EIGEN_STRONG_INLINE Packet2f pmin<Packet2f>(const Packet2f& a, const Packet2f& b) { return vmin_f32(a,b); }
1181template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return vminq_f32(a,b); }
1182
1183#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1184// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
1185template<> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) { return vminnmq_f32(a, b); }
1186template<> EIGEN_STRONG_INLINE Packet2f pmin<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) { return vminnm_f32(a, b); }
1187#endif
1188
1189template<> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) { return pmin<Packet4f>(a, b); }
1190
1191template<> EIGEN_STRONG_INLINE Packet2f pmin<PropagateNaN, Packet2f>(const Packet2f& a, const Packet2f& b) { return pmin<Packet2f>(a, b); }
1192
1199template<> EIGEN_STRONG_INLINE Packet8c pmin<Packet8c>(const Packet8c& a, const Packet8c& b) { return vmin_s8(a,b); }
1200template<> EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) { return vminq_s8(a,b); }
1207template<> EIGEN_STRONG_INLINE Packet8uc pmin<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vmin_u8(a,b); }
1208template<> EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vminq_u8(a,b); }
1209template<> EIGEN_STRONG_INLINE Packet4s pmin<Packet4s>(const Packet4s& a, const Packet4s& b) { return vmin_s16(a,b); }
1210template<> EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) { return vminq_s16(a,b); }
1211template<> EIGEN_STRONG_INLINE Packet4us pmin<Packet4us>(const Packet4us& a, const Packet4us& b) { return vmin_u16(a,b); }
1212template<> EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) { return vminq_u16(a,b); }
1213template<> EIGEN_STRONG_INLINE Packet2i pmin<Packet2i>(const Packet2i& a, const Packet2i& b) { return vmin_s32(a,b); }
1214template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vminq_s32(a,b); }
1215template<> EIGEN_STRONG_INLINE Packet2ui pmin<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vmin_u32(a,b); }
1216template<> EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vminq_u32(a,b); }
1217template<> EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
1218 return vcombine_s64(
1219 vdup_n_s64((std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
1220 vdup_n_s64((std::min)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
1221}
1222template<> EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1223 return vcombine_u64(
1224 vdup_n_u64((std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
1225 vdup_n_u64((std::min)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
1226}
1227
1228template<> EIGEN_STRONG_INLINE Packet2f pmax<Packet2f>(const Packet2f& a, const Packet2f& b) { return vmax_f32(a,b); }
1229template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); }
1230
1231#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1232// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
1233template<> EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxnmq_f32(a, b); }
1234template<> EIGEN_STRONG_INLINE Packet2f pmax<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) { return vmaxnm_f32(a, b); }
1235#endif
1236
1237template<> EIGEN_STRONG_INLINE Packet4f pmax<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) { return pmax<Packet4f>(a, b); }
1238
1239template<> EIGEN_STRONG_INLINE Packet2f pmax<PropagateNaN, Packet2f>(const Packet2f& a, const Packet2f& b) { return pmax<Packet2f>(a, b); }
1240
1247template<> EIGEN_STRONG_INLINE Packet8c pmax<Packet8c>(const Packet8c& a, const Packet8c& b) { return vmax_s8(a,b); }
1248template<> EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) { return vmaxq_s8(a,b); }
1255template<> EIGEN_STRONG_INLINE Packet8uc pmax<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vmax_u8(a,b); }
1256template<> EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vmaxq_u8(a,b); }
1257template<> EIGEN_STRONG_INLINE Packet4s pmax<Packet4s>(const Packet4s& a, const Packet4s& b) { return vmax_s16(a,b); }
1258template<> EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) { return vmaxq_s16(a,b); }
1259template<> EIGEN_STRONG_INLINE Packet4us pmax<Packet4us>(const Packet4us& a, const Packet4us& b) { return vmax_u16(a,b); }
1260template<> EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) { return vmaxq_u16(a,b); }
1261template<> EIGEN_STRONG_INLINE Packet2i pmax<Packet2i>(const Packet2i& a, const Packet2i& b) { return vmax_s32(a,b); }
1262template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); }
1263template<> EIGEN_STRONG_INLINE Packet2ui pmax<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vmax_u32(a,b); }
1264template<> EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vmaxq_u32(a,b); }
1265template<> EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
1266 return vcombine_s64(
1267 vdup_n_s64((std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
1268 vdup_n_s64((std::max)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
1269}
1270template<> EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1271 return vcombine_u64(
1272 vdup_n_u64((std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
1273 vdup_n_u64((std::max)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
1274}
1275
1296template<> EIGEN_STRONG_INLINE Packet8uc pcmp_le<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1297{ return vcle_u8(a,b); }
1304template<> EIGEN_STRONG_INLINE Packet4us pcmp_le<Packet4us>(const Packet4us& a, const Packet4us& b)
1305{ return vcle_u16(a,b); }
1312template<> EIGEN_STRONG_INLINE Packet2ui pcmp_le<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1313{ return vcle_u32(a,b); }
1316template<> EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b)
1317{
1318#if EIGEN_ARCH_ARM64
1319 return vreinterpretq_s64_u64(vcleq_s64(a,b));
1320#else
1321 return vcombine_s64(
1322 vdup_n_s64(vgetq_lane_s64(a, 0) <= vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1323 vdup_n_s64(vgetq_lane_s64(a, 1) <= vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1324#endif
1325}
1326template<> EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1327{
1328#if EIGEN_ARCH_ARM64
1329 return vcleq_u64(a,b);
1330#else
1331 return vcombine_u64(
1332 vdup_n_u64(vgetq_lane_u64(a, 0) <= vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1333 vdup_n_u64(vgetq_lane_u64(a, 1) <= vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1334#endif
1335}
1336
1357template<> EIGEN_STRONG_INLINE Packet8uc pcmp_lt<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1358{ return vclt_u8(a,b); }
1365template<> EIGEN_STRONG_INLINE Packet4us pcmp_lt<Packet4us>(const Packet4us& a, const Packet4us& b)
1366{ return vclt_u16(a,b); }
1373template<> EIGEN_STRONG_INLINE Packet2ui pcmp_lt<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1374{ return vclt_u32(a,b); }
1377template<> EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b)
1378{
1379#if EIGEN_ARCH_ARM64
1380 return vreinterpretq_s64_u64(vcltq_s64(a,b));
1381#else
1382 return vcombine_s64(
1383 vdup_n_s64(vgetq_lane_s64(a, 0) < vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1384 vdup_n_s64(vgetq_lane_s64(a, 1) < vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1385#endif
1386}
1387template<> EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1388{
1389#if EIGEN_ARCH_ARM64
1390 return vcltq_u64(a,b);
1391#else
1392 return vcombine_u64(
1393 vdup_n_u64(vgetq_lane_u64(a, 0) < vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1394 vdup_n_u64(vgetq_lane_u64(a, 1) < vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1395#endif
1396}
1397
1418template<> EIGEN_STRONG_INLINE Packet8uc pcmp_eq<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1419{ return vceq_u8(a,b); }
1426template<> EIGEN_STRONG_INLINE Packet4us pcmp_eq<Packet4us>(const Packet4us& a, const Packet4us& b)
1427{ return vceq_u16(a,b); }
1434template<> EIGEN_STRONG_INLINE Packet2ui pcmp_eq<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1435{ return vceq_u32(a,b); }
1438template<> EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b)
1439{
1440#if EIGEN_ARCH_ARM64
1441 return vreinterpretq_s64_u64(vceqq_s64(a,b));
1442#else
1443 return vcombine_s64(
1444 vdup_n_s64(vgetq_lane_s64(a, 0) == vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1445 vdup_n_s64(vgetq_lane_s64(a, 1) == vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1446#endif
1447}
1448template<> EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1449{
1450#if EIGEN_ARCH_ARM64
1451 return vceqq_u64(a,b);
1452#else
1453 return vcombine_u64(
1454 vdup_n_u64(vgetq_lane_u64(a, 0) == vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1455 vdup_n_u64(vgetq_lane_u64(a, 1) == vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1456#endif
1457}
1458
1463
1464// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
1469template<> EIGEN_STRONG_INLINE Packet4c pand<Packet4c>(const Packet4c& a, const Packet4c& b)
1470{ return a & b; }
1471template<> EIGEN_STRONG_INLINE Packet8c pand<Packet8c>(const Packet8c& a, const Packet8c& b)
1472{ return vand_s8(a,b); }
1473template<> EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(const Packet16c& a, const Packet16c& b)
1474{ return vandq_s8(a,b); }
1475template<> EIGEN_STRONG_INLINE Packet4uc pand<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1476{ return a & b; }
1477template<> EIGEN_STRONG_INLINE Packet8uc pand<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1478{ return vand_u8(a,b); }
1481template<> EIGEN_STRONG_INLINE Packet4s pand<Packet4s>(const Packet4s& a, const Packet4s& b) { return vand_s16(a,b); }
1482template<> EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(const Packet8s& a, const Packet8s& b) { return vandq_s16(a,b); }
1483template<> EIGEN_STRONG_INLINE Packet4us pand<Packet4us>(const Packet4us& a, const Packet4us& b)
1484{ return vand_u16(a,b); }
1485template<> EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b)
1486{ return vandq_u16(a,b); }
1487template<> EIGEN_STRONG_INLINE Packet2i pand<Packet2i>(const Packet2i& a, const Packet2i& b) { return vand_s32(a,b); }
1488template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vandq_s32(a,b); }
1489template<> EIGEN_STRONG_INLINE Packet2ui pand<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1490{ return vand_u32(a,b); }
1491template<> EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1492{ return vandq_u32(a,b); }
1493template<> EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) { return vandq_s64(a,b); }
1494template<> EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1495{ return vandq_u64(a,b); }
1496
1501template<> EIGEN_STRONG_INLINE Packet4c por<Packet4c>(const Packet4c& a, const Packet4c& b)
1502{ return a | b; }
1503template<> EIGEN_STRONG_INLINE Packet8c por<Packet8c>(const Packet8c& a, const Packet8c& b) { return vorr_s8(a,b); }
1504template<> EIGEN_STRONG_INLINE Packet16c por<Packet16c>(const Packet16c& a, const Packet16c& b)
1505{ return vorrq_s8(a,b); }
1506template<> EIGEN_STRONG_INLINE Packet4uc por<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1507{ return a | b; }
1508template<> EIGEN_STRONG_INLINE Packet8uc por<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1509{ return vorr_u8(a,b); }
1510template<> EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1511{ return vorrq_u8(a,b); }
1512template<> EIGEN_STRONG_INLINE Packet4s por<Packet4s>(const Packet4s& a, const Packet4s& b)
1513{ return vorr_s16(a,b); }
1514template<> EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b)
1515{ return vorrq_s16(a,b); }
1516template<> EIGEN_STRONG_INLINE Packet4us por<Packet4us>(const Packet4us& a, const Packet4us& b)
1517{ return vorr_u16(a,b); }
1518template<> EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b)
1519{ return vorrq_u16(a,b); }
1520template<> EIGEN_STRONG_INLINE Packet2i por<Packet2i>(const Packet2i& a, const Packet2i& b) { return vorr_s32(a,b); }
1521template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vorrq_s32(a,b); }
1522template<> EIGEN_STRONG_INLINE Packet2ui por<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1523{ return vorr_u32(a,b); }
1524template<> EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1525{ return vorrq_u32(a,b); }
1526template<> EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b)
1527{ return vorrq_s64(a,b); }
1528template<> EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1529{ return vorrq_u64(a,b); }
1530
1535template<> EIGEN_STRONG_INLINE Packet4c pxor<Packet4c>(const Packet4c& a, const Packet4c& b)
1536{ return a ^ b; }
1537template<> EIGEN_STRONG_INLINE Packet8c pxor<Packet8c>(const Packet8c& a, const Packet8c& b)
1538{ return veor_s8(a,b); }
1539template<> EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(const Packet16c& a, const Packet16c& b)
1540{ return veorq_s8(a,b); }
1541template<> EIGEN_STRONG_INLINE Packet4uc pxor<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1542{ return a ^ b; }
1543template<> EIGEN_STRONG_INLINE Packet8uc pxor<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1544{ return veor_u8(a,b); }
1547template<> EIGEN_STRONG_INLINE Packet4s pxor<Packet4s>(const Packet4s& a, const Packet4s& b) { return veor_s16(a,b); }
1548template<> EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(const Packet8s& a, const Packet8s& b) { return veorq_s16(a,b); }
1549template<> EIGEN_STRONG_INLINE Packet4us pxor<Packet4us>(const Packet4us& a, const Packet4us& b)
1550{ return veor_u16(a,b); }
1551template<> EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b)
1552{ return veorq_u16(a,b); }
1553template<> EIGEN_STRONG_INLINE Packet2i pxor<Packet2i>(const Packet2i& a, const Packet2i& b) { return veor_s32(a,b); }
1554template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return veorq_s32(a,b); }
1555template<> EIGEN_STRONG_INLINE Packet2ui pxor<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1556{ return veor_u32(a,b); }
1557template<> EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1558{ return veorq_u32(a,b); }
1559template<> EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b)
1560{ return veorq_s64(a,b); }
1561template<> EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1562{ return veorq_u64(a,b); }
1563
1568template<> EIGEN_STRONG_INLINE Packet4c pandnot<Packet4c>(const Packet4c& a, const Packet4c& b)
1569{ return a & ~b; }
1570template<> EIGEN_STRONG_INLINE Packet8c pandnot<Packet8c>(const Packet8c& a, const Packet8c& b) { return vbic_s8(a,b); }
1571template<> EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(const Packet16c& a, const Packet16c& b) { return vbicq_s8(a,b); }
1572template<> EIGEN_STRONG_INLINE Packet4uc pandnot<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1573{ return a & ~b; }
1574template<> EIGEN_STRONG_INLINE Packet8uc pandnot<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1575{ return vbic_u8(a,b); }
1578template<> EIGEN_STRONG_INLINE Packet4s pandnot<Packet4s>(const Packet4s& a, const Packet4s& b)
1579{ return vbic_s16(a,b); }
1580template<> EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(const Packet8s& a, const Packet8s& b)
1581{ return vbicq_s16(a,b); }
1582template<> EIGEN_STRONG_INLINE Packet4us pandnot<Packet4us>(const Packet4us& a, const Packet4us& b)
1583{ return vbic_u16(a,b); }
1586template<> EIGEN_STRONG_INLINE Packet2i pandnot<Packet2i>(const Packet2i& a, const Packet2i& b)
1587{ return vbic_s32(a,b); }
1588template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b)
1589{ return vbicq_s32(a,b); }
1590template<> EIGEN_STRONG_INLINE Packet2ui pandnot<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1591{ return vbic_u32(a,b); }
1594template<> EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b)
1595{ return vbicq_s64(a,b); }
1598
1599
1602template<int N> EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(Packet8c a) { return vshr_n_s8(a,N); }
1603template<int N> EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(Packet16c a) { return vshrq_n_s8(a,N); }
1606template<int N> EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(Packet8uc a) { return vshr_n_u8(a,N); }
1607template<int N> EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(Packet16uc a) { return vshrq_n_u8(a,N); }
1608template<int N> EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(Packet4s a) { return vshr_n_s16(a,N); }
1609template<int N> EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) { return vshrq_n_s16(a,N); }
1610template<int N> EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(Packet4us a) { return vshr_n_u16(a,N); }
1611template<int N> EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(Packet8us a) { return vshrq_n_u16(a,N); }
1612template<int N> EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(Packet2i a) { return vshr_n_s32(a,N); }
1613template<int N> EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(Packet4i a) { return vshrq_n_s32(a,N); }
1614template<int N> EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(Packet2ui a) { return vshr_n_u32(a,N); }
1615template<int N> EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(Packet4ui a) { return vshrq_n_u32(a,N); }
1616template<int N> EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(Packet2l a) { return vshrq_n_s64(a,N); }
1617template<int N> EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(Packet2ul a) { return vshrq_n_u64(a,N); }
1618
1627template<int N> EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(Packet8uc a) { return vshr_n_u8(a,N); }
1628template<int N> EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(Packet16uc a) { return vshrq_n_u8(a,N); }
1633template<int N> EIGEN_STRONG_INLINE Packet4us plogical_shift_right(Packet4us a) { return vshr_n_u16(a,N); }
1634template<int N> EIGEN_STRONG_INLINE Packet8us plogical_shift_right(Packet8us a) { return vshrq_n_u16(a,N); }
1639template<int N> EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(Packet2ui a) { return vshr_n_u32(a,N); }
1640template<int N> EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(Packet4ui a) { return vshrq_n_u32(a,N); }
1643template<int N> EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(Packet2ul a) { return vshrq_n_u64(a,N); }
1644
1647template<int N> EIGEN_STRONG_INLINE Packet8c plogical_shift_left(Packet8c a) { return vshl_n_s8(a,N); }
1648template<int N> EIGEN_STRONG_INLINE Packet16c plogical_shift_left(Packet16c a) { return vshlq_n_s8(a,N); }
1651template<int N> EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(Packet8uc a) { return vshl_n_u8(a,N); }
1652template<int N> EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(Packet16uc a) { return vshlq_n_u8(a,N); }
1653template<int N> EIGEN_STRONG_INLINE Packet4s plogical_shift_left(Packet4s a) { return vshl_n_s16(a,N); }
1654template<int N> EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) { return vshlq_n_s16(a,N); }
1655template<int N> EIGEN_STRONG_INLINE Packet4us plogical_shift_left(Packet4us a) { return vshl_n_u16(a,N); }
1656template<int N> EIGEN_STRONG_INLINE Packet8us plogical_shift_left(Packet8us a) { return vshlq_n_u16(a,N); }
1657template<int N> EIGEN_STRONG_INLINE Packet2i plogical_shift_left(Packet2i a) { return vshl_n_s32(a,N); }
1658template<int N> EIGEN_STRONG_INLINE Packet4i plogical_shift_left(Packet4i a) { return vshlq_n_s32(a,N); }
1659template<int N> EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(Packet2ui a) { return vshl_n_u32(a,N); }
1660template<int N> EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(Packet4ui a) { return vshlq_n_u32(a,N); }
1661template<int N> EIGEN_STRONG_INLINE Packet2l plogical_shift_left(Packet2l a) { return vshlq_n_s64(a,N); }
1662template<int N> EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(Packet2ul a) { return vshlq_n_u64(a,N); }
1663
1668template<> EIGEN_STRONG_INLINE Packet4c pload<Packet4c>(const int8_t* from)
1669{
1670 Packet4c res;
1671 memcpy(&res, from, sizeof(Packet4c));
1672 return res;
1673}
1678template<> EIGEN_STRONG_INLINE Packet4uc pload<Packet4uc>(const uint8_t* from)
1679{
1680 Packet4uc res;
1681 memcpy(&res, from, sizeof(Packet4uc));
1682 return res;
1683}
1708
1713template<> EIGEN_STRONG_INLINE Packet4c ploadu<Packet4c>(const int8_t* from)
1714{
1715 Packet4c res;
1716 memcpy(&res, from, sizeof(Packet4c));
1717 return res;
1718}
1723template<> EIGEN_STRONG_INLINE Packet4uc ploadu<Packet4uc>(const uint8_t* from)
1724{
1725 Packet4uc res;
1726 memcpy(&res, from, sizeof(Packet4uc));
1727 return res;
1728}
1753
1754template<> EIGEN_STRONG_INLINE Packet2f ploaddup<Packet2f>(const float* from)
1755{ return vld1_dup_f32(from); }
1763template<> EIGEN_STRONG_INLINE Packet8c ploaddup<Packet8c>(const int8_t* from)
1764{
1765 const int8x8_t a = vld1_s8(from);
1766 return vzip_s8(a,a).val[0];
1767}
1768template<> EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const int8_t* from)
1769{
1770 const int8x8_t a = vld1_s8(from);
1771 const int8x8x2_t b = vzip_s8(a,a);
1772 return vcombine_s8(b.val[0], b.val[1]);
1773}
1779template<> EIGEN_STRONG_INLINE Packet8uc ploaddup<Packet8uc>(const uint8_t* from)
1780{
1781 const uint8x8_t a = vld1_u8(from);
1782 return vzip_u8(a,a).val[0];
1783}
1784template<> EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(const uint8_t* from)
1785{
1786 const uint8x8_t a = vld1_u8(from);
1787 const uint8x8x2_t b = vzip_u8(a,a);
1788 return vcombine_u8(b.val[0], b.val[1]);
1789}
1795template<> EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(const int16_t* from)
1796{
1797 const int16x4_t a = vld1_s16(from);
1798 const int16x4x2_t b = vzip_s16(a,a);
1799 return vcombine_s16(b.val[0], b.val[1]);
1800}
1806template<> EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(const uint16_t* from)
1807{
1808 const uint16x4_t a = vld1_u16(from);
1809 const uint16x4x2_t b = vzip_u16(a,a);
1810 return vcombine_u16(b.val[0], b.val[1]);
1811}
1812template<> EIGEN_STRONG_INLINE Packet2i ploaddup<Packet2i>(const int32_t* from)
1813{ return vld1_dup_s32(from); }
1816template<> EIGEN_STRONG_INLINE Packet2ui ploaddup<Packet2ui>(const uint32_t* from)
1817{ return vld1_dup_u32(from); }
1820template<> EIGEN_STRONG_INLINE Packet2l ploaddup<Packet2l>(const int64_t* from)
1821{ return vld1q_dup_s64(from); }
1822template<> EIGEN_STRONG_INLINE Packet2ul ploaddup<Packet2ul>(const uint64_t* from)
1823{ return vld1q_dup_u64(from); }
1824
1825template<> EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(const float* from) { return vld1q_dup_f32(from); }
1866template<> EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(const int32_t* from) { return vld1q_dup_s32(from); }
1867template<> EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(const uint32_t* from) { return vld1q_dup_u32(from); }
1868
1873template<> EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet4c& from)
1874{ memcpy(to, &from, sizeof(from)); }
1879template<> EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet4uc& from)
1880{ memcpy(to, &from, sizeof(from)); }
1905
1910template<> EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet4c& from)
1911{ memcpy(to, &from, sizeof(from)); }
1916template<> EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet4uc& from)
1917{ memcpy(to, &from, sizeof(from)); }
1942
1943template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather<float, Packet2f>(const float* from, Index stride)
1944{
1945 Packet2f res = vld1_dup_f32(from);
1946 res = vld1_lane_f32(from + 1*stride, res, 1);
1947 return res;
1948}
1949template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride)
1950{
1951 Packet4f res = vld1q_dup_f32(from);
1952 res = vld1q_lane_f32(from + 1*stride, res, 1);
1953 res = vld1q_lane_f32(from + 2*stride, res, 2);
1954 res = vld1q_lane_f32(from + 3*stride, res, 3);
1955 return res;
1956}
1957template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather<int8_t, Packet4c>(const int8_t* from, Index stride)
1958{
1959 Packet4c res;
1960 for (int i = 0; i != 4; i++)
1961 reinterpret_cast<int8_t*>(&res)[i] = *(from + i * stride);
1962 return res;
1963}
1964template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather<int8_t, Packet8c>(const int8_t* from, Index stride)
1965{
1966 Packet8c res = vld1_dup_s8(from);
1967 res = vld1_lane_s8(from + 1*stride, res, 1);
1968 res = vld1_lane_s8(from + 2*stride, res, 2);
1969 res = vld1_lane_s8(from + 3*stride, res, 3);
1970 res = vld1_lane_s8(from + 4*stride, res, 4);
1971 res = vld1_lane_s8(from + 5*stride, res, 5);
1972 res = vld1_lane_s8(from + 6*stride, res, 6);
1973 res = vld1_lane_s8(from + 7*stride, res, 7);
1974 return res;
1975}
1976template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(const int8_t* from, Index stride)
1977{
1978 Packet16c res = vld1q_dup_s8(from);
1979 res = vld1q_lane_s8(from + 1*stride, res, 1);
1980 res = vld1q_lane_s8(from + 2*stride, res, 2);
1981 res = vld1q_lane_s8(from + 3*stride, res, 3);
1982 res = vld1q_lane_s8(from + 4*stride, res, 4);
1983 res = vld1q_lane_s8(from + 5*stride, res, 5);
1984 res = vld1q_lane_s8(from + 6*stride, res, 6);
1985 res = vld1q_lane_s8(from + 7*stride, res, 7);
1986 res = vld1q_lane_s8(from + 8*stride, res, 8);
1987 res = vld1q_lane_s8(from + 9*stride, res, 9);
1988 res = vld1q_lane_s8(from + 10*stride, res, 10);
1989 res = vld1q_lane_s8(from + 11*stride, res, 11);
1990 res = vld1q_lane_s8(from + 12*stride, res, 12);
1991 res = vld1q_lane_s8(from + 13*stride, res, 13);
1992 res = vld1q_lane_s8(from + 14*stride, res, 14);
1993 res = vld1q_lane_s8(from + 15*stride, res, 15);
1994 return res;
1995}
1996template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather<uint8_t, Packet4uc>(const uint8_t* from, Index stride)
1997{
1998 Packet4uc res;
1999 for (int i = 0; i != 4; i++)
2000 reinterpret_cast<uint8_t*>(&res)[i] = *(from + i * stride);
2001 return res;
2002}
2003template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather<uint8_t, Packet8uc>(const uint8_t* from, Index stride)
2004{
2005 Packet8uc res = vld1_dup_u8(from);
2006 res = vld1_lane_u8(from + 1*stride, res, 1);
2007 res = vld1_lane_u8(from + 2*stride, res, 2);
2008 res = vld1_lane_u8(from + 3*stride, res, 3);
2009 res = vld1_lane_u8(from + 4*stride, res, 4);
2010 res = vld1_lane_u8(from + 5*stride, res, 5);
2011 res = vld1_lane_u8(from + 6*stride, res, 6);
2012 res = vld1_lane_u8(from + 7*stride, res, 7);
2013 return res;
2014}
2015template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(const uint8_t* from, Index stride)
2016{
2017 Packet16uc res = vld1q_dup_u8(from);
2018 res = vld1q_lane_u8(from + 1*stride, res, 1);
2019 res = vld1q_lane_u8(from + 2*stride, res, 2);
2020 res = vld1q_lane_u8(from + 3*stride, res, 3);
2021 res = vld1q_lane_u8(from + 4*stride, res, 4);
2022 res = vld1q_lane_u8(from + 5*stride, res, 5);
2023 res = vld1q_lane_u8(from + 6*stride, res, 6);
2024 res = vld1q_lane_u8(from + 7*stride, res, 7);
2025 res = vld1q_lane_u8(from + 8*stride, res, 8);
2026 res = vld1q_lane_u8(from + 9*stride, res, 9);
2027 res = vld1q_lane_u8(from + 10*stride, res, 10);
2028 res = vld1q_lane_u8(from + 11*stride, res, 11);
2029 res = vld1q_lane_u8(from + 12*stride, res, 12);
2030 res = vld1q_lane_u8(from + 13*stride, res, 13);
2031 res = vld1q_lane_u8(from + 14*stride, res, 14);
2032 res = vld1q_lane_u8(from + 15*stride, res, 15);
2033 return res;
2034}
2035template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather<int16_t, Packet4s>(const int16_t* from, Index stride)
2036{
2037 Packet4s res = vld1_dup_s16(from);
2038 res = vld1_lane_s16(from + 1*stride, res, 1);
2039 res = vld1_lane_s16(from + 2*stride, res, 2);
2040 res = vld1_lane_s16(from + 3*stride, res, 3);
2041 return res;
2042}
2043template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(const int16_t* from, Index stride)
2044{
2045 Packet8s res = vld1q_dup_s16(from);
2046 res = vld1q_lane_s16(from + 1*stride, res, 1);
2047 res = vld1q_lane_s16(from + 2*stride, res, 2);
2048 res = vld1q_lane_s16(from + 3*stride, res, 3);
2049 res = vld1q_lane_s16(from + 4*stride, res, 4);
2050 res = vld1q_lane_s16(from + 5*stride, res, 5);
2051 res = vld1q_lane_s16(from + 6*stride, res, 6);
2052 res = vld1q_lane_s16(from + 7*stride, res, 7);
2053 return res;
2054}
2055template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather<uint16_t, Packet4us>(const uint16_t* from, Index stride)
2056{
2057 Packet4us res = vld1_dup_u16(from);
2058 res = vld1_lane_u16(from + 1*stride, res, 1);
2059 res = vld1_lane_u16(from + 2*stride, res, 2);
2060 res = vld1_lane_u16(from + 3*stride, res, 3);
2061 return res;
2062}
2063template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(const uint16_t* from, Index stride)
2064{
2065 Packet8us res = vld1q_dup_u16(from);
2066 res = vld1q_lane_u16(from + 1*stride, res, 1);
2067 res = vld1q_lane_u16(from + 2*stride, res, 2);
2068 res = vld1q_lane_u16(from + 3*stride, res, 3);
2069 res = vld1q_lane_u16(from + 4*stride, res, 4);
2070 res = vld1q_lane_u16(from + 5*stride, res, 5);
2071 res = vld1q_lane_u16(from + 6*stride, res, 6);
2072 res = vld1q_lane_u16(from + 7*stride, res, 7);
2073 return res;
2074}
2075template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather<int32_t, Packet2i>(const int32_t* from, Index stride)
2076{
2077 Packet2i res = vld1_dup_s32(from);
2078 res = vld1_lane_s32(from + 1*stride, res, 1);
2079 return res;
2080}
2081template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride)
2082{
2083 Packet4i res = vld1q_dup_s32(from);
2084 res = vld1q_lane_s32(from + 1*stride, res, 1);
2085 res = vld1q_lane_s32(from + 2*stride, res, 2);
2086 res = vld1q_lane_s32(from + 3*stride, res, 3);
2087 return res;
2088}
2089template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather<uint32_t, Packet2ui>(const uint32_t* from, Index stride)
2090{
2091 Packet2ui res = vld1_dup_u32(from);
2092 res = vld1_lane_u32(from + 1*stride, res, 1);
2093 return res;
2094}
2095template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(const uint32_t* from, Index stride)
2096{
2097 Packet4ui res = vld1q_dup_u32(from);
2098 res = vld1q_lane_u32(from + 1*stride, res, 1);
2099 res = vld1q_lane_u32(from + 2*stride, res, 2);
2100 res = vld1q_lane_u32(from + 3*stride, res, 3);
2101 return res;
2102}
2103template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(const int64_t* from, Index stride)
2104{
2105 Packet2l res = vld1q_dup_s64(from);
2106 res = vld1q_lane_s64(from + 1*stride, res, 1);
2107 return res;
2108}
2109template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(const uint64_t* from, Index stride)
2110{
2111 Packet2ul res = vld1q_dup_u64(from);
2112 res = vld1q_lane_u64(from + 1*stride, res, 1);
2113 return res;
2114}
2115
2116template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet2f>(float* to, const Packet2f& from, Index stride)
2117{
2118 vst1_lane_f32(to + stride*0, from, 0);
2119 vst1_lane_f32(to + stride*1, from, 1);
2120}
2121template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
2122{
2123 vst1q_lane_f32(to + stride*0, from, 0);
2124 vst1q_lane_f32(to + stride*1, from, 1);
2125 vst1q_lane_f32(to + stride*2, from, 2);
2126 vst1q_lane_f32(to + stride*3, from, 3);
2127}
2128template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet4c>(int8_t* to, const Packet4c& from, Index stride)
2129{
2130 for (int i = 0; i != 4; i++)
2131 *(to + i * stride) = reinterpret_cast<const int8_t*>(&from)[i];
2132}
2133template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet8c>(int8_t* to, const Packet8c& from, Index stride)
2134{
2135 vst1_lane_s8(to + stride*0, from, 0);
2136 vst1_lane_s8(to + stride*1, from, 1);
2137 vst1_lane_s8(to + stride*2, from, 2);
2138 vst1_lane_s8(to + stride*3, from, 3);
2139 vst1_lane_s8(to + stride*4, from, 4);
2140 vst1_lane_s8(to + stride*5, from, 5);
2141 vst1_lane_s8(to + stride*6, from, 6);
2142 vst1_lane_s8(to + stride*7, from, 7);
2143}
2144template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet16c>(int8_t* to, const Packet16c& from, Index stride)
2145{
2146 vst1q_lane_s8(to + stride*0, from, 0);
2147 vst1q_lane_s8(to + stride*1, from, 1);
2148 vst1q_lane_s8(to + stride*2, from, 2);
2149 vst1q_lane_s8(to + stride*3, from, 3);
2150 vst1q_lane_s8(to + stride*4, from, 4);
2151 vst1q_lane_s8(to + stride*5, from, 5);
2152 vst1q_lane_s8(to + stride*6, from, 6);
2153 vst1q_lane_s8(to + stride*7, from, 7);
2154 vst1q_lane_s8(to + stride*8, from, 8);
2155 vst1q_lane_s8(to + stride*9, from, 9);
2156 vst1q_lane_s8(to + stride*10, from, 10);
2157 vst1q_lane_s8(to + stride*11, from, 11);
2158 vst1q_lane_s8(to + stride*12, from, 12);
2159 vst1q_lane_s8(to + stride*13, from, 13);
2160 vst1q_lane_s8(to + stride*14, from, 14);
2161 vst1q_lane_s8(to + stride*15, from, 15);
2162}
2163template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet4uc>(uint8_t* to, const Packet4uc& from, Index stride)
2164{
2165 for (int i = 0; i != 4; i++)
2166 *(to + i * stride) = reinterpret_cast<const uint8_t*>(&from)[i];
2167}
2168template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet8uc>(uint8_t* to, const Packet8uc& from, Index stride)
2169{
2170 vst1_lane_u8(to + stride*0, from, 0);
2171 vst1_lane_u8(to + stride*1, from, 1);
2172 vst1_lane_u8(to + stride*2, from, 2);
2173 vst1_lane_u8(to + stride*3, from, 3);
2174 vst1_lane_u8(to + stride*4, from, 4);
2175 vst1_lane_u8(to + stride*5, from, 5);
2176 vst1_lane_u8(to + stride*6, from, 6);
2177 vst1_lane_u8(to + stride*7, from, 7);
2178}
2179template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet16uc>(uint8_t* to, const Packet16uc& from, Index stride)
2180{
2181 vst1q_lane_u8(to + stride*0, from, 0);
2182 vst1q_lane_u8(to + stride*1, from, 1);
2183 vst1q_lane_u8(to + stride*2, from, 2);
2184 vst1q_lane_u8(to + stride*3, from, 3);
2185 vst1q_lane_u8(to + stride*4, from, 4);
2186 vst1q_lane_u8(to + stride*5, from, 5);
2187 vst1q_lane_u8(to + stride*6, from, 6);
2188 vst1q_lane_u8(to + stride*7, from, 7);
2189 vst1q_lane_u8(to + stride*8, from, 8);
2190 vst1q_lane_u8(to + stride*9, from, 9);
2191 vst1q_lane_u8(to + stride*10, from, 10);
2192 vst1q_lane_u8(to + stride*11, from, 11);
2193 vst1q_lane_u8(to + stride*12, from, 12);
2194 vst1q_lane_u8(to + stride*13, from, 13);
2195 vst1q_lane_u8(to + stride*14, from, 14);
2196 vst1q_lane_u8(to + stride*15, from, 15);
2197}
2198template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet4s>(int16_t* to, const Packet4s& from, Index stride)
2199{
2200 vst1_lane_s16(to + stride*0, from, 0);
2201 vst1_lane_s16(to + stride*1, from, 1);
2202 vst1_lane_s16(to + stride*2, from, 2);
2203 vst1_lane_s16(to + stride*3, from, 3);
2204}
2205template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet8s>(int16_t* to, const Packet8s& from, Index stride)
2206{
2207 vst1q_lane_s16(to + stride*0, from, 0);
2208 vst1q_lane_s16(to + stride*1, from, 1);
2209 vst1q_lane_s16(to + stride*2, from, 2);
2210 vst1q_lane_s16(to + stride*3, from, 3);
2211 vst1q_lane_s16(to + stride*4, from, 4);
2212 vst1q_lane_s16(to + stride*5, from, 5);
2213 vst1q_lane_s16(to + stride*6, from, 6);
2214 vst1q_lane_s16(to + stride*7, from, 7);
2215}
2216template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet4us>(uint16_t* to, const Packet4us& from, Index stride)
2217{
2218 vst1_lane_u16(to + stride*0, from, 0);
2219 vst1_lane_u16(to + stride*1, from, 1);
2220 vst1_lane_u16(to + stride*2, from, 2);
2221 vst1_lane_u16(to + stride*3, from, 3);
2222}
2223template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet8us>(uint16_t* to, const Packet8us& from, Index stride)
2224{
2225 vst1q_lane_u16(to + stride*0, from, 0);
2226 vst1q_lane_u16(to + stride*1, from, 1);
2227 vst1q_lane_u16(to + stride*2, from, 2);
2228 vst1q_lane_u16(to + stride*3, from, 3);
2229 vst1q_lane_u16(to + stride*4, from, 4);
2230 vst1q_lane_u16(to + stride*5, from, 5);
2231 vst1q_lane_u16(to + stride*6, from, 6);
2232 vst1q_lane_u16(to + stride*7, from, 7);
2233}
2234template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet2i>(int32_t* to, const Packet2i& from, Index stride)
2235{
2236 vst1_lane_s32(to + stride*0, from, 0);
2237 vst1_lane_s32(to + stride*1, from, 1);
2238}
2239template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from, Index stride)
2240{
2241 vst1q_lane_s32(to + stride*0, from, 0);
2242 vst1q_lane_s32(to + stride*1, from, 1);
2243 vst1q_lane_s32(to + stride*2, from, 2);
2244 vst1q_lane_s32(to + stride*3, from, 3);
2245}
2246template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet2ui>(uint32_t* to, const Packet2ui& from, Index stride)
2247{
2248 vst1_lane_u32(to + stride*0, from, 0);
2249 vst1_lane_u32(to + stride*1, from, 1);
2250}
2251template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from, Index stride)
2252{
2253 vst1q_lane_u32(to + stride*0, from, 0);
2254 vst1q_lane_u32(to + stride*1, from, 1);
2255 vst1q_lane_u32(to + stride*2, from, 2);
2256 vst1q_lane_u32(to + stride*3, from, 3);
2257}
2258template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int64_t, Packet2l>(int64_t* to, const Packet2l& from, Index stride)
2259{
2260 vst1q_lane_s64(to + stride*0, from, 0);
2261 vst1q_lane_s64(to + stride*1, from, 1);
2262}
2263template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint64_t, Packet2ul>(uint64_t* to, const Packet2ul& from, Index stride)
2264{
2265 vst1q_lane_u64(to + stride*0, from, 0);
2266 vst1q_lane_u64(to + stride*1, from, 1);
2267}
2268
2269template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_ARM_PREFETCH(addr); }
2270template<> EIGEN_STRONG_INLINE void prefetch<int8_t>(const int8_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2271template<> EIGEN_STRONG_INLINE void prefetch<uint8_t>(const uint8_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2272template<> EIGEN_STRONG_INLINE void prefetch<int16_t>(const int16_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2273template<> EIGEN_STRONG_INLINE void prefetch<uint16_t>(const uint16_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2274template<> EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2275template<> EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2276template<> EIGEN_STRONG_INLINE void prefetch<int64_t>(const int64_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2277template<> EIGEN_STRONG_INLINE void prefetch<uint64_t>(const uint64_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2278
2279template<> EIGEN_STRONG_INLINE float pfirst<Packet2f>(const Packet2f& a) { return vget_lane_f32(a,0); }
2280template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return vgetq_lane_f32(a,0); }
2281template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet4c>(const Packet4c& a) { return static_cast<int8_t>(a & 0xff); }
2282template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet8c>(const Packet8c& a) { return vget_lane_s8(a,0); }
2283template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(const Packet16c& a) { return vgetq_lane_s8(a,0); }
2284template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet4uc>(const Packet4uc& a) { return static_cast<uint8_t>(a & 0xff); }
2285template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet8uc>(const Packet8uc& a) { return vget_lane_u8(a,0); }
2286template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(const Packet16uc& a) { return vgetq_lane_u8(a,0); }
2287template<> EIGEN_STRONG_INLINE int16_t pfirst<Packet4s>(const Packet4s& a) { return vget_lane_s16(a,0); }
2288template<> EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(const Packet8s& a) { return vgetq_lane_s16(a,0); }
2289template<> EIGEN_STRONG_INLINE uint16_t pfirst<Packet4us>(const Packet4us& a) { return vget_lane_u16(a,0); }
2290template<> EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(const Packet8us& a) { return vgetq_lane_u16(a,0); }
2291template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet2i>(const Packet2i& a) { return vget_lane_s32(a,0); }
2292template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) { return vgetq_lane_s32(a,0); }
2293template<> EIGEN_STRONG_INLINE uint32_t pfirst<Packet2ui>(const Packet2ui& a) { return vget_lane_u32(a,0); }
2294template<> EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(const Packet4ui& a) { return vgetq_lane_u32(a,0); }
2295template<> EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(const Packet2l& a) { return vgetq_lane_s64(a,0); }
2296template<> EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(const Packet2ul& a) { return vgetq_lane_u64(a,0); }
2297
2298template<> EIGEN_STRONG_INLINE Packet2f preverse(const Packet2f& a) { return vrev64_f32(a); }
2306template<> EIGEN_STRONG_INLINE Packet8c preverse(const Packet8c& a) { return vrev64_s8(a); }
2314template<> EIGEN_STRONG_INLINE Packet8uc preverse(const Packet8uc& a) { return vrev64_u8(a); }
2320template<> EIGEN_STRONG_INLINE Packet4s preverse(const Packet4s& a) { return vrev64_s16(a); }
2326template<> EIGEN_STRONG_INLINE Packet4us preverse(const Packet4us& a) { return vrev64_u16(a); }
2332template<> EIGEN_STRONG_INLINE Packet2i preverse(const Packet2i& a) { return vrev64_s32(a); }
2338template<> EIGEN_STRONG_INLINE Packet2ui preverse(const Packet2ui& a) { return vrev64_u32(a); }
2348
2349template<> EIGEN_STRONG_INLINE Packet2f pabs(const Packet2f& a) { return vabs_f32(a); }
2350template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); }
2353template<> EIGEN_STRONG_INLINE Packet8c pabs(const Packet8c& a) { return vabs_s8(a); }
2354template<> EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) { return vabsq_s8(a); }
2355template<> EIGEN_STRONG_INLINE Packet4uc pabs(const Packet4uc& a) { return a; }
2356template<> EIGEN_STRONG_INLINE Packet8uc pabs(const Packet8uc& a) { return a; }
2357template<> EIGEN_STRONG_INLINE Packet16uc pabs(const Packet16uc& a) { return a; }
2358template<> EIGEN_STRONG_INLINE Packet4s pabs(const Packet4s& a) { return vabs_s16(a); }
2359template<> EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) { return vabsq_s16(a); }
2360template<> EIGEN_STRONG_INLINE Packet4us pabs(const Packet4us& a) { return a; }
2361template<> EIGEN_STRONG_INLINE Packet8us pabs(const Packet8us& a) { return a; }
2362template<> EIGEN_STRONG_INLINE Packet2i pabs(const Packet2i& a) { return vabs_s32(a); }
2363template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); }
2364template<> EIGEN_STRONG_INLINE Packet2ui pabs(const Packet2ui& a) { return a; }
2365template<> EIGEN_STRONG_INLINE Packet4ui pabs(const Packet4ui& a) { return a; }
2366template<> EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) {
2367#if EIGEN_ARCH_ARM64
2368 return vabsq_s64(a);
2369#else
2370 return vcombine_s64(
2371 vdup_n_s64((std::abs)(vgetq_lane_s64(a, 0))),
2372 vdup_n_s64((std::abs)(vgetq_lane_s64(a, 1))));
2373#endif
2374}
2375template<> EIGEN_STRONG_INLINE Packet2ul pabs(const Packet2ul& a) { return a; }
2376
2381
2386
2387template<> EIGEN_STRONG_INLINE float predux<Packet2f>(const Packet2f& a) { return vget_lane_f32(vpadd_f32(a,a), 0); }
2388template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
2389{
2390 const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a));
2391 return vget_lane_f32(vpadd_f32(sum, sum), 0);
2392}
2393template<> EIGEN_STRONG_INLINE int8_t predux<Packet4c>(const Packet4c& a)
2394{
2395 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2396 int8x8_t sum = vpadd_s8(a_dup, a_dup);
2397 sum = vpadd_s8(sum, sum);
2398 return vget_lane_s8(sum, 0);
2399}
2400template<> EIGEN_STRONG_INLINE int8_t predux<Packet8c>(const Packet8c& a)
2401{
2402 int8x8_t sum = vpadd_s8(a,a);
2403 sum = vpadd_s8(sum, sum);
2404 sum = vpadd_s8(sum, sum);
2405 return vget_lane_s8(sum, 0);
2406}
2407template<> EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a)
2408{
2409 int8x8_t sum = vadd_s8(vget_low_s8(a), vget_high_s8(a));
2410 sum = vpadd_s8(sum, sum);
2411 sum = vpadd_s8(sum, sum);
2412 sum = vpadd_s8(sum, sum);
2413 return vget_lane_s8(sum, 0);
2414}
2415template<> EIGEN_STRONG_INLINE uint8_t predux<Packet4uc>(const Packet4uc& a)
2416{
2417 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2418 uint8x8_t sum = vpadd_u8(a_dup, a_dup);
2419 sum = vpadd_u8(sum, sum);
2420 return vget_lane_u8(sum, 0);
2421}
2422template<> EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(const Packet8uc& a)
2423{
2424 uint8x8_t sum = vpadd_u8(a,a);
2425 sum = vpadd_u8(sum, sum);
2426 sum = vpadd_u8(sum, sum);
2427 return vget_lane_u8(sum, 0);
2428}
2429template<> EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a)
2430{
2431 uint8x8_t sum = vadd_u8(vget_low_u8(a), vget_high_u8(a));
2432 sum = vpadd_u8(sum, sum);
2433 sum = vpadd_u8(sum, sum);
2434 sum = vpadd_u8(sum, sum);
2435 return vget_lane_u8(sum, 0);
2436}
2437template<> EIGEN_STRONG_INLINE int16_t predux<Packet4s>(const Packet4s& a)
2438{
2439 const int16x4_t sum = vpadd_s16(a,a);
2440 return vget_lane_s16(vpadd_s16(sum, sum), 0);
2441}
2442template<> EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a)
2443{
2444 int16x4_t sum = vadd_s16(vget_low_s16(a), vget_high_s16(a));
2445 sum = vpadd_s16(sum, sum);
2446 sum = vpadd_s16(sum, sum);
2447 return vget_lane_s16(sum, 0);
2448}
2449template<> EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(const Packet4us& a)
2450{
2451 const uint16x4_t sum = vpadd_u16(a,a);
2452 return vget_lane_u16(vpadd_u16(sum, sum), 0);
2453}
2454template<> EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a)
2455{
2456 uint16x4_t sum = vadd_u16(vget_low_u16(a), vget_high_u16(a));
2457 sum = vpadd_u16(sum, sum);
2458 sum = vpadd_u16(sum, sum);
2459 return vget_lane_u16(sum, 0);
2460}
2461template<> EIGEN_STRONG_INLINE int32_t predux<Packet2i>(const Packet2i& a) { return vget_lane_s32(vpadd_s32(a,a), 0); }
2462template<> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a)
2463{
2464 const int32x2_t sum = vadd_s32(vget_low_s32(a), vget_high_s32(a));
2465 return vget_lane_s32(vpadd_s32(sum, sum), 0);
2466}
2467template<> EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(const Packet2ui& a) { return vget_lane_u32(vpadd_u32(a,a), 0); }
2468template<> EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a)
2469{
2470 const uint32x2_t sum = vadd_u32(vget_low_u32(a), vget_high_u32(a));
2471 return vget_lane_u32(vpadd_u32(sum, sum), 0);
2472}
2473template<> EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a)
2474{ return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1); }
2475template<> EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a)
2476{ return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1); }
2477
2496
2497// Other reduction functions:
2498// mul
2499template<> EIGEN_STRONG_INLINE float predux_mul<Packet2f>(const Packet2f& a)
2500{ return vget_lane_f32(a, 0) * vget_lane_f32(a, 1); }
2509template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet8c>(const Packet8c& a)
2510{
2511 int8x8_t prod = vmul_s8(a, vrev16_s8(a));
2512 prod = vmul_s8(prod, vrev32_s8(prod));
2513 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4);
2514}
2523template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet8uc>(const Packet8uc& a)
2524{
2525 uint8x8_t prod = vmul_u8(a, vrev16_u8(a));
2526 prod = vmul_u8(prod, vrev32_u8(prod));
2527 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4);
2528}
2531template<> EIGEN_STRONG_INLINE int16_t predux_mul<Packet4s>(const Packet4s& a)
2532{
2533 const int16x4_t prod = vmul_s16(a, vrev32_s16(a));
2534 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
2535}
2536template<> EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(const Packet8s& a)
2537{
2538 int16x4_t prod;
2539
2540 // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8|
2541 prod = vmul_s16(vget_low_s16(a), vget_high_s16(a));
2542 // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8|
2543 prod = vmul_s16(prod, vrev32_s16(prod));
2544 // Multiply |a1*a5*a2*a6*a3*a7*a4*a8|
2545 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
2546}
2547template<> EIGEN_STRONG_INLINE uint16_t predux_mul<Packet4us>(const Packet4us& a)
2548{
2549 const uint16x4_t prod = vmul_u16(a, vrev32_u16(a));
2550 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
2551}
2552template<> EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(const Packet8us& a)
2553{
2554 uint16x4_t prod;
2555
2556 // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8|
2557 prod = vmul_u16(vget_low_u16(a), vget_high_u16(a));
2558 // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8|
2559 prod = vmul_u16(prod, vrev32_u16(prod));
2560 // Multiply |a1*a5*a2*a6*a3*a7*a4*a8|
2561 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
2562}
2563template<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet2i>(const Packet2i& a)
2564{ return vget_lane_s32(a, 0) * vget_lane_s32(a, 1); }
2567template<> EIGEN_STRONG_INLINE uint32_t predux_mul<Packet2ui>(const Packet2ui& a)
2568{ return vget_lane_u32(a, 0) * vget_lane_u32(a, 1); }
2571template<> EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(const Packet2l& a)
2572{ return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1); }
2573template<> EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(const Packet2ul& a)
2574{ return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1); }
2575
2576// min
2577template<> EIGEN_STRONG_INLINE float predux_min<Packet2f>(const Packet2f& a)
2578{ return vget_lane_f32(vpmin_f32(a,a), 0); }
2579template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
2580{
2581 const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a));
2582 return vget_lane_f32(vpmin_f32(min, min), 0);
2583}
2584template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet4c>(const Packet4c& a)
2585{
2586 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2587 int8x8_t min = vpmin_s8(a_dup, a_dup);
2588 min = vpmin_s8(min, min);
2589 return vget_lane_s8(min, 0);
2590}
2591template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(const Packet8c& a)
2592{
2593 int8x8_t min = vpmin_s8(a,a);
2594 min = vpmin_s8(min, min);
2595 min = vpmin_s8(min, min);
2596 return vget_lane_s8(min, 0);
2597}
2598template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a)
2599{
2600 int8x8_t min = vmin_s8(vget_low_s8(a), vget_high_s8(a));
2601 min = vpmin_s8(min, min);
2602 min = vpmin_s8(min, min);
2603 min = vpmin_s8(min, min);
2604 return vget_lane_s8(min, 0);
2605}
2606template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet4uc>(const Packet4uc& a)
2607{
2608 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2609 uint8x8_t min = vpmin_u8(a_dup, a_dup);
2610 min = vpmin_u8(min, min);
2611 return vget_lane_u8(min, 0);
2612}
2613template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(const Packet8uc& a)
2614{
2615 uint8x8_t min = vpmin_u8(a,a);
2616 min = vpmin_u8(min, min);
2617 min = vpmin_u8(min, min);
2618 return vget_lane_u8(min, 0);
2619}
2620template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a)
2621{
2622 uint8x8_t min = vmin_u8(vget_low_u8(a), vget_high_u8(a));
2623 min = vpmin_u8(min, min);
2624 min = vpmin_u8(min, min);
2625 min = vpmin_u8(min, min);
2626 return vget_lane_u8(min, 0);
2627}
2628template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(const Packet4s& a)
2629{
2630 const int16x4_t min = vpmin_s16(a,a);
2631 return vget_lane_s16(vpmin_s16(min, min), 0);
2632}
2633template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a)
2634{
2635 int16x4_t min = vmin_s16(vget_low_s16(a), vget_high_s16(a));
2636 min = vpmin_s16(min, min);
2637 min = vpmin_s16(min, min);
2638 return vget_lane_s16(min, 0);
2639}
2640template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(const Packet4us& a)
2641{
2642 const uint16x4_t min = vpmin_u16(a,a);
2643 return vget_lane_u16(vpmin_u16(min, min), 0);
2644}
2645template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a)
2646{
2647 uint16x4_t min = vmin_u16(vget_low_u16(a), vget_high_u16(a));
2648 min = vpmin_u16(min, min);
2649 min = vpmin_u16(min, min);
2650 return vget_lane_u16(min, 0);
2651}
2652template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(const Packet2i& a)
2653{ return vget_lane_s32(vpmin_s32(a,a), 0); }
2654template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a)
2655{
2656 const int32x2_t min = vmin_s32(vget_low_s32(a), vget_high_s32(a));
2657 return vget_lane_s32(vpmin_s32(min, min), 0);
2658}
2659template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(const Packet2ui& a)
2660{ return vget_lane_u32(vpmin_u32(a,a), 0); }
2661template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a)
2662{
2663 const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a));
2664 return vget_lane_u32(vpmin_u32(min, min), 0);
2665}
2666template<> EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(const Packet2l& a)
2667{ return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); }
2668template<> EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(const Packet2ul& a)
2669{ return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); }
2670
2671// max
2672template<> EIGEN_STRONG_INLINE float predux_max<Packet2f>(const Packet2f& a)
2673{ return vget_lane_f32(vpmax_f32(a,a), 0); }
2674template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
2675{
2676 const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a));
2677 return vget_lane_f32(vpmax_f32(max, max), 0);
2678}
2679template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet4c>(const Packet4c& a)
2680{
2681 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2682 int8x8_t max = vpmax_s8(a_dup, a_dup);
2683 max = vpmax_s8(max, max);
2684 return vget_lane_s8(max, 0);
2685}
2686template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(const Packet8c& a)
2687{
2688 int8x8_t max = vpmax_s8(a,a);
2689 max = vpmax_s8(max, max);
2690 max = vpmax_s8(max, max);
2691 return vget_lane_s8(max, 0);
2692}
2693template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a)
2694{
2695 int8x8_t max = vmax_s8(vget_low_s8(a), vget_high_s8(a));
2696 max = vpmax_s8(max, max);
2697 max = vpmax_s8(max, max);
2698 max = vpmax_s8(max, max);
2699 return vget_lane_s8(max, 0);
2700}
2701template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet4uc>(const Packet4uc& a)
2702{
2703 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2704 uint8x8_t max = vpmax_u8(a_dup, a_dup);
2705 max = vpmax_u8(max, max);
2706 return vget_lane_u8(max, 0);
2707}
2708template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(const Packet8uc& a)
2709{
2710 uint8x8_t max = vpmax_u8(a,a);
2711 max = vpmax_u8(max, max);
2712 max = vpmax_u8(max, max);
2713 return vget_lane_u8(max, 0);
2714}
2715template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a)
2716{
2717 uint8x8_t max = vmax_u8(vget_low_u8(a), vget_high_u8(a));
2718 max = vpmax_u8(max, max);
2719 max = vpmax_u8(max, max);
2720 max = vpmax_u8(max, max);
2721 return vget_lane_u8(max, 0);
2722}
2723template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(const Packet4s& a)
2724{
2725 const int16x4_t max = vpmax_s16(a,a);
2726 return vget_lane_s16(vpmax_s16(max, max), 0);
2727}
2728template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a)
2729{
2730 int16x4_t max = vmax_s16(vget_low_s16(a), vget_high_s16(a));
2731 max = vpmax_s16(max, max);
2732 max = vpmax_s16(max, max);
2733 return vget_lane_s16(max, 0);
2734}
2735template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(const Packet4us& a)
2736{
2737 const uint16x4_t max = vpmax_u16(a,a);
2738 return vget_lane_u16(vpmax_u16(max, max), 0);
2739}
2740template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a)
2741{
2742 uint16x4_t max = vmax_u16(vget_low_u16(a), vget_high_u16(a));
2743 max = vpmax_u16(max, max);
2744 max = vpmax_u16(max, max);
2745 return vget_lane_u16(max, 0);
2746}
2747template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(const Packet2i& a)
2748{ return vget_lane_s32(vpmax_s32(a,a), 0); }
2749template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a)
2750{
2751 const int32x2_t max = vmax_s32(vget_low_s32(a), vget_high_s32(a));
2752 return vget_lane_s32(vpmax_s32(max, max), 0);
2753}
2754template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(const Packet2ui& a)
2755{ return vget_lane_u32(vpmax_u32(a,a), 0); }
2756template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a)
2757{
2758 const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a));
2759 return vget_lane_u32(vpmax_u32(max, max), 0);
2760}
2761template<> EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(const Packet2l& a)
2762{ return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); }
2763template<> EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(const Packet2ul& a)
2764{ return (std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); }
2765
2772
2773// Helpers for ptranspose.
2774namespace detail {
2775
2776template<typename Packet>
2777void zip_in_place(Packet& p1, Packet& p2);
2778
2779template<>
2780EIGEN_ALWAYS_INLINE void zip_in_place<Packet2f>(Packet2f& p1, Packet2f& p2) {
2781 const float32x2x2_t tmp = vzip_f32(p1, p2);
2782 p1 = tmp.val[0];
2783 p2 = tmp.val[1];
2784}
2785
2786template<>
2787EIGEN_ALWAYS_INLINE void zip_in_place<Packet4f>(Packet4f& p1, Packet4f& p2) {
2788 const float32x4x2_t tmp = vzipq_f32(p1, p2);
2789 p1 = tmp.val[0];
2790 p2 = tmp.val[1];
2791}
2792
2793template<>
2794EIGEN_ALWAYS_INLINE void zip_in_place<Packet8c>(Packet8c& p1, Packet8c& p2) {
2795 const int8x8x2_t tmp = vzip_s8(p1, p2);
2796 p1 = tmp.val[0];
2797 p2 = tmp.val[1];
2798}
2799
2800template<>
2801EIGEN_ALWAYS_INLINE void zip_in_place<Packet16c>(Packet16c& p1, Packet16c& p2) {
2802 const int8x16x2_t tmp = vzipq_s8(p1, p2);
2803 p1 = tmp.val[0];
2804 p2 = tmp.val[1];
2805}
2806
2807template<>
2808EIGEN_ALWAYS_INLINE void zip_in_place<Packet8uc>(Packet8uc& p1, Packet8uc& p2) {
2809 const uint8x8x2_t tmp = vzip_u8(p1, p2);
2810 p1 = tmp.val[0];
2811 p2 = tmp.val[1];
2812}
2813
2814template<>
2815EIGEN_ALWAYS_INLINE void zip_in_place<Packet16uc>(Packet16uc& p1, Packet16uc& p2) {
2816 const uint8x16x2_t tmp = vzipq_u8(p1, p2);
2817 p1 = tmp.val[0];
2818 p2 = tmp.val[1];
2819}
2820
2821template<>
2822EIGEN_ALWAYS_INLINE void zip_in_place<Packet2i>(Packet2i& p1, Packet2i& p2) {
2823 const int32x2x2_t tmp = vzip_s32(p1, p2);
2824 p1 = tmp.val[0];
2825 p2 = tmp.val[1];
2826}
2827
2828template<>
2829EIGEN_ALWAYS_INLINE void zip_in_place<Packet4i>(Packet4i& p1, Packet4i& p2) {
2830 const int32x4x2_t tmp = vzipq_s32(p1, p2);
2831 p1 = tmp.val[0];
2832 p2 = tmp.val[1];
2833}
2834
2835template<>
2836EIGEN_ALWAYS_INLINE void zip_in_place<Packet2ui>(Packet2ui& p1, Packet2ui& p2) {
2837 const uint32x2x2_t tmp = vzip_u32(p1, p2);
2838 p1 = tmp.val[0];
2839 p2 = tmp.val[1];
2840}
2841
2842template<>
2843EIGEN_ALWAYS_INLINE void zip_in_place<Packet4ui>(Packet4ui& p1, Packet4ui& p2) {
2844 const uint32x4x2_t tmp = vzipq_u32(p1, p2);
2845 p1 = tmp.val[0];
2846 p2 = tmp.val[1];
2847}
2848
2849template<>
2850EIGEN_ALWAYS_INLINE void zip_in_place<Packet4s>(Packet4s& p1, Packet4s& p2) {
2851 const int16x4x2_t tmp = vzip_s16(p1, p2);
2852 p1 = tmp.val[0];
2853 p2 = tmp.val[1];
2854}
2855
2856template<>
2857EIGEN_ALWAYS_INLINE void zip_in_place<Packet8s>(Packet8s& p1, Packet8s& p2) {
2858 const int16x8x2_t tmp = vzipq_s16(p1, p2);
2859 p1 = tmp.val[0];
2860 p2 = tmp.val[1];
2861}
2862
2863template<>
2864EIGEN_ALWAYS_INLINE void zip_in_place<Packet4us>(Packet4us& p1, Packet4us& p2) {
2865 const uint16x4x2_t tmp = vzip_u16(p1, p2);
2866 p1 = tmp.val[0];
2867 p2 = tmp.val[1];
2868}
2869
2870template<>
2871EIGEN_ALWAYS_INLINE void zip_in_place<Packet8us>(Packet8us& p1, Packet8us& p2) {
2872 const uint16x8x2_t tmp = vzipq_u16(p1, p2);
2873 p1 = tmp.val[0];
2874 p2 = tmp.val[1];
2875}
2876
2877template<typename Packet>
2878EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
2879 zip_in_place(kernel.packet[0], kernel.packet[1]);
2880}
2881
2882template<typename Packet>
2883EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
2884 zip_in_place(kernel.packet[0], kernel.packet[2]);
2885 zip_in_place(kernel.packet[1], kernel.packet[3]);
2886 zip_in_place(kernel.packet[0], kernel.packet[1]);
2887 zip_in_place(kernel.packet[2], kernel.packet[3]);
2888}
2889
2890template<typename Packet>
2891EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
2892 zip_in_place(kernel.packet[0], kernel.packet[4]);
2893 zip_in_place(kernel.packet[1], kernel.packet[5]);
2894 zip_in_place(kernel.packet[2], kernel.packet[6]);
2895 zip_in_place(kernel.packet[3], kernel.packet[7]);
2896
2897 zip_in_place(kernel.packet[0], kernel.packet[2]);
2898 zip_in_place(kernel.packet[1], kernel.packet[3]);
2899 zip_in_place(kernel.packet[4], kernel.packet[6]);
2900 zip_in_place(kernel.packet[5], kernel.packet[7]);
2901
2902 zip_in_place(kernel.packet[0], kernel.packet[1]);
2903 zip_in_place(kernel.packet[2], kernel.packet[3]);
2904 zip_in_place(kernel.packet[4], kernel.packet[5]);
2905 zip_in_place(kernel.packet[6], kernel.packet[7]);
2906}
2907
2908template<typename Packet>
2909EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
2910 EIGEN_UNROLL_LOOP
2911 for (int i=0; i<4; ++i) {
2912 const int m = (1 << i);
2913 EIGEN_UNROLL_LOOP
2914 for (int j=0; j<m; ++j) {
2915 const int n = (1 << (3-i));
2916 EIGEN_UNROLL_LOOP
2917 for (int k=0; k<n; ++k) {
2918 const int idx = 2*j*n+k;
2919 zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
2920 }
2921 }
2922 }
2923}
2924
2925} // namespace detail
2926
2930EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
2931 detail::ptranspose_impl(kernel);
2932}
2933
2934EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4c, 4>& kernel)
2935{
2936 const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1));
2937 const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1));
2938
2939 const int8x8x2_t zip8 = vzip_s8(a,b);
2940 const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1]));
2941
2942 kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0);
2943 kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1);
2944 kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0);
2945 kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1);
2946}
2953EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2954 detail::ptranspose_impl(kernel);
2955}
2959EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2960 detail::ptranspose_impl(kernel);
2961}
2962
2963EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4uc, 4>& kernel)
2964{
2965 const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1));
2966 const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1));
2967
2968 const uint8x8x2_t zip8 = vzip_u8(a,b);
2969 const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1]));
2970
2971 kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0);
2972 kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1);
2973 kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0);
2974 kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1);
2975}
2982EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
2983 detail::ptranspose_impl(kernel);
2984}
2988EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2989 detail::ptranspose_impl(kernel);
2990}
2991
2995EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2996 detail::ptranspose_impl(kernel);
2997}
2998EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2999 detail::ptranspose_impl(kernel);
3000}
3001
3005EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
3006 detail::ptranspose_impl(kernel);
3007}
3008EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
3009 detail::ptranspose_impl(kernel);
3010}
3011
3015EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
3016 detail::ptranspose_impl(kernel);
3017}
3018EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2ui, 2>& kernel) {
3019 detail::zip_in_place(kernel.packet[0], kernel.packet[1]);
3020}
3024
3025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
3026ptranspose(PacketBlock<Packet2l, 2>& kernel)
3027{
3028#if EIGEN_ARCH_ARM64
3029 const int64x2_t tmp1 = vzip1q_s64(kernel.packet[0], kernel.packet[1]);
3030 kernel.packet[1] = vzip2q_s64(kernel.packet[0], kernel.packet[1]);
3031 kernel.packet[0] = tmp1;
3032#else
3033 const int64x1_t tmp[2][2] = {
3034 { vget_low_s64(kernel.packet[0]), vget_high_s64(kernel.packet[0]) },
3035 { vget_low_s64(kernel.packet[1]), vget_high_s64(kernel.packet[1]) }
3036 };
3037
3038 kernel.packet[0] = vcombine_s64(tmp[0][0], tmp[1][0]);
3039 kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]);
3040#endif
3041}
3042EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
3043ptranspose(PacketBlock<Packet2ul, 2>& kernel)
3044{
3045#if EIGEN_ARCH_ARM64
3046 const uint64x2_t tmp1 = vzip1q_u64(kernel.packet[0], kernel.packet[1]);
3047 kernel.packet[1] = vzip2q_u64(kernel.packet[0], kernel.packet[1]);
3048 kernel.packet[0] = tmp1;
3049#else
3050 const uint64x1_t tmp[2][2] = {
3051 { vget_low_u64(kernel.packet[0]), vget_high_u64(kernel.packet[0]) },
3052 { vget_low_u64(kernel.packet[1]), vget_high_u64(kernel.packet[1]) }
3053 };
3054
3055 kernel.packet[0] = vcombine_u64(tmp[0][0], tmp[1][0]);
3056 kernel.packet[1] = vcombine_u64(tmp[0][1], tmp[1][1]);
3057#endif
3058}
3059
3060template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect( const Packet2f& mask, const Packet2f& a, const Packet2f& b)
3061{ return vbsl_f32(vreinterpret_u32_f32(mask), a, b); }
3062template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b)
3063{ return vbslq_f32(vreinterpretq_u32_f32(mask), a, b); }
3064template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(const Packet8c& mask, const Packet8c& a, const Packet8c& b)
3065{ return vbsl_s8(vreinterpret_u8_s8(mask), a, b); }
3066template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b)
3067{ return vbslq_s8(vreinterpretq_u8_s8(mask), a, b); }
3068template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(const Packet8uc& mask, const Packet8uc& a, const Packet8uc& b)
3069{ return vbsl_u8(mask, a, b); }
3070template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, const Packet16uc& b)
3071{ return vbslq_u8(mask, a, b); }
3072template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(const Packet4s& mask, const Packet4s& a, const Packet4s& b)
3073{ return vbsl_s16(vreinterpret_u16_s16(mask), a, b); }
3074template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b)
3075{ return vbslq_s16(vreinterpretq_u16_s16(mask), a, b); }
3076template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(const Packet4us& mask, const Packet4us& a, const Packet4us& b)
3077{ return vbsl_u16(mask, a, b); }
3078template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b)
3079{ return vbslq_u16(mask, a, b); }
3080template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(const Packet2i& mask, const Packet2i& a, const Packet2i& b)
3081{ return vbsl_s32(vreinterpret_u32_s32(mask), a, b); }
3082template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b)
3083{ return vbslq_s32(vreinterpretq_u32_s32(mask), a, b); }
3084template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(const Packet2ui& mask, const Packet2ui& a, const Packet2ui& b)
3085{ return vbsl_u32(mask, a, b); }
3086template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b)
3087{ return vbslq_u32(mask, a, b); }
3088template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b)
3089{ return vbslq_s64(vreinterpretq_u64_s64(mask), a, b); }
3090template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b)
3091{ return vbslq_u64(mask, a, b); }
3092
3093// Use armv8 rounding intinsics if available.
3094#if EIGEN_ARCH_ARMV8
3095template<> EIGEN_STRONG_INLINE Packet2f print<Packet2f>(const Packet2f& a)
3096{ return vrndn_f32(a); }
3097
3098template<> EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a)
3099{ return vrndnq_f32(a); }
3100
3101template<> EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(const Packet2f& a)
3102{ return vrndm_f32(a); }
3103
3104template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
3105{ return vrndmq_f32(a); }
3106
3107template<> EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(const Packet2f& a)
3108{ return vrndp_f32(a); }
3109
3110template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a)
3111{ return vrndpq_f32(a); }
3112
3113#else
3114
3115template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) {
3116 // Adds and subtracts signum(a) * 2^23 to force rounding.
3117 const Packet4f limit = pset1<Packet4f>(static_cast<float>(1<<23));
3118 const Packet4f abs_a = pabs(a);
3119 Packet4f r = padd(abs_a, limit);
3120 // Don't compile-away addition and subtraction.
3121 EIGEN_OPTIMIZATION_BARRIER(r);
3122 r = psub(r, limit);
3123 // If greater than limit, simply return a. Otherwise, account for sign.
3124 r = pselect(pcmp_lt(abs_a, limit),
3125 pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
3126 return r;
3127}
3128
3129template<> EIGEN_STRONG_INLINE Packet2f print(const Packet2f& a) {
3130 // Adds and subtracts signum(a) * 2^23 to force rounding.
3131 const Packet2f limit = pset1<Packet2f>(static_cast<float>(1<<23));
3132 const Packet2f abs_a = pabs(a);
3133 Packet2f r = padd(abs_a, limit);
3134 // Don't compile-away addition and subtraction.
3135 EIGEN_OPTIMIZATION_BARRIER(r);
3136 r = psub(r, limit);
3137 // If greater than limit, simply return a. Otherwise, account for sign.
3138 r = pselect(pcmp_lt(abs_a, limit),
3139 pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
3140 return r;
3141}
3142
3143template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
3144{
3145 const Packet4f cst_1 = pset1<Packet4f>(1.0f);
3146 Packet4f tmp = print<Packet4f>(a);
3147 // If greater, subtract one.
3148 Packet4f mask = pcmp_lt(a, tmp);
3149 mask = pand(mask, cst_1);
3150 return psub(tmp, mask);
3151}
3152
3153template<> EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(const Packet2f& a)
3154{
3155 const Packet2f cst_1 = pset1<Packet2f>(1.0f);
3156 Packet2f tmp = print<Packet2f>(a);
3157 // If greater, subtract one.
3158 Packet2f mask = pcmp_lt(a, tmp);
3159 mask = pand(mask, cst_1);
3160 return psub(tmp, mask);
3161}
3162
3163template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a)
3164{
3165 const Packet4f cst_1 = pset1<Packet4f>(1.0f);
3166 Packet4f tmp = print<Packet4f>(a);
3167 // If smaller, add one.
3168 Packet4f mask = pcmp_lt(tmp, a);
3169 mask = pand(mask, cst_1);
3170 return padd(tmp, mask);
3171}
3172
3173template<> EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(const Packet2f& a)
3174{
3175 const Packet2f cst_1 = pset1<Packet2f>(1.0);
3176 Packet2f tmp = print<Packet2f>(a);
3177 // If smaller, add one.
3178 Packet2f mask = pcmp_lt(tmp, a);
3179 mask = pand(mask, cst_1);
3180 return padd(tmp, mask);
3181}
3182
3183#endif
3184
3191template<> EIGEN_STRONG_INLINE Packet4uc psqrt(const Packet4uc& a) {
3192 uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a));
3193 uint8x8_t res = vdup_n_u8(0);
3194 uint8x8_t add = vdup_n_u8(0x8);
3195 for (int i = 0; i < 4; i++)
3196 {
3197 const uint8x8_t temp = vorr_u8(res, add);
3198 res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res);
3199 add = vshr_n_u8(add, 1);
3200 }
3201 return vget_lane_u32(vreinterpret_u32_u8(res), 0);
3202}
3204template<> EIGEN_STRONG_INLINE Packet8uc psqrt(const Packet8uc& a) {
3205 uint8x8_t res = vdup_n_u8(0);
3206 uint8x8_t add = vdup_n_u8(0x8);
3207 for (int i = 0; i < 4; i++)
3208 {
3209 const uint8x8_t temp = vorr_u8(res, add);
3210 res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res);
3211 add = vshr_n_u8(add, 1);
3212 }
3213 return res;
3214}
3216template<> EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) {
3217 uint8x16_t res = vdupq_n_u8(0);
3218 uint8x16_t add = vdupq_n_u8(0x8);
3219 for (int i = 0; i < 4; i++)
3220 {
3221 const uint8x16_t temp = vorrq_u8(res, add);
3222 res = vbslq_u8(vcgeq_u8(a, vmulq_u8(temp, temp)), temp, res);
3223 add = vshrq_n_u8(add, 1);
3224 }
3225 return res;
3226}
3228template<> EIGEN_STRONG_INLINE Packet4us psqrt(const Packet4us& a) {
3229 uint16x4_t res = vdup_n_u16(0);
3230 uint16x4_t add = vdup_n_u16(0x80);
3231 for (int i = 0; i < 8; i++)
3232 {
3233 const uint16x4_t temp = vorr_u16(res, add);
3234 res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res);
3235 add = vshr_n_u16(add, 1);
3236 }
3237 return res;
3238}
3240template<> EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) {
3241 uint16x8_t res = vdupq_n_u16(0);
3242 uint16x8_t add = vdupq_n_u16(0x80);
3243 for (int i = 0; i < 8; i++)
3244 {
3245 const uint16x8_t temp = vorrq_u16(res, add);
3246 res = vbslq_u16(vcgeq_u16(a, vmulq_u16(temp, temp)), temp, res);
3247 add = vshrq_n_u16(add, 1);
3248 }
3249 return res;
3250}
3252template<> EIGEN_STRONG_INLINE Packet2ui psqrt(const Packet2ui& a) {
3253 uint32x2_t res = vdup_n_u32(0);
3254 uint32x2_t add = vdup_n_u32(0x8000);
3255 for (int i = 0; i < 16; i++)
3256 {
3257 const uint32x2_t temp = vorr_u32(res, add);
3258 res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res);
3259 add = vshr_n_u32(add, 1);
3260 }
3261 return res;
3262}
3264template<> EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) {
3265 uint32x4_t res = vdupq_n_u32(0);
3266 uint32x4_t add = vdupq_n_u32(0x8000);
3267 for (int i = 0; i < 16; i++)
3268 {
3269 const uint32x4_t temp = vorrq_u32(res, add);
3270 res = vbslq_u32(vcgeq_u32(a, vmulq_u32(temp, temp)), temp, res);
3271 add = vshrq_n_u32(add, 1);
3272 }
3273 return res;
3274}
3275
3276template<> EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) {
3277 // Compute approximate reciprocal sqrt.
3278 Packet4f x = vrsqrteq_f32(a);
3279 // Do Newton iterations for 1/sqrt(x).
3280 x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x);
3281 x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x);
3282 const Packet4f infinity = pset1<Packet4f>(NumTraits<float>::infinity());
3283 return pselect(pcmp_eq(a, pzero(a)), infinity, x);
3284}
3285
3286template<> EIGEN_STRONG_INLINE Packet2f prsqrt(const Packet2f& a) {
3287 // Compute approximate reciprocal sqrt.
3288 Packet2f x = vrsqrte_f32(a);
3289 // Do Newton iterations for 1/sqrt(x).
3290 x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x);
3291 x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x);
3292 const Packet2f infinity = pset1<Packet2f>(NumTraits<float>::infinity());
3293 return pselect(pcmp_eq(a, pzero(a)), infinity, x);
3294}
3295
3296// Unfortunately vsqrt_f32 is only available for A64.
3297#if EIGEN_ARCH_ARM64
3298template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){return vsqrtq_f32(_x);}
3299template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){return vsqrt_f32(_x); }
3300#else
3301template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
3302 const Packet4f infinity = pset1<Packet4f>(NumTraits<float>::infinity());
3303 const Packet4f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity));
3304 return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a)));
3305}
3306template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) {
3307 const Packet2f infinity = pset1<Packet2f>(NumTraits<float>::infinity());
3308 const Packet2f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity));
3309 return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a)));
3310}
3311#endif
3312
3313//---------- bfloat16 ----------
3314// TODO: Add support for native armv8.6-a bfloat16_t
3315
3316// TODO: Guard if we have native bfloat16 support
3317typedef eigen_packet_wrapper<uint16x4_t, 19> Packet4bf;
3318
3319template<> struct is_arithmetic<Packet4bf> { enum { value = true }; };
3320
3321template<> struct packet_traits<bfloat16> : default_packet_traits
3322{
3323 typedef Packet4bf type;
3324 typedef Packet4bf half;
3325 enum
3326 {
3327 Vectorizable = 1,
3328 AlignedOnScalar = 1,
3329 size = 4,
3330 HasHalfPacket = 0,
3331
3332 HasCmp = 1,
3333 HasAdd = 1,
3334 HasSub = 1,
3335 HasShift = 1,
3336 HasMul = 1,
3337 HasNegate = 1,
3338 HasAbs = 1,
3339 HasArg = 0,
3340 HasAbs2 = 1,
3341 HasAbsDiff = 1,
3342 HasMin = 1,
3343 HasMax = 1,
3344 HasConj = 1,
3345 HasSetLinear = 0,
3346 HasBlend = 0,
3347 HasDiv = 1,
3348 HasFloor = 1,
3349 HasCeil = 1,
3350 HasRint = 1,
3351
3352 HasSin = EIGEN_FAST_MATH,
3353 HasCos = EIGEN_FAST_MATH,
3354 HasLog = 1,
3355 HasExp = 1,
3356 HasSqrt = 0,
3357 HasTanh = EIGEN_FAST_MATH,
3358 HasErf = EIGEN_FAST_MATH,
3359 HasBessel = 0, // Issues with accuracy.
3360 HasNdtri = 0
3361 };
3362};
3363
3364template<> struct unpacket_traits<Packet4bf>
3365{
3366 typedef bfloat16 type;
3367 typedef Packet4bf half;
3368 enum
3369 {
3370 size = 4,
3371 alignment = Aligned16,
3372 vectorizable = true,
3373 masked_load_available = false,
3374 masked_store_available = false
3375 };
3376};
3377
3378namespace detail {
3379template<>
3380EIGEN_ALWAYS_INLINE void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) {
3381 const uint16x4x2_t tmp = vzip_u16(p1, p2);
3382 p1 = tmp.val[0];
3383 p2 = tmp.val[1];
3384}
3385} // namespace detail
3386
3387EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p)
3388{
3389 // See the scalar implemention in BFloat16.h for a comprehensible explanation
3390 // of this fast rounding algorithm
3391 Packet4ui input = reinterpret_cast<Packet4ui>(p);
3392
3393 // lsb = (input >> 16) & 1
3394 Packet4ui lsb = vandq_u32(vshrq_n_u32(input, 16), vdupq_n_u32(1));
3395
3396 // rounding_bias = 0x7fff + lsb
3397 Packet4ui rounding_bias = vaddq_u32(lsb, vdupq_n_u32(0x7fff));
3398
3399 // input += rounding_bias
3400 input = vaddq_u32(input, rounding_bias);
3401
3402 // input = input >> 16
3403 input = vshrq_n_u32(input, 16);
3404
3405 // Replace float-nans by bfloat16-nans, that is 0x7fc0
3406 const Packet4ui bf16_nan = vdupq_n_u32(0x7fc0);
3407 const Packet4ui mask = vceqq_f32(p, p);
3408 input = vbslq_u32(mask, input, bf16_nan);
3409
3410 // output = static_cast<uint16_t>(input)
3411 return vmovn_u32(input);
3412}
3413
3414EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p)
3415{
3416 return reinterpret_cast<Packet4f>(vshlq_n_u32(vmovl_u16(p), 16));
3417}
3418
3422
3423template<> EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(const bfloat16& from) {
3424 return pset1<Packet4us>(from.value);
3425}
3426
3430
3431template<> EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(const bfloat16* from)
3432{
3433 return pload<Packet4us>(reinterpret_cast<const uint16_t*>(from));
3434}
3435
3436template<> EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(const bfloat16* from)
3437{
3438 return ploadu<Packet4us>(reinterpret_cast<const uint16_t*>(from));
3439}
3440
3441template<> EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet4bf& from)
3442{
3443 EIGEN_DEBUG_ALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
3444}
3445
3446template<> EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet4bf& from)
3447{
3448 EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
3449}
3450
3451template<> EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(const bfloat16* from)
3452{
3453 return ploaddup<Packet4us>(reinterpret_cast<const uint16_t*>(from));
3454}
3455
3459
3470
3471template <> EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(const Packet4bf &a,
3472 const Packet4bf &b)
3473{
3474 return F32ToBf16(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3475}
3476
3487
3488template <> EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(const Packet4bf &a,
3489 const Packet4bf &b)
3490{
3491 return F32ToBf16(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3492}
3493
3494template<> EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(const bfloat16& a)
3495{
3496 return F32ToBf16(plset<Packet4f>(static_cast<float>(a)));
3497}
3498
3499template<> EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a,const Packet4bf& b) {
3500 return por<Packet4us>(a, b);
3501}
3502
3503template<> EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a,const Packet4bf& b) {
3504 return pxor<Packet4us>(a, b);
3505}
3506
3507template<> EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a,const Packet4bf& b) {
3508 return pand<Packet4us>(a, b);
3509}
3510
3511template<> EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a,const Packet4bf& b) {
3512 return pandnot<Packet4us>(a, b);
3513}
3514
3515template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a,
3516 const Packet4bf& b)
3517{
3518 return pselect<Packet4us>(mask, a, b);
3519}
3520
3525
3530
3535
3536template<> EIGEN_STRONG_INLINE Packet4bf pconj(const Packet4bf& a) { return a; }
3537
3541
3545
3549
3553
3554template<>
3555EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(const bfloat16* from, Index stride)
3556{
3557 return pgather<uint16_t, Packet4us>(reinterpret_cast<const uint16_t*>(from), stride);
3558}
3559
3560template<>
3561EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet4bf>(bfloat16* to, const Packet4bf& from, Index stride)
3562{
3563 pscatter<uint16_t, Packet4us>(reinterpret_cast<uint16_t*>(to), from, stride);
3564}
3565
3566template<> EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(const Packet4bf& a)
3567{
3568 return static_cast<bfloat16>(predux<Packet4f>(Bf16ToF32(a)));
3569}
3570
3571template<> EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(const Packet4bf& a)
3572{
3573 return static_cast<bfloat16>(predux_max<Packet4f>(Bf16ToF32(a)));
3574}
3575
3576template<> EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(const Packet4bf& a)
3577{
3578 return static_cast<bfloat16>(predux_min<Packet4f>(Bf16ToF32(a)));
3579}
3580
3581template<> EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(const Packet4bf& a)
3582{
3583 return static_cast<bfloat16>(predux_mul<Packet4f>(Bf16ToF32(a)));
3584}
3585
3590
3595
3600
3605
3610
3615
3620
3621template<> EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(const Packet4bf& a)
3622{
3623 return pxor<Packet4us>(a, pset1<Packet4us>(static_cast<uint16_t>(0x8000)));
3624}
3625
3626//---------- double ----------
3627
3628// Clang 3.5 in the iOS toolchain has an ICE triggered by NEON intrisics for double.
3629// Confirmed at least with __apple_build_version__ = 6000054.
3630#ifdef __apple_build_version__
3631// Let's hope that by the time __apple_build_version__ hits the 601* range, the bug will be fixed.
3632// https://gist.github.com/yamaya/2924292 suggests that the 3 first digits are only updated with
3633// major toolchain updates.
3634#define EIGEN_APPLE_DOUBLE_NEON_BUG (__apple_build_version__ < 6010000)
3635#else
3636#define EIGEN_APPLE_DOUBLE_NEON_BUG 0
3637#endif
3638
3639#if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG
3640
3641// Bug 907: workaround missing declarations of the following two functions in the ADK
3642// Defining these functions as templates ensures that if these intrinsics are
3643// already defined in arm_neon.h, then our workaround doesn't cause a conflict
3644// and has lower priority in overload resolution.
3645template <typename T> uint64x2_t vreinterpretq_u64_f64(T a) { return (uint64x2_t) a; }
3646
3647template <typename T> float64x2_t vreinterpretq_f64_u64(T a) { return (float64x2_t) a; }
3648
3649typedef float64x2_t Packet2d;
3650typedef float64x1_t Packet1d;
3651
3652// fuctionally equivalent to _mm_shuffle_pd in SSE (i.e. shuffle(m, n, mask) equals _mm_shuffle_pd(m,n,mask))
3653// Currently used in LU/arch/InverseSize4.h to enable a shared implementation
3654// for fast inversion of matrices of size 4.
3655EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask)
3656{
3657 const double* a = reinterpret_cast<const double*>(&m);
3658 const double* b = reinterpret_cast<const double*>(&n);
3659 Packet2d res = {*(a + (mask & 1)), *(b + ((mask >> 1) & 1))};
3660 return res;
3661}
3662
3663EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask)
3664{
3665 return shuffle(a, b, mask);
3666}
3667EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a,const Packet2d& b)
3668{
3669 return shuffle(a, b, 0);
3670}
3671EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a,const Packet2d& b)
3672{
3673 return shuffle(a, b, 3);
3674}
3675#define vec2d_duplane(a, p) \
3676 vdupq_laneq_f64(a, p)
3677
3678template<> struct packet_traits<double> : default_packet_traits
3679{
3680 typedef Packet2d type;
3681 typedef Packet2d half;
3682 enum
3683 {
3684 Vectorizable = 1,
3685 AlignedOnScalar = 1,
3686 size = 2,
3687 HasHalfPacket = 0,
3688
3689 HasCmp = 1,
3690 HasAdd = 1,
3691 HasSub = 1,
3692 HasShift = 1,
3693 HasMul = 1,
3694 HasNegate = 1,
3695 HasAbs = 1,
3696 HasArg = 0,
3697 HasAbs2 = 1,
3698 HasAbsDiff = 1,
3699 HasMin = 1,
3700 HasMax = 1,
3701 HasConj = 1,
3702 HasSetLinear = 0,
3703 HasBlend = 0,
3704
3705 HasDiv = 1,
3706 HasFloor = 1,
3707 HasCeil = 1,
3708 HasRint = 1,
3709
3710 HasSin = 0,
3711 HasCos = 0,
3712 HasLog = 1,
3713 HasExp = 1,
3714 HasSqrt = 1,
3715 HasRsqrt = 1,
3716 HasTanh = 0,
3717 HasErf = 0
3718 };
3719};
3720
3721template<> struct unpacket_traits<Packet2d>
3722{
3723 typedef double type;
3724 typedef Packet2d half;
3725 typedef Packet2l integer_packet;
3726 enum
3727 {
3728 size = 2,
3729 alignment = Aligned16,
3730 vectorizable = true,
3731 masked_load_available = false,
3732 masked_store_available = false
3733 };
3734};
3735
3736template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return vdupq_n_f64(from); }
3737
3738template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a)
3739{
3740 const double c[] = {0.0,1.0};
3741 return vaddq_f64(pset1<Packet2d>(a), vld1q_f64(c));
3742}
3743
3744template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return vaddq_f64(a,b); }
3745
3746template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return vsubq_f64(a,b); }
3747
3748template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& , const Packet2d& );
3749template<> EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(const Packet2d& a, const Packet2d& b){
3750 const Packet2d mask = {numext::bit_cast<double>(0x8000000000000000ull),0.0};
3751 return padd(a, pxor(mask, b));
3752}
3753
3754template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return vnegq_f64(a); }
3755
3756template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
3757
3758template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return vmulq_f64(a,b); }
3759
3760template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return vdivq_f64(a,b); }
3761
3762#ifdef __ARM_FEATURE_FMA
3763// See bug 936. See above comment about FMA for float.
3764template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c)
3765{ return vfmaq_f64(c,a,b); }
3766#else
3767template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c)
3768{ return vmlaq_f64(c,a,b); }
3769#endif
3770
3771template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vminq_f64(a,b); }
3772
3773#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
3774// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
3775template<> EIGEN_STRONG_INLINE Packet2d pmin<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) { return vminnmq_f64(a, b); }
3776template<> EIGEN_STRONG_INLINE Packet2d pmax<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) { return vmaxnmq_f64(a, b); }
3777
3778#endif
3779
3780template<> EIGEN_STRONG_INLINE Packet2d pmin<PropagateNaN, Packet2d>(const Packet2d& a, const Packet2d& b) { return pmin<Packet2d>(a, b); }
3781
3782template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return vmaxq_f64(a,b); }
3783
3784
3785template<> EIGEN_STRONG_INLINE Packet2d pmax<PropagateNaN, Packet2d>(const Packet2d& a, const Packet2d& b) { return pmax<Packet2d>(a, b); }
3786
3787// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
3788template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b)
3789{ return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3790
3791template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b)
3792{ return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3793
3794template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b)
3795{ return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3796
3797template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b)
3798{ return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3799
3800template<> EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b)
3801{ return vreinterpretq_f64_u64(vcleq_f64(a,b)); }
3802
3803template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b)
3804{ return vreinterpretq_f64_u64(vcltq_f64(a,b)); }
3805
3806template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b)
3807{ return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_u64(vcgeq_f64(a,b)))); }
3808
3809template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b)
3810{ return vreinterpretq_f64_u64(vceqq_f64(a,b)); }
3811
3812template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from)
3813{ EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(from); }
3814
3815template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
3816{ EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f64(from); }
3817
3818template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) { return vld1q_dup_f64(from); }
3819template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from)
3820{ EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(to,from); }
3821
3822template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from)
3823{ EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to,from); }
3824
3825template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride)
3826{
3827 Packet2d res = pset1<Packet2d>(0.0);
3828 res = vld1q_lane_f64(from + 0*stride, res, 0);
3829 res = vld1q_lane_f64(from + 1*stride, res, 1);
3830 return res;
3831}
3832
3833template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride)
3834{
3835 vst1q_lane_f64(to + stride*0, from, 0);
3836 vst1q_lane_f64(to + stride*1, from, 1);
3837}
3838
3839template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_ARM_PREFETCH(addr); }
3840
3841// FIXME only store the 2 first elements ?
3842template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return vgetq_lane_f64(a,0); }
3843
3844template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
3845{ return vcombine_f64(vget_high_f64(a), vget_low_f64(a)); }
3846
3847template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vabsq_f64(a); }
3848
3849#if EIGEN_COMP_CLANG && defined(__apple_build_version__)
3850// workaround ICE, see bug 907
3851template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
3852{ return (vget_low_f64(a) + vget_high_f64(a))[0]; }
3853#else
3854template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
3855{ return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); }
3856#endif
3857
3858// Other reduction functions:
3859// mul
3860#if EIGEN_COMP_CLANG && defined(__apple_build_version__)
3861template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
3862{ return (vget_low_f64(a) * vget_high_f64(a))[0]; }
3863#else
3864template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
3865{ return vget_lane_f64(vget_low_f64(a) * vget_high_f64(a), 0); }
3866#endif
3867
3868// min
3869template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
3870{ return vgetq_lane_f64(vpminq_f64(a,a), 0); }
3871
3872// max
3873template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
3874{ return vgetq_lane_f64(vpmaxq_f64(a,a), 0); }
3875
3876
3877EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
3878ptranspose(PacketBlock<Packet2d, 2>& kernel)
3879{
3880 const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]);
3881 const float64x2_t tmp2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]);
3882
3883 kernel.packet[0] = tmp1;
3884 kernel.packet[1] = tmp2;
3885}
3886
3887template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect( const Packet2d& mask, const Packet2d& a, const Packet2d& b)
3888{ return vbslq_f64(vreinterpretq_u64_f64(mask), a, b); }
3889
3890template<> EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a)
3891{ return vrndnq_f64(a); }
3892
3893template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a)
3894{ return vrndmq_f64(a); }
3895
3896template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a)
3897{ return vrndpq_f64(a); }
3898
3899template<> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent)
3900{ return pldexp_generic(a, exponent); }
3901
3902template<> EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent)
3903{ return pfrexp_generic(a,exponent); }
3904
3905template<> EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from)
3906{ return vreinterpretq_f64_u64(vdupq_n_u64(from)); }
3907
3908template<> EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) {
3909 // Compute approximate reciprocal sqrt.
3910 Packet2d x = vrsqrteq_f64(a);
3911 // Do Newton iterations for 1/sqrt(x).
3912 x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x);
3913 x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x);
3914 x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x);
3915 const Packet2d infinity = pset1<Packet2d>(NumTraits<double>::infinity());
3916 return pselect(pcmp_eq(a, pzero(a)), infinity, x);
3917}
3918
3919template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vsqrtq_f64(_x); }
3920
3921#endif // EIGEN_ARCH_ARM64
3922
3923// Do we have an fp16 types and supporting Neon intrinsics?
3924#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
3925typedef float16x4_t Packet4hf;
3926typedef float16x8_t Packet8hf;
3927
3928template <>
3929struct packet_traits<Eigen::half> : default_packet_traits {
3930 typedef Packet8hf type;
3931 typedef Packet4hf half;
3932 enum {
3933 Vectorizable = 1,
3934 AlignedOnScalar = 1,
3935 size = 8,
3936 HasHalfPacket = 1,
3937
3938 HasCmp = 1,
3939 HasCast = 1,
3940 HasAdd = 1,
3941 HasSub = 1,
3942 HasShift = 1,
3943 HasMul = 1,
3944 HasNegate = 1,
3945 HasAbs = 1,
3946 HasArg = 0,
3947 HasAbs2 = 1,
3948 HasAbsDiff = 0,
3949 HasMin = 1,
3950 HasMax = 1,
3951 HasConj = 1,
3952 HasSetLinear = 0,
3953 HasBlend = 0,
3954 HasInsert = 1,
3955 HasReduxp = 1,
3956 HasDiv = 1,
3957 HasFloor = 1,
3958 HasCeil = 1,
3959 HasRint = 1,
3960 HasSin = 0,
3961 HasCos = 0,
3962 HasLog = 0,
3963 HasExp = 0,
3964 HasSqrt = 1,
3965 HasRsqrt = 1,
3966 HasErf = EIGEN_FAST_MATH,
3967 HasBessel = 0, // Issues with accuracy.
3968 HasNdtri = 0
3969 };
3970};
3971
3972template <>
3973struct unpacket_traits<Packet4hf> {
3974 typedef Eigen::half type;
3975 typedef Packet4hf half;
3976 enum {
3977 size = 4,
3978 alignment = Aligned16,
3979 vectorizable = true,
3980 masked_load_available = false,
3981 masked_store_available = false
3982 };
3983};
3984
3985template <>
3986struct unpacket_traits<Packet8hf> {
3987 typedef Eigen::half type;
3988 typedef Packet4hf half;
3989 enum {
3990 size = 8,
3991 alignment = Aligned16,
3992 vectorizable = true,
3993 masked_load_available = false,
3994 masked_store_available = false
3995 };
3996};
3997
3998template<>
3999EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half_dowto4<Packet8hf>(const Packet8hf& a) {
4000 return vadd_f16(vget_low_f16(a), vget_high_f16(a));
4001}
4002
4003template <>
4004EIGEN_STRONG_INLINE Packet8hf pset1<Packet8hf>(const Eigen::half& from) {
4005 return vdupq_n_f16(from.x);
4006}
4007
4008template <>
4009EIGEN_STRONG_INLINE Packet4hf pset1<Packet4hf>(const Eigen::half& from) {
4010 return vdup_n_f16(from.x);
4011}
4012
4013template <>
4014EIGEN_STRONG_INLINE Packet8hf plset<Packet8hf>(const Eigen::half& a) {
4015 const float16_t f[] = {0, 1, 2, 3, 4, 5, 6, 7};
4016 Packet8hf countdown = vld1q_f16(f);
4017 return vaddq_f16(pset1<Packet8hf>(a), countdown);
4018}
4019
4020template <>
4021EIGEN_STRONG_INLINE Packet4hf plset<Packet4hf>(const Eigen::half& a) {
4022 const float16_t f[] = {0, 1, 2, 3};
4023 Packet4hf countdown = vld1_f16(f);
4024 return vadd_f16(pset1<Packet4hf>(a), countdown);
4025}
4026
4027template <>
4028EIGEN_STRONG_INLINE Packet8hf padd<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4029 return vaddq_f16(a, b);
4030}
4031
4032template <>
4033EIGEN_STRONG_INLINE Packet4hf padd<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4034 return vadd_f16(a, b);
4035}
4036
4037template <>
4038EIGEN_STRONG_INLINE Packet8hf psub<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4039 return vsubq_f16(a, b);
4040}
4041
4042template <>
4043EIGEN_STRONG_INLINE Packet4hf psub<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4044 return vsub_f16(a, b);
4045}
4046
4047template <>
4048EIGEN_STRONG_INLINE Packet8hf pnegate(const Packet8hf& a) {
4049 return vnegq_f16(a);
4050}
4051
4052template <>
4053EIGEN_STRONG_INLINE Packet4hf pnegate(const Packet4hf& a) {
4054 return vneg_f16(a);
4055}
4056
4057template <>
4058EIGEN_STRONG_INLINE Packet8hf pconj(const Packet8hf& a) {
4059 return a;
4060}
4061
4062template <>
4063EIGEN_STRONG_INLINE Packet4hf pconj(const Packet4hf& a) {
4064 return a;
4065}
4066
4067template <>
4068EIGEN_STRONG_INLINE Packet8hf pmul<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4069 return vmulq_f16(a, b);
4070}
4071
4072template <>
4073EIGEN_STRONG_INLINE Packet4hf pmul<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4074 return vmul_f16(a, b);
4075}
4076
4077template <>
4078EIGEN_STRONG_INLINE Packet8hf pdiv<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4079 return vdivq_f16(a, b);
4080}
4081
4082template <>
4083EIGEN_STRONG_INLINE Packet4hf pdiv<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4084 return vdiv_f16(a, b);
4085}
4086
4087template <>
4088EIGEN_STRONG_INLINE Packet8hf pmadd(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) {
4089 return vfmaq_f16(c, a, b);
4090}
4091
4092template <>
4093EIGEN_STRONG_INLINE Packet4hf pmadd(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) {
4094 return vfma_f16(c, a, b);
4095}
4096
4097template <>
4098EIGEN_STRONG_INLINE Packet8hf pmin<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4099 return vminq_f16(a, b);
4100}
4101
4102template <>
4103EIGEN_STRONG_INLINE Packet4hf pmin<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4104 return vmin_f16(a, b);
4105}
4106
4107#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
4108// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
4109template<> EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNumbers, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return vminnm_f16(a, b); }
4110template<> EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNumbers, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return vminnmq_f16(a, b); }
4111#endif
4112
4113template<> EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNaN, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return pmin<Packet4hf>(a, b); }
4114
4115template<> EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNaN, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return pmin<Packet8hf>(a, b); }
4116
4117template <>
4118EIGEN_STRONG_INLINE Packet8hf pmax<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4119 return vmaxq_f16(a, b);
4120}
4121
4122template <>
4123EIGEN_STRONG_INLINE Packet4hf pmax<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4124 return vmax_f16(a, b);
4125}
4126
4127#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
4128// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
4129template<> EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNumbers, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return vmaxnm_f16(a, b); }
4130template<> EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNumbers, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return vmaxnmq_f16(a, b); }
4131#endif
4132
4133template<> EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNaN, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return pmax<Packet4hf>(a, b); }
4134
4135template<> EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNaN, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return pmax<Packet8hf>(a, b); }
4136
4137#define EIGEN_MAKE_ARM_FP16_CMP_8(name) \
4138 template <> \
4139 EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \
4140 return vreinterpretq_f16_u16(vc##name##q_f16(a, b)); \
4141 }
4142
4143#define EIGEN_MAKE_ARM_FP16_CMP_4(name) \
4144 template <> \
4145 EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \
4146 return vreinterpret_f16_u16(vc##name##_f16(a, b)); \
4147 }
4148
4149EIGEN_MAKE_ARM_FP16_CMP_8(eq)
4150EIGEN_MAKE_ARM_FP16_CMP_8(lt)
4151EIGEN_MAKE_ARM_FP16_CMP_8(le)
4152
4153EIGEN_MAKE_ARM_FP16_CMP_4(eq)
4154EIGEN_MAKE_ARM_FP16_CMP_4(lt)
4155EIGEN_MAKE_ARM_FP16_CMP_4(le)
4156
4157#undef EIGEN_MAKE_ARM_FP16_CMP_8
4158#undef EIGEN_MAKE_ARM_FP16_CMP_4
4159
4160template <>
4161EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4162 return vreinterpretq_f16_u16(vmvnq_u16(vcgeq_f16(a, b)));
4163}
4164
4165template <>
4166EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4167 return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b)));
4168}
4169
4170template <>
4171EIGEN_STRONG_INLINE Packet8hf print<Packet8hf>(const Packet8hf& a)
4172{ return vrndnq_f16(a); }
4173
4174template <>
4175EIGEN_STRONG_INLINE Packet4hf print<Packet4hf>(const Packet4hf& a)
4176{ return vrndn_f16(a); }
4177
4178template <>
4179EIGEN_STRONG_INLINE Packet8hf pfloor<Packet8hf>(const Packet8hf& a)
4180{ return vrndmq_f16(a); }
4181
4182template <>
4183EIGEN_STRONG_INLINE Packet4hf pfloor<Packet4hf>(const Packet4hf& a)
4184{ return vrndm_f16(a); }
4185
4186template <>
4187EIGEN_STRONG_INLINE Packet8hf pceil<Packet8hf>(const Packet8hf& a)
4188{ return vrndpq_f16(a); }
4189
4190template <>
4191EIGEN_STRONG_INLINE Packet4hf pceil<Packet4hf>(const Packet4hf& a)
4192{ return vrndp_f16(a); }
4193
4194template <>
4195EIGEN_STRONG_INLINE Packet8hf psqrt<Packet8hf>(const Packet8hf& a) {
4196 return vsqrtq_f16(a);
4197}
4198
4199template <>
4200EIGEN_STRONG_INLINE Packet4hf psqrt<Packet4hf>(const Packet4hf& a) {
4201 return vsqrt_f16(a);
4202}
4203
4204template <>
4205EIGEN_STRONG_INLINE Packet8hf pand<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4206 return vreinterpretq_f16_u16(vandq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4207}
4208
4209template <>
4210EIGEN_STRONG_INLINE Packet4hf pand<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4211 return vreinterpret_f16_u16(vand_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4212}
4213
4214template <>
4215EIGEN_STRONG_INLINE Packet8hf por<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4216 return vreinterpretq_f16_u16(vorrq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4217}
4218
4219template <>
4220EIGEN_STRONG_INLINE Packet4hf por<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4221 return vreinterpret_f16_u16(vorr_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4222}
4223
4224template <>
4225EIGEN_STRONG_INLINE Packet8hf pxor<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4226 return vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4227}
4228
4229template <>
4230EIGEN_STRONG_INLINE Packet4hf pxor<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4231 return vreinterpret_f16_u16(veor_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4232}
4233
4234template <>
4235EIGEN_STRONG_INLINE Packet8hf pandnot<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4236 return vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4237}
4238
4239template <>
4240EIGEN_STRONG_INLINE Packet4hf pandnot<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4241 return vreinterpret_f16_u16(vbic_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4242}
4243
4244template <>
4245EIGEN_STRONG_INLINE Packet8hf pload<Packet8hf>(const Eigen::half* from) {
4246 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f16(reinterpret_cast<const float16_t*>(from));
4247}
4248
4249template <>
4250EIGEN_STRONG_INLINE Packet4hf pload<Packet4hf>(const Eigen::half* from) {
4251 EIGEN_DEBUG_ALIGNED_LOAD return vld1_f16(reinterpret_cast<const float16_t*>(from));
4252}
4253
4254template <>
4255EIGEN_STRONG_INLINE Packet8hf ploadu<Packet8hf>(const Eigen::half* from) {
4256 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f16(reinterpret_cast<const float16_t*>(from));
4257}
4258
4259template <>
4260EIGEN_STRONG_INLINE Packet4hf ploadu<Packet4hf>(const Eigen::half* from) {
4261 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f16(reinterpret_cast<const float16_t*>(from));
4262}
4263
4264template <>
4265EIGEN_STRONG_INLINE Packet8hf ploaddup<Packet8hf>(const Eigen::half* from) {
4266 Packet8hf packet;
4267 packet[0] = from[0].x;
4268 packet[1] = from[0].x;
4269 packet[2] = from[1].x;
4270 packet[3] = from[1].x;
4271 packet[4] = from[2].x;
4272 packet[5] = from[2].x;
4273 packet[6] = from[3].x;
4274 packet[7] = from[3].x;
4275 return packet;
4276}
4277
4278template <>
4279EIGEN_STRONG_INLINE Packet4hf ploaddup<Packet4hf>(const Eigen::half* from) {
4280 float16x4_t packet;
4281 float16_t* tmp;
4282 tmp = (float16_t*)&packet;
4283 tmp[0] = from[0].x;
4284 tmp[1] = from[0].x;
4285 tmp[2] = from[1].x;
4286 tmp[3] = from[1].x;
4287 return packet;
4288}
4289
4290template <>
4291EIGEN_STRONG_INLINE Packet8hf ploadquad<Packet8hf>(const Eigen::half* from) {
4292 Packet4hf lo, hi;
4293 lo = vld1_dup_f16(reinterpret_cast<const float16_t*>(from));
4294 hi = vld1_dup_f16(reinterpret_cast<const float16_t*>(from+1));
4295 return vcombine_f16(lo, hi);
4296}
4297
4298EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 0); }
4299
4300EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 0); }
4301
4302template <>
4303EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(const Packet8hf& mask, const Packet8hf& a, const Packet8hf& b) {
4304 return vbslq_f16(vreinterpretq_u16_f16(mask), a, b);
4305}
4306
4307template <>
4308EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(const Packet4hf& mask, const Packet4hf& a, const Packet4hf& b) {
4309 return vbsl_f16(vreinterpret_u16_f16(mask), a, b);
4310}
4311
4312EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 7); }
4313
4314EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 3); }
4315
4316template <>
4317EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8hf& from) {
4318 EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
4319}
4320
4321template <>
4322EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4hf& from) {
4323 EIGEN_DEBUG_ALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
4324}
4325
4326template <>
4327EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet8hf& from) {
4328 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
4329}
4330
4331template <>
4332EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4hf& from) {
4333 EIGEN_DEBUG_UNALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
4334}
4335
4336template <>
4337EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather<Eigen::half, Packet8hf>(const Eigen::half* from, Index stride) {
4338 Packet8hf res = pset1<Packet8hf>(Eigen::half(0.f));
4339 res = vsetq_lane_f16(from[0 * stride].x, res, 0);
4340 res = vsetq_lane_f16(from[1 * stride].x, res, 1);
4341 res = vsetq_lane_f16(from[2 * stride].x, res, 2);
4342 res = vsetq_lane_f16(from[3 * stride].x, res, 3);
4343 res = vsetq_lane_f16(from[4 * stride].x, res, 4);
4344 res = vsetq_lane_f16(from[5 * stride].x, res, 5);
4345 res = vsetq_lane_f16(from[6 * stride].x, res, 6);
4346 res = vsetq_lane_f16(from[7 * stride].x, res, 7);
4347 return res;
4348}
4349
4350template <>
4351EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather<Eigen::half, Packet4hf>(const Eigen::half* from, Index stride) {
4352 Packet4hf res = pset1<Packet4hf>(Eigen::half(0.f));
4353 res = vset_lane_f16(from[0 * stride].x, res, 0);
4354 res = vset_lane_f16(from[1 * stride].x, res, 1);
4355 res = vset_lane_f16(from[2 * stride].x, res, 2);
4356 res = vset_lane_f16(from[3 * stride].x, res, 3);
4357 return res;
4358}
4359
4360template <>
4361EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8hf>(Eigen::half* to, const Packet8hf& from, Index stride) {
4362 to[stride * 0].x = vgetq_lane_f16(from, 0);
4363 to[stride * 1].x = vgetq_lane_f16(from, 1);
4364 to[stride * 2].x = vgetq_lane_f16(from, 2);
4365 to[stride * 3].x = vgetq_lane_f16(from, 3);
4366 to[stride * 4].x = vgetq_lane_f16(from, 4);
4367 to[stride * 5].x = vgetq_lane_f16(from, 5);
4368 to[stride * 6].x = vgetq_lane_f16(from, 6);
4369 to[stride * 7].x = vgetq_lane_f16(from, 7);
4370}
4371
4372template <>
4373EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4hf>(Eigen::half* to, const Packet4hf& from, Index stride) {
4374 to[stride * 0].x = vget_lane_f16(from, 0);
4375 to[stride * 1].x = vget_lane_f16(from, 1);
4376 to[stride * 2].x = vget_lane_f16(from, 2);
4377 to[stride * 3].x = vget_lane_f16(from, 3);
4378}
4379
4380template <>
4381EIGEN_STRONG_INLINE void prefetch<Eigen::half>(const Eigen::half* addr) {
4382 EIGEN_ARM_PREFETCH(addr);
4383}
4384
4385template <>
4386EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8hf>(const Packet8hf& a) {
4387 float16_t x[8];
4388 vst1q_f16(x, a);
4389 Eigen::half h;
4390 h.x = x[0];
4391 return h;
4392}
4393
4394template <>
4395EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4hf>(const Packet4hf& a) {
4396 float16_t x[4];
4397 vst1_f16(x, a);
4398 Eigen::half h;
4399 h.x = x[0];
4400 return h;
4401}
4402
4403template<> EIGEN_STRONG_INLINE Packet8hf preverse(const Packet8hf& a) {
4404 float16x4_t a_lo, a_hi;
4405 Packet8hf a_r64;
4406
4407 a_r64 = vrev64q_f16(a);
4408 a_lo = vget_low_f16(a_r64);
4409 a_hi = vget_high_f16(a_r64);
4410 return vcombine_f16(a_hi, a_lo);
4411}
4412
4413template <>
4414EIGEN_STRONG_INLINE Packet4hf preverse<Packet4hf>(const Packet4hf& a) {
4415 return vrev64_f16(a);
4416}
4417
4418template <>
4419EIGEN_STRONG_INLINE Packet8hf pabs<Packet8hf>(const Packet8hf& a) {
4420 return vabsq_f16(a);
4421}
4422
4423template <>
4424EIGEN_STRONG_INLINE Packet4hf pabs<Packet4hf>(const Packet4hf& a) {
4425 return vabs_f16(a);
4426}
4427
4428template <>
4429EIGEN_STRONG_INLINE Eigen::half predux<Packet8hf>(const Packet8hf& a) {
4430 float16x4_t a_lo, a_hi, sum;
4431
4432 a_lo = vget_low_f16(a);
4433 a_hi = vget_high_f16(a);
4434 sum = vpadd_f16(a_lo, a_hi);
4435 sum = vpadd_f16(sum, sum);
4436 sum = vpadd_f16(sum, sum);
4437
4438 Eigen::half h;
4439 h.x = vget_lane_f16(sum, 0);
4440 return h;
4441}
4442
4443template <>
4444EIGEN_STRONG_INLINE Eigen::half predux<Packet4hf>(const Packet4hf& a) {
4445 float16x4_t sum;
4446
4447 sum = vpadd_f16(a, a);
4448 sum = vpadd_f16(sum, sum);
4449 Eigen::half h;
4450 h.x = vget_lane_f16(sum, 0);
4451 return h;
4452}
4453
4454template <>
4455EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8hf>(const Packet8hf& a) {
4456 float16x4_t a_lo, a_hi, prod;
4457
4458 a_lo = vget_low_f16(a);
4459 a_hi = vget_high_f16(a);
4460 prod = vmul_f16(a_lo, a_hi);
4461 prod = vmul_f16(prod, vrev64_f16(prod));
4462
4463 Eigen::half h;
4464 h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1));
4465 return h;
4466}
4467
4468template <>
4469EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4hf>(const Packet4hf& a) {
4470 float16x4_t prod;
4471 prod = vmul_f16(a, vrev64_f16(a));
4472 Eigen::half h;
4473 h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1));
4474 return h;
4475}
4476
4477template <>
4478EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8hf>(const Packet8hf& a) {
4479 float16x4_t a_lo, a_hi, min;
4480
4481 a_lo = vget_low_f16(a);
4482 a_hi = vget_high_f16(a);
4483 min = vpmin_f16(a_lo, a_hi);
4484 min = vpmin_f16(min, min);
4485 min = vpmin_f16(min, min);
4486
4487 Eigen::half h;
4488 h.x = vget_lane_f16(min, 0);
4489 return h;
4490}
4491
4492template <>
4493EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4hf>(const Packet4hf& a) {
4494 Packet4hf tmp;
4495 tmp = vpmin_f16(a, a);
4496 tmp = vpmin_f16(tmp, tmp);
4497 Eigen::half h;
4498 h.x = vget_lane_f16(tmp, 0);
4499 return h;
4500}
4501
4502template <>
4503EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8hf>(const Packet8hf& a) {
4504 float16x4_t a_lo, a_hi, max;
4505
4506 a_lo = vget_low_f16(a);
4507 a_hi = vget_high_f16(a);
4508 max = vpmax_f16(a_lo, a_hi);
4509 max = vpmax_f16(max, max);
4510 max = vpmax_f16(max, max);
4511
4512 Eigen::half h;
4513 h.x = vget_lane_f16(max, 0);
4514 return h;
4515}
4516
4517template <>
4518EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4hf>(const Packet4hf& a) {
4519 Packet4hf tmp;
4520 tmp = vpmax_f16(a, a);
4521 tmp = vpmax_f16(tmp, tmp);
4522 Eigen::half h;
4523 h.x = vget_lane_f16(tmp, 0);
4524 return h;
4525}
4526
4527EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8hf, 4>& kernel)
4528{
4529 const float16x8x2_t zip16_1 = vzipq_f16(kernel.packet[0], kernel.packet[1]);
4530 const float16x8x2_t zip16_2 = vzipq_f16(kernel.packet[2], kernel.packet[3]);
4531
4532 const float32x4x2_t zip32_1 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[0]), vreinterpretq_f32_f16(zip16_2.val[0]));
4533 const float32x4x2_t zip32_2 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[1]), vreinterpretq_f32_f16(zip16_2.val[1]));
4534
4535 kernel.packet[0] = vreinterpretq_f16_f32(zip32_1.val[0]);
4536 kernel.packet[1] = vreinterpretq_f16_f32(zip32_1.val[1]);
4537 kernel.packet[2] = vreinterpretq_f16_f32(zip32_2.val[0]);
4538 kernel.packet[3] = vreinterpretq_f16_f32(zip32_2.val[1]);
4539}
4540
4541EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4hf, 4>& kernel) {
4542 EIGEN_ALIGN16 float16x4x4_t tmp_x4;
4543 float16_t* tmp = (float16_t*)&kernel;
4544 tmp_x4 = vld4_f16(tmp);
4545
4546 kernel.packet[0] = tmp_x4.val[0];
4547 kernel.packet[1] = tmp_x4.val[1];
4548 kernel.packet[2] = tmp_x4.val[2];
4549 kernel.packet[3] = tmp_x4.val[3];
4550}
4551
4552EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8hf, 8>& kernel) {
4553 float16x8x2_t T_1[4];
4554
4555 T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]);
4556 T_1[1] = vuzpq_f16(kernel.packet[2], kernel.packet[3]);
4557 T_1[2] = vuzpq_f16(kernel.packet[4], kernel.packet[5]);
4558 T_1[3] = vuzpq_f16(kernel.packet[6], kernel.packet[7]);
4559
4560 float16x8x2_t T_2[4];
4561 T_2[0] = vuzpq_f16(T_1[0].val[0], T_1[1].val[0]);
4562 T_2[1] = vuzpq_f16(T_1[0].val[1], T_1[1].val[1]);
4563 T_2[2] = vuzpq_f16(T_1[2].val[0], T_1[3].val[0]);
4564 T_2[3] = vuzpq_f16(T_1[2].val[1], T_1[3].val[1]);
4565
4566 float16x8x2_t T_3[4];
4567 T_3[0] = vuzpq_f16(T_2[0].val[0], T_2[2].val[0]);
4568 T_3[1] = vuzpq_f16(T_2[0].val[1], T_2[2].val[1]);
4569 T_3[2] = vuzpq_f16(T_2[1].val[0], T_2[3].val[0]);
4570 T_3[3] = vuzpq_f16(T_2[1].val[1], T_2[3].val[1]);
4571
4572 kernel.packet[0] = T_3[0].val[0];
4573 kernel.packet[1] = T_3[2].val[0];
4574 kernel.packet[2] = T_3[1].val[0];
4575 kernel.packet[3] = T_3[3].val[0];
4576 kernel.packet[4] = T_3[0].val[1];
4577 kernel.packet[5] = T_3[2].val[1];
4578 kernel.packet[6] = T_3[1].val[1];
4579 kernel.packet[7] = T_3[3].val[1];
4580}
4581#endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
4582
4583} // end namespace internal
4584
4585} // end namespace Eigen
4586
4587#endif // EIGEN_PACKET_MATH_NEON_H
m
Matrix3f m
Definition AngleAxis_mimic_euler.cpp:1
a
ArrayXXi a
Definition Array_initializer_list_23_cxx11.cpp:1
n
int n
Definition BiCGSTAB_simple.cpp:1
i
int i
Definition BiCGSTAB_step_by_step.cpp:9
EIGEN_ALIGN16
#define EIGEN_ALIGN16
Definition ConfigureVectorization.h:153
EIGEN_DEBUG_ALIGNED_STORE
#define EIGEN_DEBUG_ALIGNED_STORE
Definition GenericPacketMath.h:35
EIGEN_DEBUG_ALIGNED_LOAD
#define EIGEN_DEBUG_ALIGNED_LOAD
Definition GenericPacketMath.h:27
EIGEN_DEBUG_UNALIGNED_STORE
#define EIGEN_DEBUG_UNALIGNED_STORE
Definition GenericPacketMath.h:39
EIGEN_DEBUG_UNALIGNED_LOAD
#define EIGEN_DEBUG_UNALIGNED_LOAD
Definition GenericPacketMath.h:31
EIGEN_ALWAYS_INLINE
#define EIGEN_ALWAYS_INLINE
Definition Macros.h:932
EIGEN_UNROLL_LOOP
#define EIGEN_UNROLL_LOOP
Definition Macros.h:1461
EIGEN_DEVICE_FUNC
#define EIGEN_DEVICE_FUNC
Definition Macros.h:976
eigen_assert
#define eigen_assert(x)
Definition Macros.h:1037
EIGEN_FAST_MATH
#define EIGEN_FAST_MATH
Definition Macros.h:49
EIGEN_STRONG_INLINE
#define EIGEN_STRONG_INLINE
Definition Macros.h:917
EIGEN_OPTIMIZATION_BARRIER
#define EIGEN_OPTIMIZATION_BARRIER(X)
Definition Macros.h:1144
p1
Vector3f p1
Definition MatrixBase_all.cpp:2
EIGEN_ARM_PREFETCH
#define EIGEN_ARM_PREFETCH(ADDR)
Definition PacketMath.h:162
res
cout<< "Here is the matrix m:"<< endl<< m<< endl;Matrix< ptrdiff_t, 3, 1 > res
Definition PartialRedux_count.cpp:3
vec4f_swizzle2
#define vec4f_swizzle2(a, b, p, q, r, s)
Definition PacketMath.h:70
vec4f_swizzle1
#define vec4f_swizzle1(v, p, q, r, s)
Definition PacketMath.h:61
vec2d_swizzle2
#define vec2d_swizzle2(a, b, mask)
Definition PacketMath.h:95
p
float * p
Definition Tutorial_Map_using.cpp:9
c
Scalar Scalar * c
Definition benchVecAdd.cpp:17
b
Scalar * b
Definition benchVecAdd.cpp:17
N
@ N
Definition constructor.cpp:23
min
#define min(a, b)
Definition datatypes.h:19
max
#define max(a, b)
Definition datatypes.h:20
x
set noclip points set clip one set noclip two set bar set border lt lw set xdata set ydata set zdata set x2data set y2data set boxwidth set dummy x
Definition gnuplot_common_settings.hh:12
Eigen::Unaligned
@ Unaligned
Definition Constants.h:233
Eigen::Aligned16
@ Aligned16
Definition Constants.h:235
s
RealScalar s
Definition level1_cplx_impl.h:126
Eigen::bfloat16_impl::raw_uint16_to_bfloat16
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(unsigned short value)
Eigen::internal::detail::zip_in_place< Packet8c >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet8c >(Packet8c &p1, Packet8c &p2)
Definition PacketMath.h:2794
Eigen::internal::detail::zip_in_place
void zip_in_place(Packet &p1, Packet &p2)
Eigen::internal::detail::zip_in_place< Packet2i >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet2i >(Packet2i &p1, Packet2i &p2)
Definition PacketMath.h:2822
Eigen::internal::detail::zip_in_place< Packet16c >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet16c >(Packet16c &p1, Packet16c &p2)
Definition PacketMath.h:2801
Eigen::internal::detail::zip_in_place< Packet4i >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet4i >(Packet4i &p1, Packet4i &p2)
Definition PacketMath.h:2829
Eigen::internal::detail::zip_in_place< Packet8uc >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet8uc >(Packet8uc &p1, Packet8uc &p2)
Definition PacketMath.h:2808
Eigen::internal::detail::zip_in_place< Packet2ui >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet2ui >(Packet2ui &p1, Packet2ui &p2)
Definition PacketMath.h:2836
Eigen::internal::detail::zip_in_place< Packet16uc >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet16uc >(Packet16uc &p1, Packet16uc &p2)
Definition PacketMath.h:2815
Eigen::internal::detail::zip_in_place< Packet4us >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet4us >(Packet4us &p1, Packet4us &p2)
Definition PacketMath.h:2864
Eigen::internal::detail::zip_in_place< Packet4ui >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet4ui >(Packet4ui &p1, Packet4ui &p2)
Definition PacketMath.h:2843
Eigen::internal::detail::zip_in_place< Packet8s >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet8s >(Packet8s &p1, Packet8s &p2)
Definition PacketMath.h:2857
Eigen::internal::detail::zip_in_place< Packet8us >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet8us >(Packet8us &p1, Packet8us &p2)
Definition PacketMath.h:2871
Eigen::internal::detail::zip_in_place< Packet4f >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet4f >(Packet4f &p1, Packet4f &p2)
Definition PacketMath.h:2787
Eigen::internal::detail::ptranspose_impl
EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock< Packet, 2 > &kernel)
Definition PacketMath.h:2878
Eigen::internal::detail::zip_in_place< Packet4bf >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet4bf >(Packet4bf &p1, Packet4bf &p2)
Definition PacketMath.h:3380
Eigen::internal::detail::zip_in_place< Packet2f >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet2f >(Packet2f &p1, Packet2f &p2)
Definition PacketMath.h:2780
Eigen::internal::detail::zip_in_place< Packet4s >
EIGEN_ALWAYS_INLINE void zip_in_place< Packet4s >(Packet4s &p1, Packet4s &p2)
Definition PacketMath.h:2850
Eigen::internal::predux_min< Packet2l >
EIGEN_STRONG_INLINE int64_t predux_min< Packet2l >(const Packet2l &a)
Definition PacketMath.h:2666
Eigen::internal::pdiv< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pdiv< Packet16uc >(const Packet16uc &, const Packet16uc &)
Definition PacketMath.h:1026
Eigen::internal::pmin< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pmin< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1207
Eigen::internal::pandnot< Packet4f >
EIGEN_STRONG_INLINE Packet4f pandnot< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:914
Eigen::internal::pabsdiff< Packet8s >
EIGEN_STRONG_INLINE Packet8s pabsdiff< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1165
Eigen::internal::psub< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui psub< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:859
Eigen::internal::pscatter< bfloat16, Packet4bf >
EIGEN_STRONG_INLINE void pscatter< bfloat16, Packet4bf >(bfloat16 *to, const Packet4bf &from, Index stride)
Definition PacketMath.h:3561
Eigen::internal::pgather< uint32_t, Packet2ui >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather< uint32_t, Packet2ui >(const uint32_t *from, Index stride)
Definition PacketMath.h:2089
Eigen::internal::pmax< Packet2i >
EIGEN_STRONG_INLINE Packet2i pmax< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1261
Eigen::internal::pmin< Packet16c >
EIGEN_STRONG_INLINE Packet16c pmin< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:846
Eigen::internal::pload< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pload< Packet8uc >(const uint8_t *from)
Definition PacketMath.h:1684
Eigen::internal::pabsdiff< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pabsdiff< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1175
Eigen::internal::pand< Packet8us >
EIGEN_STRONG_INLINE Packet8us pand< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:894
Eigen::internal::por< Packet8c >
EIGEN_STRONG_INLINE Packet8c por< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1503
Eigen::internal::pscatter< uint16_t, Packet4us >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint16_t, Packet4us >(uint16_t *to, const Packet4us &from, Index stride)
Definition PacketMath.h:2216
Eigen::internal::plset< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc plset< Packet4uc >(const uint8_t &a)
Definition PacketMath.h:742
Eigen::internal::pcmp_le< Packet16c >
EIGEN_STRONG_INLINE Packet16c pcmp_le< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1288
Eigen::internal::pgather< int8_t, Packet4c >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather< int8_t, Packet4c >(const int8_t *from, Index stride)
Definition PacketMath.h:1957
Eigen::internal::Packet2d
v2f64 Packet2d
Definition PacketMath.h:820
Eigen::internal::pset1< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pset1< Packet4ui >(const uint32_t &from)
Definition PacketMath.h:711
Eigen::internal::pstoreu< double >
EIGEN_STRONG_INLINE void pstoreu< double >(double *to, const Packet4d &from)
Definition PacketMath.h:627
Eigen::internal::pmax< Packet8s >
EIGEN_STRONG_INLINE Packet8s pmax< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:862
Eigen::internal::pload< Packet8c >
EIGEN_STRONG_INLINE Packet8c pload< Packet8c >(const int8_t *from)
Definition PacketMath.h:1674
Eigen::internal::predux< Packet2f >
EIGEN_STRONG_INLINE float predux< Packet2f >(const Packet2f &a)
Definition PacketMath.h:2387
Eigen::internal::psub< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui psub< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:858
Eigen::internal::predux< Packet2d >
EIGEN_STRONG_INLINE double predux< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1082
Eigen::internal::pstoreu< int8_t >
EIGEN_STRONG_INLINE void pstoreu< int8_t >(int8_t *to, const Packet4c &from)
Definition PacketMath.h:1910
Eigen::internal::prefetch< uint64_t >
EIGEN_STRONG_INLINE void prefetch< uint64_t >(const uint64_t *addr)
Definition PacketMath.h:2277
Eigen::internal::pconj
EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf &a)
Definition Complex.h:167
Eigen::internal::Packet2l
v2i64 Packet2l
Definition PacketMath.h:821
Eigen::internal::psub< Packet4c >
EIGEN_STRONG_INLINE Packet4c psub< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:836
Eigen::internal::predux< Packet16c >
EIGEN_STRONG_INLINE signed char predux< Packet16c >(const Packet16c &a)
Definition PacketMath.h:1521
Eigen::internal::psub< Packet8c >
EIGEN_STRONG_INLINE Packet8c psub< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:842
Eigen::internal::pdiv< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pdiv< Packet2ui >(const Packet2ui &, const Packet2ui &)
Definition PacketMath.h:1061
Eigen::internal::pcmp_lt< Packet4s >
EIGEN_STRONG_INLINE Packet4s pcmp_lt< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1361
Eigen::internal::pmax< Packet16c >
EIGEN_STRONG_INLINE Packet16c pmax< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:864
Eigen::internal::pgather< uint8_t, Packet16uc >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather< uint8_t, Packet16uc >(const uint8_t *from, Index stride)
Definition PacketMath.h:2015
Eigen::internal::pcmp_eq< Packet4s >
EIGEN_STRONG_INLINE Packet4s pcmp_eq< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1422
Eigen::internal::pset1< Packet4us >
EIGEN_STRONG_INLINE Packet4us pset1< Packet4us >(const uint16_t &from)
Definition PacketMath.h:706
Eigen::internal::pand< Packet2i >
EIGEN_STRONG_INLINE Packet2i pand< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1487
Eigen::internal::prefetch< int8_t >
EIGEN_STRONG_INLINE void prefetch< int8_t >(const int8_t *addr)
Definition PacketMath.h:2270
Eigen::internal::pand< Packet8c >
EIGEN_STRONG_INLINE Packet8c pand< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1471
Eigen::internal::prefetch< uint32_t >
EIGEN_STRONG_INLINE void prefetch< uint32_t >(const uint32_t *addr)
Definition PacketMath.h:2275
Eigen::internal::pandnot< Packet2l >
EIGEN_STRONG_INLINE Packet2l pandnot< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1594
Eigen::internal::predux< Packet2l >
EIGEN_STRONG_INLINE int64_t predux< Packet2l >(const Packet2l &a)
Definition PacketMath.h:2473
Eigen::internal::padd
EIGEN_DEVICE_FUNC Packet padd(const Packet &a, const Packet &b)
Definition GenericPacketMath.h:215
Eigen::internal::predux_min< Packet2f >
EIGEN_STRONG_INLINE float predux_min< Packet2f >(const Packet2f &a)
Definition PacketMath.h:2577
Eigen::internal::pmin< Packet4us >
EIGEN_STRONG_INLINE Packet4us pmin< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1211
Eigen::internal::predux_min< Packet8c >
EIGEN_STRONG_INLINE int8_t predux_min< Packet8c >(const Packet8c &a)
Definition PacketMath.h:2591
Eigen::internal::por< Packet4us >
EIGEN_STRONG_INLINE Packet4us por< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1516
Eigen::internal::ploadu< Packet8c >
EIGEN_STRONG_INLINE Packet8c ploadu< Packet8c >(const int8_t *from)
Definition PacketMath.h:1719
Eigen::internal::pabsdiff< Packet8us >
EIGEN_STRONG_INLINE Packet8us pabsdiff< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:1169
Eigen::internal::pmin< Packet4f >
EIGEN_STRONG_INLINE Packet4f pmin< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:832
Eigen::internal::padd< Packet2d >
EIGEN_STRONG_INLINE Packet2d padd< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:880
Eigen::internal::pandnot< Packet2d >
EIGEN_STRONG_INLINE Packet2d pandnot< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:960
Eigen::internal::Packet2ui
uint32x2_t Packet2ui
Definition PacketMath.h:76
Eigen::internal::padd< Packet8c >
EIGEN_STRONG_INLINE Packet8c padd< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:813
Eigen::internal::Bf16ToF32
EIGEN_STRONG_INLINE Packet8f Bf16ToF32(const Packet8bf &a)
Definition PacketMath.h:1260
Eigen::internal::pzero
EIGEN_STRONG_INLINE Packet8f pzero(const Packet8f &)
Definition PacketMath.h:247
Eigen::internal::predux_mul< Packet2i >
EIGEN_STRONG_INLINE int32_t predux_mul< Packet2i >(const Packet2i &a)
Definition PacketMath.h:2563
Eigen::internal::pand< Packet2f >
EIGEN_STRONG_INLINE Packet2f pand< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1465
Eigen::internal::psub< Packet2i >
EIGEN_STRONG_INLINE Packet2i psub< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:856
Eigen::internal::predux_max< Packet4ui >
EIGEN_STRONG_INLINE uint32_t predux_max< Packet4ui >(const Packet4ui &a)
Definition PacketMath.h:2756
Eigen::internal::pscatter< uint64_t, Packet2ul >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint64_t, Packet2ul >(uint64_t *to, const Packet2ul &from, Index stride)
Definition PacketMath.h:2263
Eigen::internal::Packet4i
__vector int Packet4i
Definition PacketMath.h:31
Eigen::internal::ploadu< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc ploadu< Packet16uc >(const unsigned char *from)
Definition PacketMath.h:992
Eigen::internal::ploadu< Packet4us >
EIGEN_STRONG_INLINE Packet4us ploadu< Packet4us >(const uint16_t *from)
Definition PacketMath.h:1737
Eigen::internal::pcmp_le< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pcmp_le< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3616
Eigen::internal::pfirst< Packet16uc >
EIGEN_STRONG_INLINE unsigned char pfirst< Packet16uc >(const Packet16uc &a)
Definition PacketMath.h:1142
Eigen::internal::vec4f_movelh
EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:121
Eigen::internal::pset1< Packet2f >
EIGEN_STRONG_INLINE Packet2f pset1< Packet2f >(const float &from)
Definition PacketMath.h:694
Eigen::internal::ploadu< Packet2l >
EIGEN_STRONG_INLINE Packet2l ploadu< Packet2l >(const int64_t *from)
Definition PacketMath.h:1749
Eigen::internal::pstore< int64_t >
EIGEN_STRONG_INLINE void pstore< int64_t >(int64_t *to, const Packet2l &from)
Definition PacketMath.h:1901
Eigen::internal::pcmp_eq< Packet4f >
EIGEN_STRONG_INLINE Packet4f pcmp_eq< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1400
Eigen::internal::pdiv< Packet8c >
EIGEN_STRONG_INLINE Packet8c pdiv< Packet8c >(const Packet8c &, const Packet8c &)
Definition PacketMath.h:1006
Eigen::internal::pgather< int64_t, Packet2l >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather< int64_t, Packet2l >(const int64_t *from, Index stride)
Definition PacketMath.h:2103
Eigen::internal::ploadquad< Packet16c >
EIGEN_STRONG_INLINE Packet16c ploadquad< Packet16c >(const int8_t *from)
Definition PacketMath.h:1834
Eigen::internal::pmin< PropagateNaN, Packet2d >
EIGEN_STRONG_INLINE Packet2d pmin< PropagateNaN, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:585
Eigen::internal::padd< Packet4f >
EIGEN_STRONG_INLINE Packet4f padd< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:774
Eigen::internal::pmul< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pmul< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:800
Eigen::internal::pset1< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pset1< Packet4bf >(const bfloat16 &from)
Definition PacketMath.h:3423
Eigen::internal::por< Packet4i >
EIGEN_STRONG_INLINE Packet4i por< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:901
Eigen::internal::pmax< PropagateNumbers, Packet2d >
EIGEN_STRONG_INLINE Packet2d pmax< PropagateNumbers, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:577
Eigen::internal::Packet8c
int8x8_t Packet8c
Definition PacketMath.h:65
Eigen::internal::ploaddup< Packet4c >
EIGEN_STRONG_INLINE Packet4c ploaddup< Packet4c >(const int8_t *from)
Definition PacketMath.h:1758
Eigen::internal::pcmp_eq< Packet8c >
EIGEN_STRONG_INLINE Packet8c pcmp_eq< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1408
Eigen::internal::pdiv< Packet16c >
EIGEN_STRONG_INLINE Packet16c pdiv< Packet16c >(const Packet16c &, const Packet16c &)
Definition PacketMath.h:1011
Eigen::internal::por< Packet16c >
EIGEN_STRONG_INLINE Packet16c por< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1504
Eigen::internal::predux_min< Packet8s >
EIGEN_STRONG_INLINE short int predux_min< Packet8s >(const Packet8s &a)
Definition PacketMath.h:1631
Eigen::internal::pmin< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pmin< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1222
Eigen::internal::pgather< uint16_t, Packet8us >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather< uint16_t, Packet8us >(const uint16_t *from, Index stride)
Definition PacketMath.h:2063
Eigen::internal::pcmp_le< Packet2i >
EIGEN_STRONG_INLINE Packet2i pcmp_le< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1308
Eigen::internal::preverse< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf preverse< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3586
Eigen::internal::pmul< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pmul< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:938
Eigen::internal::Packet16uc
__vector unsigned char Packet16uc
Definition PacketMath.h:37
Eigen::internal::pscatter< int64_t, Packet2l >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int64_t, Packet2l >(int64_t *to, const Packet2l &from, Index stride)
Definition PacketMath.h:2258
Eigen::internal::pandnot< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pandnot< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1574
Eigen::internal::pset1< Packet4i >
EIGEN_STRONG_INLINE Packet4i pset1< Packet4i >(const int &from)
Definition PacketMath.h:551
Eigen::internal::pload< Packet16c >
EIGEN_STRONG_INLINE Packet16c pload< Packet16c >(const signed char *from)
Definition PacketMath.h:463
Eigen::internal::pdiv< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pdiv< Packet4ui >(const Packet4ui &, const Packet4ui &)
Definition PacketMath.h:1066
Eigen::internal::pmin< Packet8us >
EIGEN_STRONG_INLINE Packet8us pmin< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:845
Eigen::internal::predux_max< Packet4us >
EIGEN_STRONG_INLINE uint16_t predux_max< Packet4us >(const Packet4us &a)
Definition PacketMath.h:2735
Eigen::internal::paddsub< Packet2d >
EIGEN_STRONG_INLINE Packet2d paddsub< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:312
Eigen::internal::pabsdiff< Packet4us >
EIGEN_STRONG_INLINE Packet4us pabsdiff< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1167
Eigen::internal::shuffle2
EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int mask)
Definition PacketMath.h:94
Eigen::internal::pmin< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmin< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3471
Eigen::internal::Packet4bf
eigen_packet_wrapper< uint16x4_t, 19 > Packet4bf
Definition PacketMath.h:3317
Eigen::internal::pscatter< int8_t, Packet4c >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int8_t, Packet4c >(int8_t *to, const Packet4c &from, Index stride)
Definition PacketMath.h:2128
Eigen::internal::pfirst< Packet4c >
EIGEN_STRONG_INLINE int8_t pfirst< Packet4c >(const Packet4c &a)
Definition PacketMath.h:2281
Eigen::internal::psub< Packet16c >
EIGEN_STRONG_INLINE Packet16c psub< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:786
Eigen::internal::ploaddup< Packet2i >
EIGEN_STRONG_INLINE Packet2i ploaddup< Packet2i >(const int32_t *from)
Definition PacketMath.h:1812
Eigen::internal::pgather< uint8_t, Packet8uc >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather< uint8_t, Packet8uc >(const uint8_t *from, Index stride)
Definition PacketMath.h:2003
Eigen::internal::pmax< PropagateNaN, Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmax< PropagateNaN, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3482
Eigen::internal::pload< Packet4us >
EIGEN_STRONG_INLINE Packet4us pload< Packet4us >(const uint16_t *from)
Definition PacketMath.h:1692
Eigen::internal::pload< Packet4c >
EIGEN_STRONG_INLINE Packet4c pload< Packet4c >(const int8_t *from)
Definition PacketMath.h:1668
Eigen::internal::pscatter< int32_t, Packet2i >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int32_t, Packet2i >(int32_t *to, const Packet2i &from, Index stride)
Definition PacketMath.h:2234
Eigen::internal::predux_mul< Packet8us >
EIGEN_STRONG_INLINE unsigned short int predux_mul< Packet8us >(const Packet8us &a)
Definition PacketMath.h:1558
Eigen::internal::pgather< uint32_t, Packet4ui >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather< uint32_t, Packet4ui >(const uint32_t *from, Index stride)
Definition PacketMath.h:2095
Eigen::internal::ploadquad< Packet4c >
EIGEN_STRONG_INLINE Packet4c ploadquad< Packet4c >(const int8_t *from)
Definition PacketMath.h:1826
Eigen::internal::vec2d_unpackhi
EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:102
Eigen::internal::pcmp_eq< Packet8s >
EIGEN_STRONG_INLINE Packet8s pcmp_eq< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1424
Eigen::internal::pstoreu< Eigen::half >
EIGEN_STRONG_INLINE void pstoreu< Eigen::half >(Eigen::half *to, const Packet8h &from)
Definition PacketMath.h:958
Eigen::internal::pxor< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pxor< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1561
Eigen::internal::pfirst< Packet4f >
EIGEN_STRONG_INLINE float pfirst< Packet4f >(const Packet4f &a)
Definition PacketMath.h:1120
Eigen::internal::pand< Packet2d >
EIGEN_STRONG_INLINE Packet2d pand< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:939
Eigen::internal::pmax< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pmax< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:865
Eigen::internal::pandnot< Packet4c >
EIGEN_STRONG_INLINE Packet4c pandnot< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1568
Eigen::internal::ploadquad< Packet4f >
EIGEN_STRONG_INLINE Packet4f ploadquad< Packet4f >(const float *from)
Definition PacketMath.h:1825
Eigen::internal::pset1< Packet4s >
EIGEN_STRONG_INLINE Packet4s pset1< Packet4s >(const int16_t &from)
Definition PacketMath.h:704
Eigen::internal::shuffle1
EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f &m, int mask)
Definition PacketMath.h:83
Eigen::internal::prefetch< uint16_t >
EIGEN_STRONG_INLINE void prefetch< uint16_t >(const uint16_t *addr)
Definition PacketMath.h:2273
Eigen::internal::pandnot< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pandnot< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1596
Eigen::internal::pcmp_le< Packet8us >
EIGEN_STRONG_INLINE Packet8us pcmp_le< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:1306
Eigen::internal::pmin< Packet4s >
EIGEN_STRONG_INLINE Packet4s pmin< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1209
Eigen::internal::por< Packet8s >
EIGEN_STRONG_INLINE Packet8s por< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:902
Eigen::internal::pcmp_lt< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pcmp_lt< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1357
Eigen::internal::prefetch< int64_t >
EIGEN_STRONG_INLINE void prefetch< int64_t >(const int64_t *addr)
Definition PacketMath.h:2276
Eigen::internal::psub< Packet8us >
EIGEN_STRONG_INLINE Packet8us psub< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:785
Eigen::internal::pcmp_eq< Packet2f >
EIGEN_STRONG_INLINE Packet2f pcmp_eq< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1398
Eigen::internal::print< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf print< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3521
Eigen::internal::pxor< Packet8c >
EIGEN_STRONG_INLINE Packet8c pxor< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1537
Eigen::internal::pmul< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmul< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3546
Eigen::internal::pset1< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pset1< Packet2ul >(const uint64_t &from)
Definition PacketMath.h:713
Eigen::internal::pdiv< Packet4c >
EIGEN_STRONG_INLINE Packet4c pdiv< Packet4c >(const Packet4c &, const Packet4c &)
Definition PacketMath.h:1001
Eigen::internal::padd< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui padd< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:776
Eigen::internal::pxor< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pxor< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1543
Eigen::internal::ptranspose
EIGEN_STRONG_INLINE void ptranspose(PacketBlock< Packet2cf, 2 > &kernel)
Definition Complex.h:224
Eigen::internal::plset< Packet16c >
EIGEN_STRONG_INLINE Packet16c plset< Packet16c >(const signed char &a)
Definition PacketMath.h:771
Eigen::internal::pand< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pand< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:893
Eigen::internal::ploadquad< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc ploadquad< Packet4uc >(const uint8_t *from)
Definition PacketMath.h:1844
Eigen::internal::pcmp_eq< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pcmp_eq< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1448
Eigen::internal::pmin< PropagateNaN, Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmin< PropagateNaN, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3465
Eigen::internal::pgather< int16_t, Packet8s >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather< int16_t, Packet8s >(const int16_t *from, Index stride)
Definition PacketMath.h:2043
Eigen::internal::pfirst< Packet8uc >
EIGEN_STRONG_INLINE uint8_t pfirst< Packet8uc >(const Packet8uc &a)
Definition PacketMath.h:2285
Eigen::internal::pabsdiff< Packet16c >
EIGEN_STRONG_INLINE Packet16c pabsdiff< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1151
Eigen::internal::predux_min< Packet2i >
EIGEN_STRONG_INLINE int32_t predux_min< Packet2i >(const Packet2i &a)
Definition PacketMath.h:2652
Eigen::internal::pset1< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pset1< Packet16uc >(const unsigned char &from)
Definition PacketMath.h:567
Eigen::internal::pmax< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmax< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3488
Eigen::internal::pscatter< uint32_t, Packet4ui >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint32_t, Packet4ui >(uint32_t *to, const Packet4ui &from, Index stride)
Definition PacketMath.h:2251
Eigen::internal::pdiv< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pdiv< Packet4uc >(const Packet4uc &, const Packet4uc &)
Definition PacketMath.h:1016
Eigen::internal::Packet4c
eigen_packet_wrapper< int32_t,2 > Packet4c
Definition PacketMath.h:64
Eigen::internal::ploaddup< Packet4i >
EIGEN_STRONG_INLINE Packet4i ploaddup< Packet4i >(const int *from)
Definition PacketMath.h:1008
Eigen::internal::predux_any
EIGEN_STRONG_INLINE bool predux_any(const Packet4f &x)
Definition PacketMath.h:1765
Eigen::internal::predux_mul< Packet2ui >
EIGEN_STRONG_INLINE uint32_t predux_mul< Packet2ui >(const Packet2ui &a)
Definition PacketMath.h:2567
Eigen::internal::pcmp_le< Packet4us >
EIGEN_STRONG_INLINE Packet4us pcmp_le< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1304
Eigen::internal::padd< Packet4s >
EIGEN_STRONG_INLINE Packet4s padd< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:823
Eigen::internal::pand< Packet4s >
EIGEN_STRONG_INLINE Packet4s pand< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1481
Eigen::internal::predux_max< Packet4f >
EIGEN_STRONG_INLINE float predux_max< Packet4f >(const Packet4f &a)
Definition PacketMath.h:1693
Eigen::internal::padd< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf padd< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3538
Eigen::internal::ploadu< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui ploadu< Packet2ui >(const uint32_t *from)
Definition PacketMath.h:1745
Eigen::internal::ploaddup< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul ploaddup< Packet2ul >(const uint64_t *from)
Definition PacketMath.h:1822
Eigen::internal::ploaddup< Packet2d >
EIGEN_STRONG_INLINE Packet2d ploaddup< Packet2d >(const double *from)
Definition PacketMath.h:1011
Eigen::internal::plset< Packet8us >
EIGEN_STRONG_INLINE Packet8us plset< Packet8us >(const unsigned short int &a)
Definition PacketMath.h:770
Eigen::internal::Packet8us
__vector unsigned short int Packet8us
Definition PacketMath.h:35
Eigen::internal::pcmp_eq< Packet2l >
EIGEN_STRONG_INLINE Packet2l pcmp_eq< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1438
Eigen::internal::vec4f_movehl
EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:125
Eigen::internal::pxor< Packet2d >
EIGEN_STRONG_INLINE Packet2d pxor< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:953
Eigen::internal::predux_max< Packet4uc >
EIGEN_STRONG_INLINE uint8_t predux_max< Packet4uc >(const Packet4uc &a)
Definition PacketMath.h:2701
Eigen::internal::predux_min< Packet2ui >
EIGEN_STRONG_INLINE uint32_t predux_min< Packet2ui >(const Packet2ui &a)
Definition PacketMath.h:2659
Eigen::internal::predux< Packet16uc >
EIGEN_STRONG_INLINE unsigned char predux< Packet16uc >(const Packet16uc &a)
Definition PacketMath.h:1526
Eigen::internal::predux_max< Packet16uc >
EIGEN_STRONG_INLINE unsigned char predux_max< Packet16uc >(const Packet16uc &a)
Definition PacketMath.h:1753
Eigen::internal::por< Packet2d >
EIGEN_STRONG_INLINE Packet2d por< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:946
Eigen::internal::pload< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pload< Packet2ul >(const uint64_t *from)
Definition PacketMath.h:1706
Eigen::internal::pceil< Packet2f >
EIGEN_STRONG_INLINE Packet2f pceil< Packet2f >(const Packet2f &a)
Definition PacketMath.h:3173
Eigen::internal::shuffle2< true >
EIGEN_STRONG_INLINE Packet4f shuffle2< true >(const Packet4f &m, const Packet4f &n, int mask)
Definition PacketMath.h:103
Eigen::internal::pstore< bfloat16 >
EIGEN_STRONG_INLINE void pstore< bfloat16 >(bfloat16 *to, const Packet8bf &from)
Definition PacketMath.h:511
Eigen::internal::pldexp< Packet2d >
EIGEN_STRONG_INLINE Packet2d pldexp< Packet2d >(const Packet2d &a, const Packet2d &exponent)
Definition PacketMath.h:928
Eigen::internal::pmul< Packet2f >
EIGEN_STRONG_INLINE Packet2f pmul< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:913
Eigen::internal::pmax< PropagateNaN, Packet2f >
EIGEN_STRONG_INLINE Packet2f pmax< PropagateNaN, Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1239
Eigen::internal::predux_mul< Packet4s >
EIGEN_STRONG_INLINE int16_t predux_mul< Packet4s >(const Packet4s &a)
Definition PacketMath.h:2531
Eigen::internal::pand< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pand< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1489
Eigen::internal::pmax< Packet4us >
EIGEN_STRONG_INLINE Packet4us pmax< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1259
Eigen::internal::padd< Packet2l >
EIGEN_STRONG_INLINE Packet2l padd< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:831
Eigen::internal::ploadu< Packet8s >
EIGEN_STRONG_INLINE Packet8s ploadu< Packet8s >(const short int *from)
Definition PacketMath.h:976
Eigen::internal::pmax< PropagateNumbers, Packet4f >
EIGEN_STRONG_INLINE Packet4f pmax< PropagateNumbers, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:573
Eigen::internal::pcmp_le< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pcmp_le< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1314
Eigen::internal::pdiv< Packet8us >
EIGEN_STRONG_INLINE Packet8us pdiv< Packet8us >(const Packet8us &, const Packet8us &)
Definition PacketMath.h:1046
Eigen::internal::pmin< PropagateNaN, Packet2f >
EIGEN_STRONG_INLINE Packet2f pmin< PropagateNaN, Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1191
Eigen::internal::predux_min< Packet4us >
EIGEN_STRONG_INLINE uint16_t predux_min< Packet4us >(const Packet4us &a)
Definition PacketMath.h:2640
Eigen::internal::padd< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui padd< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:829
Eigen::internal::pcmp_lt< Packet2l >
EIGEN_STRONG_INLINE Packet2l pcmp_lt< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1377
Eigen::internal::pmul< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pmul< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:923
Eigen::internal::pabsdiff< Packet4c >
EIGEN_STRONG_INLINE Packet4c pabsdiff< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1143
Eigen::internal::ploaddup< Packet4f >
EIGEN_STRONG_INLINE Packet4f ploaddup< Packet4f >(const float *from)
Definition PacketMath.h:1004
Eigen::internal::por< Packet4f >
EIGEN_STRONG_INLINE Packet4f por< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:900
Eigen::internal::pscatter< uint8_t, Packet8uc >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint8_t, Packet8uc >(uint8_t *to, const Packet8uc &from, Index stride)
Definition PacketMath.h:2168
Eigen::internal::por< Packet2l >
EIGEN_STRONG_INLINE Packet2l por< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1526
Eigen::internal::prefetch< int32_t >
EIGEN_STRONG_INLINE void prefetch< int32_t >(const int32_t *addr)
Definition PacketMath.h:516
Eigen::internal::pset1< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pset1< Packet8uc >(const uint8_t &from)
Definition PacketMath.h:702
Eigen::internal::pabsdiff< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pabsdiff< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1177
Eigen::internal::pmin< Packet2i >
EIGEN_STRONG_INLINE Packet2i pmin< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1213
Eigen::internal::plogical_shift_left
EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i &a)
Definition PacketMath.h:1191
Eigen::internal::plset< Packet8s >
EIGEN_STRONG_INLINE Packet8s plset< Packet8s >(const short int &a)
Definition PacketMath.h:769
Eigen::internal::predux_mul< Packet16uc >
EIGEN_STRONG_INLINE unsigned char predux_mul< Packet16uc >(const Packet16uc &a)
Definition PacketMath.h:1590
Eigen::internal::pcmp_eq< Packet4c >
EIGEN_STRONG_INLINE Packet4c pcmp_eq< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1402
Eigen::internal::predux_mul< Packet8s >
EIGEN_STRONG_INLINE short int predux_mul< Packet8s >(const Packet8s &a)
Definition PacketMath.h:1547
Eigen::internal::padd< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc padd< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:780
Eigen::internal::predux_min< Packet4i >
EIGEN_STRONG_INLINE int predux_min< Packet4i >(const Packet4i &a)
Definition PacketMath.h:1618
Eigen::internal::pload< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pload< Packet2ui >(const uint32_t *from)
Definition PacketMath.h:1700
Eigen::internal::psub< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc psub< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:787
Eigen::internal::plset< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc plset< Packet8uc >(const uint8_t &a)
Definition PacketMath.h:744
Eigen::internal::pcmp_lt_or_nan< Packet2f >
EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1459
Eigen::internal::pmul< Packet8c >
EIGEN_STRONG_INLINE Packet8c pmul< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:921
Eigen::internal::ploadu< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf ploadu< Packet4bf >(const bfloat16 *from)
Definition PacketMath.h:3436
Eigen::internal::pabsdiff< Packet8c >
EIGEN_STRONG_INLINE Packet8c pabsdiff< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1149
Eigen::internal::pxor< Packet4i >
EIGEN_STRONG_INLINE Packet4i pxor< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:909
Eigen::internal::print
EIGEN_STRONG_INLINE Packet4f print(const Packet4f &a)
Definition PacketMath.h:3115
Eigen::internal::padd< Packet2i >
EIGEN_STRONG_INLINE Packet2i padd< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:827
Eigen::internal::predux_max< Packet2d >
EIGEN_STRONG_INLINE double predux_max< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1116
Eigen::internal::predux_max< Packet8c >
EIGEN_STRONG_INLINE int8_t predux_max< Packet8c >(const Packet8c &a)
Definition PacketMath.h:2686
Eigen::internal::pcmp_lt< Packet4c >
EIGEN_STRONG_INLINE Packet4c pcmp_lt< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1341
Eigen::internal::pmul< Packet4f >
EIGEN_STRONG_INLINE Packet4f pmul< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:795
Eigen::internal::pcmp_lt< Packet8us >
EIGEN_STRONG_INLINE Packet8us pcmp_lt< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:1367
Eigen::internal::pcmp_lt< Packet2f >
EIGEN_STRONG_INLINE Packet2f pcmp_lt< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1337
Eigen::internal::pxor< Packet4s >
EIGEN_STRONG_INLINE Packet4s pxor< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1547
Eigen::internal::pcmp_le< Packet2l >
EIGEN_STRONG_INLINE Packet2l pcmp_le< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1316
Eigen::internal::predux< Packet4ui >
EIGEN_STRONG_INLINE uint32_t predux< Packet4ui >(const Packet4ui &a)
Definition PacketMath.h:2468
Eigen::internal::pcmp_eq< Packet4i >
EIGEN_STRONG_INLINE Packet4i pcmp_eq< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:1432
Eigen::internal::por< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui por< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1524
Eigen::internal::plset< Packet4c >
EIGEN_STRONG_INLINE Packet4c plset< Packet4c >(const int8_t &a)
Definition PacketMath.h:730
Eigen::internal::ploaddup< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc ploaddup< Packet8uc >(const uint8_t *from)
Definition PacketMath.h:1779
Eigen::internal::pmin< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pmin< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1216
Eigen::internal::pmin< Packet4c >
EIGEN_STRONG_INLINE Packet4c pmin< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1193
Eigen::internal::pgather< float, Packet4f >
EIGEN_DEVICE_FUNC Packet4f pgather< float, Packet4f >(const float *from, Index stride)
Definition PacketMath.h:613
Eigen::internal::pmax< Packet8us >
EIGEN_STRONG_INLINE Packet8us pmax< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:863
Eigen::internal::pfirst< Packet8s >
EIGEN_STRONG_INLINE short int pfirst< Packet8s >(const Packet8s &a)
Definition PacketMath.h:1129
Eigen::internal::pcmp_le
EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:867
Eigen::internal::paddsub< Packet4f >
EIGEN_STRONG_INLINE Packet4f paddsub< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:869
Eigen::internal::plset< Packet8c >
EIGEN_STRONG_INLINE Packet8c plset< Packet8c >(const int8_t &a)
Definition PacketMath.h:732
Eigen::internal::pset1< Packet2d >
EIGEN_STRONG_INLINE Packet2d pset1< Packet2d >(const double &from)
Definition PacketMath.h:872
Eigen::internal::plogical_shift_right
EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i &a)
Definition PacketMath.h:1189
Eigen::internal::pcmp_eq< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pcmp_eq< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1412
Eigen::internal::pgather< uint16_t, Packet4us >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather< uint16_t, Packet4us >(const uint16_t *from, Index stride)
Definition PacketMath.h:2055
Eigen::internal::pcmp_eq< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pcmp_eq< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1434
Eigen::internal::pscatter< uint32_t, Packet2ui >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint32_t, Packet2ui >(uint32_t *to, const Packet2ui &from, Index stride)
Definition PacketMath.h:2246
Eigen::internal::pload< Packet4f >
EIGEN_STRONG_INLINE Packet4f pload< Packet4f >(const float *from)
Definition PacketMath.h:443
Eigen::internal::Packet16c
__vector signed char Packet16c
Definition PacketMath.h:36
Eigen::internal::pstore< int8_t >
EIGEN_STRONG_INLINE void pstore< int8_t >(int8_t *to, const Packet4c &from)
Definition PacketMath.h:1873
Eigen::internal::predux_mul< Packet4i >
EIGEN_STRONG_INLINE int predux_mul< Packet4i >(const Packet4i &a)
Definition PacketMath.h:1540
Eigen::internal::pload< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pload< Packet4uc >(const uint8_t *from)
Definition PacketMath.h:1678
Eigen::internal::plset< Packet4us >
EIGEN_STRONG_INLINE Packet4us plset< Packet4us >(const uint16_t &a)
Definition PacketMath.h:759
Eigen::internal::pload< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pload< Packet16uc >(const unsigned char *from)
Definition PacketMath.h:468
Eigen::internal::ploaddup< Packet4us >
EIGEN_STRONG_INLINE Packet4us ploaddup< Packet4us >(const uint16_t *from)
Definition PacketMath.h:1801
Eigen::internal::por< Packet4s >
EIGEN_STRONG_INLINE Packet4s por< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1512
Eigen::internal::ploadu< Packet8us >
EIGEN_STRONG_INLINE Packet8us ploadu< Packet8us >(const unsigned short int *from)
Definition PacketMath.h:980
Eigen::internal::por< Packet4c >
EIGEN_STRONG_INLINE Packet4c por< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1501
Eigen::internal::pldexp< Packet2f >
EIGEN_STRONG_INLINE Packet2f pldexp< Packet2f >(const Packet2f &a, const Packet2f &exponent)
Definition PacketMath.h:2382
Eigen::internal::predux< Packet4s >
EIGEN_STRONG_INLINE int16_t predux< Packet4s >(const Packet4s &a)
Definition PacketMath.h:2437
Eigen::internal::predux_mul< Packet16c >
EIGEN_STRONG_INLINE signed char predux_mul< Packet16c >(const Packet16c &a)
Definition PacketMath.h:1578
Eigen::internal::por
EIGEN_STRONG_INLINE Packet8h por(const Packet8h &a, const Packet8h &b)
Definition PacketMath.h:1042
Eigen::internal::pmin< Packet2f >
EIGEN_STRONG_INLINE Packet2f pmin< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1180
Eigen::internal::pmin< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pmin< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1215
Eigen::internal::pmul< Packet8us >
EIGEN_STRONG_INLINE Packet8us pmul< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:798
Eigen::internal::predux_max< Packet2ul >
EIGEN_STRONG_INLINE uint64_t predux_max< Packet2ul >(const Packet2ul &a)
Definition PacketMath.h:2763
Eigen::internal::pand< Packet8s >
EIGEN_STRONG_INLINE Packet8s pand< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1482
Eigen::internal::Packet4ui
__vector unsigned int Packet4ui
Definition PacketMath.h:32
Eigen::internal::pmax< PropagateNaN, Packet2d >
EIGEN_STRONG_INLINE Packet2d pmax< PropagateNaN, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:593
Eigen::internal::predux_min< Packet16uc >
EIGEN_STRONG_INLINE unsigned char predux_min< Packet16uc >(const Packet16uc &a)
Definition PacketMath.h:1673
Eigen::internal::pcmp_lt_or_nan< Packet4f >
EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1461
Eigen::internal::pmin< PropagateNaN, Packet4f >
EIGEN_STRONG_INLINE Packet4f pmin< PropagateNaN, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1189
Eigen::internal::pload< Packet2i >
EIGEN_STRONG_INLINE Packet2i pload< Packet2i >(const int32_t *from)
Definition PacketMath.h:1696
Eigen::internal::pstore< uint8_t >
EIGEN_STRONG_INLINE void pstore< uint8_t >(uint8_t *to, const Packet4uc &from)
Definition PacketMath.h:1879
Eigen::internal::preverse
EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf &a)
Definition Complex.h:184
Eigen::internal::pscatter< int16_t, Packet4s >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int16_t, Packet4s >(int16_t *to, const Packet4s &from, Index stride)
Definition PacketMath.h:2198
Eigen::internal::pmax< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pmax< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1263
Eigen::internal::pcmp_le< Packet8s >
EIGEN_STRONG_INLINE Packet8s pcmp_le< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1302
Eigen::internal::predux< Packet4us >
EIGEN_STRONG_INLINE uint16_t predux< Packet4us >(const Packet4us &a)
Definition PacketMath.h:2449
Eigen::internal::pstore< double >
EIGEN_STRONG_INLINE void pstore< double >(double *to, const Packet4d &from)
Definition PacketMath.h:623
Eigen::internal::pcmp_le< Packet4f >
EIGEN_STRONG_INLINE Packet4f pcmp_le< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1278
Eigen::internal::padd< Packet4i >
EIGEN_STRONG_INLINE Packet4i padd< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:775
Eigen::internal::predux_mul
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type predux_mul(const Packet &a)
Definition GenericPacketMath.h:882
Eigen::internal::pfloor< Packet4f >
EIGEN_STRONG_INLINE Packet4f pfloor< Packet4f >(const Packet4f &a)
Definition PacketMath.h:939
Eigen::internal::pcmp_eq< Packet16c >
EIGEN_STRONG_INLINE Packet16c pcmp_eq< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1410
Eigen::internal::pfirst< Packet4ui >
EIGEN_STRONG_INLINE uint32_t pfirst< Packet4ui >(const Packet4ui &a)
Definition PacketMath.h:2294
Eigen::internal::pscatter< float, Packet2f >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< float, Packet2f >(float *to, const Packet2f &from, Index stride)
Definition PacketMath.h:2116
Eigen::internal::pand< Packet4c >
EIGEN_STRONG_INLINE Packet4c pand< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1469
Eigen::internal::pmadd
EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f &a, const Packet4f &b, const Packet4f &c)
Definition PacketMath.h:827
Eigen::internal::pcmp_le< Packet8c >
EIGEN_STRONG_INLINE Packet8c pcmp_le< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1286
Eigen::internal::pcmp_lt< Packet16c >
EIGEN_STRONG_INLINE Packet16c pcmp_lt< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1349
Eigen::internal::padd< Packet4us >
EIGEN_STRONG_INLINE Packet4us padd< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:825
Eigen::internal::pmul< Packet4us >
EIGEN_STRONG_INLINE Packet4us pmul< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:933
Eigen::internal::pandnot< Packet4i >
EIGEN_STRONG_INLINE Packet4i pandnot< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:915
Eigen::internal::pmul
EIGEN_DEVICE_FUNC Packet pmul(const Packet &a, const Packet &b)
Definition GenericPacketMath.h:237
Eigen::internal::predux_min< Packet8uc >
EIGEN_STRONG_INLINE uint8_t predux_min< Packet8uc >(const Packet8uc &a)
Definition PacketMath.h:2613
Eigen::internal::pandnot< Packet8us >
EIGEN_STRONG_INLINE Packet8us pandnot< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:1584
Eigen::internal::pmul< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pmul< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:944
Eigen::internal::pandnot< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pandnot< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1592
Eigen::internal::ploaddup< Packet8s >
EIGEN_STRONG_INLINE Packet8s ploaddup< Packet8s >(const short int *from)
Definition PacketMath.h:1013
Eigen::internal::F32MaskToBf16Mask
EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f &p)
Definition PacketMath.h:3419
Eigen::internal::predux< Packet2ui >
EIGEN_STRONG_INLINE uint32_t predux< Packet2ui >(const Packet2ui &a)
Definition PacketMath.h:2467
Eigen::internal::pmax< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pmax< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1270
Eigen::internal::pdiv< Packet4f >
EIGEN_STRONG_INLINE Packet4f pdiv< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:803
Eigen::internal::pmin< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pmin< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1201
Eigen::internal::pandnot
EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h &a, const Packet8h &b)
Definition PacketMath.h:1053
Eigen::internal::pmax< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pmax< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1255
Eigen::internal::pload< Packet2d >
EIGEN_STRONG_INLINE Packet2d pload< Packet2d >(const double *from)
Definition PacketMath.h:967
Eigen::internal::pcmp_lt< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pcmp_lt< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1351
Eigen::internal::pmul< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pmul< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:929
Eigen::internal::predux_max< Packet4bf >
EIGEN_STRONG_INLINE bfloat16 predux_max< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3571
Eigen::internal::pload< Packet8us >
EIGEN_STRONG_INLINE Packet8us pload< Packet8us >(const unsigned short int *from)
Definition PacketMath.h:458
Eigen::internal::pcmp_lt< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pcmp_lt< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1359
Eigen::internal::pset1frombits< Packet2f >
EIGEN_STRONG_INLINE Packet2f pset1frombits< Packet2f >(unsigned int from)
Definition PacketMath.h:715
Eigen::internal::pcmp_le< Packet4c >
EIGEN_STRONG_INLINE Packet4c pcmp_le< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1280
Eigen::internal::pmul< Packet2d >
EIGEN_STRONG_INLINE Packet2d pmul< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:916
Eigen::internal::pabsdiff< Packet4i >
EIGEN_STRONG_INLINE Packet4i pabsdiff< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:1173
Eigen::internal::psub< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc psub< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:850
Eigen::internal::pfirst< Packet4bf >
EIGEN_STRONG_INLINE bfloat16 pfirst< Packet4bf >(const Packet4bf &from)
Definition PacketMath.h:3427
Eigen::internal::pmax< Packet4s >
EIGEN_STRONG_INLINE Packet4s pmax< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1257
Eigen::internal::pnegate
EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf &a)
Definition Complex.h:166
Eigen::internal::pandnot< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pandnot< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1590
Eigen::internal::pfrexp< Packet4f >
EIGEN_STRONG_INLINE Packet4f pfrexp< Packet4f >(const Packet4f &a, Packet4f &exponent)
Definition PacketMath.h:1361
Eigen::internal::predux_mul< Packet4f >
EIGEN_STRONG_INLINE float predux_mul< Packet4f >(const Packet4f &a)
Definition PacketMath.h:1533
Eigen::internal::plset< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul plset< Packet2ul >(const uint64_t &a)
Definition PacketMath.h:799
Eigen::internal::pcmp_lt
EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:868
Eigen::internal::pmin< PropagateNumbers, Packet2d >
EIGEN_STRONG_INLINE Packet2d pmin< PropagateNumbers, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:569
Eigen::internal::pandnot< Packet2f >
EIGEN_STRONG_INLINE Packet2f pandnot< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1564
Eigen::internal::pselect
EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f &mask, const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:917
Eigen::internal::pcmp_lt< Packet2i >
EIGEN_STRONG_INLINE Packet2i pcmp_lt< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1369
Eigen::internal::pcmp_le< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pcmp_le< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1296
Eigen::internal::pmax< Packet2f >
EIGEN_STRONG_INLINE Packet2f pmax< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1228
Eigen::internal::prefetch< float >
EIGEN_STRONG_INLINE void prefetch< float >(const float *addr)
Definition PacketMath.h:1117
Eigen::internal::pscatter< int16_t, Packet8s >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int16_t, Packet8s >(int16_t *to, const Packet8s &from, Index stride)
Definition PacketMath.h:2205
Eigen::internal::pstoreu< bfloat16 >
EIGEN_STRONG_INLINE void pstoreu< bfloat16 >(bfloat16 *to, const Packet8bf &from)
Definition PacketMath.h:1104
Eigen::internal::parithmetic_shift_right
EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i &a)
Definition PacketMath.h:1187
Eigen::internal::pcmp_eq< Packet4us >
EIGEN_STRONG_INLINE Packet4us pcmp_eq< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1426
Eigen::internal::pxor< Packet4us >
EIGEN_STRONG_INLINE Packet4us pxor< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1549
Eigen::internal::pfirst< Packet2i >
EIGEN_STRONG_INLINE int32_t pfirst< Packet2i >(const Packet2i &a)
Definition PacketMath.h:2291
Eigen::internal::pcmp_lt< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pcmp_lt< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1375
Eigen::internal::pfirst< Packet16c >
EIGEN_STRONG_INLINE signed char pfirst< Packet16c >(const Packet16c &a)
Definition PacketMath.h:1137
Eigen::internal::padd< Packet8s >
EIGEN_STRONG_INLINE Packet8s padd< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:777
Eigen::internal::pscatter< double, Packet2d >
EIGEN_DEVICE_FUNC void pscatter< double, Packet2d >(double *to, const Packet2d &from, Index stride)
Definition PacketMath.h:1044
Eigen::internal::psub< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc psub< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:844
Eigen::internal::predux_min< Packet4bf >
EIGEN_STRONG_INLINE bfloat16 predux_min< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3576
Eigen::internal::pload< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pload< Packet4ui >(const uint32_t *from)
Definition PacketMath.h:1702
Eigen::internal::pmul< Packet2i >
EIGEN_STRONG_INLINE Packet2i pmul< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:935
Eigen::internal::pandnot< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pandnot< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1576
Eigen::internal::ploaddup< Packet8us >
EIGEN_STRONG_INLINE Packet8us ploaddup< Packet8us >(const unsigned short int *from)
Definition PacketMath.h:1021
Eigen::internal::pmax< PropagateNaN, Packet4f >
EIGEN_STRONG_INLINE Packet4f pmax< PropagateNaN, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1237
Eigen::internal::ploadu< Packet4i >
EIGEN_STRONG_INLINE Packet4i ploadu< Packet4i >(const int *from)
Definition PacketMath.h:972
Eigen::internal::predux_mul< Packet2d >
EIGEN_STRONG_INLINE double predux_mul< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1092
Eigen::internal::Packet8s
__vector short int Packet8s
Definition PacketMath.h:34
Eigen::internal::pdiv< Packet2d >
EIGEN_STRONG_INLINE Packet2d pdiv< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:923
Eigen::internal::ploadu< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul ploadu< Packet2ul >(const uint64_t *from)
Definition PacketMath.h:1751
Eigen::internal::pcmp_lt< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pcmp_lt< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1373
Eigen::internal::pset1< Packet2i >
EIGEN_STRONG_INLINE Packet2i pset1< Packet2i >(const int32_t &from)
Definition PacketMath.h:708
Eigen::internal::pabsdiff< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pabsdiff< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1153
Eigen::internal::pset1< Packet4c >
EIGEN_STRONG_INLINE Packet4c pset1< Packet4c >(const int8_t &from)
Definition PacketMath.h:696
Eigen::internal::predux_min< Packet2ul >
EIGEN_STRONG_INLINE uint64_t predux_min< Packet2ul >(const Packet2ul &a)
Definition PacketMath.h:2668
Eigen::internal::pandnot< Packet4s >
EIGEN_STRONG_INLINE Packet4s pandnot< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1578
Eigen::internal::pdiv< Packet2i >
EIGEN_STRONG_INLINE Packet2i pdiv< Packet2i >(const Packet2i &, const Packet2i &)
Definition PacketMath.h:1051
Eigen::internal::predux_min< Packet2d >
EIGEN_STRONG_INLINE double predux_min< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1101
Eigen::internal::pcmp_lt< Packet8c >
EIGEN_STRONG_INLINE Packet8c pcmp_lt< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1347
Eigen::internal::ploadu< Packet4s >
EIGEN_STRONG_INLINE Packet4s ploadu< Packet4s >(const int16_t *from)
Definition PacketMath.h:1733
Eigen::internal::pset1< Packet4f >
EIGEN_STRONG_INLINE Packet4f pset1< Packet4f >(const float &from)
Definition PacketMath.h:547
Eigen::internal::pfloor< Packet2f >
EIGEN_STRONG_INLINE Packet2f pfloor< Packet2f >(const Packet2f &a)
Definition PacketMath.h:3153
Eigen::internal::pstore< int16_t >
EIGEN_STRONG_INLINE void pstore< int16_t >(int16_t *to, const Packet4s &from)
Definition PacketMath.h:1885
Eigen::internal::por< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui por< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1522
Eigen::internal::pand< Packet4us >
EIGEN_STRONG_INLINE Packet4us pand< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1483
Eigen::internal::pmax< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pmax< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1264
Eigen::internal::predux_min< Packet4c >
EIGEN_STRONG_INLINE int8_t predux_min< Packet4c >(const Packet4c &a)
Definition PacketMath.h:2584
Eigen::internal::ploadquad< Packet8c >
EIGEN_STRONG_INLINE Packet8c ploadquad< Packet8c >(const int8_t *from)
Definition PacketMath.h:1828
Eigen::internal::Packet4us
uint16x4_t Packet4us
Definition PacketMath.h:72
Eigen::internal::predux_min< Packet4ui >
EIGEN_STRONG_INLINE uint32_t predux_min< Packet4ui >(const Packet4ui &a)
Definition PacketMath.h:2661
Eigen::internal::predux_mul< Packet4uc >
EIGEN_STRONG_INLINE uint8_t predux_mul< Packet4uc >(const Packet4uc &a)
Definition PacketMath.h:2517
Eigen::internal::ploaddup< Packet4s >
EIGEN_STRONG_INLINE Packet4s ploaddup< Packet4s >(const int16_t *from)
Definition PacketMath.h:1790
Eigen::internal::psub< Packet4i >
EIGEN_STRONG_INLINE Packet4i psub< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:783
Eigen::internal::pmax< Packet8c >
EIGEN_STRONG_INLINE Packet8c pmax< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1247
Eigen::internal::pmin< PropagateNumbers, Packet4f >
EIGEN_STRONG_INLINE Packet4f pmin< PropagateNumbers, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:565
Eigen::internal::plset< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui plset< Packet4ui >(const uint32_t &a)
Definition PacketMath.h:789
Eigen::internal::pnegate< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pnegate< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3621
Eigen::internal::plset< Packet4s >
EIGEN_STRONG_INLINE Packet4s plset< Packet4s >(const int16_t &a)
Definition PacketMath.h:754
Eigen::internal::pscatter< float, Packet4f >
EIGEN_DEVICE_FUNC void pscatter< float, Packet4f >(float *to, const Packet4f &from, Index stride)
Definition PacketMath.h:695
Eigen::internal::pfirst< Packet4us >
EIGEN_STRONG_INLINE uint16_t pfirst< Packet4us >(const Packet4us &a)
Definition PacketMath.h:2289
Eigen::internal::plset< Packet2d >
EIGEN_STRONG_INLINE Packet2d plset< Packet2d >(const double &a)
Definition PacketMath.h:887
Eigen::internal::predux_mul< Packet2ul >
EIGEN_STRONG_INLINE uint64_t predux_mul< Packet2ul >(const Packet2ul &a)
Definition PacketMath.h:2573
Eigen::internal::ploadu< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc ploadu< Packet8uc >(const uint8_t *from)
Definition PacketMath.h:1729
Eigen::internal::pload< Packet8s >
EIGEN_STRONG_INLINE Packet8s pload< Packet8s >(const short int *from)
Definition PacketMath.h:453
Eigen::internal::ploadquad< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc ploadquad< Packet8uc >(const uint8_t *from)
Definition PacketMath.h:1846
Eigen::internal::padd< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul padd< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:832
Eigen::internal::Packet8uc
uint8x8_t Packet8uc
Definition PacketMath.h:68
Eigen::internal::pxor< Packet8us >
EIGEN_STRONG_INLINE Packet8us pxor< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:1551
Eigen::internal::plset< Packet2f >
EIGEN_STRONG_INLINE Packet2f plset< Packet2f >(const float &a)
Definition PacketMath.h:720
Eigen::internal::predux_mul< Packet2l >
EIGEN_STRONG_INLINE int64_t predux_mul< Packet2l >(const Packet2l &a)
Definition PacketMath.h:2571
Eigen::internal::pcmp_lt< Packet4f >
EIGEN_STRONG_INLINE Packet4f pcmp_lt< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1339
Eigen::internal::padd< Packet8us >
EIGEN_STRONG_INLINE Packet8us padd< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:778
Eigen::internal::predux_min< Packet4s >
EIGEN_STRONG_INLINE int16_t predux_min< Packet4s >(const Packet4s &a)
Definition PacketMath.h:2628
Eigen::internal::pceil< Packet4f >
EIGEN_STRONG_INLINE Packet4f pceil< Packet4f >(const Packet4f &a)
Definition PacketMath.h:938
Eigen::internal::pgather< int8_t, Packet16c >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather< int8_t, Packet16c >(const int8_t *from, Index stride)
Definition PacketMath.h:1976
Eigen::internal::pand< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pand< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1479
Eigen::internal::pstore< float >
EIGEN_STRONG_INLINE void pstore< float >(float *to, const Packet4f &from)
Definition PacketMath.h:491
Eigen::internal::pset1< Packet2l >
EIGEN_STRONG_INLINE Packet2l pset1< Packet2l >(const int64_t &from)
Definition PacketMath.h:712
Eigen::internal::pcmp_lt< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pcmp_lt< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3606
Eigen::internal::por< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc por< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1508
Eigen::internal::pabs
EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f &a)
Definition PacketMath.h:1176
Eigen::internal::pmul< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pmul< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:937
Eigen::internal::pxor< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pxor< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1555
Eigen::internal::pscatter< uint8_t, Packet4uc >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint8_t, Packet4uc >(uint8_t *to, const Packet4uc &from, Index stride)
Definition PacketMath.h:2163
Eigen::internal::pand< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pand< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1475
Eigen::internal::ploaddup< Packet2l >
EIGEN_STRONG_INLINE Packet2l ploaddup< Packet2l >(const int64_t *from)
Definition PacketMath.h:1820
Eigen::internal::padd< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc padd< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:821
Eigen::internal::por< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul por< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1528
Eigen::internal::por< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc por< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1510
Eigen::internal::pand< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pand< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1477
Eigen::internal::pcmp_le< Packet4i >
EIGEN_STRONG_INLINE Packet4i pcmp_le< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:1310
Eigen::internal::predux< Packet2ul >
EIGEN_STRONG_INLINE uint64_t predux< Packet2ul >(const Packet2ul &a)
Definition PacketMath.h:2475
Eigen::internal::por< Packet8us >
EIGEN_STRONG_INLINE Packet8us por< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:903
Eigen::internal::Packet2f
float32x2_t Packet2f
Definition PacketMath.h:62
Eigen::internal::pstoreu< uint32_t >
EIGEN_STRONG_INLINE void pstoreu< uint32_t >(uint32_t *to, const Packet2ui &from)
Definition PacketMath.h:1934
Eigen::internal::predux< Packet8us >
EIGEN_STRONG_INLINE unsigned short int predux< Packet8us >(const Packet8us &a)
Definition PacketMath.h:1494
Eigen::internal::por< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc por< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1506
Eigen::internal::pset1frombits< Packet4f >
EIGEN_STRONG_INLINE Packet4f pset1frombits< Packet4f >(unsigned int from)
Definition PacketMath.h:571
Eigen::internal::plset< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf plset< Packet4bf >(const bfloat16 &a)
Definition PacketMath.h:3494
Eigen::internal::pcmp_lt< Packet4us >
EIGEN_STRONG_INLINE Packet4us pcmp_lt< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1365
Eigen::internal::predux_min< Packet4uc >
EIGEN_STRONG_INLINE uint8_t predux_min< Packet4uc >(const Packet4uc &a)
Definition PacketMath.h:2606
Eigen::internal::pandnot< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pandnot< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1572
Eigen::internal::pxor< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pxor< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1541
Eigen::internal::pldexp< Packet4f >
EIGEN_STRONG_INLINE Packet4f pldexp< Packet4f >(const Packet4f &a, const Packet4f &exponent)
Definition PacketMath.h:1354
Eigen::internal::ploadu< Packet2d >
EIGEN_STRONG_INLINE Packet2d ploadu< Packet2d >(const double *from)
Definition PacketMath.h:1004
Eigen::internal::pscatter< int8_t, Packet8c >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int8_t, Packet8c >(int8_t *to, const Packet8c &from, Index stride)
Definition PacketMath.h:2133
Eigen::internal::pdiv< Packet4us >
EIGEN_STRONG_INLINE Packet4us pdiv< Packet4us >(const Packet4us &, const Packet4us &)
Definition PacketMath.h:1041
Eigen::internal::vec4f_unpackhi
EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:133
Eigen::internal::pfirst< Packet2f >
EIGEN_STRONG_INLINE float pfirst< Packet2f >(const Packet2f &a)
Definition PacketMath.h:2279
Eigen::internal::pfrexp< Packet2f >
EIGEN_STRONG_INLINE Packet2f pfrexp< Packet2f >(const Packet2f &a, Packet2f &exponent)
Definition PacketMath.h:2377
Eigen::internal::pabsdiff< Packet2i >
EIGEN_STRONG_INLINE Packet2i pabsdiff< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1171
Eigen::internal::pcmp_le< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pcmp_le< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1326
Eigen::internal::pmin< Packet8c >
EIGEN_STRONG_INLINE Packet8c pmin< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1199
Eigen::internal::predux_mul< Packet4c >
EIGEN_STRONG_INLINE int8_t predux_mul< Packet4c >(const Packet4c &a)
Definition PacketMath.h:2503
Eigen::internal::pxor< Packet4f >
EIGEN_STRONG_INLINE Packet4f pxor< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:908
Eigen::internal::pstoreu< int32_t >
EIGEN_STRONG_INLINE void pstoreu< int32_t >(int32_t *to, const Packet4i &from)
Definition PacketMath.h:450
Eigen::internal::pscatter< int8_t, Packet16c >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< int8_t, Packet16c >(int8_t *to, const Packet16c &from, Index stride)
Definition PacketMath.h:2144
Eigen::internal::pmin< Packet4i >
EIGEN_STRONG_INLINE Packet4i pmin< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:843
Eigen::internal::pfirst< Packet2ui >
EIGEN_STRONG_INLINE uint32_t pfirst< Packet2ui >(const Packet2ui &a)
Definition PacketMath.h:2293
Eigen::internal::pmax< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pmax< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1249
Eigen::internal::predux_half_dowto4
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half_dowto4(const Packet8c &a)
Definition PacketMath.h:2478
Eigen::internal::pcmp_le< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pcmp_le< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:1290
Eigen::internal::prefetch< uint8_t >
EIGEN_STRONG_INLINE void prefetch< uint8_t >(const uint8_t *addr)
Definition PacketMath.h:2271
Eigen::internal::ploadu< Packet2f >
EIGEN_STRONG_INLINE Packet2f ploadu< Packet2f >(const float *from)
Definition PacketMath.h:1709
Eigen::internal::psqrt
EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f &a)
Definition PacketMath.h:723
Eigen::internal::ploadu< Packet2i >
EIGEN_STRONG_INLINE Packet2i ploadu< Packet2i >(const int32_t *from)
Definition PacketMath.h:1741
Eigen::internal::plset< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc plset< Packet16uc >(const unsigned char &a)
Definition PacketMath.h:772
Eigen::internal::ploadu< Packet16c >
EIGEN_STRONG_INLINE Packet16c ploadu< Packet16c >(const signed char *from)
Definition PacketMath.h:988
Eigen::internal::pmax< Packet2l >
EIGEN_STRONG_INLINE Packet2l pmax< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1265
Eigen::internal::psub< Packet2l >
EIGEN_STRONG_INLINE Packet2l psub< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:860
Eigen::internal::pfirst< Packet4uc >
EIGEN_STRONG_INLINE uint8_t pfirst< Packet4uc >(const Packet4uc &a)
Definition PacketMath.h:2284
Eigen::internal::pfrexp< Packet2d >
EIGEN_STRONG_INLINE Packet2d pfrexp< Packet2d >(const Packet2d &a, Packet2d &exponent)
Definition PacketMath.h:918
Eigen::internal::pcmp_eq
EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf &a, const Packet2cf &b)
Definition Complex.h:231
Eigen::internal::pgather< int32_t, Packet2i >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather< int32_t, Packet2i >(const int32_t *from, Index stride)
Definition PacketMath.h:2075
Eigen::internal::pmin< Packet8s >
EIGEN_STRONG_INLINE Packet8s pmin< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:844
Eigen::internal::pset1< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc pset1< Packet4uc >(const uint8_t &from)
Definition PacketMath.h:700
Eigen::internal::pstoreu< uint16_t >
EIGEN_STRONG_INLINE void pstoreu< uint16_t >(uint16_t *to, const Packet4us &from)
Definition PacketMath.h:1926
Eigen::internal::pset1< Packet8s >
EIGEN_STRONG_INLINE Packet8s pset1< Packet8s >(const short int &from)
Definition PacketMath.h:555
Eigen::internal::pldexp_generic
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_generic(const Packet &a, const Packet &exponent)
Definition GenericPacketMathFunctions.h:85
Eigen::internal::pset1< Packet8c >
EIGEN_STRONG_INLINE Packet8c pset1< Packet8c >(const int8_t &from)
Definition PacketMath.h:698
Eigen::internal::pmul< Packet8s >
EIGEN_STRONG_INLINE Packet8s pmul< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:797
Eigen::internal::pabs< Packet4c >
EIGEN_STRONG_INLINE Packet4c pabs< Packet4c >(const Packet4c &a)
Definition PacketMath.h:2351
Eigen::internal::vec4f_unpacklo
EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:129
Eigen::internal::pmax< PropagateNumbers, Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmax< PropagateNumbers, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3477
Eigen::internal::pcmp_le< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pcmp_le< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1298
Eigen::internal::pand< Packet2l >
EIGEN_STRONG_INLINE Packet2l pand< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1493
Eigen::internal::pxor< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pxor< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1557
Eigen::internal::pand
EIGEN_STRONG_INLINE Packet8h pand(const Packet8h &a, const Packet8h &b)
Definition PacketMath.h:1050
Eigen::internal::predux< Packet4c >
EIGEN_STRONG_INLINE int8_t predux< Packet4c >(const Packet4c &a)
Definition PacketMath.h:2393
Eigen::internal::pcmp_eq< Packet2i >
EIGEN_STRONG_INLINE Packet2i pcmp_eq< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1430
Eigen::internal::pcmp_lt< Packet8s >
EIGEN_STRONG_INLINE Packet8s pcmp_lt< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1363
Eigen::internal::paddsub< Packet2f >
EIGEN_STRONG_INLINE Packet2f paddsub< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:864
Eigen::internal::predux_max< Packet8us >
EIGEN_STRONG_INLINE unsigned short int predux_max< Packet8us >(const Packet8us &a)
Definition PacketMath.h:1726
Eigen::internal::pstore< int32_t >
EIGEN_STRONG_INLINE void pstore< int32_t >(int32_t *to, const Packet4i &from)
Definition PacketMath.h:436
Eigen::internal::pand< Packet16c >
EIGEN_STRONG_INLINE Packet16c pand< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1473
Eigen::internal::pabsdiff< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pabsdiff< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3596
Eigen::internal::pload< Packet4s >
EIGEN_STRONG_INLINE Packet4s pload< Packet4s >(const int16_t *from)
Definition PacketMath.h:1688
Eigen::internal::pandnot< Packet16c >
EIGEN_STRONG_INLINE Packet16c pandnot< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1571
Eigen::internal::pmin< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pmin< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:847
Eigen::internal::pxor
EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h &a, const Packet8h &b)
Definition PacketMath.h:1047
Eigen::internal::pfloor< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pfloor< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3526
Eigen::internal::pfirst< Packet4i >
EIGEN_STRONG_INLINE int pfirst< Packet4i >(const Packet4i &a)
Definition PacketMath.h:1121
Eigen::internal::plset< Packet4i >
EIGEN_STRONG_INLINE Packet4i plset< Packet4i >(const int &a)
Definition PacketMath.h:768
Eigen::internal::vec2d_unpacklo
EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:98
Eigen::internal::pcmp_le< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pcmp_le< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition PacketMath.h:1312
Eigen::internal::predux< Packet4f >
EIGEN_STRONG_INLINE float predux< Packet4f >(const Packet4f &a)
Definition PacketMath.h:1444
Eigen::internal::predux_mul< Packet4bf >
EIGEN_STRONG_INLINE bfloat16 predux_mul< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3581
Eigen::internal::pdiv< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pdiv< Packet8uc >(const Packet8uc &, const Packet8uc &)
Definition PacketMath.h:1021
Eigen::internal::psub< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf psub< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3542
Eigen::internal::predux_mul< Packet2f >
EIGEN_STRONG_INLINE float predux_mul< Packet2f >(const Packet2f &a)
Definition PacketMath.h:2499
Eigen::internal::pcmp_lt< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pcmp_lt< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1387
Eigen::internal::pceil< Packet2d >
EIGEN_STRONG_INLINE Packet2d pceil< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1182
Eigen::internal::pload< Packet2l >
EIGEN_STRONG_INLINE Packet2l pload< Packet2l >(const int64_t *from)
Definition PacketMath.h:1704
Eigen::internal::padd< Packet2f >
EIGEN_STRONG_INLINE Packet2f padd< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:805
Eigen::internal::pmul< Packet4s >
EIGEN_STRONG_INLINE Packet4s pmul< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:931
Eigen::internal::ploadu< Packet4f >
EIGEN_STRONG_INLINE Packet4f ploadu< Packet4f >(const float *from)
Definition PacketMath.h:968
Eigen::internal::pcmp_eq< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pcmp_eq< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1420
Eigen::internal::por< Packet2f >
EIGEN_STRONG_INLINE Packet2f por< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1497
Eigen::internal::pmul< Packet4i >
EIGEN_STRONG_INLINE Packet4i pmul< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:796
Eigen::internal::predux_max< Packet2i >
EIGEN_STRONG_INLINE int32_t predux_max< Packet2i >(const Packet2i &a)
Definition PacketMath.h:2747
Eigen::internal::pmin< PropagateNumbers, Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pmin< PropagateNumbers, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3460
Eigen::internal::predux_max< Packet8s >
EIGEN_STRONG_INLINE short int predux_max< Packet8s >(const Packet8s &a)
Definition PacketMath.h:1711
Eigen::internal::pdiv< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pdiv< Packet2ul >(const Packet2ul &, const Packet2ul &)
Definition PacketMath.h:1076
Eigen::internal::pcmp_lt_or_nan< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3611
Eigen::internal::predux< Packet8s >
EIGEN_STRONG_INLINE short int predux< Packet8s >(const Packet8s &a)
Definition PacketMath.h:1489
Eigen::internal::ploadquad< Packet4i >
EIGEN_STRONG_INLINE Packet4i ploadquad< Packet4i >(const int32_t *from)
Definition PacketMath.h:1866
Eigen::internal::pand< Packet4i >
EIGEN_STRONG_INLINE Packet4i pand< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:892
Eigen::internal::pset1< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui pset1< Packet2ui >(const uint32_t &from)
Definition PacketMath.h:710
Eigen::internal::pload< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pload< Packet4bf >(const bfloat16 *from)
Definition PacketMath.h:3431
Eigen::internal::pmin< Packet2d >
EIGEN_STRONG_INLINE Packet2d pmin< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:974
Eigen::internal::ploaddup< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc ploaddup< Packet16uc >(const unsigned char *from)
Definition PacketMath.h:1058
Eigen::internal::predux< Packet4i >
EIGEN_STRONG_INLINE int predux< Packet4i >(const Packet4i &a)
Definition PacketMath.h:1454
Eigen::internal::pcmp_eq< Packet8us >
EIGEN_STRONG_INLINE Packet8us pcmp_eq< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition PacketMath.h:1428
Eigen::internal::pgather< uint64_t, Packet2ul >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather< uint64_t, Packet2ul >(const uint64_t *from, Index stride)
Definition PacketMath.h:2109
Eigen::internal::pdiv< Packet2f >
EIGEN_STRONG_INLINE Packet2f pdiv< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:950
Eigen::internal::pfrexp_generic
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pfrexp_generic(const Packet &a, Packet &exponent)
Definition GenericPacketMathFunctions.h:40
Eigen::internal::pstoreu< int16_t >
EIGEN_STRONG_INLINE void pstoreu< int16_t >(int16_t *to, const Packet4s &from)
Definition PacketMath.h:1922
Eigen::internal::pceil< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pceil< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3531
Eigen::internal::pand< Packet4f >
EIGEN_STRONG_INLINE Packet4f pand< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:891
Eigen::internal::pstoreu< int64_t >
EIGEN_STRONG_INLINE void pstoreu< int64_t >(int64_t *to, const Packet2l &from)
Definition PacketMath.h:1938
Eigen::internal::psub
EIGEN_DEVICE_FUNC Packet psub(const Packet &a, const Packet &b)
Definition GenericPacketMath.h:222
Eigen::internal::predux_min< Packet8us >
EIGEN_STRONG_INLINE unsigned short int predux_min< Packet8us >(const Packet8us &a)
Definition PacketMath.h:1646
Eigen::internal::predux_mul< Packet8c >
EIGEN_STRONG_INLINE int8_t predux_mul< Packet8c >(const Packet8c &a)
Definition PacketMath.h:2509
Eigen::internal::predux_max< Packet2f >
EIGEN_STRONG_INLINE float predux_max< Packet2f >(const Packet2f &a)
Definition PacketMath.h:2672
Eigen::internal::predux_max< Packet4i >
EIGEN_STRONG_INLINE int predux_max< Packet4i >(const Packet4i &a)
Definition PacketMath.h:1698
Eigen::internal::predux< Packet8uc >
EIGEN_STRONG_INLINE uint8_t predux< Packet8uc >(const Packet8uc &a)
Definition PacketMath.h:2422
Eigen::internal::pmin< Packet2l >
EIGEN_STRONG_INLINE Packet2l pmin< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1217
Eigen::internal::pscatter< uint8_t, Packet16uc >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint8_t, Packet16uc >(uint8_t *to, const Packet16uc &from, Index stride)
Definition PacketMath.h:2179
Eigen::internal::pset1< Packet8us >
EIGEN_STRONG_INLINE Packet8us pset1< Packet8us >(const unsigned short int &from)
Definition PacketMath.h:559
Eigen::internal::predux< Packet4bf >
EIGEN_STRONG_INLINE bfloat16 predux< Packet4bf >(const Packet4bf &a)
Definition PacketMath.h:3566
Eigen::internal::ploaddup< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui ploaddup< Packet2ui >(const uint32_t *from)
Definition PacketMath.h:1816
Eigen::internal::pandnot< Packet8s >
EIGEN_STRONG_INLINE Packet8s pandnot< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1580
Eigen::internal::pgather< double, Packet2d >
EIGEN_DEVICE_FUNC Packet2d pgather< double, Packet2d >(const double *from, Index stride)
Definition PacketMath.h:1033
Eigen::internal::pgather< int16_t, Packet4s >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather< int16_t, Packet4s >(const int16_t *from, Index stride)
Definition PacketMath.h:2035
Eigen::internal::ploaddup< Packet2f >
EIGEN_STRONG_INLINE Packet2f ploaddup< Packet2f >(const float *from)
Definition PacketMath.h:1754
Eigen::internal::pabsdiff< Packet4s >
EIGEN_STRONG_INLINE Packet4s pabsdiff< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1163
Eigen::internal::ploaddup< Packet16c >
EIGEN_STRONG_INLINE Packet16c ploaddup< Packet16c >(const signed char *from)
Definition PacketMath.h:1050
Eigen::internal::predux< Packet8c >
EIGEN_STRONG_INLINE int8_t predux< Packet8c >(const Packet8c &a)
Definition PacketMath.h:2400
Eigen::internal::ploaddup< Packet8c >
EIGEN_STRONG_INLINE Packet8c ploaddup< Packet8c >(const int8_t *from)
Definition PacketMath.h:1763
Eigen::internal::pdiv< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pdiv< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3550
Eigen::internal::pcmp_lt< Packet4i >
EIGEN_STRONG_INLINE Packet4i pcmp_lt< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:1371
Eigen::internal::predux_max< Packet2l >
EIGEN_STRONG_INLINE int64_t predux_max< Packet2l >(const Packet2l &a)
Definition PacketMath.h:2761
Eigen::internal::pxor< Packet4c >
EIGEN_STRONG_INLINE Packet4c pxor< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1535
Eigen::internal::pfirst< Packet8c >
EIGEN_STRONG_INLINE int8_t pfirst< Packet8c >(const Packet8c &a)
Definition PacketMath.h:2282
Eigen::internal::pmax< Packet4i >
EIGEN_STRONG_INLINE Packet4i pmax< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition PacketMath.h:861
Eigen::internal::pfirst< Packet4s >
EIGEN_STRONG_INLINE int16_t pfirst< Packet4s >(const Packet4s &a)
Definition PacketMath.h:2287
Eigen::internal::pandnot< Packet8c >
EIGEN_STRONG_INLINE Packet8c pandnot< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition PacketMath.h:1570
Eigen::internal::pdiv< Packet4i >
EIGEN_STRONG_INLINE Packet4i pdiv< Packet4i >(const Packet4i &, const Packet4i &)
Definition PacketMath.h:821
Eigen::internal::pxor< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pxor< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1545
Eigen::internal::pabsdiff< Packet4f >
EIGEN_STRONG_INLINE Packet4f pabsdiff< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:1141
Eigen::internal::pgather< int8_t, Packet8c >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather< int8_t, Packet8c >(const int8_t *from, Index stride)
Definition PacketMath.h:1964
Eigen::internal::padd< Packet16c >
EIGEN_STRONG_INLINE Packet16c padd< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:779
Eigen::internal::padd< Packet4c >
EIGEN_STRONG_INLINE Packet4c padd< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:807
Eigen::internal::pdiv< Packet8s >
EIGEN_STRONG_INLINE Packet8s pdiv< Packet8s >(const Packet8s &, const Packet8s &)
Definition PacketMath.h:1036
Eigen::internal::predux_max< Packet16c >
EIGEN_STRONG_INLINE signed char predux_max< Packet16c >(const Packet16c &a)
Definition PacketMath.h:1741
Eigen::internal::predux_max< Packet4c >
EIGEN_STRONG_INLINE int8_t predux_max< Packet4c >(const Packet4c &a)
Definition PacketMath.h:2679
Eigen::internal::ploadquad< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc ploadquad< Packet16uc >(const uint8_t *from)
Definition PacketMath.h:1852
Eigen::internal::predux_mul< Packet8uc >
EIGEN_STRONG_INLINE uint8_t predux_mul< Packet8uc >(const Packet8uc &a)
Definition PacketMath.h:2523
Eigen::internal::predux< Packet4uc >
EIGEN_STRONG_INLINE uint8_t predux< Packet4uc >(const Packet4uc &a)
Definition PacketMath.h:2415
Eigen::internal::pmul< Packet2l >
EIGEN_STRONG_INLINE Packet2l pmul< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:939
Eigen::internal::predux_max< Packet4s >
EIGEN_STRONG_INLINE int16_t predux_max< Packet4s >(const Packet4s &a)
Definition PacketMath.h:2723
Eigen::internal::pxor< Packet2i >
EIGEN_STRONG_INLINE Packet2i pxor< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1553
Eigen::internal::pset1frombits< Packet2d >
EIGEN_STRONG_INLINE Packet2d pset1frombits< Packet2d >(uint64_t from)
Definition PacketMath.h:264
Eigen::internal::pload< Packet4i >
EIGEN_STRONG_INLINE Packet4i pload< Packet4i >(const int *from)
Definition PacketMath.h:448
Eigen::internal::predux< Packet2i >
EIGEN_STRONG_INLINE int32_t predux< Packet2i >(const Packet2i &a)
Definition PacketMath.h:2461
Eigen::internal::Packet4f
__vector float Packet4f
Definition PacketMath.h:30
Eigen::internal::pgather< int32_t, Packet4i >
EIGEN_DEVICE_FUNC Packet4i pgather< int32_t, Packet4i >(const int32_t *from, Index stride)
Definition PacketMath.h:469
Eigen::internal::psub< Packet2d >
EIGEN_STRONG_INLINE Packet2d psub< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:895
Eigen::internal::psub< Packet4s >
EIGEN_STRONG_INLINE Packet4s psub< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:852
Eigen::internal::psub< Packet4f >
EIGEN_STRONG_INLINE Packet4f psub< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:782
Eigen::internal::pmul< Packet4c >
EIGEN_STRONG_INLINE Packet4c pmul< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:915
Eigen::internal::psub< Packet8s >
EIGEN_STRONG_INLINE Packet8s psub< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:784
Eigen::internal::pxor< Packet2f >
EIGEN_STRONG_INLINE Packet2f pxor< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1531
Eigen::internal::prsqrt
EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f &a)
Definition PacketMath.h:730
Eigen::internal::padd< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc padd< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition PacketMath.h:815
Eigen::internal::pstore< uint32_t >
EIGEN_STRONG_INLINE void pstore< uint32_t >(uint32_t *to, const Packet2ui &from)
Definition PacketMath.h:1897
Eigen::internal::Packet2i
int32x2_t Packet2i
Definition PacketMath.h:74
Eigen::internal::plset< Packet4f >
EIGEN_STRONG_INLINE Packet4f plset< Packet4f >(const float &a)
Definition PacketMath.h:767
Eigen::internal::predux_mul< Packet4ui >
EIGEN_STRONG_INLINE uint32_t predux_mul< Packet4ui >(const Packet4ui &a)
Definition PacketMath.h:2569
Eigen::internal::psub< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul psub< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:861
Eigen::internal::pcmp_eq< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pcmp_eq< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1418
Eigen::internal::pscatter< uint16_t, Packet8us >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter< uint16_t, Packet8us >(uint16_t *to, const Packet8us &from, Index stride)
Definition PacketMath.h:2223
Eigen::internal::ploadquad< Packet8s >
EIGEN_STRONG_INLINE Packet8s ploadquad< Packet8s >(const short int *from)
Definition PacketMath.h:1029
Eigen::internal::pfirst< Packet2ul >
EIGEN_STRONG_INLINE uint64_t pfirst< Packet2ul >(const Packet2ul &a)
Definition PacketMath.h:2296
Eigen::internal::pdiv< Packet4s >
EIGEN_STRONG_INLINE Packet4s pdiv< Packet4s >(const Packet4s &, const Packet4s &)
Definition PacketMath.h:1031
Eigen::internal::F32ToBf16
EIGEN_STRONG_INLINE Packet8bf F32ToBf16(Packet4f p4f)
Definition PacketMath.h:1252
Eigen::internal::ploadu< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc ploadu< Packet4uc >(const uint8_t *from)
Definition PacketMath.h:1723
Eigen::internal::pstoreu< float >
EIGEN_STRONG_INLINE void pstoreu< float >(float *to, const Packet4f &from)
Definition PacketMath.h:1088
Eigen::internal::pmax< Packet2d >
EIGEN_STRONG_INLINE Packet2d pmax< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition PacketMath.h:989
Eigen::internal::pcmp_lt_or_nan
EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:870
Eigen::internal::Packet4uc
eigen_packet_wrapper< uint32_t,5 > Packet4uc
Definition PacketMath.h:67
Eigen::internal::prefetch< int16_t >
EIGEN_STRONG_INLINE void prefetch< int16_t >(const int16_t *addr)
Definition PacketMath.h:2272
Eigen::internal::pgather< bfloat16, Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pgather< bfloat16, Packet4bf >(const bfloat16 *from, Index stride)
Definition PacketMath.h:3555
Eigen::internal::pset1< Packet16c >
EIGEN_STRONG_INLINE Packet16c pset1< Packet16c >(const signed char &from)
Definition PacketMath.h:563
Eigen::internal::pstore< uint64_t >
EIGEN_STRONG_INLINE void pstore< uint64_t >(uint64_t *to, const Packet2ul &from)
Definition PacketMath.h:1903
Eigen::internal::predux_max< Packet8uc >
EIGEN_STRONG_INLINE uint8_t predux_max< Packet8uc >(const Packet8uc &a)
Definition PacketMath.h:2708
Eigen::internal::plset< Packet2l >
EIGEN_STRONG_INLINE Packet2l plset< Packet2l >(const int64_t &a)
Definition PacketMath.h:794
Eigen::internal::pdiv< Packet2l >
EIGEN_STRONG_INLINE Packet2l pdiv< Packet2l >(const Packet2l &, const Packet2l &)
Definition PacketMath.h:1071
Eigen::internal::pandnot< Packet2i >
EIGEN_STRONG_INLINE Packet2i pandnot< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1586
Eigen::internal::ploadu< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui ploadu< Packet4ui >(const uint32_t *from)
Definition PacketMath.h:1747
Eigen::internal::pfirst< Packet2l >
EIGEN_STRONG_INLINE int64_t pfirst< Packet2l >(const Packet2l &a)
Definition PacketMath.h:2295
Eigen::internal::pandnot< Packet4us >
EIGEN_STRONG_INLINE Packet4us pandnot< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:1582
Eigen::internal::pcmp_eq< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf pcmp_eq< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition PacketMath.h:3601
Eigen::internal::pstore< Eigen::half >
EIGEN_STRONG_INLINE void pstore< Eigen::half >(Eigen::half *to, const Packet8h &from)
Definition PacketMath.h:954
Eigen::internal::psub< Packet4us >
EIGEN_STRONG_INLINE Packet4us psub< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition PacketMath.h:854
Eigen::internal::pxor< Packet16c >
EIGEN_STRONG_INLINE Packet16c pxor< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:1539
Eigen::internal::ploadquad< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui ploadquad< Packet4ui >(const uint32_t *from)
Definition PacketMath.h:1867
Eigen::internal::por< Packet2i >
EIGEN_STRONG_INLINE Packet2i por< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition PacketMath.h:1520
Eigen::internal::pstoreu< uint64_t >
EIGEN_STRONG_INLINE void pstoreu< uint64_t >(uint64_t *to, const Packet2ul &from)
Definition PacketMath.h:1940
Eigen::internal::pstore< uint16_t >
EIGEN_STRONG_INLINE void pstore< uint16_t >(uint16_t *to, const Packet4us &from)
Definition PacketMath.h:1889
Eigen::internal::pfirst< Packet2d >
EIGEN_STRONG_INLINE double pfirst< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1061
Eigen::internal::pgather< float, Packet2f >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather< float, Packet2f >(const float *from, Index stride)
Definition PacketMath.h:1943
Eigen::internal::predux_mul< Packet4us >
EIGEN_STRONG_INLINE uint16_t predux_mul< Packet4us >(const Packet4us &a)
Definition PacketMath.h:2547
Eigen::internal::pabsdiff< Packet16uc >
EIGEN_STRONG_INLINE Packet16uc pabsdiff< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition PacketMath.h:1161
Eigen::internal::pfirst< Packet8us >
EIGEN_STRONG_INLINE unsigned short int pfirst< Packet8us >(const Packet8us &a)
Definition PacketMath.h:1133
Eigen::internal::pabsdiff< Packet2f >
EIGEN_STRONG_INLINE Packet2f pabsdiff< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1139
Eigen::internal::pcmp_eq< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui pcmp_eq< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition PacketMath.h:1436
Eigen::internal::Packet2ul
v2u64 Packet2ul
Definition PacketMath.h:822
Eigen::internal::pxor< Packet2l >
EIGEN_STRONG_INLINE Packet2l pxor< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition PacketMath.h:1559
Eigen::internal::Packet4s
int16x4_t Packet4s
Definition PacketMath.h:70
Eigen::internal::pcmp_le< Packet4s >
EIGEN_STRONG_INLINE Packet4s pcmp_le< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition PacketMath.h:1300
Eigen::internal::ploaddup< Packet4uc >
EIGEN_STRONG_INLINE Packet4uc ploaddup< Packet4uc >(const uint8_t *from)
Definition PacketMath.h:1774
Eigen::internal::plset< Packet2ui >
EIGEN_STRONG_INLINE Packet2ui plset< Packet2ui >(const uint32_t &a)
Definition PacketMath.h:784
Eigen::internal::pmul< Packet16c >
EIGEN_STRONG_INLINE Packet16c pmul< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition PacketMath.h:799
Eigen::internal::pstoreu< uint8_t >
EIGEN_STRONG_INLINE void pstoreu< uint8_t >(uint8_t *to, const Packet4uc &from)
Definition PacketMath.h:1916
Eigen::internal::ploaddup< Packet4bf >
EIGEN_STRONG_INLINE Packet4bf ploaddup< Packet4bf >(const bfloat16 *from)
Definition PacketMath.h:3451
Eigen::internal::pxor< Packet8s >
EIGEN_STRONG_INLINE Packet8s pxor< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition PacketMath.h:1548
Eigen::internal::pgather< uint8_t, Packet4uc >
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather< uint8_t, Packet4uc >(const uint8_t *from, Index stride)
Definition PacketMath.h:1996
Eigen::internal::ploaddup< Packet4ui >
EIGEN_STRONG_INLINE Packet4ui ploaddup< Packet4ui >(const uint32_t *from)
Definition PacketMath.h:1818
Eigen::internal::pcmp_le< Packet2f >
EIGEN_STRONG_INLINE Packet2f pcmp_le< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:1276
Eigen::internal::pload< Packet2f >
EIGEN_STRONG_INLINE Packet2f pload< Packet2f >(const float *from)
Definition PacketMath.h:1664
Eigen::internal::ploadquad< Packet8us >
EIGEN_STRONG_INLINE Packet8us ploadquad< Packet8us >(const unsigned short int *from)
Definition PacketMath.h:1037
Eigen::internal::print< Packet4f >
EIGEN_STRONG_INLINE Packet4f print< Packet4f >(const Packet4f &a)
Definition PacketMath.h:940
Eigen::internal::pfloor< Packet2d >
EIGEN_STRONG_INLINE Packet2d pfloor< Packet2d >(const Packet2d &a)
Definition PacketMath.h:1163
Eigen::internal::psub< Packet2f >
EIGEN_STRONG_INLINE Packet2f psub< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition PacketMath.h:834
Eigen::internal::predux_min< Packet4f >
EIGEN_STRONG_INLINE float predux_min< Packet4f >(const Packet4f &a)
Definition PacketMath.h:1613
Eigen::internal::prefetch< double >
EIGEN_STRONG_INLINE void prefetch< double >(const double *addr)
Definition PacketMath.h:692
Eigen::internal::predux_max< Packet2ui >
EIGEN_STRONG_INLINE uint32_t predux_max< Packet2ui >(const Packet2ui &a)
Definition PacketMath.h:2754
Eigen::internal::pscatter< int32_t, Packet4i >
EIGEN_DEVICE_FUNC void pscatter< int32_t, Packet4i >(int32_t *to, const Packet4i &from, Index stride)
Definition PacketMath.h:495
Eigen::internal::pmax< Packet4f >
EIGEN_STRONG_INLINE Packet4f pmax< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition PacketMath.h:850
Eigen::internal::pand< Packet2ul >
EIGEN_STRONG_INLINE Packet2ul pand< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition PacketMath.h:1494
Eigen::internal::pmax< Packet4c >
EIGEN_STRONG_INLINE Packet4c pmax< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition PacketMath.h:1241
Eigen::internal::pabsdiff< Packet8uc >
EIGEN_STRONG_INLINE Packet8uc pabsdiff< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition PacketMath.h:1159
Eigen::internal::ploadu< Packet4c >
EIGEN_STRONG_INLINE Packet4c ploadu< Packet4c >(const int8_t *from)
Definition PacketMath.h:1713
Eigen::internal::predux_min< Packet16c >
EIGEN_STRONG_INLINE signed char predux_min< Packet16c >(const Packet16c &a)
Definition PacketMath.h:1661
Eigen::internal::plset< Packet2i >
EIGEN_STRONG_INLINE Packet2i plset< Packet2i >(const int32_t &a)
Definition PacketMath.h:774
Eigen::numext::uint64_t
::uint64_t uint64_t
Definition Meta.h:58
Eigen::numext::int64_t
::int64_t int64_t
Definition Meta.h:59
Eigen
Namespace containing all symbols from the Eigen library.
Definition bench_norm.cpp:85
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:74
Eigen::prod
const Product< Lhs, Rhs > prod(const Lhs &lhs, const Rhs &rhs)
Definition evaluators.cpp:8
internal
Definition BandTriangularSolver.h:13
Eigen::NumTraits
Holds information about the various numeric (i.e. scalar) types allowed by Eigen.
Definition NumTraits.h:233
Eigen::bfloat16
Definition BFloat16.h:58
Eigen::half_impl::__half_raw::x
numext::uint16_t x
Definition Half.h:104
Eigen::half
Definition Half.h:142
Eigen::internal::default_packet_traits
Definition GenericPacketMath.h:43
Eigen::internal::default_packet_traits::HasRsqrt
@ HasRsqrt
Definition GenericPacketMath.h:67
Eigen::internal::default_packet_traits::HasSin
@ HasSin
Definition GenericPacketMath.h:75
Eigen::internal::default_packet_traits::HasBlend
@ HasBlend
Definition GenericPacketMath.h:60
Eigen::internal::default_packet_traits::HasAbsDiff
@ HasAbsDiff
Definition GenericPacketMath.h:55
Eigen::internal::default_packet_traits::HasArg
@ HasArg
Definition GenericPacketMath.h:53
Eigen::internal::default_packet_traits::HasNdtri
@ HasNdtri
Definition GenericPacketMath.h:90
Eigen::internal::default_packet_traits::HasCos
@ HasCos
Definition GenericPacketMath.h:76
Eigen::internal::default_packet_traits::HasCmp
@ HasCmp
Definition GenericPacketMath.h:63
Eigen::internal::default_packet_traits::HasShift
@ HasShift
Definition GenericPacketMath.h:49
Eigen::internal::default_packet_traits::HasCeil
@ HasCeil
Definition GenericPacketMath.h:101
Eigen::internal::default_packet_traits::HasExp
@ HasExp
Definition GenericPacketMath.h:68
Eigen::internal::default_packet_traits::HasRint
@ HasRint
Definition GenericPacketMath.h:99
Eigen::internal::default_packet_traits::HasSqrt
@ HasSqrt
Definition GenericPacketMath.h:66
Eigen::internal::default_packet_traits::HasErf
@ HasErf
Definition GenericPacketMath.h:88
Eigen::internal::default_packet_traits::HasBessel
@ HasBessel
Definition GenericPacketMath.h:91
Eigen::internal::default_packet_traits::HasLog
@ HasLog
Definition GenericPacketMath.h:70
Eigen::internal::default_packet_traits::HasTanh
@ HasTanh
Definition GenericPacketMath.h:83
Eigen::internal::default_packet_traits::HasFloor
@ HasFloor
Definition GenericPacketMath.h:100
Eigen::internal::default_packet_traits::HasDiv
@ HasDiv
Definition GenericPacketMath.h:65
Eigen::internal::eigen_packet_wrapper
Definition GenericPacketMath.h:160
Eigen::internal::is_arithmetic
Definition Meta.h:133
Eigen::internal::is_arithmetic::value
@ value
Definition Meta.h:133
Eigen::internal::packet_traits< bfloat16 >::half
Packet4bf half
Definition PacketMath.h:3324
Eigen::internal::packet_traits< bfloat16 >::type
Packet4bf type
Definition PacketMath.h:3323
Eigen::internal::packet_traits< float >::half
Packet2f half
Definition PacketMath.h:169
Eigen::internal::packet_traits< float >::type
Packet4f type
Definition PacketMath.h:168
Eigen::internal::packet_traits< int16_t >::type
Packet8s type
Definition PacketMath.h:273
Eigen::internal::packet_traits< int16_t >::half
Packet4s half
Definition PacketMath.h:274
Eigen::internal::packet_traits< int32_t >::half
Packet2i half
Definition PacketMath.h:333
Eigen::internal::packet_traits< int32_t >::type
Packet4i type
Definition PacketMath.h:332
Eigen::internal::packet_traits< int64_t >::half
Packet2l half
Definition PacketMath.h:393
Eigen::internal::packet_traits< int64_t >::type
Packet2l type
Definition PacketMath.h:392
Eigen::internal::packet_traits< int8_t >::half
Packet8c half
Definition PacketMath.h:214
Eigen::internal::packet_traits< int8_t >::type
Packet16c type
Definition PacketMath.h:213
Eigen::internal::packet_traits< uint16_t >::half
Packet4us half
Definition PacketMath.h:303
Eigen::internal::packet_traits< uint16_t >::type
Packet8us type
Definition PacketMath.h:302
Eigen::internal::packet_traits< uint32_t >::half
Packet2ui half
Definition PacketMath.h:362
Eigen::internal::packet_traits< uint32_t >::type
Packet4ui type
Definition PacketMath.h:361
Eigen::internal::packet_traits< uint64_t >::type
Packet2ul type
Definition PacketMath.h:422
Eigen::internal::packet_traits< uint64_t >::half
Packet2ul half
Definition PacketMath.h:423
Eigen::internal::packet_traits< uint8_t >::type
Packet16uc type
Definition PacketMath.h:242
Eigen::internal::packet_traits< uint8_t >::half
Packet8uc half
Definition PacketMath.h:243
Eigen::internal::packet_traits
Definition GenericPacketMath.h:107
Eigen::internal::packet_traits::HasSub
@ HasSub
Definition GenericPacketMath.h:118
Eigen::internal::packet_traits::HasMax
@ HasMax
Definition GenericPacketMath.h:124
Eigen::internal::packet_traits::HasNegate
@ HasNegate
Definition GenericPacketMath.h:120
Eigen::internal::packet_traits::HasMul
@ HasMul
Definition GenericPacketMath.h:119
Eigen::internal::packet_traits::HasAdd
@ HasAdd
Definition GenericPacketMath.h:117
Eigen::internal::packet_traits::HasSetLinear
@ HasSetLinear
Definition GenericPacketMath.h:126
Eigen::internal::packet_traits::HasMin
@ HasMin
Definition GenericPacketMath.h:123
Eigen::internal::packet_traits::HasConj
@ HasConj
Definition GenericPacketMath.h:125
Eigen::internal::packet_traits::HasAbs2
@ HasAbs2
Definition GenericPacketMath.h:122
Eigen::internal::packet_traits::HasAbs
@ HasAbs
Definition GenericPacketMath.h:121
Eigen::internal::packet_traits::type
T type
Definition GenericPacketMath.h:108
Eigen::internal::packet_traits::half
T half
Definition GenericPacketMath.h:109
Eigen::internal::packet_traits::HasHalfPacket
@ HasHalfPacket
Definition GenericPacketMath.h:114
Eigen::internal::packet_traits::size
@ size
Definition GenericPacketMath.h:112
Eigen::internal::packet_traits::AlignedOnScalar
@ AlignedOnScalar
Definition GenericPacketMath.h:113
Eigen::internal::packet_traits::Vectorizable
@ Vectorizable
Definition GenericPacketMath.h:111
Eigen::internal::traits
Definition ForwardDeclarations.h:17
Eigen::internal::unpacket_traits< Packet16c >::type
int8_t type
Definition PacketMath.h:514
Eigen::internal::unpacket_traits< Packet16c >::half
Packet8c half
Definition PacketMath.h:515
Eigen::internal::unpacket_traits< Packet16uc >::type
uint8_t type
Definition PacketMath.h:553
Eigen::internal::unpacket_traits< Packet16uc >::half
Packet8uc half
Definition PacketMath.h:554
Eigen::internal::unpacket_traits< Packet2f >::half
Packet2f half
Definition PacketMath.h:461
Eigen::internal::unpacket_traits< Packet2f >::integer_packet
Packet2i integer_packet
Definition PacketMath.h:462
Eigen::internal::unpacket_traits< Packet2f >::type
float type
Definition PacketMath.h:460
Eigen::internal::unpacket_traits< Packet2i >::type
int32_t type
Definition PacketMath.h:617
Eigen::internal::unpacket_traits< Packet2i >::half
Packet2i half
Definition PacketMath.h:618
Eigen::internal::unpacket_traits< Packet2l >::half
Packet2l half
Definition PacketMath.h:670
Eigen::internal::unpacket_traits< Packet2l >::type
int64_t type
Definition PacketMath.h:669
Eigen::internal::unpacket_traits< Packet2ui >::type
uint32_t type
Definition PacketMath.h:643
Eigen::internal::unpacket_traits< Packet2ui >::half
Packet2ui half
Definition PacketMath.h:644
Eigen::internal::unpacket_traits< Packet2ul >::half
Packet2ul half
Definition PacketMath.h:683
Eigen::internal::unpacket_traits< Packet2ul >::type
uint64_t type
Definition PacketMath.h:682
Eigen::internal::unpacket_traits< Packet4bf >::half
Packet4bf half
Definition PacketMath.h:3367
Eigen::internal::unpacket_traits< Packet4bf >::type
bfloat16 type
Definition PacketMath.h:3366
Eigen::internal::unpacket_traits< Packet4c >::half
Packet4c half
Definition PacketMath.h:489
Eigen::internal::unpacket_traits< Packet4c >::type
int8_t type
Definition PacketMath.h:488
Eigen::internal::unpacket_traits< Packet4f >::integer_packet
Packet4i integer_packet
Definition PacketMath.h:476
Eigen::internal::unpacket_traits< Packet4f >::half
Packet2f half
Definition PacketMath.h:475
Eigen::internal::unpacket_traits< Packet4f >::type
float type
Definition PacketMath.h:474
Eigen::internal::unpacket_traits< Packet4i >::type
int32_t type
Definition PacketMath.h:630
Eigen::internal::unpacket_traits< Packet4i >::half
Packet2i half
Definition PacketMath.h:631
Eigen::internal::unpacket_traits< Packet4s >::type
int16_t type
Definition PacketMath.h:565
Eigen::internal::unpacket_traits< Packet4s >::half
Packet4s half
Definition PacketMath.h:566
Eigen::internal::unpacket_traits< Packet4uc >::half
Packet4uc half
Definition PacketMath.h:528
Eigen::internal::unpacket_traits< Packet4uc >::type
uint8_t type
Definition PacketMath.h:527
Eigen::internal::unpacket_traits< Packet4ui >::type
uint32_t type
Definition PacketMath.h:656
Eigen::internal::unpacket_traits< Packet4ui >::half
Packet2ui half
Definition PacketMath.h:657
Eigen::internal::unpacket_traits< Packet4us >::type
uint16_t type
Definition PacketMath.h:591
Eigen::internal::unpacket_traits< Packet4us >::half
Packet4us half
Definition PacketMath.h:592
Eigen::internal::unpacket_traits< Packet8c >::type
int8_t type
Definition PacketMath.h:501
Eigen::internal::unpacket_traits< Packet8c >::half
Packet4c half
Definition PacketMath.h:502
Eigen::internal::unpacket_traits< Packet8s >::type
int16_t type
Definition PacketMath.h:578
Eigen::internal::unpacket_traits< Packet8s >::half
Packet4s half
Definition PacketMath.h:579
Eigen::internal::unpacket_traits< Packet8uc >::type
uint8_t type
Definition PacketMath.h:540
Eigen::internal::unpacket_traits< Packet8uc >::half
Packet4uc half
Definition PacketMath.h:541
Eigen::internal::unpacket_traits< Packet8us >::half
Packet4us half
Definition PacketMath.h:605
Eigen::internal::unpacket_traits< Packet8us >::type
uint16_t type
Definition PacketMath.h:604
Eigen::internal::unpacket_traits
Definition GenericPacketMath.h:133
Eigen::internal::unpacket_traits::type
T type
Definition GenericPacketMath.h:134
Eigen::internal::unpacket_traits::half
T half
Definition GenericPacketMath.h:135
Eigen::internal::unpacket_traits::masked_load_available
@ masked_load_available
Definition GenericPacketMath.h:141
Eigen::internal::unpacket_traits::size
@ size
Definition GenericPacketMath.h:138
Eigen::internal::unpacket_traits::masked_store_available
@ masked_store_available
Definition GenericPacketMath.h:142
Eigen::internal::unpacket_traits::vectorizable
@ vectorizable
Definition GenericPacketMath.h:140
Eigen::internal::unpacket_traits::alignment
@ alignment
Definition GenericPacketMath.h:139
j
std::ptrdiff_t j
Definition tut_arithmetic_redux_minmax.cpp:2
Eigen::internal::Packet
Definition PacketMath.h:47
Generated by doxygen 1.9.8